dwmac: print Tx error statuses

Cargo.toml

@@ -39,6 +39,7 @@ ahash = { version = "0.8.11", default-features = false, features = ["no-rng"] }
 
 # acpi
 acpi = { git = "https://github.com/alnyan/acpi.git", package = "acpi", branch = "acpi-system" }
+rsdp = { git = "https://github.com/alnyan/acpi.git", package = "rsdp", branch = "acpi-system" }
 aml = { git = "https://github.com/alnyan/acpi.git", branch = "acpi-system" }
 acpi-system = { git = "https://github.com/alnyan/acpi-system.git" }
 

kernel/Cargo.toml

@@ -30,12 +30,15 @@ ygg_driver_net_core = { path = "driver/net/core" }
 ygg_driver_net_loopback = { path = "driver/net/loopback" }
 ygg_driver_virtio_net = { path = "driver/virtio/net", features = ["pci"] }
 ygg_driver_virtio_gpu = { path = "driver/virtio/gpu", features = ["pci"] }
+ygg_driver_nvme = { path = "driver/block/nvme" }
 ygg_driver_ahci = { path = "driver/block/ahci" }
-ygg_driver_usb_xhci = { path = "driver/usb/xhci" }
 ygg_driver_input = { path = "driver/input" }
+ygg_driver_usb_xhci.path = "driver/usb/xhci"
+ygg_driver_net_rtl81xx.path = "driver/net/rtl81xx"
 
 memfs = { path = "driver/fs/memfs" }
 ext2 = { path = "driver/fs/ext2" }
+ygg_driver_fat32.path = "driver/fs/fat32"
 
 log.workspace = true
 bitflags.workspace = true
@@ -45,6 +48,7 @@ bytemuck.workspace = true
 futures-util.workspace = true
 crossbeam-queue.workspace = true
 async-trait.workspace = true
+cfg-if.workspace = true
 
 git-version = "0.3.9"
 
@@ -56,17 +60,17 @@ kernel-arch-aarch64.workspace = true
 [target.'cfg(target_arch = "riscv64")'.dependencies]
 device-tree.workspace = true
 kernel-arch-riscv64.workspace = true
+ygg_driver_net_stmmac.path = "driver/net/stmmac"
 
 [target.'cfg(target_arch = "x86_64")'.dependencies]
 yboot-proto.workspace = true
 kernel-arch-x86_64.workspace = true
 kernel-arch-x86.workspace = true
 
-ygg_driver_nvme = { path = "driver/block/nvme" }
+ygg_driver_acpi.path = "driver/acpi"
+ygg_driver_net_igbe.path = "driver/net/igbe"
 
 acpi.workspace = true
-aml.workspace = true
-acpi-system.workspace = true
 
 [target.'cfg(target_arch = "x86")'.dependencies]
 kernel-arch-i686.workspace = true
@@ -87,6 +91,9 @@ kernel-arch-x86.workspace = true
 kernel-arch-aarch64.workspace = true
 kernel-arch-riscv64.workspace = true
 
+ygg_driver_acpi.path = "driver/acpi"
+ygg_driver_net_stmmac.path = "driver/net/stmmac"
+
 [features]
 default = ["fb_console"]
 fb_console = []

kernel/arch/aarch64/src/lib.rs

@@ -6,9 +6,12 @@ extern crate alloc;
 
 use core::sync::atomic::{AtomicUsize, Ordering};
 
-use aarch64_cpu::registers::{DAIF, MPIDR_EL1, TPIDR_EL1};
+use aarch64_cpu::{
+    asm::barrier,
+    registers::{DAIF, MPIDR_EL1, TPIDR_EL1},
+};
 use alloc::{boxed::Box, sync::Arc, vec::Vec};
-use device_api::interrupt::{LocalInterruptController, MessageInterruptController};
+use device_api::interrupt::LocalInterruptController;
 use kernel_arch_interface::{
     cpu::{CpuData, CpuImpl, IpiQueue},
     guard::IrqGuard,
@@ -123,24 +126,10 @@ impl Architecture for ArchitectureImpl {
         CPU_COUNT.load(Ordering::Acquire)
     }
 
-    fn message_interrupt_controller() -> Option<&'static dyn MessageInterruptController> {
-        None
-    }
-
     fn local_interrupt_controller() -> Option<&'static dyn LocalInterruptController> {
         None
     }
 
-    // fn local_interrupt_controller() -> Option<&'static dyn LocalInterruptController> {
-    //     let local = Self::local_cpu_data()?;
-    //     let intc = *local.gic.try_get()?;
-    //     Some(intc)
-    // }
-
-    // fn message_interrupt_controller() -> &'static dyn MessageInterruptController {
-    //     todo!()
-    // }
-
     fn cpu_available_features<S: Scheduler>(_cpu: &CpuImpl<Self, S>) -> Option<&Self::CpuFeatures> {
         None
     }
@@ -148,4 +137,30 @@ impl Architecture for ArchitectureImpl {
     fn cpu_enabled_features<S: Scheduler>(_cpu: &CpuImpl<Self, S>) -> Option<&Self::CpuFeatures> {
         None
     }
+
+    // Cache/barrier operation
+    fn load_barrier() {
+        barrier::dmb(barrier::ISHLD);
+    }
+
+    fn store_barrier() {
+        barrier::dmb(barrier::ISHST);
+    }
+
+    fn memory_barrier() {
+        barrier::dsb(barrier::SY);
+    }
+
+    fn flush_virtual_range(range: core::ops::Range<usize>) {
+        // TODO cache line assumed to be 64 bytes
+        const CLSIZE: usize = 64;
+        let start = range.start & !(CLSIZE - 1);
+        let end = (range.end + (CLSIZE - 1)) & !(CLSIZE - 1);
+
+        for line in (start..end).step_by(CLSIZE) {
+            unsafe {
+                core::arch::asm!("dc ivac, {address}", address = in(reg) line);
+            }
+        }
+    }
 }

kernel/arch/aarch64/src/mem/mod.rs

@@ -322,6 +322,7 @@ pub(crate) unsafe fn map_device_memory(
         let address = base_address + l2_offset;
 
         Ok(RawDeviceMemoryMapping::from_raw_parts(
+            l2_aligned.into_u64(),
             address,
             base_address,
             page_count,
@@ -333,6 +334,7 @@ pub(crate) unsafe fn map_device_memory(
         let address = base_address + l3_offset;
 
         Ok(RawDeviceMemoryMapping::from_raw_parts(
+            l3_aligned.into_u64(),
             address,
             base_address,
             page_count,

kernel/arch/hosted/src/lib.rs

@@ -5,7 +5,7 @@ use std::{
 };
 
 use kernel_arch_interface::{
-    cpu::IpiQueue,
+    cpu::{CpuData, IpiQueue},
     mem::{
         DeviceMemoryAttributes, KernelTableManager, PhysicalMemoryAllocator, RawDeviceMemoryMapping,
     },
@@ -36,17 +36,21 @@ pub struct TaskContextImpl<K: KernelTableManager, PA: PhysicalMemoryAllocator>(
 
 static DUMMY_INTERRUPT_MASK: AtomicBool = AtomicBool::new(true);
 
+pub struct DummyCpuData;
+
+impl CpuData for DummyCpuData {}
+
 impl Architecture for ArchitectureImpl {
-    type PerCpuData = ();
+    type PerCpuData = DummyCpuData;
     type CpuFeatures = ();
     type BreakpointType = u8;
     const BREAKPOINT_VALUE: Self::BreakpointType = 0x00;
 
-    fn local_cpu() -> *mut Self::PerCpuData {
+    fn local_cpu() -> *mut () {
         unimplemented!()
     }
 
-    unsafe fn set_local_cpu(_cpu: *mut Self::PerCpuData) {
+    unsafe fn set_local_cpu(_cpu: *mut ()) {
         unimplemented!()
     }
 
@@ -154,7 +158,7 @@ impl<TA: TableAllocator> ProcessAddressSpaceManager<TA> for ProcessAddressSpaceI
         unimplemented!()
     }
 
-    fn as_address_with_asid(&self) -> u64 {
+    fn as_address_with_asid(&self) -> (u64, u64) {
         unimplemented!()
     }
 }

kernel/arch/interface/src/lib.rs

@@ -2,9 +2,11 @@
 #![feature(step_trait, const_trait_impl, never_type, decl_macro)]
 #![allow(clippy::new_without_default)]
 
+use core::ops::Range;
+
 use alloc::vec::Vec;
 use cpu::{CpuData, CpuFeatureSet, CpuImpl, IpiQueue};
-use device_api::interrupt::{LocalInterruptController, MessageInterruptController};
+use device_api::interrupt::LocalInterruptController;
 use task::Scheduler;
 
 extern crate alloc;
@@ -72,10 +74,6 @@ pub trait Architecture: Sized + 'static {
         None
     }
 
-    fn message_interrupt_controller() -> Option<&'static dyn MessageInterruptController> {
-        None
-    }
-
     #[allow(unused)]
     fn cpu_available_features<S: Scheduler>(cpu: &CpuImpl<Self, S>) -> Option<&Self::CpuFeatures> {
         None
@@ -84,4 +82,16 @@ pub trait Architecture: Sized + 'static {
     fn cpu_enabled_features<S: Scheduler>(cpu: &CpuImpl<Self, S>) -> Option<&Self::CpuFeatures> {
         None
     }
+
+    // Cache/barrier operation
+
+    fn load_barrier();
+    fn store_barrier();
+    fn memory_barrier() {
+        Self::store_barrier();
+        Self::load_barrier();
+    }
+
+    /// Flushes/invalidates a range of virtual memory from the CPU's data cache.
+    fn flush_virtual_range(range: Range<usize>);
 }

kernel/arch/interface/src/mem/mod.rs

@@ -35,6 +35,8 @@ pub struct DeviceMemoryAttributes {
 /// Describes a single device memory mapping
 #[derive(Debug)]
 pub struct RawDeviceMemoryMapping<A: KernelTableManager> {
+    /// Physical base address of the object
+    pub physical_base: u64,
     /// Virtual address of the mapped object
     pub address: usize,
     /// Base address of the mapping start
@@ -98,7 +100,8 @@ impl<A: KernelTableManager> RawDeviceMemoryMapping<A> {
         address
     }
 
-    pub fn into_raw_parts(self) -> (usize, usize, usize, usize) {
+    pub fn into_raw_parts(self) -> (u64, usize, usize, usize, usize) {
+        let physical_base = self.physical_base;
         let address = self.address;
         let base_address = self.base_address;
         let page_count = self.page_count;
@@ -106,7 +109,7 @@ impl<A: KernelTableManager> RawDeviceMemoryMapping<A> {
 
         core::mem::forget(self);
 
-        (address, base_address, page_count, page_size)
+        (physical_base, address, base_address, page_count, page_size)
     }
 
     /// # Safety
@@ -114,12 +117,14 @@ impl<A: KernelTableManager> RawDeviceMemoryMapping<A> {
     /// Preconditions: all the fields must come from a [RawDeviceMemoryMapping::into_raw_parts]
     /// call.
     pub unsafe fn from_raw_parts(
+        physical_base: u64,
         address: usize,
         base_address: usize,
         page_count: usize,
         page_size: usize,
     ) -> Self {
         Self {
+            physical_base,
             address,
             base_address,
             page_count,

kernel/arch/riscv64/src/lib.rs

@@ -3,10 +3,13 @@
 
 extern crate alloc;
 
-use core::sync::atomic::{AtomicUsize, Ordering};
+use core::{
+    ops::Range,
+    sync::atomic::{AtomicUsize, Ordering},
+};
 
 use alloc::{boxed::Box, collections::btree_map::BTreeMap, vec::Vec};
-use device_api::interrupt::{LocalInterruptController, MessageInterruptController};
+use device_api::interrupt::LocalInterruptController;
 use kernel_arch_interface::{
     cpu::{CpuData, CpuImpl, IpiQueue},
     sync::IrqSafeSpinlock,
@@ -160,11 +163,24 @@ impl Architecture for ArchitectureImpl {
         todo!()
     }
 
-    fn message_interrupt_controller() -> Option<&'static dyn MessageInterruptController> {
-        todo!()
-    }
-
     fn idle_task() -> extern "C" fn(usize) -> ! {
         idle_task
     }
+
+    // Cache/barrier operation
+    fn load_barrier() {
+        unsafe { core::arch::asm!("fence r, w") };
+    }
+
+    fn store_barrier() {
+        unsafe { core::arch::asm!("fence w, r") };
+    }
+
+    fn memory_barrier() {
+        unsafe { core::arch::asm!("fence rw, rw") };
+    }
+
+    fn flush_virtual_range(_range: Range<usize>) {
+        // TODO
+    }
 }

kernel/arch/riscv64/src/mem/mod.rs

@@ -189,6 +189,7 @@ pub(crate) unsafe fn map_device_memory(
             let address = base_address + l2_offset;
 
             Ok(RawDeviceMemoryMapping::from_raw_parts(
+                l2_aligned.into_u64(),
                 address,
                 base_address,
                 page_count,
@@ -202,6 +203,7 @@ pub(crate) unsafe fn map_device_memory(
             let address = base_address + l3_offset;
 
             Ok(RawDeviceMemoryMapping::from_raw_parts(
+                l3_aligned.into_u64(),
                 address,
                 base_address,
                 page_count,

kernel/arch/x86/src/lib.rs

@@ -6,4 +6,10 @@ extern crate alloc;
 
 pub mod cpuid;
 pub mod gdt;
+pub mod intrinsics;
 pub mod registers;
+
+#[cfg(any(target_arch = "x86_64", rust_analyzer))]
+pub const ISA_IRQ_OFFSET: u32 = 1024;
+#[cfg(any(target_arch = "x86", rust_analyzer))]
+pub const ISA_IRQ_OFFSET: u32 = 0;

kernel/arch/x86_64/src/lib.rs

@@ -5,11 +5,11 @@
 extern crate alloc;
 
 use core::{
-    ops::DerefMut,
+    ops::{DerefMut, Range},
     sync::atomic::{AtomicUsize, Ordering},
 };
 
-use alloc::{boxed::Box, vec::Vec};
+use alloc::{sync::Arc, vec::Vec};
 use device_api::interrupt::{LocalInterruptController, MessageInterruptController};
 use kernel_arch_interface::{
     cpu::{CpuData, CpuImpl, IpiQueue},
@@ -52,8 +52,7 @@ pub struct PerCpuData {
     // 0x10, used in assembly
     pub tmp_address: usize,
 
-    pub local_apic: Box<dyn LocalApicInterface>,
-    // pub local_apic: &'static dyn LocalApicInterface,
+    pub local_apic: Arc<dyn LocalApicInterface>,
     pub available_features: CpuFeatures,
     pub enabled_features: CpuFeatures,
 }
@@ -189,11 +188,6 @@ impl Architecture for ArchitectureImpl {
         Some(cpu.local_apic.as_ref())
     }
 
-    fn message_interrupt_controller() -> Option<&'static dyn MessageInterruptController> {
-        let cpu = Self::local_cpu_data()?;
-        Some(cpu.local_apic.as_ref())
-    }
-
     fn cpu_enabled_features<S: Scheduler>(cpu: &CpuImpl<Self, S>) -> Option<&Self::CpuFeatures> {
         Some(&cpu.enabled_features)
     }
@@ -201,4 +195,30 @@ impl Architecture for ArchitectureImpl {
     fn cpu_available_features<S: Scheduler>(cpu: &CpuImpl<Self, S>) -> Option<&Self::CpuFeatures> {
         Some(&cpu.available_features)
     }
+
+    // Cache/barrier
+
+    fn load_barrier() {
+        unsafe { core::arch::x86_64::_mm_lfence() };
+    }
+
+    fn store_barrier() {
+        unsafe { core::arch::x86_64::_mm_sfence() };
+    }
+
+    fn memory_barrier() {
+        unsafe { core::arch::x86_64::_mm_mfence() };
+    }
+
+    fn flush_virtual_range(range: Range<usize>) {
+        // TODO I assume 64-byte cache line on all CPUs
+        // TODO clflush instruction may not be available, test for it
+        const CLSIZE: usize = 64;
+        let start = range.start & !(CLSIZE - 1);
+        let end = (range.end + (CLSIZE - 1)) & !(CLSIZE - 1);
+
+        for line in (start..end).step_by(CLSIZE) {
+            unsafe { core::arch::x86_64::_mm_clflush(line as _) };
+        }
+    }
 }

kernel/arch/x86_64/src/mem/mod.rs

@@ -222,6 +222,7 @@ unsafe fn map_device_memory(
         let address = base_address + l2_offset;
 
         Ok(RawDeviceMemoryMapping::from_raw_parts(
+            l2_aligned.into_u64(),
             address,
             base_address,
             page_count,
@@ -233,6 +234,7 @@ unsafe fn map_device_memory(
         let address = base_address + l3_offset;
 
         Ok(RawDeviceMemoryMapping::from_raw_parts(
+            l3_aligned.into_u64(),
             address,
             base_address,
             page_count,

kernel/driver/acpi/Cargo.toml

@@ -0,0 +1,18 @@
+[package]
+name = "ygg_driver_acpi"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+libk-util.workspace = true
+libk-mm.workspace = true
+libk.workspace = true
+device-api.workspace = true
+kernel-arch-x86.path = "../../arch/x86"
+
+acpi.workspace = true
+rsdp.workspace = true
+aml.workspace = true
+acpi-system.workspace = true
+
+log.workspace = true

kernel/driver/acpi/src/aml_handler.rs

@@ -0,0 +1,131 @@
+use core::time::Duration;
+
+use crate::AcpiHandlerImpl;
+
+impl aml::Handler for AcpiHandlerImpl {
+    fn read_io_u8(&self, port: u16) -> u8 {
+        <Self as acpi_system::Handler>::io_read_u8(port)
+    }
+
+    fn read_io_u16(&self, port: u16) -> u16 {
+        <Self as acpi_system::Handler>::io_read_u16(port)
+    }
+
+    fn read_io_u32(&self, port: u16) -> u32 {
+        <Self as acpi_system::Handler>::io_read_u32(port)
+    }
+
+    fn write_io_u8(&self, port: u16, value: u8) {
+        <Self as acpi_system::Handler>::io_write_u8(port, value)
+    }
+
+    fn write_io_u16(&self, port: u16, value: u16) {
+        <Self as acpi_system::Handler>::io_write_u16(port, value)
+    }
+
+    fn write_io_u32(&self, port: u16, value: u32) {
+        <Self as acpi_system::Handler>::io_write_u32(port, value)
+    }
+
+    fn read_u8(&self, address: usize) -> u8 {
+        <Self as acpi_system::Handler>::mem_read_u8(address as u64)
+    }
+
+    fn read_u16(&self, address: usize) -> u16 {
+        <Self as acpi_system::Handler>::mem_read_u16(address as u64)
+    }
+
+    fn read_u32(&self, address: usize) -> u32 {
+        <Self as acpi_system::Handler>::mem_read_u32(address as u64)
+    }
+
+    fn read_u64(&self, address: usize) -> u64 {
+        <Self as acpi_system::Handler>::mem_read_u64(address as u64)
+    }
+
+    fn write_u8(&self, address: usize, value: u8) {
+        <Self as acpi_system::Handler>::mem_write_u8(address as u64, value)
+    }
+
+    fn write_u16(&self, address: usize, value: u16) {
+        <Self as acpi_system::Handler>::mem_write_u16(address as u64, value)
+    }
+
+    fn write_u32(&self, address: usize, value: u32) {
+        <Self as acpi_system::Handler>::mem_write_u32(address as u64, value)
+    }
+
+    fn write_u64(&self, address: usize, value: u64) {
+        <Self as acpi_system::Handler>::mem_write_u64(address as u64, value)
+    }
+
+    fn read_pci_u8(&self, _segment: u16, _bus: u8, _device: u8, _function: u8, _offset: u16) -> u8 {
+        0xFF
+    }
+
+    fn read_pci_u16(
+        &self,
+        _segment: u16,
+        _bus: u8,
+        _device: u8,
+        _function: u8,
+        _offset: u16,
+    ) -> u16 {
+        0xFFFF
+    }
+
+    fn read_pci_u32(
+        &self,
+        _segment: u16,
+        _bus: u8,
+        _device: u8,
+        _function: u8,
+        _offset: u16,
+    ) -> u32 {
+        0xFFFFFFFF
+    }
+
+    fn write_pci_u8(
+        &self,
+        _segment: u16,
+        _bus: u8,
+        _device: u8,
+        _function: u8,
+        _offset: u16,
+        _value: u8,
+    ) {
+    }
+
+    fn write_pci_u16(
+        &self,
+        _segment: u16,
+        _bus: u8,
+        _device: u8,
+        _function: u8,
+        _offset: u16,
+        _value: u16,
+    ) {
+    }
+
+    fn write_pci_u32(
+        &self,
+        _segment: u16,
+        _bus: u8,
+        _device: u8,
+        _function: u8,
+        _offset: u16,
+        _value: u32,
+    ) {
+    }
+
+    fn read_ec_u8(&self, _address: u64) -> u8 {
+        0x00
+    }
+
+    fn write_ec_u8(&self, _address: u64, _value: u8) {}
+
+    fn sleep(&self, _duration: Duration) {
+        todo!()
+        // util::polling_sleep(duration).unwrap();
+    }
+}

kernel/driver/acpi/src/handler.rs

@@ -0,0 +1,171 @@
+use core::{ptr::NonNull, time::Duration};
+
+use acpi::PhysicalMapping;
+use acpi_system::AcpiSystemError;
+use alloc::sync::Arc;
+use device_api::{
+    device::Device,
+    interrupt::{InterruptHandler, Irq, IrqVector},
+};
+use kernel_arch_x86::{intrinsics, ISA_IRQ_OFFSET};
+use libk::device::external_interrupt_controller;
+use libk_mm::{
+    address::{PhysicalAddress, Virtualize},
+    pointer::PhysicalRef,
+};
+
+use crate::{
+    mem::{read_memory, write_memory},
+    ACPI_SYSTEM,
+};
+
+#[derive(Clone, Copy)]
+#[doc(hidden)]
+pub struct AcpiHandlerImpl;
+
+struct SciHandler;
+
+impl acpi_system::Handler for AcpiHandlerImpl {
+    type MappedSlice = PhysicalRef<'static, [u8]>;
+
+    unsafe fn map_slice(address: u64, length: u64) -> Self::MappedSlice {
+        unsafe {
+            PhysicalRef::map_slice(
+                PhysicalAddress::from_u64(address),
+                length.try_into().unwrap(),
+            )
+        }
+    }
+
+    fn io_read_u8(port: u16) -> u8 {
+        let value = unsafe { intrinsics::inb(port) };
+        log::trace!("io_read_u8 {:#x} <- {:#x}", port, value);
+        value
+    }
+
+    fn io_read_u16(port: u16) -> u16 {
+        let value = unsafe { intrinsics::inw(port) };
+        log::trace!("io_read_u16 {:#x} <- {:#x}", port, value);
+        value
+    }
+
+    fn io_read_u32(port: u16) -> u32 {
+        let value = unsafe { intrinsics::inl(port) };
+        log::trace!("io_read_u32 {:#x} <- {:#x}", port, value);
+        value
+    }
+
+    fn io_write_u8(port: u16, value: u8) {
+        log::trace!("io_write_u8 {:#x}, {:#x}", port, value);
+        unsafe { intrinsics::outb(port, value) }
+    }
+
+    fn io_write_u16(port: u16, value: u16) {
+        log::trace!("io_write_u16 {:#x}, {:#x}", port, value);
+        unsafe { intrinsics::outw(port, value) }
+    }
+
+    fn io_write_u32(port: u16, value: u32) {
+        log::trace!("io_write_u32 {:#x}, {:#x}", port, value);
+        unsafe { intrinsics::outl(port, value) }
+    }
+
+    fn mem_read_u8(address: u64) -> u8 {
+        let value = unsafe { read_memory(PhysicalAddress::from_u64(address)) };
+        log::trace!("mem_read_u8 {:#x} -> {:#x}", address, value);
+        value
+    }
+
+    fn mem_read_u16(address: u64) -> u16 {
+        let value = unsafe { read_memory(PhysicalAddress::from_u64(address)) };
+        log::trace!("mem_read_u16 {:#x} -> {:#x}", address, value);
+        value
+    }
+
+    fn mem_read_u32(address: u64) -> u32 {
+        let value = unsafe { read_memory(PhysicalAddress::from_u64(address)) };
+        log::trace!("mem_read_u32 {:#x} -> {:#x}", address, value);
+        value
+    }
+
+    fn mem_read_u64(address: u64) -> u64 {
+        let value = unsafe { read_memory(PhysicalAddress::from_u64(address)) };
+        log::trace!("mem_read_u64 {:#x} -> {:#x}", address, value);
+        value
+    }
+
+    fn mem_write_u8(address: u64, value: u8) {
+        log::trace!("mem_write_u8 {:#x}, {:#x}", address, value);
+        unsafe { write_memory(PhysicalAddress::from_u64(address), value) }
+    }
+
+    fn mem_write_u16(address: u64, value: u16) {
+        log::trace!("mem_write_u16 {:#x}, {:#x}", address, value);
+        unsafe { write_memory(PhysicalAddress::from_u64(address), value) }
+    }
+
+    fn mem_write_u32(address: u64, value: u32) {
+        log::trace!("mem_write_u32 {:#x}, {:#x}", address, value);
+        unsafe { write_memory(PhysicalAddress::from_u64(address), value) }
+    }
+
+    fn mem_write_u64(address: u64, value: u64) {
+        log::trace!("mem_write_u64 {:#x}, {:#x}", address, value);
+        unsafe { write_memory(PhysicalAddress::from_u64(address), value) }
+    }
+
+    fn install_interrupt_handler(irq: u32) -> Result<(), AcpiSystemError> {
+        log::info!("Installing ACPI SCI handler at IRQ #{}", irq);
+
+        let intc = external_interrupt_controller().expect("No external intc");
+        let handler = Arc::new(SciHandler);
+        let irq = Irq::External(irq + ISA_IRQ_OFFSET);
+
+        intc.register_irq(irq, Default::default(), handler).unwrap();
+        intc.enable_irq(irq).unwrap();
+
+        Ok(())
+    }
+
+    fn stall(_duration: Duration) {
+        // TODO polling_sleep is not yet implemented properly
+        todo!()
+        // util::polling_sleep(duration).ok();
+    }
+}
+
+impl rsdp::handler::AcpiHandler for AcpiHandlerImpl {
+    unsafe fn map_physical_region<T>(
+        &self,
+        physical_address: usize,
+        size: usize,
+    ) -> PhysicalMapping<Self, T> {
+        unsafe {
+            PhysicalMapping::new(
+                physical_address,
+                NonNull::new_unchecked(
+                    PhysicalAddress::from_usize(physical_address).virtualize() as *mut T
+                ),
+                size,
+                size,
+                *self,
+            )
+        }
+    }
+
+    fn unmap_physical_region<T>(_region: &acpi::PhysicalMapping<Self, T>) {}
+}
+
+impl InterruptHandler for SciHandler {
+    fn handle_irq(self: Arc<Self>, _vector: IrqVector) -> bool {
+        log::trace!("ACPI SCI received");
+        ACPI_SYSTEM.get().lock().handle_sci();
+        true
+    }
+}
+
+impl Device for SciHandler {
+    fn display_name(&self) -> &str {
+        "ACPI SCI handler"
+    }
+}

kernel/driver/acpi/src/lib.rs

@@ -0,0 +1,89 @@
+#![feature(allocator_api)]
+#![no_std]
+
+use acpi::AcpiTables;
+use acpi_system::{AcpiInterruptMethod, AcpiSystem};
+use alloc::boxed::Box;
+use libk::error::Error;
+use libk_util::{sync::IrqSafeSpinlock, OneTimeInit};
+
+extern crate alloc;
+
+pub mod mem;
+pub use mem::AcpiAllocator;
+pub mod handler;
+pub use handler::AcpiHandlerImpl;
+pub mod aml_handler;
+
+pub use acpi_system::{
+    EventAction, FixedEvent, InterruptPolarity, InterruptTrigger, IrqDescriptor, PciPin,
+};
+
+static ACPI_SYSTEM: OneTimeInit<IrqSafeSpinlock<AcpiSystem<AcpiHandlerImpl>>> = OneTimeInit::new();
+
+pub fn add_event_handler<F: Fn(&AcpiSystem<AcpiHandlerImpl>) -> EventAction + 'static>(
+    event: &FixedEvent,
+    handler: F,
+) -> Result<(), Error> {
+    ACPI_SYSTEM
+        .get()
+        .lock()
+        .enable_fixed_event(event, Box::new(handler))
+        .map_err(|_| Error::InvalidArgument)
+}
+
+pub fn get_pci_route(
+    aml_path: &str,
+    device: u16,
+    function: u16,
+    pin: PciPin,
+) -> Option<IrqDescriptor> {
+    ACPI_SYSTEM
+        .get()
+        .lock()
+        .pci_route(aml_path, device, function, pin)
+        .ok()
+}
+
+/// Initializes ACPI management
+pub fn switch_to_acpi(tables: &'static AcpiTables<AcpiHandlerImpl>) -> Result<(), Error> {
+    // NOTE mostly broken for real HW
+    let mut system = AcpiSystem::new(tables, Box::new(AcpiHandlerImpl)).unwrap();
+
+    system.initialize(AcpiInterruptMethod::Apic).unwrap();
+
+    // system
+    //     .enable_fixed_event(
+    //         &FixedEvent::POWER_BUTTON,
+    //         Box::new(|_| {
+    //             log::info!("Power button was pressed");
+
+    //             // TODO the correct way would be to
+    //             // 1. Nicely ask all the processes to quit
+    //             // 2. Wait for some time
+    //             // 3. Kill the remaining ones
+    //             // 4. Halt other cores
+    //             // 5. Sync filesystem
+    //             // 6. Do something with the devices
+    //             // 7. Actually enter the S5 state
+
+    //             unsafe {
+    //                 PLATFORM
+    //                     .send_ipi(IpiDeliveryTarget::OtherCpus, IpiMessage::Shutdown)
+    //                     .unwrap();
+    //             }
+
+    //             SHUTDOWN_FENCE.signal();
+    //             SHUTDOWN_FENCE.wait_all(CPU_COUNT.load(Ordering::Acquire));
+
+    //             log::info!("CPUs are parked, can shutdown now");
+
+    //             EventAction::EnterSleepState(AcpiSleepState::S5)
+    //         }),
+    //     )
+    //     .unwrap();
+
+    ACPI_SYSTEM.init(IrqSafeSpinlock::new(system));
+
+    Ok(())
+}

kernel/driver/acpi/src/mem.rs

@@ -0,0 +1,64 @@
+//! ACPI memory IO and management functions
+
+use core::{
+    alloc::{AllocError, Allocator, GlobalAlloc, Layout},
+    ptr::NonNull,
+};
+
+use libk_mm::{address::PhysicalAddress, device::DeviceMemoryMapping, heap::GLOBAL_HEAP};
+
+#[derive(Clone, Copy)]
+#[doc(hidden)]
+pub struct AcpiAllocator;
+
+unsafe impl Allocator for AcpiAllocator {
+    fn allocate(&self, layout: Layout) -> Result<NonNull<[u8]>, AllocError> {
+        let ptr = unsafe { GLOBAL_HEAP.alloc(layout) };
+        log::trace!("ACPI alloc: {:?} -> {:p}", layout, ptr);
+
+        if ptr.is_null() {
+            Err(AllocError)
+        } else {
+            unsafe {
+                Ok(NonNull::slice_from_raw_parts(
+                    NonNull::new_unchecked(ptr),
+                    layout.size(),
+                ))
+            }
+        }
+    }
+
+    unsafe fn deallocate(&self, ptr: NonNull<u8>, layout: Layout) {
+        log::trace!("ACPI dealloc: {:?}, {:?}", ptr, layout);
+        unsafe { GLOBAL_HEAP.dealloc(ptr.as_ptr(), layout) };
+    }
+}
+
+// TODO don't map memory as device if not necessary
+pub unsafe fn read_memory<T>(address: PhysicalAddress) -> T {
+    let io =
+        unsafe { DeviceMemoryMapping::map(address, size_of::<T>(), Default::default()).unwrap() };
+    let address = io.address();
+
+    unsafe {
+        if address % align_of::<T>() == 0 {
+            (address as *const T).read_volatile()
+        } else {
+            (address as *const T).read_unaligned()
+        }
+    }
+}
+
+pub unsafe fn write_memory<T>(address: PhysicalAddress, value: T) {
+    let io =
+        unsafe { DeviceMemoryMapping::map(address, size_of::<T>(), Default::default()).unwrap() };
+    let address = io.address();
+
+    unsafe {
+        if address % align_of::<T>() == 0 {
+            (address as *mut T).write_volatile(value)
+        } else {
+            (address as *mut T).write_unaligned(value)
+        }
+    }
+}

kernel/driver/block/ahci/src/command.rs

@@ -1,12 +1,10 @@
 use core::mem::{size_of, MaybeUninit};
 
-use libk_mm::{
-    address::{AsPhysicalAddress, PhysicalAddress},
-    PageBox, PageSlice,
-};
+use device_api::dma::DmaAllocator;
+use libk::dma::{BusAddress, DmaBuffer, DmaSliceMut};
 use tock_registers::register_structs;
 
-use crate::{data::AtaString, error::AhciError, MAX_PRD_SIZE, SECTOR_SIZE};
+use crate::{data::AtaString, error::AhciError, MAX_PRD_SIZE};
 
 #[derive(Clone, Copy, PartialEq, Eq, Debug)]
 #[repr(u8)]
@@ -22,7 +20,7 @@ pub trait AtaCommand {
 
     fn lba(&self) -> u64;
     fn sector_count(&self) -> usize;
-    fn buffer(&self) -> Option<(PhysicalAddress, usize)>;
+    fn buffer(&self) -> Option<(BusAddress, usize)>;
     unsafe fn into_response(self) -> Self::Response;
 
     fn prd_count(&self) -> usize {
@@ -64,44 +62,41 @@ register_structs! {
 }
 
 pub struct AtaIdentify {
-    buffer: PageBox<MaybeUninit<AtaIdentifyResponse>>,
+    buffer: DmaBuffer<MaybeUninit<AtaIdentifyResponse>>,
 }
 
 pub struct AtaReadDmaEx {
     lba: u64,
     sector_count: usize,
-    buffer_base: PhysicalAddress,
+    buffer_base: BusAddress,
     buffer_size: usize,
 }
 
 impl AtaIdentify {
-    pub fn create() -> Result<Self, AhciError> {
-        PageBox::new_uninit()
+    pub fn create(dma: &dyn DmaAllocator) -> Result<Self, AhciError> {
+        DmaBuffer::new_uninit(dma)
             .map(Self::with_data)
             .map_err(AhciError::MemoryError)
     }
 
-    pub fn with_data(buffer: PageBox<MaybeUninit<AtaIdentifyResponse>>) -> Self {
+    pub fn with_data(buffer: DmaBuffer<MaybeUninit<AtaIdentifyResponse>>) -> Self {
         Self { buffer }
     }
 }
 
 impl AtaReadDmaEx {
-    pub fn new(lba: u64, sector_count: usize, buffer: &PageSlice<MaybeUninit<u8>>) -> Self {
-        assert_eq!(buffer.len() % SECTOR_SIZE, 0);
-        assert_ne!(buffer.len(), 0);
-
+    pub fn new(lba: u64, sector_count: usize, buffer: DmaSliceMut<MaybeUninit<u8>>) -> Self {
         Self {
             lba,
             sector_count,
-            buffer_base: unsafe { buffer.as_physical_address() },
+            buffer_base: buffer.bus_address(),
             buffer_size: buffer.len(),
         }
     }
 }
 
 impl AtaCommand for AtaIdentify {
-    type Response = PageBox<AtaIdentifyResponse>;
+    type Response = DmaBuffer<AtaIdentifyResponse>;
 
     const COMMAND_ID: AtaCommandId = AtaCommandId::Identify;
 
@@ -113,14 +108,14 @@ impl AtaCommand for AtaIdentify {
         0
     }
 
-    fn buffer(&self) -> Option<(PhysicalAddress, usize)> {
-        let base = unsafe { self.buffer.as_physical_address() };
+    fn buffer(&self) -> Option<(BusAddress, usize)> {
+        let base = self.buffer.bus_address();
         let size = size_of::<AtaIdentifyResponse>();
         Some((base, size))
     }
 
     unsafe fn into_response(self) -> Self::Response {
-        self.buffer.assume_init()
+        DmaBuffer::assume_init(self.buffer)
     }
 }
 
@@ -137,7 +132,7 @@ impl AtaCommand for AtaReadDmaEx {
         self.sector_count
     }
 
-    fn buffer(&self) -> Option<(PhysicalAddress, usize)> {
+    fn buffer(&self) -> Option<(BusAddress, usize)> {
         Some((self.buffer_base, self.buffer_size))
     }
 

kernel/driver/block/ahci/src/data.rs

@@ -2,7 +2,7 @@ use core::mem::size_of;
 
 use alloc::string::String;
 use bytemuck::{Pod, Zeroable};
-use libk_mm::address::PhysicalAddress;
+use libk::dma::BusAddress;
 use libk_util::{ConstAssert, IsTrue};
 use static_assertions::const_assert_eq;
 
@@ -174,7 +174,7 @@ impl CommandTable {
 }
 
 impl CommandListEntry {
-    pub fn new(command_table_entry: PhysicalAddress, prd_count: usize) -> Result<Self, AhciError> {
+    pub fn new(command_table_entry: BusAddress, prd_count: usize) -> Result<Self, AhciError> {
         if prd_count > 0xFFFF {
             todo!()
         }
@@ -183,7 +183,7 @@ impl CommandListEntry {
             attr: (size_of::<RegisterHostToDeviceFis>() / size_of::<u32>()) as _,
             prdtl: prd_count as _,
             prdbc: 0,
-            ctba: command_table_entry.into(),
+            ctba: command_table_entry.into_u64(),
             _0: [0; 4],
         })
     }
@@ -201,18 +201,14 @@ unsafe impl Zeroable for CommandTable {
 }
 
 impl PhysicalRegionDescriptor {
-    pub fn new(
-        address: PhysicalAddress,
-        byte_count: usize,
-        is_last: bool,
-    ) -> Result<Self, AhciError> {
+    pub fn new(address: BusAddress, byte_count: usize, is_last: bool) -> Result<Self, AhciError> {
         if byte_count > MAX_PRD_SIZE {
             return Err(AhciError::RegionTooLarge);
         }
 
         let dbc_mask = (is_last as u32) << 31;
         Ok(Self {
-            buffer_address: address.into(),
+            buffer_address: address.into_u64(),
             _0: 0,
             dbc: ((byte_count as u32 - 1) << 1) | 1 | dbc_mask,
         })

kernel/driver/block/ahci/src/lib.rs

@@ -8,18 +8,20 @@ use alloc::{format, sync::Arc, vec::Vec};
 use bytemuck::Zeroable;
 use data::ReceivedFis;
 use device_api::{
-    device::Device,
-    interrupt::{InterruptAffinity, InterruptHandler},
+    device::{Device, DeviceInitContext},
+    dma::DmaAllocator,
+    interrupt::{InterruptAffinity, InterruptHandler, IrqVector},
 };
 use error::AhciError;
-use libk::{device::manager::probe_partitions, fs::devfs, task::runtime};
-use libk_mm::{address::AsPhysicalAddress, device::DeviceMemoryIo, PageBox};
+use libk::{device::manager::probe_partitions, dma::DmaBuffer, fs::devfs, task::runtime};
+use libk_mm::device::DeviceMemoryIo;
 use libk_util::{sync::IrqSafeSpinlock, OneTimeInit};
 use port::AhciPort;
 use regs::{PortRegs, Regs};
 use tock_registers::interfaces::{ReadWriteable, Readable, Writeable};
 use ygg_driver_pci::{
     device::{PciDeviceInfo, PreferredInterruptMode},
+    macros::pci_driver,
     PciCommandRegister, PciConfigurationSpace,
 };
 use yggdrasil_abi::{error::Error, io::FileMode};
@@ -39,8 +41,9 @@ const MAX_DRIVES: usize = (b'z' - b'a') as usize;
 
 pub struct AhciController {
     regs: IrqSafeSpinlock<DeviceMemoryIo<'static, Regs>>,
+    dma: Arc<dyn DmaAllocator>,
     ports: OneTimeInit<Vec<Arc<AhciPort>>>,
-    received_fis_buffers: OneTimeInit<[Option<PageBox<ReceivedFis>>; 16]>,
+    received_fis_buffers: OneTimeInit<[Option<DmaBuffer<ReceivedFis>>; 16]>,
 
     version: Version,
     max_port_count: usize,
@@ -80,8 +83,9 @@ impl AhciController {
             let regs = self.regs.lock();
             let port = &regs.PORTS[i];
 
-            let buffer = PageBox::new(ReceivedFis::zeroed()).map_err(AhciError::MemoryError)?;
-            port.set_received_fis_address_64(unsafe { buffer.as_physical_address() });
+            let buffer = DmaBuffer::new(&*self.dma, ReceivedFis::zeroed())
+                .map_err(AhciError::MemoryError)?;
+            port.set_received_fis_address_64(buffer.bus_address());
             *fis_buffer_slot = Some(buffer);
         }
 
@@ -158,7 +162,7 @@ impl AhciController {
 }
 
 impl InterruptHandler for AhciController {
-    fn handle_irq(self: Arc<Self>, _vector: Option<usize>) -> bool {
+    fn handle_irq(self: Arc<Self>, _vector: IrqVector) -> bool {
         let regs = self.regs.lock();
 
         let is = regs.IS.get();
@@ -180,7 +184,7 @@ impl InterruptHandler for AhciController {
 }
 
 impl Device for AhciController {
-    unsafe fn init(self: Arc<Self>) -> Result<(), Error> {
+    unsafe fn init(self: Arc<Self>, _cx: DeviceInitContext) -> Result<(), Error> {
         // Do the init in background
         runtime::spawn(self.late_init())?;
         Ok(())
@@ -191,51 +195,6 @@ impl Device for AhciController {
     }
 }
 
-static SATA_DRIVES: IrqSafeSpinlock<Vec<Arc<AhciPort>>> = IrqSafeSpinlock::new(Vec::new());
-
-pub fn probe(info: &PciDeviceInfo) -> Result<Arc<dyn Device>, Error> {
-    let bar5 = info.config_space.bar(5).ok_or(Error::InvalidOperation)?;
-    let bar5 = bar5.as_memory().ok_or(Error::InvalidOperation)?;
-
-    let mut cmd = PciCommandRegister::from_bits_retain(info.config_space.command());
-    cmd &= !(PciCommandRegister::DISABLE_INTERRUPTS | PciCommandRegister::ENABLE_IO);
-    cmd |= PciCommandRegister::ENABLE_MEMORY | PciCommandRegister::BUS_MASTER;
-    info.config_space.set_command(cmd.bits());
-
-    info.init_interrupts(PreferredInterruptMode::Msi)?;
-
-    // // TODO support regular PCI interrupts (ACPI dependency)
-    // let Some(mut msi) = info.config_space.capability::<MsiCapability>() else {
-    //     log::warn!("Ignoring AHCI: does not support MSI (and the OS doesn't yet support PCI IRQ)");
-    //     return Err(Error::InvalidOperation);
-    // };
-
-    // Map the registers
-    let regs = unsafe { DeviceMemoryIo::<Regs>::map(bar5, Default::default()) }?;
-
-    let version = Version::try_from(regs.VS.get())?;
-    let ahci_only = regs.CAP.matches_all(CAP::SAM::SET);
-    let max_port_count = regs.CAP.read(CAP::NP) as usize;
-    let has_64_bit = regs.CAP.matches_all(CAP::S64A::SET);
-
-    // TODO extract Number of Command Slots
-
-    let ahci = Arc::new(AhciController {
-        regs: IrqSafeSpinlock::new(regs),
-        ports: OneTimeInit::new(),
-        received_fis_buffers: OneTimeInit::new(),
-        version,
-        max_port_count,
-        ahci_only,
-        has_64_bit,
-    });
-
-    // TODO use multiple vectors if capable
-    info.map_interrupt(InterruptAffinity::Any, ahci.clone())?;
-
-    Ok(ahci)
-}
-
 pub fn register_sata_drive(drive: Arc<AhciPort>, probe: bool) {
     let index = {
         let mut drives = SATA_DRIVES.lock();
@@ -273,3 +232,58 @@ pub fn register_sata_drive(drive: Arc<AhciPort>, probe: bool) {
         .ok();
     }
 }
+
+static SATA_DRIVES: IrqSafeSpinlock<Vec<Arc<AhciPort>>> = IrqSafeSpinlock::new(Vec::new());
+
+pci_driver! {
+    matches: [class (0x01:0x06:0x01)],
+    driver: {
+        fn driver_name(&self) -> &str {
+            "ahci"
+        }
+
+        fn probe(&self, info: &PciDeviceInfo, dma: &Arc<dyn DmaAllocator>) -> Result<Arc<dyn Device>, Error> {
+            let bar5 = info.config_space.bar(5).ok_or(Error::InvalidOperation)?;
+            let bar5 = bar5.as_memory().ok_or(Error::InvalidOperation)?;
+
+            let mut cmd = PciCommandRegister::from_bits_retain(info.config_space.command());
+            cmd &= !(PciCommandRegister::DISABLE_INTERRUPTS | PciCommandRegister::ENABLE_IO);
+            cmd |= PciCommandRegister::ENABLE_MEMORY | PciCommandRegister::BUS_MASTER;
+            info.config_space.set_command(cmd.bits());
+
+            info.init_interrupts(PreferredInterruptMode::Msi(true))?;
+
+            // // TODO support regular PCI interrupts (ACPI dependency)
+            // let Some(mut msi) = info.config_space.capability::<MsiCapability>() else {
+            //     log::warn!("Ignoring AHCI: does not support MSI (and the OS doesn't yet support PCI IRQ)");
+            //     return Err(Error::InvalidOperation);
+            // };
+
+            // Map the registers
+            let regs = unsafe { DeviceMemoryIo::<Regs>::map(bar5, Default::default()) }?;
+
+            let version = Version::try_from(regs.VS.get())?;
+            let ahci_only = regs.CAP.matches_all(CAP::SAM::SET);
+            let max_port_count = regs.CAP.read(CAP::NP) as usize;
+            let has_64_bit = regs.CAP.matches_all(CAP::S64A::SET);
+
+            // TODO extract Number of Command Slots
+
+            let ahci = Arc::new(AhciController {
+                regs: IrqSafeSpinlock::new(regs),
+                dma: dma.clone(),
+                ports: OneTimeInit::new(),
+                received_fis_buffers: OneTimeInit::new(),
+                version,
+                max_port_count,
+                ahci_only,
+                has_64_bit,
+            });
+
+            // TODO use multiple vectors if capable
+            info.map_interrupt(InterruptAffinity::Any, ahci.clone())?;
+
+            Ok(ahci)
+        }
+    }
+}

kernel/driver/block/ahci/src/port.rs

@@ -8,14 +8,15 @@ use core::{
 use alloc::{boxed::Box, string::String, sync::Arc};
 use async_trait::async_trait;
 use bytemuck::Zeroable;
-use device_api::device::Device;
+use device_api::{device::Device, dma::DmaAllocator};
 use futures_util::task::AtomicWaker;
-use libk::{device::block::BlockDevice, error::Error};
+use libk::{
+    device::block::BlockDevice,
+    dma::{DmaBuffer, DmaSlice, DmaSliceMut},
+    error::Error,
+};
 use libk_mm::{
-    address::{AsPhysicalAddress, PhysicalAddress},
-    device::DeviceMemoryIo,
-    table::MapAttributes,
-    PageBox, PageProvider, PageSlice,
+    address::PhysicalAddress, device::DeviceMemoryIo, table::MapAttributes, PageProvider,
 };
 use libk_util::{sync::IrqSafeSpinlock, waker::QueueWaker, OneTimeInit};
 use tock_registers::interfaces::{Readable, Writeable};
@@ -37,8 +38,8 @@ struct PortInner {
     regs: DeviceMemoryIo<'static, PortRegs>,
 
     #[allow(unused)]
-    received_fis: PageBox<ReceivedFis>,
-    command_list: PageBox<[CommandListEntry]>,
+    received_fis: DmaBuffer<ReceivedFis>,
+    command_list: DmaBuffer<[CommandListEntry]>,
 }
 
 pub struct PortInfo {
@@ -90,18 +91,16 @@ impl Drop for SubmittedCommand<'_> {
 impl PortInner {
     fn submit_command<C: AtaCommand>(
         &mut self,
+        dma: &dyn DmaAllocator,
         index: usize,
         command: &C,
     ) -> Result<(), AhciError> {
         let list_entry = &mut self.command_list[index];
         let mut table_entry =
-            PageBox::new(CommandTable::zeroed()).map_err(AhciError::MemoryError)?;
+            DmaBuffer::new(dma, CommandTable::zeroed()).map_err(AhciError::MemoryError)?;
 
         table_entry.setup_command(command)?;
-        *list_entry = CommandListEntry::new(
-            unsafe { table_entry.as_physical_address() },
-            command.prd_count(),
-        )?;
+        *list_entry = CommandListEntry::new(table_entry.bus_address(), command.prd_count())?;
 
         // Sync before send
         // XXX do this properly
@@ -137,12 +136,14 @@ impl AhciPort {
             return Err(AhciError::DeviceError);
         }
 
-        let received_fis = PageBox::new(ReceivedFis::zeroed()).map_err(AhciError::MemoryError)?;
-        let command_list = PageBox::new_slice(CommandListEntry::zeroed(), COMMAND_LIST_LENGTH)
-            .map_err(AhciError::MemoryError)?;
+        let received_fis =
+            DmaBuffer::new(&*ahci.dma, ReceivedFis::zeroed()).map_err(AhciError::MemoryError)?;
+        let command_list =
+            DmaBuffer::new_slice(&*ahci.dma, CommandListEntry::zeroed(), COMMAND_LIST_LENGTH)
+                .map_err(AhciError::MemoryError)?;
 
-        regs.set_received_fis_address_64(unsafe { received_fis.as_physical_address() });
-        regs.set_command_list_address_64(unsafe { command_list.as_physical_address() });
+        regs.set_received_fis_address_64(received_fis.bus_address());
+        regs.set_command_list_address_64(command_list.bus_address());
 
         regs.IE.write(
             IE::DPE::SET
@@ -182,7 +183,9 @@ impl AhciPort {
     }
 
     pub async fn init_inner(&self) -> Result<(), AhciError> {
-        let identify = self.perform_command(AtaIdentify::create()?).await?;
+        let identify = self
+            .perform_command(AtaIdentify::create(&*self.ahci.dma)?)
+            .await?;
 
         let model = identify.model_number.to_string();
         let serial = identify.serial_number.to_string();
@@ -237,7 +240,11 @@ impl AhciPort {
             return Err(AhciError::RegionTooLarge);
         }
         let index = self.allocate_command().await;
-        if let Err(error) = self.inner.lock().submit_command(index, command) {
+        if let Err(error) = self
+            .inner
+            .lock()
+            .submit_command(&*self.ahci.dma, index, command)
+        {
             self.free_command(index);
             return Err(error);
         }
@@ -302,28 +309,38 @@ impl AhciPort {
 
 #[async_trait]
 impl BlockDevice for AhciPort {
+    fn allocate_buffer(&self, size: usize) -> Result<DmaBuffer<[MaybeUninit<u8>]>, Error> {
+        DmaBuffer::new_uninit_slice(&*self.ahci.dma, size)
+    }
+
     async fn read_aligned(
         &self,
         position: u64,
-        buffer: &mut PageSlice<MaybeUninit<u8>>,
+        buffer: DmaSliceMut<'_, MaybeUninit<u8>>,
     ) -> Result<(), Error> {
-        if position % SECTOR_SIZE as u64 != 0 {
+        if buffer.len() % SECTOR_SIZE != 0 {
+            log::warn!("ahci: misaligned buffer size: {}", buffer.len());
             return Err(Error::InvalidOperation);
         }
-        if buffer.len() % SECTOR_SIZE != 0 {
+        if position % SECTOR_SIZE as u64 != 0 {
+            log::warn!("ahci: misaligned read");
             return Err(Error::InvalidOperation);
         }
 
         let lba = position / SECTOR_SIZE as u64;
-        let command = AtaReadDmaEx::new(lba, buffer.len() / SECTOR_SIZE, buffer);
-        self.submit(&command)
-            .await?
-            .wait_for_completion()
-            .await
-            .map_err(AhciError::into)
+        let lba_count = buffer.len() / SECTOR_SIZE;
+        if lba + lba_count as u64 >= self.block_count() {
+            log::warn!("ahci: read crosses medium end");
+            return Err(Error::InvalidOperation);
+        }
+
+        let command = AtaReadDmaEx::new(lba, lba_count, buffer);
+        self.submit(&command).await?.wait_for_completion().await?;
+
+        Ok(())
     }
 
-    async fn write_aligned(&self, _position: u64, _buffer: &PageSlice<u8>) -> Result<(), Error> {
+    async fn write_aligned(&self, _position: u64, _buffer: DmaSlice<'_, u8>) -> Result<(), Error> {
         // TODO AtaWriteDmaEx
         Err(Error::NotImplemented)
     }

kernel/driver/block/ahci/src/regs.rs

@@ -1,4 +1,4 @@
-use libk_mm::address::PhysicalAddress;
+use libk::dma::BusAddress;
 use tock_registers::{
     interfaces::{ReadWriteable, Readable, Writeable},
     register_bitfields, register_structs,
@@ -141,14 +141,14 @@ impl PortRegs {
         Ok(())
     }
 
-    pub fn set_received_fis_address_64(&self, address: PhysicalAddress) {
-        let address: u64 = address.into();
+    pub fn set_received_fis_address_64(&self, address: BusAddress) {
+        let address: u64 = address.into_u64();
         self.FB.set(address as u32);
         self.FBU.set((address >> 32) as u32);
     }
 
-    pub fn set_command_list_address_64(&self, address: PhysicalAddress) {
-        let address: u64 = address.into();
+    pub fn set_command_list_address_64(&self, address: BusAddress) {
+        let address: u64 = address.into_u64();
         self.CLB.set(address as u32);
         self.CLBU.set((address >> 32) as u32);
     }

kernel/driver/block/nvme/Cargo.toml

@@ -10,6 +10,7 @@ libk-util.workspace = true
 libk-mm.workspace = true
 libk.workspace = true
 device-api = { workspace = true, features = ["derive"] }
+kernel-arch.workspace = true
 
 ygg_driver_pci = { path = "../../bus/pci" }
 kernel-fs = { path = "../../fs/kernel-fs" }

kernel/driver/block/nvme/src/command.rs

@@ -2,7 +2,7 @@
 
 use core::fmt::{self, Write};
 
-use libk_mm::address::PhysicalAddress;
+use libk::dma::BusAddress;
 use tock_registers::{interfaces::Readable, register_structs, registers::ReadOnly, UIntLike};
 
 use crate::queue::PhysicalRegionPage;
@@ -74,7 +74,7 @@ pub struct CreateIoCompletionQueue {
     pub id: u32,
     pub size: usize,
     pub vector: u32,
-    pub data: PhysicalAddress,
+    pub data: BusAddress,
 }
 
 #[derive(Clone, Copy, Debug)]
@@ -82,7 +82,7 @@ pub struct CreateIoSubmissionQueue {
     pub id: u32,
     pub cq_id: u32,
     pub size: usize,
-    pub data: PhysicalAddress,
+    pub data: BusAddress,
 }
 
 // Replies

kernel/driver/block/nvme/src/drive.rs

@@ -3,7 +3,11 @@ use core::mem::MaybeUninit;
 use alloc::{boxed::Box, sync::Arc};
 use async_trait::async_trait;
 use device_api::device::Device;
-use libk::{device::block::BlockDevice, error::Error};
+use libk::{
+    device::block::BlockDevice,
+    dma::{DmaBuffer, DmaSlice, DmaSliceMut},
+    error::Error,
+};
 use libk_mm::{
     address::{AsPhysicalAddress, PhysicalAddress},
     table::MapAttributes,
@@ -30,7 +34,9 @@ impl NvmeNamespace {
         max_transfer_size: usize,
     ) -> Result<Arc<NvmeNamespace>, NvmeError> {
         let admin_q = controller.admin_q.get();
-        let identify = admin_q.request(IdentifyNamespaceRequest { nsid }).await?;
+        let identify = admin_q
+            .request(&*controller.dma, IdentifyNamespaceRequest { nsid })
+            .await?;
 
         let current_lba_format_idx = identify.current_lba_fmt_idx();
         let current_lba_format = identify.lba_fmt(current_lba_format_idx).unwrap();
@@ -76,62 +82,57 @@ impl Device for NvmeNamespace {
 
 #[async_trait]
 impl BlockDevice for NvmeNamespace {
+    fn allocate_buffer(&self, size: usize) -> Result<DmaBuffer<[MaybeUninit<u8>]>, Error> {
+        DmaBuffer::new_uninit_slice(&*self.controller.dma, size)
+    }
+
+    // TODO read directly to cache
     async fn read_aligned(
         &self,
         position: u64,
-        buffer: &mut PageSlice<MaybeUninit<u8>>,
+        buffer: DmaSliceMut<'_, MaybeUninit<u8>>,
     ) -> Result<(), Error> {
-        debug_assert_eq!(position % self.block_size() as u64, 0);
+        if position % self.block_size() as u64 != 0 {
+            return Err(Error::InvalidOperation);
+        }
+        if buffer.len() % self.block_size() != 0 || buffer.is_empty() {
+            return Err(Error::InvalidOperation);
+        }
         let lba = position / self.block_size() as u64;
-        debug_assert_eq!(buffer.len() % self.block_size(), 0);
-        let buffer_address = unsafe { buffer.as_physical_address() };
-        debug_assert_eq!(buffer_address.into_u64() % self.block_size() as u64, 0);
-        let lba_count = buffer.len() / self.block_size();
-
-        let result = self
-            .controller
-            .perform_io(
-                self.nsid,
-                lba,
-                lba_count,
-                buffer_address,
-                buffer.len(),
-                IoDirection::Read,
-            )
-            .await;
-
-        log::info!(target: "io", "read #{lba}, {lba_count} blocks -> {result:?} @ {buffer_address:#x}");
-
-        result.map_err(NvmeError::into)
-    }
-
-    async fn write_aligned(&self, position: u64, buffer: &PageSlice<u8>) -> Result<(), Error> {
-        debug_assert_eq!(position % self.block_size() as u64, 0);
-        let lba = position / self.block_size() as u64;
-        debug_assert_eq!(buffer.len() % self.block_size(), 0);
-        let buffer_address = unsafe { buffer.as_physical_address() };
-        debug_assert_eq!(buffer_address.into_u64() % self.block_size() as u64, 0);
-        let lba_count = buffer.len() / self.block_size();
-
-        // TODO ArchitectureImpl::flush_data_cache()
-        #[cfg(target_arch = "x86_64")]
-        unsafe {
-            core::arch::asm!("wbinvd");
+        let lba_count = buffer.len().div_ceil(self.block_size());
+        if lba + lba_count as u64 > self.block_count() {
+            return Err(Error::InvalidOperation);
         }
 
         let result = self
             .controller
-            .perform_io(
-                self.nsid,
-                lba,
-                lba_count,
-                buffer_address,
-                buffer.len(),
-                IoDirection::Write,
-            )
+            .perform_read(self.nsid, lba, lba_count, buffer)
             .await;
 
-        log::info!(target: "io", "write -> #{lba}, {lba_count} blocks -> {result:?} @ {buffer_address:#x}");
+        log::trace!(target: "io", "read #{lba}, {lba_count} blocks -> {result:?}");
+
+        result.map_err(NvmeError::into)
+    }
+
+    async fn write_aligned(&self, position: u64, buffer: DmaSlice<'_, u8>) -> Result<(), Error> {
+        if position % self.block_size() as u64 != 0 {
+            return Err(Error::InvalidOperation);
+        }
+        if buffer.len() % self.block_size() != 0 || buffer.is_empty() {
+            return Err(Error::InvalidOperation);
+        }
+        let lba = position / self.block_size() as u64;
+        let lba_count = buffer.len().div_ceil(self.block_size());
+        if lba + lba_count as u64 > self.block_count() {
+            return Err(Error::InvalidOperation);
+        }
+
+        let result = self
+            .controller
+            .perform_write(self.nsid, lba, lba_count, buffer)
+            .await;
+
+        log::trace!(target: "io", "write -> #{lba}, {lba_count} blocks -> {result:?}");
 
         result.map_err(NvmeError::into)
     }

kernel/driver/block/nvme/src/error.rs

@@ -5,6 +5,7 @@ use super::queue::CommandError;
 
 #[derive(Debug)]
 pub enum NvmeError {
+    InitError(Error),
     InvalidBuffer(PhysicalAddress, usize),
     RequestTooLarge(usize),
     MemoryError(Error),
@@ -20,6 +21,7 @@ impl From<CommandError> for NvmeError {
 impl From<NvmeError> for Error {
     fn from(value: NvmeError) -> Self {
         match value {
+            NvmeError::InitError(error) => error,
             NvmeError::RequestTooLarge(_) => Error::InvalidArgument,
             NvmeError::InvalidBuffer(_, _) => Error::InvalidArgument,
             NvmeError::MemoryError(err) => err,

kernel/driver/block/nvme/src/lib.rs

@@ -1,11 +1,13 @@
 #![feature(const_trait_impl, let_chains, if_let_guard, maybe_uninit_slice)]
 #![allow(missing_docs)]
 #![no_std]
+// TODO
+#![allow(unused)]
 
 extern crate alloc;
 
 use core::{
-    mem::size_of,
+    mem::{size_of, MaybeUninit},
     sync::atomic::{AtomicUsize, Ordering},
     time::Duration,
 };
@@ -13,12 +15,15 @@ use core::{
 use alloc::{collections::BTreeMap, format, sync::Arc, vec::Vec};
 use command::{IdentifyActiveNamespaceIdListRequest, IdentifyControllerRequest};
 use device_api::{
-    device::Device,
-    interrupt::{InterruptAffinity, InterruptHandler},
+    device::{Device, DeviceInitContext},
+    dma::DmaAllocator,
+    interrupt::{InterruptAffinity, InterruptHandler, IrqVector},
 };
 use drive::NvmeNamespace;
+use kernel_arch::{Architecture, ArchitectureImpl};
 use libk::{
     device::manager::probe_partitions,
+    dma::{BusAddress, DmaSlice, DmaSliceMut},
     fs::devfs,
     task::{cpu_count, cpu_index, runtime},
 };
@@ -28,6 +33,7 @@ use libk_util::{
     OneTimeInit,
 };
 use queue::PrpList;
+use regs::{CAP, CC};
 use tock_registers::{
     interfaces::{ReadWriteable, Readable, Writeable},
     register_bitfields, register_structs,
@@ -35,6 +41,7 @@ use tock_registers::{
 };
 use ygg_driver_pci::{
     device::{PciDeviceInfo, PreferredInterruptMode},
+    macros::pci_driver,
     PciCommandRegister, PciConfigurationSpace,
 };
 use yggdrasil_abi::{error::Error, io::FileMode};
@@ -48,83 +55,19 @@ use self::{
     command::{CreateIoCompletionQueue, CreateIoSubmissionQueue, SetFeatureRequest},
     error::NvmeError,
     queue::QueuePair,
+    regs::Regs,
 };
 
 mod command;
 mod drive;
 mod error;
 mod queue;
+mod regs;
 
 pub const MAX_PAGES_PER_REQUEST: usize = 256;
 // Use host page
 pub const PAGE_SIZE: usize = L3_PAGE_SIZE;
 
-register_bitfields! {
-    u32,
-    CC [
-        IOCQES OFFSET(20) NUMBITS(4) [],
-        IOSQES OFFSET(16) NUMBITS(4) [],
-        AMS OFFSET(11) NUMBITS(3) [],
-        MPS OFFSET(7) NUMBITS(4) [],
-        CSS OFFSET(4) NUMBITS(3) [
-            NvmCommandSet = 0
-        ],
-        ENABLE OFFSET(0) NUMBITS(1) [],
-    ],
-    CSTS [
-        CFS OFFSET(1) NUMBITS(1) [],
-        RDY OFFSET(0) NUMBITS(1) [],
-    ],
-    AQA [
-        /// Admin Completion Queue Size in entries - 1
-        ACQS OFFSET(16) NUMBITS(12) [],
-        /// Admin Submission Queue Size in entries - 1
-        ASQS OFFSET(0) NUMBITS(12) [],
-    ]
-}
-
-register_bitfields! {
-    u64,
-    CAP [
-        /// Maximum Queue Entries Supported - 1. i.e., 0 means maximum queue len of 1, 1 = 2 etc.
-        MQES OFFSET(0) NUMBITS(16) [],
-        /// Timeout. Represents the worst-case time the host software should wait for CSTS.RDY to
-        /// change its state.
-        TO OFFSET(24) NUMBITS(8) [],
-        /// Doorbell stride. Stride in bytes = pow(2, 2 + DSTRD).
-        DSTRD OFFSET(32) NUMBITS(4) [],
-        /// NVM Subsystem Reset Supported (see NVMe BS Section 3.7.1)
-        NSSRS OFFSET(36) NUMBITS(1) [],
-        /// Controller supports one or more I/O command sets
-        CSS_IO_COMMANDS OFFSET(43) NUMBITS(1) [],
-        /// Controller only supports admin commands and no I/O commands
-        CSS_ADMIN_ONLY OFFSET(44) NUMBITS(1) [],
-        /// Memory page size minimum (bytes = pow(2, 12 + MPSMIN))
-        MPSMIN OFFSET(48) NUMBITS(4) [],
-        /// Memory page size maximum -|-
-        MPSMAX OFFSET(52) NUMBITS(4) [],
-    ]
-}
-
-register_structs! {
-    #[allow(non_snake_case)]
-    Regs {
-        (0x00 => CAP: ReadOnly<u64, CAP::Register>),
-        (0x08 => VS: ReadOnly<u32>),
-        (0x0C => INTMS: WriteOnly<u32>),
-        (0x10 => INTMC: WriteOnly<u32>),
-        (0x14 => CC: ReadWrite<u32, CC::Register>),
-        (0x18 => _0),
-        (0x1C => CSTS: ReadOnly<u32, CSTS::Register>),
-        (0x20 => _1),
-        (0x24 => AQA: ReadWrite<u32, AQA::Register>),
-        (0x28 => ASQ: ReadWrite<u64>),
-        (0x30 => ACQ: ReadWrite<u64>),
-        (0x38 => _2),
-        (0x2000 => @END),
-    }
-}
-
 pub struct NvmeController {
     regs: IrqSafeSpinlock<DeviceMemoryIo<'static, Regs>>,
     admin_q: OneTimeInit<QueuePair>,
@@ -134,6 +77,7 @@ pub struct NvmeController {
     controller_id: OneTimeInit<u32>,
 
     pci: PciDeviceInfo,
+    dma: Arc<dyn DmaAllocator>,
 
     doorbell_shift: usize,
     min_page_size: usize,
@@ -180,15 +124,22 @@ impl NvmeController {
             let id = i as u32;
 
             let (sq_doorbell, cq_doorbell) = unsafe { self.doorbell_pair(i) };
-            let queue = QueuePair::new(id, i, Self::IO_QUEUE_SIZE, sq_doorbell, cq_doorbell)
-                .map_err(NvmeError::MemoryError)?;
+            let queue = QueuePair::new(
+                &*self.dma,
+                id,
+                i,
+                Self::IO_QUEUE_SIZE,
+                sq_doorbell,
+                cq_doorbell,
+            )
+            .map_err(NvmeError::MemoryError)?;
 
             admin_q
                 .request_no_data(CreateIoCompletionQueue {
                     id,
                     vector: id,
                     size: Self::IO_QUEUE_SIZE,
-                    data: queue.cq_physical_pointer(),
+                    data: queue.cq_bus_pointer(),
                 })
                 .await?;
 
@@ -197,7 +148,7 @@ impl NvmeController {
                     id,
                     cq_id: id,
                     size: Self::IO_QUEUE_SIZE,
-                    data: queue.sq_physical_pointer(),
+                    data: queue.sq_bus_pointer(),
                 })
                 .await?;
 
@@ -219,7 +170,7 @@ impl NvmeController {
             let range = self
                 .pci
                 .map_interrupt_multiple(0..io_queue_count + 1, InterruptAffinity::Any, self.clone())
-                .unwrap();
+                .map_err(NvmeError::InitError)?;
 
             // TODO handle different MSI range allocations
             for (i, msi) in range.iter().enumerate() {
@@ -230,7 +181,9 @@ impl NvmeController {
         let admin_q = self.admin_q.get();
 
         // Identify the controller
-        let identify = admin_q.request(IdentifyControllerRequest).await?;
+        let identify = admin_q
+            .request(&*self.dma, IdentifyControllerRequest)
+            .await?;
 
         let max_transfer_size = if identify.mdts == 0 {
             // Pick some sane default value
@@ -254,7 +207,10 @@ impl NvmeController {
         let admin_q = self.admin_q.get();
 
         let namespaces = admin_q
-            .request(IdentifyActiveNamespaceIdListRequest { start_id: 0 })
+            .request(
+                &*self.dma,
+                IdentifyActiveNamespaceIdListRequest { start_id: 0 },
+            )
             .await?;
 
         let count = namespaces.entries.iter().position(|&x| x == 0).unwrap();
@@ -274,42 +230,53 @@ impl NvmeController {
         Ok(())
     }
 
-    pub async fn perform_io(
+    pub async fn perform_read(
         &self,
         nsid: u32,
         lba: u64,
         lba_count: usize,
-        buffer_address: PhysicalAddress,
-        transfer_size: usize,
-        direction: IoDirection,
+        buffer: DmaSliceMut<'_, MaybeUninit<u8>>,
     ) -> Result<(), NvmeError> {
-        let prp_list = PrpList::from_buffer(buffer_address, transfer_size)?;
-
-        let _guard = IrqGuard::acquire();
+        let prp_list = PrpList::from_buffer(&*self.dma, buffer.bus_address(), buffer.len())?;
         let cpu_index = cpu_index();
         let ioq = &self.ioqs.get()[cpu_index as usize];
+        let cmd_id = ioq.submit(
+            IoRead {
+                nsid,
+                lba,
+                count: lba_count as _,
+            },
+            &prp_list,
+            true,
+        )?;
+        ioq.wait_for_completion(cmd_id, ()).await?;
 
-        let cmd_id = match direction {
-            IoDirection::Read => ioq.submit(
-                IoRead {
-                    nsid,
-                    lba,
-                    count: lba_count as _,
-                },
-                &prp_list,
-                true,
-            )?,
-            IoDirection::Write => ioq.submit(
-                IoWrite {
-                    nsid,
-                    lba,
-                    count: lba_count as _,
-                },
-                &prp_list,
-                true,
-            )?,
-        };
+        buffer.cache_flush_all(false);
 
+        Ok(())
+    }
+
+    pub async fn perform_write(
+        &self,
+        nsid: u32,
+        lba: u64,
+        lba_count: usize,
+        buffer: DmaSlice<'_, u8>,
+    ) -> Result<(), NvmeError> {
+        buffer.cache_flush_all(true);
+
+        let prp_list = PrpList::from_buffer(&*self.dma, buffer.bus_address(), buffer.len())?;
+        let cpu_index = cpu_index();
+        let ioq = &self.ioqs.get()[cpu_index as usize];
+        let cmd_id = ioq.submit(
+            IoWrite {
+                nsid,
+                lba,
+                count: lba_count as _,
+            },
+            &prp_list,
+            true,
+        )?;
         ioq.wait_for_completion(cmd_id, ()).await?;
 
         Ok(())
@@ -324,8 +291,10 @@ impl NvmeController {
 }
 
 impl InterruptHandler for NvmeController {
-    fn handle_irq(self: Arc<Self>, vector: Option<usize>) -> bool {
-        let vector = vector.expect("Only MSI-X interrupts are supported");
+    fn handle_irq(self: Arc<Self>, vector: IrqVector) -> bool {
+        let IrqVector::Msi(vector) = vector else {
+            unreachable!("Only MSI-x interrupts are supported for NVMe");
+        };
 
         if vector == 0 {
             self.admin_q.get().process_completions() != 0
@@ -340,66 +309,34 @@ impl InterruptHandler for NvmeController {
 }
 
 impl Device for NvmeController {
-    unsafe fn init(self: Arc<Self>) -> Result<(), Error> {
+    unsafe fn init(self: Arc<Self>, _cx: DeviceInitContext) -> Result<(), Error> {
         let regs = self.regs.lock();
 
         let timeout = Duration::from_millis(regs.CAP.read(CAP::TO) * 500);
         log::debug!("Worst-case timeout: {:?}", timeout);
 
-        while regs.CSTS.matches_all(CSTS::RDY::SET) {
-            core::hint::spin_loop();
-        }
-
-        if Self::ADMIN_QUEUE_SIZE as u64 > regs.CAP.read(CAP::MQES) + 1 {
-            todo!(
-                "queue_slots too big, max = {}",
-                regs.CAP.read(CAP::MQES) + 1
-            );
-        }
-
         // Setup the admin queue (index 0)
         let admin_sq_doorbell = unsafe { regs.doorbell_ptr(self.doorbell_shift, false, 0) };
         let admin_cq_doorbell = unsafe { regs.doorbell_ptr(self.doorbell_shift, true, 0) };
-        log::debug!("sq_doorbell for adminq = {:p}", admin_sq_doorbell);
         let admin_q = QueuePair::new(
+            &*self.dma,
             0,
             0,
             Self::ADMIN_QUEUE_SIZE,
             admin_sq_doorbell,
             admin_cq_doorbell,
-        )
-        .unwrap();
+        )?;
 
-        regs.AQA.modify(
-            AQA::ASQS.val(Self::ADMIN_QUEUE_SIZE as u32 - 1)
-                + AQA::ACQS.val(Self::ADMIN_QUEUE_SIZE as u32 - 1),
-        );
-        regs.ASQ.set(admin_q.sq_physical_pointer().into());
-        regs.ACQ.set(admin_q.cq_physical_pointer().into());
+        regs.configure_admin_queue(
+            admin_q.sq_bus_pointer(),
+            admin_q.cq_bus_pointer(),
+            Self::ADMIN_QUEUE_SIZE,
+            Self::ADMIN_QUEUE_SIZE,
+        )?;
 
         // Configure the controller
-        const IOSQES: u32 = size_of::<SubmissionQueueEntry>().ilog2();
-        const IOCQES: u32 = size_of::<CompletionQueueEntry>().ilog2();
-
-        regs.CC.modify(
-            CC::IOCQES.val(IOCQES)
-                + CC::IOSQES.val(IOSQES)
-                + CC::MPS.val(0)
-                + CC::CSS::NvmCommandSet,
-        );
-
-        // Enable the controller
-        regs.CC.modify(CC::ENABLE::SET);
-
-        log::debug!("Reset the controller");
-
-        while !regs.CSTS.matches_any(&[CSTS::RDY::SET, CSTS::CFS::SET]) {
-            core::hint::spin_loop();
-        }
-
-        if regs.CSTS.matches_all(CSTS::CFS::SET) {
-            todo!("CFS set after reset!");
-        }
+        regs.configure_controller();
+        regs.enable_controller(10000000)?;
 
         self.admin_q.init(admin_q);
 
@@ -417,54 +354,6 @@ impl Device for NvmeController {
 // TODO
 unsafe impl Sync for NvmeController {}
 
-static NVME_CONTROLLERS: IrqSafeSpinlock<Vec<Arc<NvmeController>>> =
-    IrqSafeSpinlock::new(Vec::new());
-
-pub fn probe(info: &PciDeviceInfo) -> Result<Arc<dyn Device>, Error> {
-    let bar0 = info
-        .config_space
-        .bar(0)
-        .unwrap()
-        .as_memory()
-        .expect("Expected a memory BAR0");
-
-    info.init_interrupts(PreferredInterruptMode::Msi)?;
-
-    let mut cmd = PciCommandRegister::from_bits_retain(info.config_space.command());
-    cmd &= !(PciCommandRegister::DISABLE_INTERRUPTS | PciCommandRegister::ENABLE_IO);
-    cmd |= PciCommandRegister::ENABLE_MEMORY | PciCommandRegister::BUS_MASTER;
-    info.config_space.set_command(cmd.bits());
-
-    let regs = unsafe { DeviceMemoryIo::<Regs>::map(bar0, Default::default()) }?;
-
-    // Disable the controller
-    regs.CC.modify(CC::ENABLE::CLEAR);
-
-    let doorbell_shift = regs.CAP.read(CAP::DSTRD) as usize + 1;
-    let min_page_size = 1 << (regs.CAP.read(CAP::MPSMIN) + 12);
-
-    if min_page_size > PAGE_SIZE {
-        log::error!("Cannot support NVMe HC: min page size ({min_page_size}) > host page size ({PAGE_SIZE})");
-        return Err(Error::InvalidArgument);
-    }
-
-    let device = NvmeController {
-        regs: IrqSafeSpinlock::new(regs),
-        admin_q: OneTimeInit::new(),
-        ioqs: OneTimeInit::new(),
-        drive_table: IrqSafeSpinlock::new(BTreeMap::new()),
-        controller_id: OneTimeInit::new(),
-
-        pci: info.clone(),
-
-        io_queue_count: AtomicUsize::new(1),
-        doorbell_shift,
-        min_page_size,
-    };
-
-    Ok(Arc::new(device))
-}
-
 pub fn register_nvme_controller(controller: Arc<NvmeController>) {
     let mut list = NVME_CONTROLLERS.lock();
     let id = list.len();
@@ -496,3 +385,61 @@ pub fn register_nvme_namespace(namespace: Arc<NvmeNamespace>, probe: bool) {
         .ok();
     }
 }
+
+static NVME_CONTROLLERS: IrqSafeSpinlock<Vec<Arc<NvmeController>>> =
+    IrqSafeSpinlock::new(Vec::new());
+
+pci_driver! {
+    matches: [class (0x01:0x08:0x02)],
+    driver: {
+        fn driver_name(&self) -> &str {
+            "nvme"
+        }
+
+        fn probe(&self, info: &PciDeviceInfo, dma: &Arc<dyn DmaAllocator>) -> Result<Arc<dyn Device>, Error> {
+            let bar0 = info
+                .config_space
+                .bar(0)
+                .unwrap()
+                .as_memory()
+                .expect("Expected a memory BAR0");
+
+            info.init_interrupts(PreferredInterruptMode::Msi(true))?;
+
+            let mut cmd = PciCommandRegister::from_bits_retain(info.config_space.command());
+            cmd &= !(PciCommandRegister::DISABLE_INTERRUPTS | PciCommandRegister::ENABLE_IO);
+            cmd |= PciCommandRegister::ENABLE_MEMORY | PciCommandRegister::BUS_MASTER;
+            info.config_space.set_command(cmd.bits());
+
+            let regs = unsafe { DeviceMemoryIo::<Regs>::map(bar0, Default::default()) }?;
+
+            // Disable the controller
+            regs.disable_controller(10000000)?;
+
+            let doorbell_shift = regs.CAP.read(CAP::DSTRD) as usize + 1;
+            let min_page_size = 1 << (regs.CAP.read(CAP::MPSMIN) + 12);
+
+            if min_page_size > PAGE_SIZE {
+                log::error!("Cannot support NVMe HC: min page size ({min_page_size}) > host page size ({PAGE_SIZE})");
+                return Err(Error::InvalidArgument);
+            }
+
+            let device = NvmeController {
+                regs: IrqSafeSpinlock::new(regs),
+                admin_q: OneTimeInit::new(),
+                ioqs: OneTimeInit::new(),
+                drive_table: IrqSafeSpinlock::new(BTreeMap::new()),
+                controller_id: OneTimeInit::new(),
+
+                pci: info.clone(),
+                dma: dma.clone(),
+
+                io_queue_count: AtomicUsize::new(1),
+                doorbell_shift,
+                min_page_size,
+            };
+
+            Ok(Arc::new(device))
+        }
+    }
+}

kernel/driver/block/nvme/src/queue.rs

@@ -2,10 +2,10 @@ use core::{future::poll_fn, mem::size_of, ptr::null_mut, task::Poll};
 
 use alloc::collections::{BTreeMap, BTreeSet};
 use bytemuck::{Pod, Zeroable};
-use libk_mm::{
-    address::{AsPhysicalAddress, PhysicalAddress},
-    PageBox,
-};
+use device_api::dma::DmaAllocator;
+use kernel_arch::{Architecture, ArchitectureImpl};
+use libk::dma::{BusAddress, DmaBuffer};
+use libk_mm::address::AsPhysicalAddress;
 use libk_util::{sync::IrqSafeSpinlock, waker::QueueWaker};
 use static_assertions::const_assert;
 use yggdrasil_abi::error::Error;
@@ -58,7 +58,7 @@ pub struct CompletionQueueEntry {
 }
 
 pub struct Queue<T> {
-    data: PageBox<[T]>,
+    data: DmaBuffer<[T]>,
     mask: usize,
     head: usize,
     tail: usize,
@@ -82,8 +82,8 @@ pub struct QueuePair {
     #[allow(unused)]
     vector: usize,
 
-    sq_base: PhysicalAddress,
-    cq_base: PhysicalAddress,
+    sq_base: BusAddress,
+    cq_base: BusAddress,
 
     pub completion_notify: QueueWaker,
 
@@ -94,7 +94,7 @@ pub struct PrpList {
     prp1: PhysicalRegionPage,
     prp2: PhysicalRegionPage,
     #[allow(unused)]
-    list: Option<PageBox<[PhysicalAddress]>>,
+    list: Option<DmaBuffer<[BusAddress]>>,
 }
 
 impl PrpList {
@@ -106,7 +106,11 @@ impl PrpList {
         }
     }
 
-    pub fn from_buffer(base: PhysicalAddress, size: usize) -> Result<Self, NvmeError> {
+    pub fn from_buffer(
+        dma: &dyn DmaAllocator,
+        base: BusAddress,
+        size: usize,
+    ) -> Result<Self, NvmeError> {
         // TODO hardcoded page size
         if base.into_u64() % 0x1000 != 0 {
             todo!();
@@ -126,12 +130,13 @@ impl PrpList {
             }),
             _ => {
                 let count = (size + 0xFFF) / 0x1000;
-                let list = PageBox::new_slice_with(|i| base.add((i + 1) * 0x1000), count - 1)
-                    .map_err(NvmeError::MemoryError)?;
+                let list =
+                    DmaBuffer::new_slice_with(dma, |i| base.add((i + 1) * 0x1000), count - 1)
+                        .map_err(NvmeError::MemoryError)?;
 
                 Ok(Self {
                     prp1: PhysicalRegionPage::with_addr(base),
-                    prp2: PhysicalRegionPage::with_addr(unsafe { list.as_physical_address() }),
+                    prp2: PhysicalRegionPage::with_addr(list.bus_address()),
                     list: Some(list),
                 })
             }
@@ -146,7 +151,7 @@ impl PhysicalRegionPage {
         Self(0)
     }
 
-    pub const fn with_addr(address: PhysicalAddress) -> Self {
+    pub const fn with_addr(address: BusAddress) -> Self {
         Self(address.into_u64())
     }
 }
@@ -197,7 +202,7 @@ impl CompletionQueueEntry {
 
 impl<T> Queue<T> {
     pub fn new(
-        data: PageBox<[T]>,
+        data: DmaBuffer<[T]>,
         head_doorbell: *mut u32,
         tail_doorbell: *mut u32,
         phase: bool,
@@ -250,10 +255,12 @@ impl<T> Queue<T> {
         self.tail = new_tail;
 
         if !self.tail_doorbell.is_null() {
+            self.data.cache_flush_element(self.tail, true);
             unsafe {
                 self.tail_doorbell
                     .write_volatile(self.tail.try_into().unwrap());
             }
+            ArchitectureImpl::memory_barrier();
         }
 
         wrapped
@@ -277,17 +284,18 @@ impl<T> Queue<T> {
 
 impl QueuePair {
     pub fn new(
+        dma: &dyn DmaAllocator,
         id: u32,
         vector: usize,
         capacity: usize,
         sq_doorbell: *mut u32,
         cq_doorbell: *mut u32,
     ) -> Result<Self, Error> {
-        let sq_data = PageBox::new_slice(SubmissionQueueEntry::zeroed(), capacity)?;
-        let cq_data = PageBox::new_slice(CompletionQueueEntry::zeroed(), capacity)?;
+        let sq_data = DmaBuffer::new_slice(dma, SubmissionQueueEntry::zeroed(), capacity)?;
+        let cq_data = DmaBuffer::new_slice(dma, CompletionQueueEntry::zeroed(), capacity)?;
 
-        let sq_base = unsafe { sq_data.as_physical_address() };
-        let cq_base = unsafe { cq_data.as_physical_address() };
+        let sq_base = sq_data.bus_address();
+        let cq_base = cq_data.bus_address();
 
         log::debug!("Allocated queue pair: sq={:p}, cq={:p}", sq_data, cq_data);
 
@@ -313,12 +321,12 @@ impl QueuePair {
     }
 
     #[inline]
-    pub fn sq_physical_pointer(&self) -> PhysicalAddress {
+    pub fn sq_bus_pointer(&self) -> BusAddress {
         self.sq_base
     }
 
     #[inline]
-    pub fn cq_physical_pointer(&self) -> PhysicalAddress {
+    pub fn cq_bus_pointer(&self) -> BusAddress {
         self.cq_base
     }
 
@@ -385,16 +393,17 @@ impl QueuePair {
 
     pub async fn request<'r, R: Request>(
         &'r self,
+        dma: &dyn DmaAllocator,
         req: R,
-    ) -> Result<PageBox<R::Response>, NvmeError>
+    ) -> Result<DmaBuffer<R::Response>, NvmeError>
     where
         R::Response: 'r,
     {
-        let response = PageBox::new_uninit().map_err(NvmeError::MemoryError)?;
-        let list = PrpList::from_buffer(unsafe { response.as_physical_address() }, size_of::<R>())?;
+        let response = DmaBuffer::new_uninit(dma).map_err(NvmeError::MemoryError)?;
+        let list = PrpList::from_buffer(dma, response.bus_address(), size_of::<R>())?;
         let command_id = self.submit(req, &list, true)?;
         let result = self.wait_for_completion(command_id, response).await?;
-        Ok(unsafe { result.assume_init() })
+        Ok(unsafe { DmaBuffer::assume_init(result) })
     }
 
     pub fn process_completions(&self) -> usize {

kernel/driver/block/nvme/src/regs.rs

@@ -0,0 +1,150 @@
+use libk::{dma::BusAddress, error::Error};
+use tock_registers::{
+    interfaces::{ReadWriteable, Readable, Writeable},
+    register_bitfields, register_structs,
+    registers::{ReadOnly, ReadWrite, WriteOnly},
+};
+
+use crate::queue::{CompletionQueueEntry, SubmissionQueueEntry};
+
+register_bitfields! {
+    u32,
+    pub CC [
+        IOCQES OFFSET(20) NUMBITS(4) [],
+        IOSQES OFFSET(16) NUMBITS(4) [],
+        AMS OFFSET(11) NUMBITS(3) [],
+        MPS OFFSET(7) NUMBITS(4) [],
+        CSS OFFSET(4) NUMBITS(3) [
+            NvmCommandSet = 0
+        ],
+        ENABLE OFFSET(0) NUMBITS(1) [],
+    ],
+    pub CSTS [
+        CFS OFFSET(1) NUMBITS(1) [],
+        RDY OFFSET(0) NUMBITS(1) [],
+    ],
+    pub AQA [
+        /// Admin Completion Queue Size in entries - 1
+        ACQS OFFSET(16) NUMBITS(12) [],
+        /// Admin Submission Queue Size in entries - 1
+        ASQS OFFSET(0) NUMBITS(12) [],
+    ]
+}
+
+register_bitfields! {
+    u64,
+    pub CAP [
+        /// Maximum Queue Entries Supported - 1. i.e., 0 means maximum queue len of 1, 1 = 2 etc.
+        MQES OFFSET(0) NUMBITS(16) [],
+        /// Timeout. Represents the worst-case time the host software should wait for CSTS.RDY to
+        /// change its state.
+        TO OFFSET(24) NUMBITS(8) [],
+        /// Doorbell stride. Stride in bytes = pow(2, 2 + DSTRD).
+        DSTRD OFFSET(32) NUMBITS(4) [],
+        /// NVM Subsystem Reset Supported (see NVMe BS Section 3.7.1)
+        NSSRS OFFSET(36) NUMBITS(1) [],
+        /// Controller supports one or more I/O command sets
+        CSS_IO_COMMANDS OFFSET(43) NUMBITS(1) [],
+        /// Controller only supports admin commands and no I/O commands
+        CSS_ADMIN_ONLY OFFSET(44) NUMBITS(1) [],
+        /// Memory page size minimum (bytes = pow(2, 12 + MPSMIN))
+        MPSMIN OFFSET(48) NUMBITS(4) [],
+        /// Memory page size maximum -|-
+        MPSMAX OFFSET(52) NUMBITS(4) [],
+    ]
+}
+
+register_structs! {
+    #[allow(non_snake_case)]
+    pub Regs {
+        (0x00 => pub CAP: ReadOnly<u64, CAP::Register>),
+        (0x08 => pub VS: ReadOnly<u32>),
+        (0x0C => pub INTMS: WriteOnly<u32>),
+        (0x10 => pub INTMC: WriteOnly<u32>),
+        (0x14 => pub CC: ReadWrite<u32, CC::Register>),
+        (0x18 => _0),
+        (0x1C => pub CSTS: ReadOnly<u32, CSTS::Register>),
+        (0x20 => _1),
+        (0x24 => AQA: ReadWrite<u32, AQA::Register>),
+        (0x28 => ASQ: ReadWrite<u64>),
+        (0x30 => ACQ: ReadWrite<u64>),
+        (0x38 => _2),
+        (0x2000 => @END),
+    }
+}
+
+impl Regs {
+    pub fn configure_admin_queue(
+        &self,
+        submission_queue_pointer: BusAddress,
+        completion_queue_pointer: BusAddress,
+        submission_queue_size: usize,
+        completion_queue_size: usize,
+    ) -> Result<(), Error> {
+        let max_queue_size = self.CAP.read(CAP::MQES) + 1;
+        if submission_queue_size as u64 > max_queue_size {
+            log::error!("admin submission queue too large");
+            return Err(Error::InvalidArgument);
+        }
+        if completion_queue_size as u64 > max_queue_size {
+            log::error!("admin completion queue too large");
+            return Err(Error::InvalidArgument);
+        }
+
+        self.AQA.write(
+            AQA::ASQS.val(submission_queue_size as u32 - 1)
+                + AQA::ACQS.val(completion_queue_size as u32 - 1),
+        );
+        self.ASQ.set(submission_queue_pointer.into_u64());
+        self.ACQ.set(completion_queue_pointer.into_u64());
+
+        Ok(())
+    }
+
+    pub fn configure_controller(&self) {
+        const IOSQES: u32 = size_of::<SubmissionQueueEntry>().ilog2();
+        const IOCQES: u32 = size_of::<CompletionQueueEntry>().ilog2();
+
+        self.CC.modify(
+            CC::IOCQES.val(IOCQES)
+                + CC::IOSQES.val(IOSQES)
+                + CC::MPS.val(0)
+                + CC::CSS::NvmCommandSet,
+        );
+    }
+
+    pub fn enable_controller(&self, mut timeout_cycles: u64) -> Result<(), Error> {
+        self.CC.modify(CC::ENABLE::SET);
+
+        while timeout_cycles > 0 && !self.CSTS.matches_any(&[CSTS::RDY::SET, CSTS::CFS::SET]) {
+            timeout_cycles -= 1;
+            core::hint::spin_loop();
+        }
+
+        if timeout_cycles == 0 {
+            return Err(Error::TimedOut);
+        }
+
+        if self.CSTS.matches_all(CSTS::CFS::SET) {
+            log::error!("nvme: controller fatal status after enable");
+            return Err(Error::InvalidArgument);
+        }
+
+        Ok(())
+    }
+
+    pub fn disable_controller(&self, mut timeout_cycles: u64) -> Result<(), Error> {
+        self.CC.modify(CC::ENABLE::CLEAR);
+
+        while timeout_cycles > 0 && self.CSTS.matches_all(CSTS::RDY::SET) {
+            timeout_cycles -= 1;
+            core::hint::spin_loop();
+        }
+
+        if timeout_cycles > 0 {
+            Ok(())
+        } else {
+            Err(Error::TimedOut)
+        }
+    }
+}

kernel/driver/block/scsi/Cargo.toml

@@ -0,0 +1,14 @@
+[package]
+name = "ygg_driver_scsi"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+yggdrasil-abi.workspace = true
+device-api.workspace = true
+libk-util.workspace = true
+libk-mm.workspace = true
+libk.workspace = true
+
+async-trait.workspace = true
+log.workspace = true

kernel/driver/block/scsi/src/command.rs

@@ -0,0 +1,102 @@
+use libk::error::Error;
+
+use crate::device::ScsiDeviceType;
+
+pub trait ScsiCommand {
+    type Response;
+    const REQUEST_LEN: usize;
+    const RESPONSE_LEN: usize;
+
+    fn into_bytes(self) -> [u8; Self::REQUEST_LEN];
+    fn parse_response(bytes: &[u8]) -> Result<Self::Response, Error>;
+}
+
+// Add more info when needed
+pub struct ScsiInquiry;
+#[derive(Debug)]
+pub struct ScsiInquiryResponse {
+    pub device_type: ScsiDeviceType,
+}
+
+impl ScsiCommand for ScsiInquiry {
+    type Response = ScsiInquiryResponse;
+    const REQUEST_LEN: usize = 6;
+    const RESPONSE_LEN: usize = 36;
+
+    fn into_bytes(self) -> [u8; Self::REQUEST_LEN] {
+        [0x12, 0x00, 0x00, 0x00, 0x00, 0x00]
+    }
+
+    fn parse_response(bytes: &[u8]) -> Result<Self::Response, Error> {
+        if bytes.len() != 36 {
+            return Err(Error::InvalidArgument);
+        }
+        let device_type = ScsiDeviceType::try_from(bytes[0] & 0x1F).unwrap_or_default();
+        Ok(ScsiInquiryResponse { device_type })
+    }
+}
+
+pub struct ScsiTestUnitReady;
+#[derive(Debug)]
+pub struct ScsiTestUnitReadyResponse;
+
+impl ScsiCommand for ScsiTestUnitReady {
+    type Response = ScsiTestUnitReadyResponse;
+    const RESPONSE_LEN: usize = 0;
+    const REQUEST_LEN: usize = 6;
+
+    fn into_bytes(self) -> [u8; Self::REQUEST_LEN] {
+        [0x00, 0x00, 0x00, 0x00, 0x00, 0x00]
+    }
+
+    fn parse_response(_bytes: &[u8]) -> Result<Self::Response, Error> {
+        Ok(ScsiTestUnitReadyResponse)
+    }
+}
+
+pub struct ScsiRequestSense;
+#[derive(Debug)]
+pub struct ScsiRequestSenseResponse;
+
+impl ScsiCommand for ScsiRequestSense {
+    type Response = ScsiRequestSenseResponse;
+    const RESPONSE_LEN: usize = 0;
+    const REQUEST_LEN: usize = 6;
+
+    fn into_bytes(self) -> [u8; Self::REQUEST_LEN] {
+        [0x03, 0x00, 0x00, 0x00, 0x00, 0x00]
+    }
+
+    fn parse_response(_bytes: &[u8]) -> Result<Self::Response, Error> {
+        Ok(ScsiRequestSenseResponse)
+    }
+}
+
+pub struct ScsiReadCapacity;
+#[derive(Debug)]
+pub struct ScsiReadCapacityResponse {
+    pub block_size: u32,
+    pub block_count: u32,
+}
+
+impl ScsiCommand for ScsiReadCapacity {
+    type Response = ScsiReadCapacityResponse;
+    const REQUEST_LEN: usize = 10;
+    const RESPONSE_LEN: usize = 8;
+
+    fn into_bytes(self) -> [u8; Self::REQUEST_LEN] {
+        [0x25, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]
+    }
+
+    fn parse_response(bytes: &[u8]) -> Result<Self::Response, Error> {
+        if bytes.len() != 8 {
+            return Err(Error::InvalidArgument);
+        }
+        let block_count = u32::from_be_bytes(bytes[0..4].try_into().unwrap());
+        let block_size = u32::from_be_bytes(bytes[4..8].try_into().unwrap());
+        Ok(ScsiReadCapacityResponse {
+            block_size,
+            block_count,
+        })
+    }
+}

kernel/driver/block/scsi/src/device.rs

@@ -0,0 +1,24 @@
+use yggdrasil_abi::primitive_enum;
+
+primitive_enum! {
+    #[derive(Default)]
+    pub enum ScsiDeviceType: u8 {
+        DirectAccessBlock = 0x00,
+        SequentialAccess = 0x01,
+        Printer = 0x02,
+        Processor = 0x03,
+        WriteOnce = 0x04,
+        CdDvd = 0x05,
+        OpticalMemory = 0x07,
+        MediumChanger = 0x08,
+        StorageArrayController = 0x0C,
+        EnclosureServices = 0x0D,
+        SimplifiedDirectAccess = 0x0E,
+        OpticalCard = 0x0F,
+        BridgeController = 0x10,
+        ObjectBasedStorage = 0x11,
+        AutomationDriveInterface = 0x12,
+        #[default]
+        Other = 0x1F,
+    }
+}

kernel/driver/block/scsi/src/lib.rs

@@ -0,0 +1,370 @@
+#![feature(generic_const_exprs, maybe_uninit_slice)]
+#![allow(incomplete_features)]
+#![no_std]
+
+use core::{
+    mem::MaybeUninit,
+    sync::atomic::{AtomicBool, Ordering},
+    time::Duration,
+};
+
+use alloc::{
+    boxed::Box, collections::btree_map::BTreeMap, format, string::String, sync::Arc, vec::Vec,
+};
+use async_trait::async_trait;
+use command::{ScsiReadCapacity, ScsiRequestSense, ScsiTestUnitReady};
+use device_api::device::Device;
+use libk::{
+    block,
+    device::{block::BlockDevice, manager::probe_partitions},
+    dma::{DmaBuffer, DmaSlice, DmaSliceMut},
+    error::Error,
+    fs::devfs,
+    task::{runtime, sync::AsyncMutex},
+};
+use libk_mm::{address::PhysicalAddress, table::MapAttributes, PageProvider};
+use libk_util::{
+    sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock},
+    OneTimeInit,
+};
+use transport::{ScsiTransport, ScsiTransportWrapper};
+use yggdrasil_abi::io::FileMode;
+
+extern crate alloc;
+
+pub mod command;
+pub mod device;
+pub mod transport;
+
+pub struct ScsiEnclosure {
+    transport: AsyncMutex<ScsiTransportWrapper>,
+    units: Vec<IrqSafeRwLock<Option<Arc<ScsiUnit>>>>,
+    index: OneTimeInit<u32>,
+    shutdown: AtomicBool,
+}
+
+pub struct ScsiUnit {
+    enclosure: Arc<ScsiEnclosure>,
+    lun: u8,
+    lba_count: u64,
+    lba_size: usize,
+    max_lba_per_request: usize,
+    names: IrqSafeRwLock<Vec<String>>,
+}
+
+impl ScsiEnclosure {
+    pub async fn setup(
+        transport: Box<dyn ScsiTransport>,
+        lun_count: usize,
+    ) -> Result<Arc<Self>, Error> {
+        let transport = AsyncMutex::new(ScsiTransportWrapper::new(transport));
+        let units = (0..lun_count).map(|_| IrqSafeRwLock::new(None)).collect();
+        let this = Arc::new(Self {
+            transport,
+            units,
+            index: OneTimeInit::new(),
+            shutdown: AtomicBool::new(false),
+        });
+        register_enclosure(this.clone())?;
+
+        // Probe LUNs
+        for i in 0..lun_count {
+            if this.probe_lun(i as u8).await {
+                if let Ok(unit) = ScsiUnit::setup(this.clone(), i as u8).await {
+                    *this.units[i].write() = Some(unit);
+                }
+            }
+        }
+
+        // Start enclosure poll task
+        let enclosure = this.clone();
+        runtime::spawn(async move {
+            while !enclosure.shutdown.load(Ordering::Acquire) {
+                enclosure.poll().await;
+                runtime::sleep(Duration::from_millis(100)).await;
+            }
+        })
+        .ok();
+
+        Ok(this)
+    }
+
+    async fn probe_lun(self: &Arc<Self>, lun: u8) -> bool {
+        let mut attempts = 3;
+        let mut timeout = 10;
+        // TODO get statuses to better see if there's a real error or the LUN is not present
+        while attempts > 0 {
+            let mut transport = self.transport.lock().await;
+
+            // TEST UNIT READY (6)
+            if transport
+                .perform_command(lun, ScsiTestUnitReady)
+                .await
+                .is_ok()
+            {
+                break;
+            }
+
+            // If not, send a REQUEST SENSE (6)
+            transport.perform_command(lun, ScsiRequestSense).await.ok();
+
+            drop(transport);
+
+            runtime::sleep(Duration::from_millis(timeout)).await;
+            timeout *= 2;
+            attempts -= 1;
+        }
+
+        if attempts == 0 {
+            false
+        } else {
+            true
+        }
+    }
+
+    async fn poll(self: &Arc<Self>) {
+        let index = *self.index.get();
+        for lun in 0..self.units.len() {
+            let mut slot = self.units[lun].write();
+            let present = self.probe_lun(lun as u8).await;
+
+            if let Some(unit) = slot.as_ref() {
+                if !present {
+                    log::warn!("scsi{index}u{lun} lost");
+                    unit.detach();
+                    *slot = None;
+                }
+            } else if present {
+                if let Ok(unit) = ScsiUnit::setup(self.clone(), lun as u8).await {
+                    log::info!("scsi{index}u{lun} attached");
+                    *slot = Some(unit);
+                } else {
+                    log::warn!("scsi{index}u{lun} attached, but could not setup");
+                }
+            }
+        }
+    }
+
+    pub fn detach(&self) {
+        self.shutdown.store(true, Ordering::Release);
+        let index = self.index.try_get().copied();
+
+        for unit in self.units.iter() {
+            if let Some(unit) = unit.write().take() {
+                unit.detach();
+            }
+        }
+
+        // Deregister the enclosure
+        if let Some(index) = index {
+            remove_enclosure(index);
+        }
+    }
+}
+
+impl ScsiUnit {
+    pub async fn setup(enclosure: Arc<ScsiEnclosure>, lun: u8) -> Result<Arc<Self>, Error> {
+        let enclosure_index = *enclosure.index.get();
+        let mut transport = enclosure.transport.lock().await;
+
+        // TODO INQUIRY fails for real USB flash drives
+        // transport.perform_command(0, ScsiInquiry).await?;
+
+        let capacity_info = transport.perform_command(lun, ScsiReadCapacity).await?;
+        let max_lba_per_request =
+            transport.max_bytes_per_request() / capacity_info.block_size as usize;
+        log::info!(
+            "scsi{enclosure_index}u{lun}: lba_size={}, lba_count={}, max_lba_per_request={}",
+            capacity_info.block_size,
+            capacity_info.block_count,
+            max_lba_per_request
+        );
+
+        drop(transport);
+
+        let unit = Arc::new(Self {
+            enclosure,
+            lun,
+            lba_count: capacity_info.block_count.into(),
+            lba_size: capacity_info.block_size as usize,
+            max_lba_per_request,
+            names: IrqSafeRwLock::new(Vec::new()),
+        });
+
+        register_unit(enclosure_index, lun, unit.clone());
+
+        Ok(unit)
+    }
+
+    fn detach(&self) {
+        let id = *self.enclosure.index.get();
+        log::info!("scsi{id}u{} detached", self.lun);
+        for name in self.names.read().iter() {
+            devfs::remove_node(name).ok();
+        }
+    }
+}
+
+#[async_trait]
+impl BlockDevice for ScsiUnit {
+    fn allocate_buffer(&self, size: usize) -> Result<DmaBuffer<[MaybeUninit<u8>]>, Error> {
+        block!(self.enclosure.transport.lock().await.allocate_buffer(size))?
+    }
+
+    async fn read_aligned(
+        &self,
+        position: u64,
+        buffer: DmaSliceMut<'_, MaybeUninit<u8>>,
+    ) -> Result<(), Error> {
+        if position % self.lba_size as u64 != 0 {
+            log::warn!("scsi: misaligned read");
+            return Err(Error::InvalidArgument);
+        }
+        if buffer.len() % self.lba_size != 0 {
+            log::warn!("scsi: misaligned buffer size");
+            return Err(Error::InvalidArgument);
+        }
+        let lba_start = position / self.lba_size as u64;
+        let lba_count = buffer.len() / self.lba_size;
+
+        if lba_start.saturating_add(lba_count as u64) >= self.lba_count {
+            log::warn!("scsi: read beyond medium end");
+            return Err(Error::InvalidArgument);
+        }
+
+        let lba_end = lba_start + lba_count as u64;
+
+        let mut transport = self.enclosure.transport.lock().await;
+
+        // TODO DmaSliceMut subslicing
+        let (buffer, range) = buffer.into_parts();
+        let mut offset = range.start;
+
+        for i in (0..lba_count).step_by(self.max_lba_per_request) {
+            let lba = lba_start + i as u64;
+            let end = (lba + self.max_lba_per_request as u64).min(lba_end);
+            let count = (end - lba) as usize;
+            let amount = count * self.lba_size;
+
+            let dst_slice = buffer.slice_mut(offset..offset + amount);
+            let len = transport
+                .read(self.lun, lba, count as u16, dst_slice)
+                .await?;
+            if len != amount {
+                return Err(Error::InvalidArgument);
+            }
+
+            offset += amount;
+        }
+
+        Ok(())
+    }
+
+    async fn write_aligned(&self, _position: u64, _buffer: DmaSlice<'_, u8>) -> Result<(), Error> {
+        Err(Error::NotImplemented)
+    }
+
+    fn block_size(&self) -> usize {
+        self.lba_size
+    }
+
+    fn block_count(&self) -> u64 {
+        self.lba_count
+    }
+
+    fn max_blocks_per_request(&self) -> usize {
+        self.max_lba_per_request
+    }
+}
+
+impl PageProvider for ScsiUnit {
+    fn get_page(&self, _offset: u64) -> Result<PhysicalAddress, Error> {
+        Err(Error::NotImplemented)
+    }
+
+    fn release_page(&self, _offset: u64, _phys: PhysicalAddress) -> Result<(), Error> {
+        Err(Error::NotImplemented)
+    }
+
+    fn clone_page(
+        &self,
+        _offset: u64,
+        _src_phys: PhysicalAddress,
+        _src_attrs: MapAttributes,
+    ) -> Result<PhysicalAddress, Error> {
+        Err(Error::NotImplemented)
+    }
+}
+
+impl Device for ScsiUnit {
+    fn display_name(&self) -> &str {
+        "SCSI Unit"
+    }
+}
+
+impl Drop for ScsiUnit {
+    fn drop(&mut self) {
+        if let Some(index) = self.enclosure.index.try_get() {
+            log::info!("scsi{index}u{} dropped", self.lun);
+        }
+    }
+}
+
+// TODO this is crap
+static SCSI_ENCLOSURES: IrqSafeSpinlock<BTreeMap<u32, Arc<ScsiEnclosure>>> =
+    IrqSafeSpinlock::new(BTreeMap::new());
+static SCSI_BITMAP: IrqSafeSpinlock<u32> = IrqSafeSpinlock::new(0);
+
+fn register_enclosure(enclosure: Arc<ScsiEnclosure>) -> Result<(), Error> {
+    let index = {
+        let mut bitmap = SCSI_BITMAP.lock();
+        let index = (0..8)
+            .position(|p| *bitmap & (1 << p) == 0)
+            .ok_or(Error::InvalidOperation)
+            .inspect_err(|_| log::warn!("Cannot attach SCSI enclosure: too many of them"))?
+            as u32;
+        let mut devices = SCSI_ENCLOSURES.lock();
+        *bitmap |= 1 << index;
+        assert!(!devices.contains_key(&index));
+        devices.insert(index, enclosure.clone());
+        index
+    };
+
+    enclosure.index.init(index);
+
+    Ok(())
+}
+
+fn register_unit(enclosure_index: u32, lun: u8, unit: Arc<ScsiUnit>) {
+    let name = format!("scsi{enclosure_index}u{lun}");
+    unit.names.write().push(name.clone());
+    devfs::add_named_block_device(unit.clone(), name.clone(), FileMode::new(0o600)).ok();
+
+    // TODO this code is repeated everywhere
+    runtime::spawn(async move {
+        let name = name;
+        probe_partitions(unit.clone(), |index, partition| {
+            let partition_name = format!("{name}p{}", index + 1);
+            log::info!("{name}: partition {partition_name}");
+            unit.names.write().push(partition_name.clone());
+            devfs::add_named_block_device(
+                Arc::new(partition),
+                partition_name,
+                FileMode::new(0o600),
+            )
+            .ok();
+        })
+        .await
+        .ok();
+    })
+    .ok();
+}
+
+fn remove_enclosure(index: u32) {
+    let mut devices = SCSI_ENCLOSURES.lock();
+    let mut bitmap = SCSI_BITMAP.lock();
+
+    *bitmap &= !(1 << index);
+    devices.remove(&index);
+    log::info!("scsi: enclosure {index} detached");
+}

kernel/driver/block/scsi/src/transport.rs

@@ -0,0 +1,100 @@
+use core::{mem::MaybeUninit, ops::Deref};
+
+use alloc::boxed::Box;
+use async_trait::async_trait;
+use libk::{
+    dma::{DmaBuffer, DmaSliceMut},
+    error::Error,
+};
+
+use crate::command::ScsiCommand;
+
+#[async_trait]
+pub trait ScsiTransport: Send + Sync {
+    fn allocate_buffer(&self, size: usize) -> Result<DmaBuffer<[MaybeUninit<u8>]>, Error>;
+
+    /// Perform a no-data request
+    async fn perform_request_raw(
+        &mut self,
+        lun: u8,
+        request_data: &[u8],
+        response_buffer: DmaSliceMut<'_, MaybeUninit<u8>>,
+    ) -> Result<usize, Error>;
+
+    fn max_bytes_per_request(&self) -> usize;
+}
+
+pub struct ScsiTransportWrapper {
+    inner: Box<dyn ScsiTransport>,
+}
+
+impl ScsiTransportWrapper {
+    pub fn new(inner: Box<dyn ScsiTransport>) -> Self {
+        Self { inner }
+    }
+
+    pub async fn read(
+        &mut self,
+        lun: u8,
+        lba: u64,
+        lba_count: u16,
+        buffer: DmaSliceMut<'_, MaybeUninit<u8>>,
+    ) -> Result<usize, Error> {
+        if lba >= u32::MAX as u64 {
+            return Err(Error::InvalidArgument);
+        }
+        let lba_bytes = (lba as u32).to_be_bytes();
+        let lba_count = (lba_count as u16).to_be_bytes();
+        // Issue a READ (10) command
+        let request_buffer = [
+            0x28,
+            0x00,
+            lba_bytes[0],
+            lba_bytes[1],
+            lba_bytes[2],
+            lba_bytes[3],
+            0x00,
+            lba_count[0],
+            lba_count[1],
+            0x00,
+        ];
+
+        self.inner
+            .perform_request_raw(lun, &request_buffer, buffer)
+            .await
+    }
+
+    pub async fn perform_command<R: ScsiCommand>(
+        &mut self,
+        lun: u8,
+        request: R,
+    ) -> Result<R::Response, Error>
+    where
+        [u8; R::RESPONSE_LEN]: Sized,
+        [u8; R::REQUEST_LEN]: Sized,
+    {
+        let mut response_buffer = self.allocate_buffer(R::RESPONSE_LEN)?;
+        let request_buffer = request.into_bytes();
+
+        let response_len = self
+            .inner
+            .perform_request_raw(
+                lun,
+                &request_buffer,
+                response_buffer.slice_mut(0..R::RESPONSE_LEN),
+            )
+            .await?;
+        let response_bytes =
+            unsafe { MaybeUninit::slice_assume_init_ref(&response_buffer[..response_len]) };
+
+        R::parse_response(response_bytes)
+    }
+}
+
+impl Deref for ScsiTransportWrapper {
+    type Target = dyn ScsiTransport;
+
+    fn deref(&self) -> &Self::Target {
+        self.inner.as_ref()
+    }
+}

kernel/driver/bus/pci/Cargo.toml

@@ -16,7 +16,9 @@ bitflags.workspace = true
 tock-registers.workspace = true
 
 [target.'cfg(target_arch = "x86_64")'.dependencies]
+ygg_driver_acpi.path = "../../acpi"
 acpi.workspace = true
+kernel-arch-x86.workspace = true
 
 [lints]
 workspace = true

kernel/driver/bus/pci/src/capability.rs

@@ -1,6 +1,7 @@
 //! PCI capability structures and queries
 
 use alloc::{sync::Arc, vec, vec::Vec};
+use bitflags::bitflags;
 use device_api::interrupt::{
     InterruptAffinity, InterruptHandler, MessageInterruptController, MsiInfo,
 };
@@ -11,8 +12,25 @@ use tock_registers::{
 };
 use yggdrasil_abi::error::Error;
 
+use crate::PciBaseAddress;
+
 use super::{PciCapability, PciCapabilityId, PciConfigurationSpace};
 
+bitflags! {
+    pub struct PcieLinkControl: u16 {
+        const ASPM_DISABLE = 0 << 0;
+        // Active state power management control
+        const ASPM_MASK = 0x3 << 0;
+        // Enable clock power management
+        const ECPM = 1 << 8;
+    }
+}
+
+#[cfg(any(target_arch = "x86", target_arch = "x86_64", rust_analyzer))]
+use core::mem::offset_of;
+#[cfg(any(target_arch = "x86", target_arch = "x86_64", rust_analyzer))]
+use kernel_arch_x86::intrinsics;
+
 pub trait VirtioCapabilityData<'s, S: PciConfigurationSpace + ?Sized + 's>: Sized {
     fn from_space_offset(space: &'s S, offset: usize) -> Self;
 
@@ -41,11 +59,14 @@ pub trait VirtioCapability {
     type Output<'a, S: PciConfigurationSpace + ?Sized + 'a>: VirtioCapabilityData<'a, S>;
 }
 
+/// Power management capability entry
+pub struct PowerManagementCapability;
 /// MSI-X capability query
 pub struct MsiXCapability;
-
 /// MSI capability query
 pub struct MsiCapability;
+/// PCIe capability
+pub struct PciExpressCapability;
 
 // VirtIO-over-PCI capabilities
 /// VirtIO PCI configuration access
@@ -57,6 +78,15 @@ pub struct VirtioNotifyConfigCapability;
 /// VirtIO interrupt status
 pub struct VirtioInterruptStatusCapability;
 
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum DevicePowerState {
+    D0,
+    D1,
+    D2,
+    D3Cold,
+    D3Hot,
+}
+
 /// Represents an entry in MSI-X vector table
 #[repr(C)]
 pub struct MsiXEntry {
@@ -68,8 +98,21 @@ pub struct MsiXEntry {
     pub control: ReadWrite<u32>,
 }
 
+enum MsiXVectorTableAccess<'a> {
+    Memory(DeviceMemoryIoMut<'a, [MsiXEntry]>),
+    #[cfg(any(target_arch = "x86", target_arch = "x86_64", rust_analyzer))]
+    Io(u16),
+}
+
 pub struct MsiXVectorTable<'a> {
-    vectors: DeviceMemoryIoMut<'a, [MsiXEntry]>,
+    access: MsiXVectorTableAccess<'a>,
+    len: usize,
+}
+
+/// PCI Power Management capability data structure
+pub struct PowerManagementData<'s, S: PciConfigurationSpace + ?Sized + 's> {
+    space: &'s S,
+    offset: usize,
 }
 
 /// MSI-X capability data structure
@@ -84,6 +127,12 @@ pub struct MsiData<'s, S: PciConfigurationSpace + ?Sized + 's> {
     offset: usize,
 }
 
+/// PCI Express capability data structure
+pub struct PcieData<'s, S: PciConfigurationSpace + ?Sized + 's> {
+    space: &'s S,
+    offset: usize,
+}
+
 pub struct VirtioDeviceConfigData<'s, S: PciConfigurationSpace + ?Sized + 's> {
     space: &'s S,
     offset: usize,
@@ -122,6 +171,19 @@ impl<T: VirtioCapability> PciCapability for T {
     }
 }
 
+impl PciCapability for PowerManagementCapability {
+    const ID: PciCapabilityId = PciCapabilityId::PowerManagement;
+    type CapabilityData<'a, S: PciConfigurationSpace + ?Sized + 'a> = PowerManagementData<'a, S>;
+
+    fn data<'s, S: PciConfigurationSpace + ?Sized + 's>(
+        space: &'s S,
+        offset: usize,
+        _len: usize,
+    ) -> Self::CapabilityData<'s, S> {
+        PowerManagementData { space, offset }
+    }
+}
+
 impl PciCapability for MsiXCapability {
     const ID: PciCapabilityId = PciCapabilityId::MsiX;
     type CapabilityData<'a, S: PciConfigurationSpace + ?Sized + 'a> = MsiXData<'a, S>;
@@ -148,6 +210,19 @@ impl PciCapability for MsiCapability {
     }
 }
 
+impl PciCapability for PciExpressCapability {
+    const ID: PciCapabilityId = PciCapabilityId::PciExpress;
+    type CapabilityData<'a, S: PciConfigurationSpace + ?Sized + 'a> = PcieData<'a, S>;
+
+    fn data<'s, S: PciConfigurationSpace + ?Sized + 's>(
+        space: &'s S,
+        offset: usize,
+        _len: usize,
+    ) -> Self::CapabilityData<'s, S> {
+        PcieData { space, offset }
+    }
+}
+
 impl VirtioCapability for VirtioDeviceConfigCapability {
     const CFG_TYPE: u8 = 0x04;
     type Output<'a, S: PciConfigurationSpace + ?Sized + 'a> = VirtioDeviceConfigData<'a, S>;
@@ -246,6 +321,56 @@ impl<'s, S: PciConfigurationSpace + ?Sized + 's> VirtioCapabilityData<'s, S>
     }
 }
 
+impl<'s, S: PciConfigurationSpace + ?Sized + 's> PowerManagementData<'s, S> {
+    pub fn set_device_power_state(&self, state: DevicePowerState) {
+        let pmcsr = self.space.read_u16(self.offset + 4) & !0x3;
+        let current = self.get_device_power_state();
+
+        if state == current {
+            return;
+        }
+
+        log::info!("Set device power state: {state:?}");
+
+        match state {
+            DevicePowerState::D0 => {
+                // power = 0b00 | PME_EN
+                self.space.write_u16(self.offset + 4, pmcsr);
+            }
+            _ => {
+                log::warn!("TODO: {state:?} power state");
+            }
+        }
+    }
+
+    pub fn set_pme_en(&self, state: bool) {
+        let pmcsr = self.space.read_u16(self.offset + 4);
+        let new = if state {
+            pmcsr | (1 << 8)
+        } else {
+            pmcsr & !(1 << 8)
+        };
+        if pmcsr == new {
+            return;
+        }
+
+        log::info!("Set PMCSR.PME_En = {state}");
+
+        self.space.write_u16(self.offset + 4, new);
+    }
+
+    pub fn get_device_power_state(&self) -> DevicePowerState {
+        let pmcsr = self.space.read_u16(self.offset + 4);
+        match pmcsr & 0x3 {
+            0b00 => DevicePowerState::D0,
+            0b01 => DevicePowerState::D1,
+            0b10 => DevicePowerState::D2,
+            0b11 => DevicePowerState::D3Hot,
+            _ => unreachable!(),
+        }
+    }
+}
+
 impl<'s, S: PciConfigurationSpace + ?Sized + 's> MsiXData<'s, S> {
     // TODO use pending bits as well
     /// Maps and returns the vector table associated with the device's MSI-X capability
@@ -260,13 +385,30 @@ impl<'s, S: PciConfigurationSpace + ?Sized + 's> MsiXData<'s, S> {
         let Some(base) = self.space.bar(bir) else {
             return Err(Error::DoesNotExist);
         };
-        let Some(base) = base.as_memory() else {
-            return Err(Error::InvalidOperation);
-        };
 
-        log::debug!("MSI-X table address: {:#x}", base.add(table_offset));
-
-        unsafe { MsiXVectorTable::from_raw_parts(base.add(table_offset), table_size) }
+        match base {
+            PciBaseAddress::Memory32(mem32) => unsafe {
+                log::info!("MSI-X table address: {:#x}", mem32 + table_offset as u32);
+                MsiXVectorTable::memory_from_raw_parts(
+                    PhysicalAddress::from_u32(mem32).add(table_offset),
+                    table_size,
+                )
+            },
+            PciBaseAddress::Memory64(mem64) => unsafe {
+                log::info!("MSI-X table address: {:#x}", mem64 + table_offset as u64);
+                MsiXVectorTable::memory_from_raw_parts(
+                    PhysicalAddress::from_u64(mem64).add(table_offset),
+                    table_size,
+                )
+            },
+            #[cfg(any(target_arch = "x86", target_arch = "x86_64", rust_analyzer))]
+            PciBaseAddress::Io(io) => unsafe {
+                log::info!("MSI-X table I/O: {:#x}", io + table_offset as u16);
+                MsiXVectorTable::io_from_raw_parts(io + table_offset as u16, table_size)
+            },
+            #[cfg(any(not(any(target_arch = "x86", target_arch = "x86_64")), rust_analyzer))]
+            PciBaseAddress::Io(_) => Err(Error::DoesNotExist),
+        }
     }
 
     /// Changes the global enable status for the device's MSI-X capability. If set, regular IRQs
@@ -292,23 +434,96 @@ impl<'s, S: PciConfigurationSpace + ?Sized + 's> MsiXData<'s, S> {
     }
 }
 
-impl MsiXVectorTable<'_> {
-    unsafe fn from_raw_parts(base: PhysicalAddress, len: usize) -> Result<Self, Error> {
-        let vectors = DeviceMemoryIoMut::map_slice(base, len, Default::default())?;
-        Ok(Self { vectors })
-    }
-
-    pub fn mask_all(&mut self) {
-        for vector in self.vectors.iter_mut() {
-            vector.set_masked(true);
+impl MsiXVectorTableAccess<'_> {
+    fn set_vector_masked(&mut self, vector: usize, masked: bool) {
+        let old = self.read_control(vector);
+        let new = if masked { old | 1 } else { old & !1 };
+        if old != new {
+            self.write_control(vector, new);
         }
     }
 
-    pub fn register_range<C: MessageInterruptController + ?Sized>(
+    fn read_control(&mut self, vector: usize) -> u32 {
+        match self {
+            #[cfg(any(target_arch = "x86", target_arch = "x86_64", rust_analyzer))]
+            &mut Self::Io(base) => unsafe {
+                let a = base
+                    + (vector * size_of::<MsiXEntry>() + offset_of!(MsiXEntry, control)) as u16;
+                intrinsics::inl(a)
+            },
+
+            Self::Memory(vectors) => vectors[vector].control.get(),
+        }
+    }
+
+    fn write_address(&mut self, vector: usize, value: u64) {
+        match self {
+            #[cfg(any(target_arch = "x86", target_arch = "x86_64", rust_analyzer))]
+            &mut Self::Io(base) => unsafe {
+                let a = base + (vector * size_of::<MsiXEntry>()) as u16;
+                intrinsics::outl(a, value as u32);
+                intrinsics::outl(a + 4, (value >> 32) as u32);
+            },
+
+            Self::Memory(vectors) => vectors[vector].address.set(value),
+        }
+    }
+
+    fn write_data(&mut self, vector: usize, value: u32) {
+        match self {
+            #[cfg(any(target_arch = "x86", target_arch = "x86_64", rust_analyzer))]
+            &mut Self::Io(base) => unsafe {
+                let a =
+                    base + (vector * size_of::<MsiXEntry>() + offset_of!(MsiXEntry, data)) as u16;
+                intrinsics::outl(a, value)
+            },
+
+            Self::Memory(vectors) => vectors[vector].data.set(value),
+        }
+    }
+
+    fn write_control(&mut self, vector: usize, value: u32) {
+        match self {
+            #[cfg(any(target_arch = "x86", target_arch = "x86_64", rust_analyzer))]
+            &mut Self::Io(base) => unsafe {
+                let a = base
+                    + (vector * size_of::<MsiXEntry>() + offset_of!(MsiXEntry, control)) as u16;
+                intrinsics::outl(a, value)
+            },
+
+            Self::Memory(vectors) => vectors[vector].control.set(value),
+        }
+    }
+}
+
+impl MsiXVectorTable<'_> {
+    unsafe fn memory_from_raw_parts(base: PhysicalAddress, len: usize) -> Result<Self, Error> {
+        let vectors = DeviceMemoryIoMut::map_slice(base, len, Default::default())?;
+        Ok(Self {
+            access: MsiXVectorTableAccess::Memory(vectors),
+            len,
+        })
+    }
+
+    #[cfg(any(target_arch = "x86", target_arch = "x86_64", rust_analyzer))]
+    unsafe fn io_from_raw_parts(base: u16, len: usize) -> Result<Self, Error> {
+        Ok(Self {
+            access: MsiXVectorTableAccess::Io(base),
+            len,
+        })
+    }
+
+    pub fn mask_all(&mut self) {
+        for i in 0..self.len {
+            self.access.set_vector_masked(i, true);
+        }
+    }
+
+    pub fn register_range(
         &mut self,
         start: usize,
         end: usize,
-        ic: &C,
+        ic: &Arc<dyn MessageInterruptController>,
         affinity: InterruptAffinity,
         handler: Arc<dyn InterruptHandler>,
     ) -> Result<Vec<MsiInfo>, Error> {
@@ -320,38 +535,37 @@ impl MsiXVectorTable<'_> {
             };
             end - start
         ];
-        ic.register_msi_range(&mut range, handler)?;
+        ic.clone().register_msi_range(&mut range, handler)?;
 
         for (i, info) in range.iter().enumerate() {
             let index = i + start;
-            self.vectors[index].address.set(info.address as _);
-            self.vectors[index].data.set(info.value);
-            self.vectors[index].set_masked(false);
+            self.access.write_address(index, info.address as _);
+            self.access.write_data(index, info.value);
+            self.access.set_vector_masked(index, false);
         }
 
         Ok(range)
     }
 }
 
-impl MsiXEntry {
-    /// If set, prevents the MSI-X interrupt from being delivered
-    fn set_masked(&mut self, masked: bool) {
-        if masked {
-            self.control.set(self.control.get() | 1);
-        } else {
-            self.control.set(self.control.get() & !1);
-        }
-    }
-}
-
 impl<'s, S: PciConfigurationSpace + ?Sized + 's> MsiData<'s, S> {
-    pub fn register<C: MessageInterruptController + ?Sized>(
+    pub fn set_enabled(&mut self, enabled: bool) {
+        let mut w0 = self.space.read_u16(self.offset + 2);
+        if enabled {
+            w0 |= 1 << 0;
+        } else {
+            w0 &= !(1 << 0);
+        }
+        self.space.write_u16(self.offset + 2, w0);
+    }
+
+    pub fn register(
         &mut self,
-        ic: &C,
+        ic: &Arc<dyn MessageInterruptController>,
         affinity: InterruptAffinity,
         handler: Arc<dyn InterruptHandler>,
     ) -> Result<MsiInfo, Error> {
-        let info = ic.register_msi(affinity, handler)?;
+        let info = ic.clone().register_msi(affinity, handler)?;
 
         let mut w0 = self.space.read_u16(self.offset + 2);
         // Enable the vector first
@@ -387,3 +601,13 @@ impl<'s, S: PciConfigurationSpace + ?Sized + 's> MsiData<'s, S> {
         Ok(info)
     }
 }
+
+impl<'s, S: PciConfigurationSpace + ?Sized + 's> PcieData<'s, S> {
+    pub fn link_control(&self) -> PcieLinkControl {
+        PcieLinkControl::from_bits_retain(self.space.read_u16(self.offset + 0x10))
+    }
+
+    pub fn set_link_control(&mut self, value: PcieLinkControl) {
+        self.space.write_u16(self.offset + 0x10, value.bits());
+    }
+}

kernel/driver/bus/pci/src/device.rs

@@ -3,15 +3,19 @@ use core::ops::Range;
 use alloc::{sync::Arc, vec::Vec};
 use device_api::{
     device::Device,
-    interrupt::{InterruptAffinity, InterruptHandler, IrqOptions, MsiInfo},
+    interrupt::{
+        ExternalInterruptController, InterruptAffinity, InterruptHandler, Irq, IrqOptions,
+        MessageInterruptController, MsiInfo,
+    },
 };
-use libk::device::{message_interrupt_controller, register_global_interrupt};
+use libk::device::external_interrupt_controller;
 use libk_util::{sync::spin_rwlock::IrqSafeRwLock, OneTimeInit};
 use yggdrasil_abi::error::Error;
 
 use crate::{
     capability::{MsiCapability, MsiXCapability, MsiXVectorTable},
-    PciAddress, PciConfigSpace, PciConfigurationSpace, PciSegmentInfo,
+    driver::PciDriver,
+    PciAddress, PciCommandRegister, PciConfigSpace, PciConfigurationSpace, PciSegmentInfo,
 };
 
 /// Describes a PCI device
@@ -19,6 +23,16 @@ use crate::{
 pub struct PciDeviceInfo {
     /// Address of the device
     pub address: PciAddress,
+    /// Class field of the configuration space
+    pub class: u8,
+    /// Subclass field of the configuration space
+    pub subclass: u8,
+    /// Prog IF field of the configuration space
+    pub prog_if: u8,
+    /// Vendor ID field of the configuration space
+    pub vendor_id: u16,
+    /// Device ID field of the configuration space
+    pub device_id: u16,
     /// Configuration space access method
     pub config_space: PciConfigSpace,
     /// Describes the PCI segment this device is a part of
@@ -43,16 +57,19 @@ pub enum PciInterruptPin {
 
 #[derive(Clone, Copy, PartialEq, Eq, Debug)]
 pub enum PreferredInterruptMode {
-    Msi,
+    Msi(bool),
     Legacy,
 }
 
 enum ConfiguredInterruptMode {
-    MsiX(MsiXVectorTable<'static>),
-    Msi,
-    LegacyPin(PciInterruptPin),
+    MsiX(
+        Arc<dyn MessageInterruptController>,
+        MsiXVectorTable<'static>,
+    ),
+    Msi(Arc<dyn MessageInterruptController>),
+    LegacyPin(Arc<dyn ExternalInterruptController>, PciInterruptPin),
     #[cfg_attr(not(target_arch = "x86"), allow(unused))]
-    LegacyLine(u8),
+    LegacyLine(Arc<dyn ExternalInterruptController>, u8),
     None,
 }
 
@@ -68,69 +85,133 @@ pub struct PciInterruptRoute {
     pub options: IrqOptions,
 }
 
-pub enum PciMatch {
-    Generic(fn(&PciDeviceInfo) -> bool),
-    Vendor(u16, u16),
-    Class(u8, Option<u8>, Option<u8>),
+#[derive(Clone)]
+pub struct PciMsiRoute {
+    // TODO `msi-base`
+    pub controller: Arc<dyn MessageInterruptController>,
 }
 
-pub struct PciDriver {
-    pub(crate) name: &'static str,
-    pub(crate) check: PciMatch,
-    pub(crate) probe: fn(&PciDeviceInfo) -> Result<Arc<dyn Device>, Error>,
+#[derive(Clone, Copy, PartialEq, Debug)]
+pub enum PciDeviceState {
+    None,
+    Probed,
+    Initialized,
+    Failed,
 }
 
 /// Used to store PCI bus devices which were enumerated by the kernel
 pub struct PciBusDevice {
     pub(crate) info: PciDeviceInfo,
-    pub(crate) driver: Option<Arc<dyn Device>>,
+    pub(crate) device: Option<Arc<dyn Device>>,
+    pub(crate) driver: Option<&'static dyn PciDriver>,
+    pub(crate) state: PciDeviceState,
 }
 
 impl PciDeviceInfo {
+    pub fn set_command(
+        &self,
+        enable_irq: bool,
+        enable_mem: bool,
+        enable_io: bool,
+        enable_bus_master: bool,
+    ) {
+        let command = PciCommandRegister::from_bits_retain(self.config_space.command());
+        let mut new = command;
+        if enable_irq {
+            new &= !PciCommandRegister::DISABLE_INTERRUPTS;
+        } else {
+            new |= PciCommandRegister::DISABLE_INTERRUPTS;
+        }
+        if enable_mem {
+            new |= PciCommandRegister::ENABLE_MEMORY;
+        } else {
+            new &= !PciCommandRegister::ENABLE_MEMORY;
+        }
+        if enable_io {
+            new |= PciCommandRegister::ENABLE_IO;
+        } else {
+            new &= !PciCommandRegister::ENABLE_IO;
+        }
+        if enable_bus_master {
+            new |= PciCommandRegister::BUS_MASTER;
+        } else {
+            new &= !PciCommandRegister::BUS_MASTER;
+        }
+        if new != command {
+            self.config_space.set_command(new.bits());
+        }
+    }
+
     pub fn init_interrupts(&self, preferred_mode: PreferredInterruptMode) -> Result<(), Error> {
         self.interrupt_config
             .try_init_with(|| {
-                let configured_mode =
-                    if self.segment.has_msi && preferred_mode == PreferredInterruptMode::Msi {
-                        if let Some(mut msix) = self.config_space.capability::<MsiXCapability>() {
-                            let mut vt = msix.vector_table().unwrap();
+                let configured_mode = if let PreferredInterruptMode::Msi(want_msix) = preferred_mode
+                    && let Some(msi_route) = self.segment.msi_translation_map.map_msi(self.address)
+                {
+                    // Try to setup MSI (or MSI-x, if requested)
+                    let mut result = None;
+                    if want_msix
+                        && let Some(mut msix) = self.config_space.capability::<MsiXCapability>()
+                    {
+                        if let Ok(mut vt) = msix.vector_table() {
+                            if let Some(mut msi) = self.config_space.capability::<MsiCapability>() {
+                                msi.set_enabled(false);
+                            }
 
                             vt.mask_all();
 
                             msix.set_function_mask(false);
                             msix.set_enabled(true);
 
-                            ConfiguredInterruptMode::MsiX(vt)
-                        } else if self.config_space.capability::<MsiCapability>().is_some() {
-                            ConfiguredInterruptMode::Msi
-                        } else {
-                            self.legacy_interrupt_mode()
+                            result = Some(ConfiguredInterruptMode::MsiX(
+                                msi_route.controller.clone(),
+                                vt,
+                            ));
                         }
+                    }
+
+                    // Fall back to MSI if MSI-x is not available or not requested
+                    if result.is_none() && self.config_space.capability::<MsiCapability>().is_some()
+                    {
+                        result = Some(ConfiguredInterruptMode::Msi(msi_route.controller));
+                    }
+
+                    // Fall back to legacy IRQ if nothing else works
+                    if let Some(result) = result {
+                        result
                     } else {
-                        // Ignore preferred_mode, the only supported is Legacy
                         self.legacy_interrupt_mode()
-                    };
+                    }
+                } else {
+                    // MSI not requested or segment does not have MSI functionality
+                    self.legacy_interrupt_mode()
+                };
 
                 IrqSafeRwLock::new(InterruptConfig {
                     preferred_mode,
                     configured_mode,
                 })
             })
-            .expect("Attempted to double-configure interrupts for a PCI device");
+            .expect("Possible bug: double-initialization of PCI(e) interrupt config");
 
         Ok(())
     }
 
     fn legacy_interrupt_mode(&self) -> ConfiguredInterruptMode {
+        let Ok(intc) = external_interrupt_controller() else {
+            return ConfiguredInterruptMode::None;
+        };
+
+        // TODO this should be retrieved from interrupt map
         #[cfg(any(target_arch = "x86", rust_analyzer))]
         {
             if let Some(irq) = self.config_space.interrupt_line() {
-                return ConfiguredInterruptMode::LegacyLine(irq);
+                return ConfiguredInterruptMode::LegacyLine(intc.clone(), irq);
             }
         }
 
         match self.config_space.interrupt_pin() {
-            Some(pin) => ConfiguredInterruptMode::LegacyPin(pin),
+            Some(pin) => ConfiguredInterruptMode::LegacyPin(intc.clone(), pin),
             None => ConfiguredInterruptMode::None,
         }
     }
@@ -143,27 +224,26 @@ impl PciDeviceInfo {
         let mut irq = self.interrupt_config.get().write();
 
         match &mut irq.configured_mode {
-            ConfiguredInterruptMode::MsiX(msix) => {
-                let info =
-                    msix.register_range(0, 1, message_interrupt_controller()?, affinity, handler)?;
-                Ok(Some(info[0]))
-            }
-            ConfiguredInterruptMode::Msi => {
+            ConfiguredInterruptMode::Msi(controller) => {
                 let mut msi = self
                     .config_space
                     .capability::<MsiCapability>()
                     .ok_or(Error::InvalidOperation)?;
 
-                let info = msi.register(message_interrupt_controller()?, affinity, handler)?;
+                let info = msi.register(controller, affinity, handler)?;
 
                 Ok(Some(info))
             }
-            ConfiguredInterruptMode::LegacyPin(pin) => {
-                self.try_map_legacy(*pin, handler)?;
+            ConfiguredInterruptMode::MsiX(controller, msix) => {
+                let info = msix.register_range(0, 1, controller, affinity, handler)?;
+                Ok(Some(info[0]))
+            }
+            ConfiguredInterruptMode::LegacyPin(intc, pin) => {
+                self.try_map_legacy(intc.as_ref(), *pin, handler)?;
                 Ok(None)
             }
-            ConfiguredInterruptMode::LegacyLine(irq) => {
-                self.try_map_legacy_line(*irq, handler)?;
+            ConfiguredInterruptMode::LegacyLine(intc, irq) => {
+                self.try_map_legacy_line(intc.as_ref(), *irq, handler)?;
                 Ok(None)
             }
             ConfiguredInterruptMode::None => Err(Error::InvalidOperation),
@@ -181,19 +261,16 @@ impl PciDeviceInfo {
         let end = vector_range.end;
 
         match &mut irq.configured_mode {
-            ConfiguredInterruptMode::MsiX(msix) => msix.register_range(
-                start,
-                end,
-                message_interrupt_controller()?,
-                affinity,
-                handler,
-            ),
+            ConfiguredInterruptMode::MsiX(controller, msix) => {
+                msix.register_range(start, end, controller, affinity, handler)
+            }
             _ => Err(Error::InvalidOperation),
         }
     }
 
     fn try_map_legacy(
         &self,
+        intc: &dyn ExternalInterruptController,
         pin: PciInterruptPin,
         handler: Arc<dyn InterruptHandler>,
     ) -> Result<(), Error> {
@@ -204,8 +281,8 @@ impl PciDeviceInfo {
         let route = self
             .segment
             .irq_translation_map
-            .get(&src)
-            .ok_or(Error::InvalidOperation)?;
+            .map_interrupt(&src)
+            .inspect_err(|e| log::warn!("Could not map PCI IRQ {pin:?}: {e:?}"))?;
 
         log::debug!(
             "PCI {} pin {:?} -> system IRQ #{}",
@@ -214,17 +291,22 @@ impl PciDeviceInfo {
             route.number
         );
 
-        register_global_interrupt(route.number, route.options, handler)
+        let irq = Irq::External(route.number);
+        intc.register_irq(irq, route.options, handler)?;
+        intc.enable_irq(irq)
     }
 
     fn try_map_legacy_line(
         &self,
+        intc: &dyn ExternalInterruptController,
         line: u8,
         handler: Arc<dyn InterruptHandler>,
     ) -> Result<(), Error> {
         log::debug!("PCI {} -> IRQ#{}", self.address, line);
 
-        register_global_interrupt(line as _, Default::default(), handler)
+        let irq = Irq::External(line as u32);
+        intc.register_irq(irq, Default::default(), handler)?;
+        intc.enable_irq(irq)
     }
 }
 

kernel/driver/bus/pci/src/driver.rs

@@ -0,0 +1,63 @@
+use alloc::{sync::Arc, vec::Vec};
+use device_api::{device::Device, dma::DmaAllocator};
+use libk::error::Error;
+use libk_util::sync::spin_rwlock::IrqSafeRwLock;
+
+use crate::device::PciDeviceInfo;
+
+pub enum PciMatch {
+    Generic(fn(&PciDeviceInfo) -> bool),
+    Vendor(u16, u16),
+    Class(u8, Option<u8>, Option<u8>),
+}
+
+pub struct PciDriverMatch {
+    pub driver: &'static dyn PciDriver,
+    pub check: PciMatch,
+}
+
+pub trait PciDriver: Sync {
+    fn probe(
+        &self,
+        info: &PciDeviceInfo,
+        dma: &Arc<dyn DmaAllocator>,
+    ) -> Result<Arc<dyn Device>, Error>;
+    fn driver_name(&self) -> &str;
+}
+
+impl PciMatch {
+    pub fn check_device(&self, info: &PciDeviceInfo) -> bool {
+        match self {
+            Self::Generic(f) => f(info),
+            &Self::Vendor(vendor_, device_) => {
+                info.vendor_id == vendor_ && info.device_id == device_
+            }
+            &Self::Class(class_, Some(subclass_), Some(prog_if_)) => {
+                class_ == info.class && subclass_ == info.subclass && prog_if_ == info.prog_if
+            }
+            &Self::Class(class_, Some(subclass_), _) => {
+                class_ == info.class && subclass_ == info.subclass
+            }
+            &Self::Class(class_, _, _) => class_ == info.class,
+        }
+    }
+}
+
+pub fn register_match(pmatch: PciMatch, driver: &'static dyn PciDriver) {
+    DRIVERS.write().push(PciDriverMatch {
+        check: pmatch,
+        driver,
+    });
+}
+
+pub fn lookup_driver(info: &PciDeviceInfo) -> Option<&'static dyn PciDriver> {
+    DRIVERS.read().iter().find_map(|pmatch| {
+        if pmatch.check.check_device(info) {
+            Some(pmatch.driver)
+        } else {
+            None
+        }
+    })
+}
+
+static DRIVERS: IrqSafeRwLock<Vec<PciDriverMatch>> = IrqSafeRwLock::new(Vec::new());

kernel/driver/bus/pci/src/interrupt.rs

@@ -0,0 +1,141 @@
+use core::fmt;
+
+use alloc::{collections::btree_map::BTreeMap, sync::Arc, vec::Vec};
+use device_api::interrupt::MessageInterruptController;
+use libk::error::Error;
+
+use crate::{
+    device::{PciInterrupt, PciInterruptRoute, PciMsiRoute},
+    PciAddress,
+};
+
+#[derive(Debug)]
+pub enum PciInterruptMap {
+    Fixed(BTreeMap<PciInterrupt, PciInterruptRoute>),
+    #[cfg(any(target_arch = "x86_64", rust_analyzer))]
+    Acpi(alloc::string::String),
+    Legacy,
+}
+
+// TODO device-tree also provides a "msi-base" value, which is ignored and assumed to be zero for
+//      now
+pub struct PciFixedMsiMapping {
+    pub start_address: PciAddress,
+    pub end_address: PciAddress,
+    pub controller: Arc<dyn MessageInterruptController>,
+}
+
+pub struct PciFixedMsiMap {
+    pub entries: Vec<PciFixedMsiMapping>,
+}
+
+pub enum PciMsiMap {
+    Fixed(PciFixedMsiMap),
+    Identity(Arc<dyn MessageInterruptController>),
+    Legacy,
+}
+
+impl PciInterruptMap {
+    pub fn map_interrupt(&self, interrupt: &PciInterrupt) -> Result<PciInterruptRoute, Error> {
+        match self {
+            Self::Fixed(map) => map.get(interrupt).cloned().ok_or(Error::DoesNotExist),
+            #[cfg(any(target_arch = "x86_64", rust_analyzer))]
+            Self::Acpi(aml_object_name) => {
+                use device_api::interrupt::{IrqLevel, IrqOptions, IrqTrigger};
+
+                use crate::device::PciInterruptPin;
+
+                let aml_pin = match interrupt.pin {
+                    PciInterruptPin::A => ygg_driver_acpi::PciPin::IntA,
+                    PciInterruptPin::B => ygg_driver_acpi::PciPin::IntB,
+                    PciInterruptPin::C => ygg_driver_acpi::PciPin::IntC,
+                    PciInterruptPin::D => ygg_driver_acpi::PciPin::IntD,
+                };
+                let aml_route = ygg_driver_acpi::get_pci_route(
+                    aml_object_name.as_str(),
+                    interrupt.address.device as u16,
+                    interrupt.address.function as u16,
+                    aml_pin,
+                )
+                .or_else(|| {
+                    ygg_driver_acpi::get_pci_route(
+                        aml_object_name.as_str(),
+                        interrupt.address.device as u16,
+                        0xFFFF,
+                        aml_pin,
+                    )
+                })
+                .ok_or(Error::DoesNotExist)?;
+
+                let trigger = match aml_route.trigger {
+                    ygg_driver_acpi::InterruptTrigger::Edge => IrqTrigger::Edge,
+                    ygg_driver_acpi::InterruptTrigger::Level => IrqTrigger::Level,
+                };
+                let level = match aml_route.polarity {
+                    ygg_driver_acpi::InterruptPolarity::ActiveLow => IrqLevel::ActiveLow,
+                    ygg_driver_acpi::InterruptPolarity::ActiveHigh => IrqLevel::ActiveHigh,
+                };
+
+                Ok(PciInterruptRoute {
+                    options: IrqOptions { trigger, level },
+                    number: aml_route.irq,
+                })
+            }
+            Self::Legacy => todo!(),
+        }
+    }
+}
+
+impl PciMsiMap {
+    pub fn map_msi(&self, address: PciAddress) -> Option<PciMsiRoute> {
+        match self {
+            Self::Fixed(map) => map.map_msi(address),
+            Self::Identity(controller) => Some(PciMsiRoute {
+                controller: controller.clone(),
+            }),
+            Self::Legacy => None,
+        }
+    }
+}
+
+impl fmt::Debug for PciMsiMap {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::Legacy => f.debug_struct("Legacy").finish(),
+            Self::Fixed(map) => f
+                .debug_struct("Fixed")
+                .field("entries", &map.entries)
+                .finish(),
+            Self::Identity(_) => f.debug_struct("Identity").finish(),
+        }
+    }
+}
+
+impl PciFixedMsiMap {
+    pub fn map_msi(&self, address: PciAddress) -> Option<PciMsiRoute> {
+        for entry in self.entries.iter() {
+            if entry.contains(address) {
+                let route = PciMsiRoute {
+                    controller: entry.controller.clone(),
+                };
+                return Some(route);
+            }
+        }
+        None
+    }
+}
+
+impl PciFixedMsiMapping {
+    pub fn contains(&self, address: PciAddress) -> bool {
+        self.start_address <= address && self.end_address > address
+    }
+}
+
+impl fmt::Debug for PciFixedMsiMapping {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("PciFixedMsiMapping")
+            .field("start_address", &self.start_address)
+            .field("end_address", &self.end_address)
+            .finish()
+    }
+}

kernel/driver/bus/pci/src/lib.rs

@@ -1,24 +1,38 @@
 //! PCI/PCIe bus interfaces
 #![no_std]
+#![feature(let_chains, decl_macro)]
 #![allow(clippy::missing_transmute_annotations)]
 
 extern crate alloc;
 
 use core::fmt;
 
-#[cfg(target_arch = "x86_64")]
+#[cfg(any(target_arch = "x86_64", rust_analyzer))]
 use acpi::mcfg::McfgEntry;
-use alloc::{collections::BTreeMap, sync::Arc, vec::Vec};
+use alloc::{format, sync::Arc, vec::Vec};
+
 use bitflags::bitflags;
-use device::{PciBusDevice, PciDeviceInfo, PciDriver, PciInterrupt, PciInterruptRoute, PciMatch};
-use device_api::device::Device;
+use device::{PciBusDevice, PciDeviceInfo, PciDeviceState};
+use device_api::{device::DeviceInitContext, dma::DmaAllocator};
+use interrupt::{PciInterruptMap, PciMsiMap};
+use libk::{
+    dma::DummyDmaAllocator,
+    fs::sysfs::{self, object::KObject},
+};
 use libk_mm::address::PhysicalAddress;
 use libk_util::{sync::IrqSafeSpinlock, OneTimeInit};
 use space::legacy;
-use yggdrasil_abi::error::Error;
+use yggdrasil_abi::{error::Error, primitive_enum};
+
+#[cfg(target_arch = "x86_64")]
+use device_api::interrupt::MessageInterruptController;
 
 pub mod capability;
 pub mod device;
+pub mod driver;
+pub mod interrupt;
+pub mod macros;
+mod nodes;
 mod space;
 
 pub use space::{
@@ -29,6 +43,7 @@ pub use space::{
 
 bitflags! {
     /// Command register of the PCI configuration space
+    #[derive(PartialEq, Clone, Copy)]
     pub struct PciCommandRegister: u16 {
         /// If set, I/O access to the device is enabled
         const ENABLE_IO = 1 << 0;
@@ -73,19 +88,14 @@ pub enum PciBaseAddress {
     Io(u16),
 }
 
-/// Unique ID assigned to PCI capability structures
-#[derive(Clone, Copy, PartialEq, Eq, Debug)]
-#[non_exhaustive]
-#[repr(u8)]
-pub enum PciCapabilityId {
-    /// MSI (32-bit or 64-bit)
-    Msi = 0x05,
-    /// Vendor-specific capability
-    VendorSpecific = 0x09,
-    /// MSI-X
-    MsiX = 0x11,
-    /// Unknown capability missing from this list
-    Unknown,
+primitive_enum! {
+    pub enum PciCapabilityId: u8 {
+        PowerManagement = 0x01,
+        Msi = 0x05,
+        VendorSpecific = 0x09,
+        PciExpress = 0x10,
+        MsiX = 0x11,
+    }
 }
 
 /// Interface used for querying PCI capabilities
@@ -122,6 +132,45 @@ struct BusAddressAllocator {
     offset_32: u32,
 }
 
+#[derive(Debug)]
+pub struct PciSegmentInfo {
+    pub segment_number: u8,
+    pub bus_number_start: u8,
+    pub bus_number_end: u8,
+    pub ecam_phys_base: Option<PhysicalAddress>,
+
+    pub irq_translation_map: PciInterruptMap,
+    pub msi_translation_map: PciMsiMap,
+}
+
+/// Represents a single PCIe bus segment
+pub struct PciBusSegment {
+    allocator: Option<BusAddressAllocator>,
+    info: Arc<PciSegmentInfo>,
+    devices: Vec<Arc<KObject<IrqSafeSpinlock<PciBusDevice>>>>,
+}
+
+#[derive(Debug)]
+pub enum PciRangeType {
+    Configuration,
+    Io,
+    Memory32,
+    Memory64,
+}
+
+pub struct PciAddressRange {
+    pub ty: PciRangeType,
+    pub bus_number: u8,
+    pub pci_base: u64,
+    pub host_base: PhysicalAddress,
+    pub size: usize,
+}
+
+/// Manager struct to store and control all PCI devices in the system
+pub struct PciBusManager {
+    segments: Vec<PciBusSegment>,
+}
+
 #[cfg_attr(
     any(target_arch = "x86_64", target_arch = "x86", target_arch = "riscv64"),
     allow(dead_code)
@@ -193,45 +242,6 @@ impl BusAddressAllocator {
     }
 }
 
-#[derive(Debug)]
-pub struct PciSegmentInfo {
-    pub segment_number: u8,
-    pub bus_number_start: u8,
-    pub bus_number_end: u8,
-    pub ecam_phys_base: Option<PhysicalAddress>,
-
-    pub irq_translation_map: BTreeMap<PciInterrupt, PciInterruptRoute>,
-    pub has_msi: bool,
-}
-
-/// Represents a single PCIe bus segment
-pub struct PciBusSegment {
-    allocator: Option<BusAddressAllocator>,
-    info: Arc<PciSegmentInfo>,
-    devices: Vec<PciBusDevice>,
-}
-
-#[derive(Debug)]
-pub enum PciRangeType {
-    Configuration,
-    Io,
-    Memory32,
-    Memory64,
-}
-
-pub struct PciAddressRange {
-    pub ty: PciRangeType,
-    pub bus_number: u8,
-    pub pci_base: u64,
-    pub host_base: PhysicalAddress,
-    pub size: usize,
-}
-
-/// Manager struct to store and control all PCI devices in the system
-pub struct PciBusManager {
-    segments: Vec<PciBusSegment>,
-}
-
 impl PciBaseAddress {
     pub fn as_memory(self) -> Option<PhysicalAddress> {
         match self {
@@ -240,6 +250,14 @@ impl PciBaseAddress {
             _ => None,
         }
     }
+
+    pub fn is_zero(&self) -> bool {
+        match *self {
+            Self::Memory32(base) => base == 0,
+            Self::Memory64(base) => base == 0,
+            Self::Io(base) => base == 0,
+        }
+    }
 }
 
 impl PciBusSegment {
@@ -348,13 +366,40 @@ impl PciBusSegment {
             }
         }
 
+        let vendor_id = config.vendor_id();
+        let device_id = config.device_id();
+        let class = config.class_code();
+        let subclass = config.subclass();
+        let prog_if = config.prog_if();
+
         let info = PciDeviceInfo {
             address,
+            vendor_id,
+            device_id,
+            class,
+            subclass,
+            prog_if,
             segment: self.info.clone(),
             config_space: config,
             interrupt_config: Arc::new(OneTimeInit::new()),
         };
-        self.devices.push(PciBusDevice { info, driver: None });
+
+        let object = nodes::make_sysfs_object(PciBusDevice {
+            info,
+            driver: None,
+            device: None,
+            state: PciDeviceState::None,
+        });
+        let pci_object = PCI_SYSFS_NODE.or_init_with(|| {
+            let bus_object = sysfs::bus().unwrap();
+            let pci_object = KObject::new(());
+            bus_object.add_object("pci", pci_object.clone()).ok();
+            pci_object
+        });
+
+        let name = format!("{address}");
+        pci_object.add_object(name, object.clone()).ok();
+        self.devices.push(object);
 
         Ok(())
     }
@@ -380,6 +425,12 @@ impl PciBusSegment {
     }
 }
 
+impl PciSegmentInfo {
+    pub fn has_msi(&self) -> bool {
+        !matches!(self.msi_translation_map, PciMsiMap::Legacy)
+    }
+}
+
 impl PciBusManager {
     const fn new() -> Self {
         Self {
@@ -389,11 +440,16 @@ impl PciBusManager {
 
     /// Walks the bus device list and calls init/init_irq functions on any devices with associated
     /// drivers
-    pub fn setup_bus_devices() -> Result<(), Error> {
-        log::info!("Setting up bus devices");
+    pub fn probe_bus_devices() -> Result<(), Error> {
         Self::walk_bus_devices(|device| {
-            log::info!("Set up {}", device.info.address);
-            setup_bus_device(device)?;
+            probe_bus_device(device, false)?;
+            Ok(true)
+        })
+    }
+
+    pub fn setup_bus_devices(rescan: bool) -> Result<(), Error> {
+        Self::walk_bus_devices(|device| {
+            setup_bus_device(device, rescan)?;
             Ok(true)
         })
     }
@@ -407,7 +463,8 @@ impl PciBusManager {
 
         for segment in this.segments.iter_mut() {
             for device in segment.devices.iter_mut() {
-                if !f(device)? {
+                let mut device = device.lock();
+                if !f(&mut *device)? {
                     return Ok(());
                 }
             }
@@ -425,8 +482,8 @@ impl PciBusManager {
                 bus_number_start: 0,
                 bus_number_end: 255,
                 ecam_phys_base: None,
-                irq_translation_map: BTreeMap::new(),
-                has_msi: false,
+                irq_translation_map: PciInterruptMap::Legacy,
+                msi_translation_map: PciMsiMap::Legacy,
             }),
             allocator: None,
             devices: Vec::new(),
@@ -441,7 +498,12 @@ impl PciBusManager {
 
     /// Enumerates a bus segment provided by ACPI MCFG table entry
     #[cfg(target_arch = "x86_64")]
-    pub fn add_segment_from_mcfg(entry: &McfgEntry) -> Result<(), Error> {
+    pub fn add_segment_from_mcfg(
+        entry: &McfgEntry,
+        msi_controller: Arc<dyn MessageInterruptController>,
+    ) -> Result<(), Error> {
+        let msi_translation_map = PciMsiMap::Identity(msi_controller);
+
         let mut bus_segment = PciBusSegment {
             info: Arc::new(PciSegmentInfo {
                 segment_number: entry.pci_segment_group as u8,
@@ -449,9 +511,9 @@ impl PciBusManager {
                 bus_number_end: entry.bus_number_end,
                 ecam_phys_base: Some(PhysicalAddress::from_u64(entry.base_address)),
 
-                // TODO obtain this from ACPI SSDT
-                irq_translation_map: BTreeMap::new(),
-                has_msi: true,
+                // TODO get the segment's PCI root bridge AML name
+                irq_translation_map: PciInterruptMap::Acpi("\\_SB.PCI0._PRT".into()),
+                msi_translation_map,
             }),
             // Firmware done this for us
             allocator: None,
@@ -471,7 +533,8 @@ impl PciBusManager {
         cfg_base: PhysicalAddress,
         bus_range: core::ops::Range<u8>,
         ranges: Vec<PciAddressRange>,
-        interrupt_map: BTreeMap<PciInterrupt, PciInterruptRoute>,
+        irq_translation_map: PciInterruptMap,
+        msi_translation_map: PciMsiMap,
     ) -> Result<(), Error> {
         let mut bus_segment = PciBusSegment {
             info: Arc::new(PciSegmentInfo {
@@ -480,8 +543,8 @@ impl PciBusManager {
                 bus_number_end: bus_range.end,
                 ecam_phys_base: Some(cfg_base),
 
-                irq_translation_map: interrupt_map,
-                has_msi: false,
+                irq_translation_map,
+                msi_translation_map,
             }),
             allocator: Some(BusAddressAllocator::from_ranges(&ranges)),
 
@@ -554,100 +617,66 @@ impl PciConfigurationSpace for PciConfigSpace {
     }
 }
 
-fn setup_bus_device(device: &mut PciBusDevice) -> Result<(), Error> {
-    if device.driver.is_some() {
+fn probe_bus_device(device: &mut PciBusDevice, _rescan: bool) -> Result<(), Error> {
+    // Already has a driver/device set up
+    if device.device.is_some() || device.state != PciDeviceState::None {
         return Ok(());
     }
 
-    let config = &device.info.config_space;
+    if let Some(driver) = driver::lookup_driver(&device.info) {
+        let dma: Arc<dyn DmaAllocator> = Arc::new(DummyDmaAllocator);
 
-    log::debug!(
-        "{}: {:04x}:{:04x}",
-        device.info.address,
-        config.vendor_id(),
-        config.device_id()
-    );
-
-    let class = config.class_code();
-    let subclass = config.subclass();
-    let prog_if = config.prog_if();
-
-    let drivers = PCI_DRIVERS.lock();
-    for driver in drivers.iter() {
-        if driver
-            .check
-            .check_device(&device.info, class, subclass, prog_if)
-        {
-            // TODO add the device to the bus
-            log::debug!(" -> {:?}", driver.name);
-            let instance = (driver.probe)(&device.info)?;
-
-            unsafe { instance.clone().init() }?;
-
-            device.driver.replace(instance);
-            break;
-        } else {
-            log::debug!(" -> No driver");
+        match driver.probe(&device.info, &dma) {
+            Ok(instance) => {
+                log::info!("{} -> {}", device.info.address, driver.driver_name());
+                device.device.replace(instance);
+                device.driver.replace(driver);
+                device.state = PciDeviceState::Probed;
+            }
+            Err(error) => {
+                log::error!(
+                    "{} ({}) probe error: {error:?}",
+                    device.info.address,
+                    driver.driver_name()
+                );
+            }
         }
     }
 
     Ok(())
 }
 
-impl PciMatch {
-    pub fn check_device(&self, info: &PciDeviceInfo, class: u8, subclass: u8, prog_if: u8) -> bool {
-        match self {
-            Self::Generic(f) => f(info),
-            &Self::Vendor(vendor_, device_) => {
-                info.config_space.vendor_id() == vendor_ && info.config_space.device_id() == device_
-            }
-            &Self::Class(class_, Some(subclass_), Some(prog_if_)) => {
-                class_ == class && subclass_ == subclass && prog_if_ == prog_if
-            }
-            &Self::Class(class_, Some(subclass_), _) => class_ == class && subclass_ == subclass,
-            &Self::Class(class_, _, _) => class_ == class,
+fn setup_bus_device(device: &mut PciBusDevice, _rescan: bool) -> Result<(), Error> {
+    // No driver yet (TODO probe if rescan is asked)
+    let (Some(dev), Some(driver)) = (device.device.as_ref(), device.driver) else {
+        return Ok(());
+    };
+    // Already initialized/failed
+    if device.state != PciDeviceState::Probed {
+        return Ok(());
+    }
+
+    let dma: Arc<dyn DmaAllocator> = Arc::new(DummyDmaAllocator);
+    let cx = DeviceInitContext {
+        dma_allocator: dma.clone(),
+    };
+
+    match unsafe { dev.clone().init(cx) } {
+        Ok(()) => {
+            device.state = PciDeviceState::Initialized;
+        }
+        Err(error) => {
+            log::error!(
+                "{} ({}) setup error: {error:?}",
+                device.info.address,
+                driver.driver_name()
+            );
+            device.state = PciDeviceState::Failed;
         }
     }
+
+    Ok(())
 }
 
-pub fn register_class_driver(
-    name: &'static str,
-    class: u8,
-    subclass: Option<u8>,
-    prog_if: Option<u8>,
-    probe: fn(&PciDeviceInfo) -> Result<Arc<dyn Device>, Error>,
-) {
-    PCI_DRIVERS.lock().push(PciDriver {
-        name,
-        check: PciMatch::Class(class, subclass, prog_if),
-        probe,
-    });
-}
-
-pub fn register_vendor_driver(
-    name: &'static str,
-    vendor_id: u16,
-    device_id: u16,
-    probe: fn(&PciDeviceInfo) -> Result<Arc<dyn Device>, Error>,
-) {
-    PCI_DRIVERS.lock().push(PciDriver {
-        name,
-        check: PciMatch::Vendor(vendor_id, device_id),
-        probe,
-    });
-}
-
-pub fn register_generic_driver(
-    name: &'static str,
-    check: fn(&PciDeviceInfo) -> bool,
-    probe: fn(&PciDeviceInfo) -> Result<Arc<dyn Device>, Error>,
-) {
-    PCI_DRIVERS.lock().push(PciDriver {
-        name,
-        check: PciMatch::Generic(check),
-        probe,
-    });
-}
-
-static PCI_DRIVERS: IrqSafeSpinlock<Vec<PciDriver>> = IrqSafeSpinlock::new(Vec::new());
 static PCI_MANAGER: IrqSafeSpinlock<PciBusManager> = IrqSafeSpinlock::new(PciBusManager::new());
+static PCI_SYSFS_NODE: OneTimeInit<Arc<KObject<()>>> = OneTimeInit::new();

kernel/driver/bus/pci/src/macros.rs

@@ -0,0 +1,35 @@
+pub macro pci_driver_match {
+    (class ($class:literal:$subclass:literal:$prog_if:literal)) => {
+        $crate::driver::PciMatch::Class($class, Some($subclass), Some($prog_if))
+    },
+    (class ($class:literal:$subclass:literal)) => {
+        $crate::driver::PciMatch::Class($class, Some($subclass), None)
+    },
+    (class $class:literal) => {
+        $crate::driver::PciMatch::Class($class, None, None)
+    },
+    (device ($vendor:literal:$device:literal)) => {
+        $crate::driver::PciMatch::Vendor($vendor, $device)
+    }
+}
+
+pub macro pci_driver(
+    matches: [$($kind:ident $match:tt),+ $(,)?],
+    driver: $driver:tt
+) {
+    #[link_section = ".init_array"]
+    #[used]
+    static __REGISTER_FN: extern "C" fn() = __register_fn;
+
+    extern "C" fn __register_fn() {
+        struct Driver;
+        impl $crate::driver::PciDriver for Driver $driver
+        static DRIVER: Driver = Driver;
+
+        log::info!("register pci driver: {:?}", $crate::driver::PciDriver::driver_name(&Driver));
+        $(
+            let pmatch = $crate::macros::pci_driver_match!($kind $match);
+            $crate::driver::register_match(pmatch, &DRIVER);
+        )+
+    }
+}

kernel/driver/bus/pci/src/nodes.rs

@@ -0,0 +1,139 @@
+use alloc::{format, string::String, sync::Arc};
+use libk::{
+    error::Error,
+    fs::sysfs::{
+        attribute::{StringAttribute, StringAttributeOps},
+        object::KObject,
+    },
+};
+use libk_util::sync::IrqSafeSpinlock;
+
+use crate::{device::PciBusDevice, PciBaseAddress, PciCapabilityId, PciConfigurationSpace};
+
+pub(crate) fn make_sysfs_object(
+    device: PciBusDevice,
+) -> Arc<KObject<IrqSafeSpinlock<PciBusDevice>>> {
+    struct Resources;
+    struct Capabilities;
+    struct Driver;
+    struct Class;
+    struct Id;
+
+    impl StringAttributeOps for Driver {
+        type Data = IrqSafeSpinlock<PciBusDevice>;
+        const NAME: &'static str = "driver";
+
+        fn read(state: &Self::Data) -> Result<String, Error> {
+            let state = state.lock();
+            if let Some(driver) = state.driver.map(|driver| driver.driver_name()) {
+                Ok(driver.into())
+            } else {
+                Ok("".into())
+            }
+        }
+    }
+
+    impl StringAttributeOps for Id {
+        type Data = IrqSafeSpinlock<PciBusDevice>;
+        const NAME: &'static str = "id";
+
+        fn read(state: &Self::Data) -> Result<String, Error> {
+            let state = state.lock();
+            Ok(format!(
+                "{:04x}:{:04x}",
+                state.info.vendor_id, state.info.device_id
+            ))
+        }
+    }
+
+    impl StringAttributeOps for Class {
+        type Data = IrqSafeSpinlock<PciBusDevice>;
+        const NAME: &'static str = "class";
+
+        fn read(state: &Self::Data) -> Result<String, Error> {
+            let state = state.lock();
+            Ok(format!(
+                "{:02x}:{:02x}:{:02x}",
+                state.info.class, state.info.subclass, state.info.prog_if
+            ))
+        }
+    }
+
+    impl StringAttributeOps for Resources {
+        type Data = IrqSafeSpinlock<PciBusDevice>;
+        const NAME: &'static str = "resources";
+        const NEWLINE: bool = false;
+
+        fn read(state: &Self::Data) -> Result<String, Error> {
+            use core::fmt::Write;
+
+            let state = state.lock();
+            let mut output = String::new();
+            for i in 0..6 {
+                if let Some(bar) = state.info.config_space.bar(i) {
+                    if bar.is_zero() {
+                        continue;
+                    }
+
+                    match bar {
+                        PciBaseAddress::Io(base) => {
+                            writeln!(output, "{i}:pio:{base:#06x}").ok();
+                        }
+                        PciBaseAddress::Memory32(base) => {
+                            writeln!(output, "{i}:m32:{base:#010x}").ok();
+                        }
+                        PciBaseAddress::Memory64(base) => {
+                            writeln!(output, "{i}:m64:{base:#018x}").ok();
+                        }
+                    }
+                }
+            }
+            if output.is_empty() {
+                output.push('\n');
+            }
+            Ok(output)
+        }
+    }
+
+    impl StringAttributeOps for Capabilities {
+        type Data = IrqSafeSpinlock<PciBusDevice>;
+        const NAME: &'static str = "capabilities";
+        const NEWLINE: bool = false;
+
+        fn read(state: &Self::Data) -> Result<String, Error> {
+            use core::fmt::Write;
+            let state = state.lock();
+            let mut output = String::new();
+            for (capability, offset, _) in state.info.config_space.capability_iter() {
+                write!(output, "{offset:04x}:").ok();
+                match capability {
+                    Some(PciCapabilityId::Msi) => write!(output, "msi").ok(),
+                    Some(PciCapabilityId::MsiX) => write!(output, "msix").ok(),
+                    Some(PciCapabilityId::VendorSpecific) => write!(output, "vendor-specific").ok(),
+                    Some(PciCapabilityId::PciExpress) => write!(output, "pcie").ok(),
+                    Some(PciCapabilityId::PowerManagement) => {
+                        write!(output, "power-management").ok()
+                    }
+                    None => write!(output, "unknown").ok(),
+                };
+                writeln!(output).ok();
+            }
+            if output.is_empty() {
+                output.push('\n');
+            }
+            Ok(output)
+        }
+    }
+
+    let object = KObject::new(IrqSafeSpinlock::new(device));
+
+    object
+        .add_attribute(StringAttribute::from(Capabilities))
+        .ok();
+    object.add_attribute(StringAttribute::from(Resources)).ok();
+    object.add_attribute(StringAttribute::from(Driver)).ok();
+    object.add_attribute(StringAttribute::from(Class)).ok();
+    object.add_attribute(StringAttribute::from(Id)).ok();
+
+    object
+}

kernel/driver/bus/pci/src/space/mod.rs

@@ -75,12 +75,12 @@ pub struct CapabilityIterator<'s, S: PciConfigurationSpace + ?Sized> {
 }
 
 impl<S: PciConfigurationSpace + ?Sized> Iterator for CapabilityIterator<'_, S> {
-    type Item = (PciCapabilityId, usize, usize);
+    type Item = (Option<PciCapabilityId>, usize, usize);
 
     fn next(&mut self) -> Option<Self::Item> {
         let offset = self.current? & !0x3;
 
-        let id = unsafe { core::mem::transmute(self.space.read_u8(offset)) };
+        let id = PciCapabilityId::try_from(self.space.read_u8(offset)).ok();
         let len = self.space.read_u8(offset + 2);
         let next_pointer = self.space.read_u8(offset + 1);
 
@@ -339,7 +339,7 @@ pub trait PciConfigurationSpace {
                         Some(PciBaseAddress::Memory32(w0 & !0xF))
                     }
                     // TODO can 64-bit BARs not be on a 64-bit boundary?
-                    2 => todo!(),
+                    2 => None,
                     _ => unimplemented!(),
                 },
                 1 => todo!(),
@@ -374,7 +374,7 @@ pub trait PciConfigurationSpace {
     /// Locates a capability within this configuration space
     fn capability<C: PciCapability>(&self) -> Option<C::CapabilityData<'_, Self>> {
         self.capability_iter().find_map(|(id, offset, len)| {
-            if id == C::ID && C::check(self, offset, len) {
+            if id.map_or(false, |id| id == C::ID) && C::check(self, offset, len) {
                 Some(C::data(self, offset, len))
             } else {
                 None

kernel/driver/bus/usb/Cargo.toml

@@ -12,7 +12,10 @@ libk-mm.workspace = true
 libk.workspace = true
 
 ygg_driver_input = { path = "../../input" }
+# For mass storage
+ygg_driver_scsi = { path = "../../block/scsi" }
 
 log.workspace = true
 bytemuck.workspace = true
 futures-util.workspace = true
+async-trait.workspace = true

kernel/driver/bus/usb/src/bus.rs

@@ -24,6 +24,7 @@ impl UsbBusManager {
     }
 
     pub fn register_device(device: Arc<UsbDeviceAccess>) {
+        log::info!("usb: register device {}", device.bus_address());
         BUS_MANAGER
             .devices
             .write()

kernel/driver/bus/usb/src/class_driver/hid_keyboard.rs

@@ -0,0 +1,176 @@
+use core::mem::MaybeUninit;
+
+use alloc::{boxed::Box, sync::Arc};
+use async_trait::async_trait;
+use yggdrasil_abi::io::{KeyboardKey, KeyboardKeyEvent};
+
+use crate::{device::UsbDeviceAccess, error::UsbError, info::UsbDeviceClass};
+
+use super::{UsbClassInfo, UsbDriver};
+
+pub struct UsbHidKeyboardDriver;
+
+const MODIFIER_MAP: &[KeyboardKey] = &[
+    KeyboardKey::LControl,
+    KeyboardKey::LShift,
+    KeyboardKey::LAlt,
+    KeyboardKey::Unknown,
+    KeyboardKey::RControl,
+    KeyboardKey::RShift,
+    KeyboardKey::RAlt,
+    KeyboardKey::Unknown,
+];
+
+#[derive(Default)]
+struct KeyboardState {
+    state: [u64; 4],
+    mods: u8,
+}
+
+impl KeyboardState {
+    pub fn new() -> Self {
+        Self::default()
+    }
+
+    pub fn translate_key(k: u8) -> KeyboardKey {
+        match k {
+            4..=29 => KeyboardKey::Char(k - 4 + b'a'),
+            30..=38 => KeyboardKey::Char(k - 30 + b'1'),
+            39 => KeyboardKey::Char(b'0'),
+
+            40 => KeyboardKey::Enter,
+            41 => KeyboardKey::Escape,
+            42 => KeyboardKey::Backspace,
+            43 => KeyboardKey::Tab,
+            44 => KeyboardKey::Char(b' '),
+            45 => KeyboardKey::Char(b'-'),
+            46 => KeyboardKey::Char(b'='),
+            47 => KeyboardKey::Char(b'['),
+            48 => KeyboardKey::Char(b']'),
+            49 => KeyboardKey::Char(b'\\'),
+            51 => KeyboardKey::Char(b';'),
+            52 => KeyboardKey::Char(b'\''),
+            53 => KeyboardKey::Char(b'`'),
+            54 => KeyboardKey::Char(b','),
+            55 => KeyboardKey::Char(b'.'),
+            56 => KeyboardKey::Char(b'/'),
+
+            58..=69 => KeyboardKey::F(k - 58),
+
+            _ => {
+                log::debug!("Unknown key: {}", k);
+                KeyboardKey::Unknown
+            }
+        }
+    }
+
+    pub fn retain_modifiers(
+        &mut self,
+        m: u8,
+        events: &mut [MaybeUninit<KeyboardKeyEvent>],
+    ) -> usize {
+        let mut count = 0;
+        let released = self.mods & !m;
+        for (i, modifier) in MODIFIER_MAP.iter().enumerate().take(8) {
+            if released & (1 << i) != 0 {
+                events[count].write(KeyboardKeyEvent::Released(*modifier));
+                count += 1;
+            }
+        }
+        self.mods &= m;
+        count
+    }
+
+    pub fn press_modifiers(
+        &mut self,
+        m: u8,
+        events: &mut [MaybeUninit<KeyboardKeyEvent>],
+    ) -> usize {
+        let mut count = 0;
+        let pressed = m & !self.mods;
+        for (i, modifier) in MODIFIER_MAP.iter().enumerate().take(8) {
+            if pressed & (1 << i) != 0 {
+                events[count].write(KeyboardKeyEvent::Pressed(*modifier));
+                count += 1;
+            }
+        }
+        self.mods = m;
+        count
+    }
+
+    pub fn retain(&mut self, keys: &[u8], events: &mut [MaybeUninit<KeyboardKeyEvent>]) -> usize {
+        let mut count = 0;
+        for i in 1..256 {
+            if self.state[i / 64] & (1 << (i % 64)) != 0 && !keys.contains(&(i as u8)) {
+                events[count].write(KeyboardKeyEvent::Released(Self::translate_key(i as u8)));
+                self.state[i / 64] &= !(1 << (i % 64));
+                count += 1;
+            }
+        }
+        count
+    }
+
+    pub fn press(&mut self, keys: &[u8], events: &mut [MaybeUninit<KeyboardKeyEvent>]) -> usize {
+        let mut count = 0;
+        for &k in keys {
+            let index = (k as usize) / 64;
+            if self.state[index] & (1 << (k % 64)) == 0 {
+                self.state[index] |= 1 << (k % 64);
+                events[count].write(KeyboardKeyEvent::Pressed(Self::translate_key(k)));
+                count += 1;
+            }
+        }
+        count
+    }
+}
+
+#[async_trait]
+impl UsbDriver for UsbHidKeyboardDriver {
+    async fn run(self: Arc<Self>, device: Arc<UsbDeviceAccess>) -> Result<(), UsbError> {
+        // TODO not sure whether to use boot protocol (easy) or GetReport
+        let config = device.select_configuration(|_| true).await?.unwrap();
+
+        log::info!("Setup HID keyboard");
+        let pipe = device
+            .open_interrupt_in_pipe(1, config.endpoints[0].max_packet_size as u16)
+            .await?;
+
+        let mut buffer = [0; 8];
+        let mut state = KeyboardState::new();
+        let mut events = [MaybeUninit::uninit(); 16];
+
+        loop {
+            let mut event_count = 0;
+
+            let len = pipe.read(&mut buffer).await?;
+            if len < 8 {
+                continue;
+            }
+
+            event_count += state.retain_modifiers(buffer[0], &mut events);
+            event_count += state.press_modifiers(buffer[0], &mut events[event_count..]);
+            event_count += state.retain(&buffer[2..], &mut events[event_count..]);
+            event_count += state.press(&buffer[2..], &mut events[event_count..]);
+
+            let events = unsafe { MaybeUninit::slice_assume_init_ref(&events[..event_count]) };
+
+            for &event in events {
+                log::trace!("Generic Keyboard: {:?}", event);
+                ygg_driver_input::send_event(event);
+            }
+        }
+    }
+
+    fn name(&self) -> &'static str {
+        "USB HID Keyboard"
+    }
+
+    fn probe(&self, class: &UsbClassInfo, _device: &UsbDeviceAccess) -> bool {
+        log::info!(
+            "class = {:?}, subclass = {:02x}",
+            class.class,
+            class.subclass
+        );
+        class.class == UsbDeviceClass::Hid && (class.subclass == 0x00 || class.subclass == 0x01)
+    }
+}

kernel/driver/bus/usb/src/class_driver/mass_storage.rs

@@ -0,0 +1,273 @@
+use core::mem::MaybeUninit;
+
+use alloc::{boxed::Box, sync::Arc};
+use async_trait::async_trait;
+use bytemuck::{Pod, Zeroable};
+use libk::{
+    dma::{DmaBuffer, DmaSliceMut},
+    error::Error,
+};
+use ygg_driver_scsi::{transport::ScsiTransport, ScsiEnclosure};
+
+use crate::{
+    communication::UsbDirection,
+    device::{UsbDeviceAccess, UsbDeviceDetachHandler},
+    error::UsbError,
+    info::{UsbDeviceClass, UsbEndpointType},
+    pipe::{
+        control::{ControlTransferSetup, UsbClassSpecificRequest},
+        normal::{UsbBulkInPipeAccess, UsbBulkOutPipeAccess},
+    },
+};
+
+use super::{UsbClassInfo, UsbDriver};
+
+pub struct UsbMassStorageDriverBulkOnly;
+
+#[derive(Debug, Clone, Copy, Zeroable, Pod)]
+#[repr(C)]
+struct Cbw {
+    signature: u32,       // 0x00
+    tag: u32,             // 0x04
+    transfer_length: u32, // 0x08
+    flags: u8,            // 0x0C
+    lun: u8,              // 0x0D
+    cb_length: u8,        // 0x0E
+    cb_data: [u8; 16],    // 0x0F
+    // Not sent
+    _0: u8,
+}
+
+#[derive(Debug, Clone, Copy, Zeroable, Pod)]
+#[repr(C)]
+struct Csw {
+    signature: u32,
+    tag: u32,
+    data_residue: u32,
+    status: u8,
+    _0: [u8; 3],
+}
+
+struct Bbb {
+    #[allow(unused)]
+    device: Arc<UsbDeviceAccess>,
+    in_pipe: UsbBulkInPipeAccess,
+    out_pipe: UsbBulkOutPipeAccess,
+    last_tag: u32,
+}
+
+struct DetachHandler(Arc<ScsiEnclosure>);
+
+impl Bbb {
+    pub fn new(
+        device: Arc<UsbDeviceAccess>,
+        in_pipe: UsbBulkInPipeAccess,
+        out_pipe: UsbBulkOutPipeAccess,
+    ) -> Result<Self, UsbError> {
+        Ok(Self {
+            device,
+            in_pipe,
+            out_pipe,
+            last_tag: 0,
+        })
+    }
+}
+
+impl Bbb {
+    async fn send_cbw(
+        &mut self,
+        lun: u8,
+        host_to_dev: bool,
+        command: &[u8],
+        response_len: usize,
+    ) -> Result<u32, Error> {
+        self.last_tag = self.last_tag.wrapping_add(1);
+
+        let flags = if !host_to_dev { 1 << 7 } else { 0 };
+        let tag = self.last_tag;
+        let mut cbw_bytes = [0; 32];
+        let cbw = bytemuck::from_bytes_mut::<Cbw>(&mut cbw_bytes);
+
+        cbw.signature = 0x43425355;
+        cbw.transfer_length = response_len as u32;
+        cbw.flags = flags;
+        cbw.tag = tag;
+        cbw.lun = lun;
+        cbw.cb_length = command.len() as u8;
+        cbw.cb_data[..command.len()].copy_from_slice(command);
+
+        self.out_pipe
+            .write(&cbw_bytes[..31])
+            .await
+            .inspect_err(|error| log::error!("msc: CBW send error: {error:?}"))?;
+
+        Ok(tag)
+    }
+
+    async fn read_csw(&mut self, tag: u32) -> Result<(), Error> {
+        let mut csw_bytes = [0; 16];
+        self.in_pipe
+            .read_exact(&mut csw_bytes[..13])
+            .await
+            .inspect_err(|error| log::error!("msc: CSW receive error: {error:?}"))?;
+        let csw = bytemuck::from_bytes::<Csw>(&csw_bytes);
+
+        if csw.signature != 0x53425355 {
+            log::warn!("msc: invalid csw signature");
+            return Err(Error::InvalidArgument);
+        }
+        if csw.tag != tag {
+            let csw_tag = csw.tag;
+            log::warn!("msc: invalid csw tag (got {}, expected {tag})", csw_tag);
+            return Err(Error::InvalidArgument);
+        }
+        if csw.status != 0x00 {
+            return Err(Error::InvalidArgument);
+        }
+        Ok(())
+    }
+
+    async fn read_response_data(
+        &mut self,
+        buffer: DmaSliceMut<'_, MaybeUninit<u8>>,
+    ) -> Result<usize, Error> {
+        if buffer.len() == 0 {
+            return Ok(0);
+        }
+        let len = self
+            .in_pipe
+            .read_dma(buffer)
+            .await
+            .inspect_err(|error| log::error!("msc: DMA read error: {error:?}"))?;
+        Ok(len)
+    }
+}
+
+#[async_trait]
+impl ScsiTransport for Bbb {
+    fn allocate_buffer(&self, size: usize) -> Result<DmaBuffer<[MaybeUninit<u8>]>, Error> {
+        Ok(self.in_pipe.allocate_dma_buffer(size)?)
+    }
+
+    async fn perform_request_raw(
+        &mut self,
+        lun: u8,
+        request_data: &[u8],
+        response_buffer: DmaSliceMut<'_, MaybeUninit<u8>>,
+    ) -> Result<usize, Error> {
+        if request_data.len() > 16 || response_buffer.len() > self.max_bytes_per_request() {
+            return Err(Error::InvalidArgument);
+        }
+
+        let tag = self
+            .send_cbw(lun, false, request_data, response_buffer.len())
+            .await?;
+        let response_len = self.read_response_data(response_buffer).await?;
+        self.read_csw(tag).await?;
+        Ok(response_len)
+    }
+
+    fn max_bytes_per_request(&self) -> usize {
+        32768
+    }
+}
+
+impl UsbDeviceDetachHandler for DetachHandler {
+    fn handle_device_detach(&self) {
+        log::info!("Mass storage detached");
+        self.0.detach();
+    }
+}
+
+#[derive(Debug, Pod, Zeroable, Clone, Copy)]
+#[repr(C)]
+pub struct BulkOnlyMassStorageReset;
+
+#[derive(Debug, Pod, Zeroable, Clone, Copy)]
+#[repr(C)]
+pub struct GetMaxLun;
+
+impl UsbClassSpecificRequest for BulkOnlyMassStorageReset {
+    const BM_REQUEST_TYPE: u8 = 0b00100001;
+    const B_REQUEST: u8 = 0b11111111;
+}
+
+impl UsbClassSpecificRequest for GetMaxLun {
+    const BM_REQUEST_TYPE: u8 = 0b10100001;
+    const B_REQUEST: u8 = 0b11111110;
+}
+
+#[async_trait]
+impl UsbDriver for UsbMassStorageDriverBulkOnly {
+    async fn run(self: Arc<Self>, device: Arc<UsbDeviceAccess>) -> Result<(), UsbError> {
+        // TODO filter to only accept BBB config
+        let config = device.select_configuration(|_| true).await?.unwrap();
+        // Bulk-in, bulk-out
+        assert_eq!(config.endpoints.len(), 2);
+        let control_pipe = device.control_pipe();
+        let (in_index, in_info) = config
+            .find_endpoint(|ep| ep.is(UsbEndpointType::Bulk, UsbDirection::In))
+            .ok_or(UsbError::InvalidConfiguration)?;
+        let (out_index, out_info) = config
+            .find_endpoint(|ep| ep.is(UsbEndpointType::Bulk, UsbDirection::Out))
+            .ok_or(UsbError::InvalidConfiguration)?;
+        let in_pipe = device
+            .open_bulk_in_pipe(in_index, in_info.max_packet_size as u16)
+            .await?;
+        let out_pipe = device
+            .open_bulk_out_pipe(out_index, out_info.max_packet_size as u16)
+            .await?;
+
+        // Perform a Bulk-Only Mass Storage Reset
+        // TODO interface id?
+        control_pipe
+            .control_transfer(ControlTransferSetup {
+                bm_request_type: BulkOnlyMassStorageReset::BM_REQUEST_TYPE,
+                b_request: BulkOnlyMassStorageReset::B_REQUEST,
+                w_value: 0,
+                w_index: 0,
+                w_length: 0,
+            })
+            .await?;
+
+        // Get max LUN
+        // TODO on devices which do not support multiple LUNs, this command may STALL
+        let mut buffer = [MaybeUninit::uninit()];
+        let len = control_pipe
+            .control_transfer_in(
+                ControlTransferSetup {
+                    bm_request_type: GetMaxLun::BM_REQUEST_TYPE,
+                    b_request: GetMaxLun::B_REQUEST,
+                    w_value: 0,
+                    w_index: 0,
+                    w_length: 1,
+                },
+                &mut buffer,
+            )
+            .await?;
+        let max_lun = if len < 1 {
+            0
+        } else {
+            unsafe { buffer[0].assume_init() }
+        };
+
+        let bbb = Bbb::new(device.clone(), in_pipe, out_pipe)?;
+        let scsi = ScsiEnclosure::setup(Box::new(bbb), max_lun as usize + 1)
+            .await
+            .inspect_err(|error| log::error!("msc: scsi error {error:?}"))
+            .map_err(|_| UsbError::DriverError)?;
+        let detach = DetachHandler(scsi.clone());
+        device.set_detach_handler(Arc::new(detach));
+
+        Ok(())
+    }
+
+    fn name(&self) -> &'static str {
+        "USB Mass Storage"
+    }
+
+    fn probe(&self, class: &UsbClassInfo, _device: &UsbDeviceAccess) -> bool {
+        // TODO support other protocols
+        class.class == UsbDeviceClass::MassStorage && class.interface_protocol_number == 0x50
+    }
+}

kernel/driver/bus/usb/src/class_driver/mod.rs

@@ -1,5 +1,5 @@
-use alloc::{sync::Arc, vec::Vec};
-use futures_util::future::BoxFuture;
+use alloc::{boxed::Box, sync::Arc, vec::Vec};
+use async_trait::async_trait;
 use libk::task::runtime;
 use libk_util::sync::spin_rwlock::IrqSafeRwLock;
 
@@ -9,30 +9,34 @@ use crate::{
     info::{UsbDeviceClass, UsbDeviceProtocol},
 };
 
+pub mod hid_keyboard;
+pub mod mass_storage;
+
+#[derive(Debug)]
 pub struct UsbClassInfo {
     pub class: UsbDeviceClass,
     pub subclass: u8,
     pub protocol: UsbDeviceProtocol,
+    pub device_protocol_number: u8,
+    pub interface_protocol_number: u8,
 }
 
+#[async_trait]
 pub trait UsbDriver: Send + Sync {
-    fn name(&self) -> &'static str;
-    fn run(
-        self: Arc<Self>,
-        device: Arc<UsbDeviceAccess>,
-    ) -> BoxFuture<'static, Result<(), UsbError>>;
+    async fn run(self: Arc<Self>, device: Arc<UsbDeviceAccess>) -> Result<(), UsbError>;
 
+    fn name(&self) -> &'static str;
     fn probe(&self, class: &UsbClassInfo, device: &UsbDeviceAccess) -> bool;
 }
 
 async fn extract_class_info(device: &UsbDeviceAccess) -> Result<Option<UsbClassInfo>, UsbError> {
-    if device.num_configurations != 1 {
+    if device.info.num_configurations != 1 {
         return Ok(None);
     }
     let device_info = &device.info;
     let config_info = device.query_configuration_info(0).await?;
 
-    if config_info.interfaces.len() == 1 {
+    if config_info.interfaces.len() >= 1 {
         let if_info = &config_info.interfaces[0];
 
         let class = if device_info.device_class == UsbDeviceClass::FromInterface {
@@ -55,6 +59,8 @@ async fn extract_class_info(device: &UsbDeviceAccess) -> Result<Option<UsbClassI
             class,
             subclass,
             protocol,
+            interface_protocol_number: if_info.interface_protocol_number,
+            device_protocol_number: device_info.device_protocol_number,
         }))
     } else {
         Ok(None)
@@ -104,191 +110,8 @@ pub fn register_driver(driver: Arc<dyn UsbDriver + 'static>) {
 
 pub fn register_default_class_drivers() {
     register_driver(Arc::new(hid_keyboard::UsbHidKeyboardDriver));
+    register_driver(Arc::new(mass_storage::UsbMassStorageDriverBulkOnly));
 }
 
 static USB_DEVICE_DRIVERS: IrqSafeRwLock<Vec<Arc<dyn UsbDriver + 'static>>> =
     IrqSafeRwLock::new(Vec::new());
-
-pub mod hid_keyboard {
-    use core::mem::MaybeUninit;
-
-    use alloc::sync::Arc;
-    use futures_util::{future::BoxFuture, FutureExt};
-    use libk_mm::PageBox;
-    use yggdrasil_abi::io::{KeyboardKey, KeyboardKeyEvent};
-
-    use crate::{device::UsbDeviceAccess, error::UsbError, info::UsbDeviceClass};
-
-    use super::{UsbClassInfo, UsbDriver};
-
-    pub struct UsbHidKeyboardDriver;
-
-    const MODIFIER_MAP: &[KeyboardKey] = &[
-        KeyboardKey::LControl,
-        KeyboardKey::LShift,
-        KeyboardKey::LAlt,
-        KeyboardKey::Unknown,
-        KeyboardKey::RControl,
-        KeyboardKey::RShift,
-        KeyboardKey::RAlt,
-        KeyboardKey::Unknown,
-    ];
-
-    #[derive(Default)]
-    struct KeyboardState {
-        state: [u64; 4],
-        mods: u8,
-    }
-
-    impl KeyboardState {
-        pub fn new() -> Self {
-            Self::default()
-        }
-
-        pub fn translate_key(k: u8) -> KeyboardKey {
-            match k {
-                4..=29 => KeyboardKey::Char(k - 4 + b'a'),
-                30..=38 => KeyboardKey::Char(k - 30 + b'1'),
-                39 => KeyboardKey::Char(b'0'),
-
-                40 => KeyboardKey::Enter,
-                41 => KeyboardKey::Escape,
-                42 => KeyboardKey::Backspace,
-                43 => KeyboardKey::Tab,
-                44 => KeyboardKey::Char(b' '),
-                45 => KeyboardKey::Char(b'-'),
-                46 => KeyboardKey::Char(b'='),
-                47 => KeyboardKey::Char(b'['),
-                48 => KeyboardKey::Char(b']'),
-                49 => KeyboardKey::Char(b'\\'),
-                51 => KeyboardKey::Char(b';'),
-                52 => KeyboardKey::Char(b'\''),
-                53 => KeyboardKey::Char(b'`'),
-                54 => KeyboardKey::Char(b','),
-                55 => KeyboardKey::Char(b'.'),
-                56 => KeyboardKey::Char(b'/'),
-
-                58..=69 => KeyboardKey::F(k - 58),
-
-                _ => {
-                    log::debug!("Unknown key: {}", k);
-                    KeyboardKey::Unknown
-                }
-            }
-        }
-
-        pub fn retain_modifiers(
-            &mut self,
-            m: u8,
-            events: &mut [MaybeUninit<KeyboardKeyEvent>],
-        ) -> usize {
-            let mut count = 0;
-            let released = self.mods & !m;
-            for (i, modifier) in MODIFIER_MAP.iter().enumerate().take(8) {
-                if released & (1 << i) != 0 {
-                    events[count].write(KeyboardKeyEvent::Released(*modifier));
-                    count += 1;
-                }
-            }
-            self.mods &= m;
-            count
-        }
-
-        pub fn press_modifiers(
-            &mut self,
-            m: u8,
-            events: &mut [MaybeUninit<KeyboardKeyEvent>],
-        ) -> usize {
-            let mut count = 0;
-            let pressed = m & !self.mods;
-            for (i, modifier) in MODIFIER_MAP.iter().enumerate().take(8) {
-                if pressed & (1 << i) != 0 {
-                    events[count].write(KeyboardKeyEvent::Pressed(*modifier));
-                    count += 1;
-                }
-            }
-            self.mods = m;
-            count
-        }
-
-        pub fn retain(
-            &mut self,
-            keys: &[u8],
-            events: &mut [MaybeUninit<KeyboardKeyEvent>],
-        ) -> usize {
-            let mut count = 0;
-            for i in 1..256 {
-                if self.state[i / 64] & (1 << (i % 64)) != 0 && !keys.contains(&(i as u8)) {
-                    events[count].write(KeyboardKeyEvent::Released(Self::translate_key(i as u8)));
-                    self.state[i / 64] &= !(1 << (i % 64));
-                    count += 1;
-                }
-            }
-            count
-        }
-
-        pub fn press(
-            &mut self,
-            keys: &[u8],
-            events: &mut [MaybeUninit<KeyboardKeyEvent>],
-        ) -> usize {
-            let mut count = 0;
-            for &k in keys {
-                let index = (k as usize) / 64;
-                if self.state[index] & (1 << (k % 64)) == 0 {
-                    self.state[index] |= 1 << (k % 64);
-                    events[count].write(KeyboardKeyEvent::Pressed(Self::translate_key(k)));
-                    count += 1;
-                }
-            }
-            count
-        }
-    }
-
-    impl UsbDriver for UsbHidKeyboardDriver {
-        fn run(
-            self: Arc<Self>,
-            device: Arc<UsbDeviceAccess>,
-        ) -> BoxFuture<'static, Result<(), UsbError>> {
-            async move {
-                // TODO not sure whether to use boot protocol (easy) or GetReport
-                let config = device.select_configuration(|_| true).await?.unwrap();
-                assert_eq!(config.endpoints.len(), 1);
-
-                let pipe = device.open_interrupt_in_pipe(1).await?;
-
-                let mut buffer = PageBox::new_slice(0, 8).map_err(UsbError::MemoryError)?;
-                let mut state = KeyboardState::new();
-                let mut events = [MaybeUninit::uninit(); 16];
-
-                loop {
-                    let mut event_count = 0;
-
-                    let data = pipe.read(&mut buffer).await?;
-
-                    event_count += state.retain_modifiers(data[0], &mut events);
-                    event_count += state.press_modifiers(data[0], &mut events[event_count..]);
-                    event_count += state.retain(&data[2..], &mut events[event_count..]);
-                    event_count += state.press(&data[2..], &mut events[event_count..]);
-
-                    let events =
-                        unsafe { MaybeUninit::slice_assume_init_ref(&events[..event_count]) };
-
-                    for &event in events {
-                        log::trace!("Generic Keyboard: {:?}", event);
-                        ygg_driver_input::send_event(event);
-                    }
-                }
-            }
-            .boxed()
-        }
-
-        fn name(&self) -> &'static str {
-            "USB HID Keyboard"
-        }
-
-        fn probe(&self, class: &UsbClassInfo, _device: &UsbDeviceAccess) -> bool {
-            class.class == UsbDeviceClass::Hid && class.subclass == 0x01
-        }
-    }
-}

kernel/driver/bus/usb/src/communication.rs

@@ -1,124 +1,11 @@
-use core::{
-    future::poll_fn,
-    sync::atomic::{AtomicU32, Ordering},
-    task::{Context, Poll},
-};
-
-use alloc::{sync::Arc, vec::Vec};
-use futures_util::task::AtomicWaker;
-use libk_mm::address::PhysicalAddress;
-
-use crate::error::UsbError;
-
 #[derive(Clone, Copy, PartialEq, Eq, Debug)]
 pub enum UsbDirection {
     Out,
     In,
 }
 
-#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
-#[repr(transparent)]
-pub struct UsbTransferToken(pub u64);
-
-#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
-#[repr(transparent)]
-pub struct UsbTransferResult(pub u32);
-
-pub struct UsbTransferStatus {
-    pub result: AtomicU32,
-    pub notify: AtomicWaker,
-}
-
-pub struct UsbControlTransfer {
-    pub id: UsbTransferToken,
-    pub length: usize,
-    pub direction: UsbDirection,
-    pub elements: Vec<PhysicalAddress>,
-    pub status: Arc<UsbTransferStatus>,
-}
-
-pub struct UsbInterruptTransfer {
-    pub address: PhysicalAddress,
-    pub length: usize,
-    pub direction: UsbDirection,
-    pub status: Arc<UsbTransferStatus>,
-}
-
 impl UsbDirection {
     pub const fn is_device_to_host(self) -> bool {
         matches!(self, UsbDirection::In)
     }
 }
-
-// TODO this is xHCI-specific
-impl UsbTransferResult {
-    pub fn is_aborted(&self) -> bool {
-        self.0 == u32::MAX
-    }
-
-    pub fn is_success(&self) -> bool {
-        (self.0 >> 24) & 0xFF == 1
-    }
-
-    pub fn sub_length(&self) -> usize {
-        (self.0 & 0xFFFFFF) as _
-    }
-}
-
-impl UsbControlTransfer {
-    pub async fn wait(&self) -> Result<usize, UsbError> {
-        let sub_length = self.status.wait().await?;
-        Ok(self.length.saturating_sub(sub_length))
-    }
-}
-
-impl UsbInterruptTransfer {
-    pub async fn wait(&self) -> Result<usize, UsbError> {
-        let sub_length = self.status.wait().await?;
-        Ok(self.length.saturating_sub(sub_length))
-    }
-}
-
-impl UsbTransferStatus {
-    pub fn new() -> Self {
-        Self {
-            result: AtomicU32::new(0),
-            notify: AtomicWaker::new(),
-        }
-    }
-
-    pub(crate) async fn wait(&self) -> Result<usize, UsbError> {
-        poll_fn(|cx| {
-            self.poll(cx).map(|v| {
-                if v.is_success() {
-                    Ok(v.sub_length())
-                } else if v.is_aborted() {
-                    Err(UsbError::DeviceDisconnected)
-                } else {
-                    Err(UsbError::TransferFailed)
-                }
-            })
-        })
-        .await
-    }
-
-    pub fn signal(&self, status: u32) {
-        self.result.store(status, Ordering::Release);
-        self.notify.wake();
-    }
-
-    pub fn abort(&self) {
-        self.result.store(u32::MAX, Ordering::Release);
-        self.notify.wake();
-    }
-
-    pub fn poll(&self, cx: &mut Context<'_>) -> Poll<UsbTransferResult> {
-        self.notify.register(cx.waker());
-        let value = self.result.load(Ordering::Acquire);
-        if value != 0 {
-            Poll::Ready(UsbTransferResult(value))
-        } else {
-            Poll::Pending
-        }
-    }
-}

kernel/driver/bus/usb/src/descriptor.rs

@@ -1,9 +1,10 @@
 use bytemuck::{Pod, Zeroable};
 
 use crate::{
+    communication::UsbDirection,
+    device::UsbSpeed,
     error::UsbError,
-    info::{UsbDeviceClass, UsbDeviceProtocol, UsbEndpointType},
-    UsbDirection,
+    info::{UsbDeviceClass, UsbDeviceProtocol, UsbEndpointType, UsbVersion},
 };
 
 #[derive(Clone, Copy, Debug, Default, Pod, Zeroable)]
@@ -134,12 +135,14 @@ impl UsbDeviceDescriptor {
         UsbDeviceProtocol::try_from(self.device_protocol).unwrap_or(UsbDeviceProtocol::Unknown)
     }
 
-    pub fn max_packet_size(&self) -> Result<usize, UsbError> {
-        match self.max_packet_size_0 {
-            8 => Ok(8),
-            16 => Ok(16),
-            32 => Ok(32),
-            64 => Ok(64),
+    pub fn max_packet_size(&self, version: UsbVersion, speed: UsbSpeed) -> Result<usize, UsbError> {
+        match (version.is_version_3(), speed, self.max_packet_size_0) {
+            (true, UsbSpeed::Super, 9) => Ok(1 << 9),
+            (true, _, _) => todo!("Non-GenX speed USB3+ maxpacketsize0"),
+            (false, _, 8) => Ok(8),
+            (false, _, 16) => Ok(16),
+            (false, _, 32) => Ok(32),
+            (false, _, 64) => Ok(64),
             _ => Err(UsbError::InvalidDescriptorField),
         }
     }

kernel/driver/bus/usb/src/device.rs

@@ -1,16 +1,21 @@
 use core::{fmt, ops::Deref};
 
-use alloc::{boxed::Box, vec::Vec};
-use futures_util::future::BoxFuture;
-use libk_mm::PageBox;
+use alloc::{boxed::Box, sync::Arc, vec::Vec};
+use async_trait::async_trait;
 use libk_util::sync::spin_rwlock::{IrqSafeRwLock, IrqSafeRwLockReadGuard};
 
 use crate::{
     error::UsbError,
-    info::{UsbConfigurationInfo, UsbDeviceInfo, UsbEndpointInfo, UsbInterfaceInfo},
+    info::{
+        UsbConfigurationInfo, UsbDeviceInfo, UsbEndpointInfo, UsbEndpointType, UsbInterfaceInfo,
+        UsbVersion,
+    },
     pipe::{
         control::{ConfigurationDescriptorEntry, UsbControlPipeAccess},
-        interrupt::UsbInterruptInPipeAccess,
+        normal::{
+            UsbBulkInPipeAccess, UsbBulkOutPipeAccess, UsbInterruptInPipeAccess, UsbNormalPipeIn,
+            UsbNormalPipeOut,
+        },
     },
     UsbHostController,
 };
@@ -24,9 +29,8 @@ pub struct UsbBusAddress {
 }
 
 pub struct UsbDeviceAccess {
-    pub device: Box<dyn UsbDevice>,
+    pub device: Arc<dyn UsbDevice>,
     pub info: UsbDeviceInfo,
-    pub num_configurations: u8,
     pub current_configuration: IrqSafeRwLock<Option<UsbConfigurationInfo>>,
 }
 
@@ -38,23 +42,36 @@ pub enum UsbSpeed {
     Super,
 }
 
+pub trait UsbDeviceDetachHandler: Send + Sync {
+    fn handle_device_detach(&self);
+}
+
+#[async_trait]
 #[allow(unused)]
 pub trait UsbDevice: Send + Sync {
     // Endpoint "0"
     fn control_pipe(&self) -> &UsbControlPipeAccess;
 
-    fn open_interrupt_in_pipe(
+    async fn open_normal_in_pipe(
         &self,
         number: u8,
-    ) -> BoxFuture<Result<UsbInterruptInPipeAccess, UsbError>> {
-        unimplemented!()
-    }
+        max_packet_size: u16,
+        ty: UsbEndpointType,
+    ) -> Result<Box<dyn UsbNormalPipeIn>, UsbError>;
+
+    async fn open_normal_out_pipe(
+        &self,
+        number: u8,
+        max_packet_size: u16,
+        ty: UsbEndpointType,
+    ) -> Result<Box<dyn UsbNormalPipeOut>, UsbError>;
 
     fn port_number(&self) -> u8;
     fn bus_address(&self) -> UsbBusAddress;
     fn speed(&self) -> UsbSpeed;
     fn controller_ref(&self) -> &dyn UsbHostController;
 
+    fn set_detach_handler(&self, handler: Arc<dyn UsbDeviceDetachHandler>);
     fn handle_detach(&self);
 
     fn debug(&self) {}
@@ -67,22 +84,29 @@ impl UsbDeviceAccess {
     /// * Device is not yet configured
     /// * Control pipe for the device has been properly set up
     /// * Device has been assigned a bus address
-    pub async fn setup(raw: Box<dyn UsbDevice>) -> Result<Self, UsbError> {
+    pub async fn setup(raw: Arc<dyn UsbDevice>) -> Result<Self, UsbError> {
         let control = raw.control_pipe();
-        let mut string_buffer = PageBox::new_uninit().map_err(UsbError::MemoryError)?;
 
         let device_desc = control.query_device_descriptor().await?;
 
-        let manufacturer = control
-            .query_string(device_desc.manufacturer_str, &mut string_buffer)
-            .await?;
-        let product = control
-            .query_string(device_desc.product_str, &mut string_buffer)
-            .await?;
+        let bcd_usb = device_desc.bcd_usb;
+        let usb_version = UsbVersion::from_bcd_usb(device_desc.bcd_usb)
+            .ok_or(UsbError::InvalidDescriptorField)
+            .inspect_err(|_| {
+                log::error!(
+                    "{}: unsupported/invalid USB version: {:#x}",
+                    raw.bus_address(),
+                    bcd_usb
+                )
+            })?;
+
+        let manufacturer = control.query_string(device_desc.manufacturer_str).await?;
+        let product = control.query_string(device_desc.product_str).await?;
 
         let info = UsbDeviceInfo {
             manufacturer,
             product,
+            usb_version,
 
             id_vendor: device_desc.id_vendor,
             id_product: device_desc.id_product,
@@ -90,18 +114,56 @@ impl UsbDeviceAccess {
             device_class: device_desc.class(),
             device_subclass: device_desc.device_subclass,
             device_protocol: device_desc.protocol(),
+            device_protocol_number: device_desc.device_protocol,
 
-            max_packet_size: device_desc.max_packet_size()?,
+            num_configurations: device_desc.num_configurations,
+
+            max_packet_size: device_desc.max_packet_size(usb_version, raw.speed())?,
         };
 
         Ok(Self {
             device: raw,
             info,
-            num_configurations: device_desc.num_configurations,
             current_configuration: IrqSafeRwLock::new(None),
         })
     }
 
+    pub async fn open_interrupt_in_pipe(
+        &self,
+        number: u8,
+        max_packet_size: u16,
+    ) -> Result<UsbInterruptInPipeAccess, UsbError> {
+        let pipe = self
+            .device
+            .open_normal_in_pipe(number, max_packet_size, UsbEndpointType::Interrupt)
+            .await?;
+        Ok(UsbInterruptInPipeAccess(pipe))
+    }
+
+    pub async fn open_bulk_in_pipe(
+        &self,
+        number: u8,
+        max_packet_size: u16,
+    ) -> Result<UsbBulkInPipeAccess, UsbError> {
+        let pipe = self
+            .device
+            .open_normal_in_pipe(number, max_packet_size, UsbEndpointType::Bulk)
+            .await?;
+        Ok(UsbBulkInPipeAccess(pipe))
+    }
+
+    pub async fn open_bulk_out_pipe(
+        &self,
+        number: u8,
+        max_packet_size: u16,
+    ) -> Result<UsbBulkOutPipeAccess, UsbError> {
+        let pipe = self
+            .device
+            .open_normal_out_pipe(number, max_packet_size, UsbEndpointType::Bulk)
+            .await?;
+        Ok(UsbBulkOutPipeAccess(pipe))
+    }
+
     pub fn read_current_configuration(
         &self,
     ) -> IrqSafeRwLockReadGuard<Option<UsbConfigurationInfo>> {
@@ -115,7 +177,7 @@ impl UsbDeviceAccess {
         let mut current_config = self.current_configuration.write();
         let control_pipe = self.control_pipe();
 
-        for i in 0..self.num_configurations {
+        for i in 0..self.info.num_configurations {
             let info = self.query_configuration_info(i).await?;
 
             if predicate(&info) {
@@ -137,15 +199,14 @@ impl UsbDeviceAccess {
         &self,
         index: u8,
     ) -> Result<UsbConfigurationInfo, UsbError> {
-        if index >= self.num_configurations {
+        if index >= self.info.num_configurations {
             return Err(UsbError::InvalidConfiguration);
         }
-        let mut string_buffer = PageBox::new_uninit().map_err(UsbError::MemoryError)?;
         let control_pipe = self.control_pipe();
         let query = control_pipe.query_configuration_descriptor(index).await?;
 
         let configuration_name = control_pipe
-            .query_string(query.configuration().config_str, &mut string_buffer)
+            .query_string(query.configuration().config_str)
             .await?;
 
         let mut endpoints = Vec::new();
@@ -162,9 +223,7 @@ impl UsbDeviceAccess {
                     });
                 }
                 ConfigurationDescriptorEntry::Interface(iface) => {
-                    let name = control_pipe
-                        .query_string(iface.interface_str, &mut string_buffer)
-                        .await?;
+                    let name = control_pipe.query_string(iface.interface_str).await?;
                     interfaces.push(UsbInterfaceInfo {
                         name,
                         number: iface.interface_number,
@@ -172,6 +231,7 @@ impl UsbDeviceAccess {
                         interface_class: iface.class(),
                         interface_subclass: iface.interface_subclass,
                         interface_protocol: iface.protocol(),
+                        interface_protocol_number: iface.interface_protocol,
                     });
                 }
                 _ => (),
@@ -187,6 +247,10 @@ impl UsbDeviceAccess {
 
         Ok(info)
     }
+
+    pub fn set_detach_handler(&self, handler: Arc<dyn UsbDeviceDetachHandler>) {
+        self.device.set_detach_handler(handler);
+    }
 }
 
 impl Deref for UsbDeviceAccess {

kernel/driver/bus/usb/src/error.rs

@@ -1,5 +1,15 @@
 use yggdrasil_abi::error::Error;
 
+#[derive(Debug, Clone, Copy)]
+pub enum TransferError {
+    InvalidTransfer,
+    ShortPacket(usize),
+    BufferError,
+    UsbTransactionError,
+    Stall,
+    Other(u8),
+}
+
 #[derive(Debug)]
 pub enum UsbError {
     /// Could not allocate memory for some device structure
@@ -8,23 +18,35 @@ pub enum UsbError {
     SystemError(Error),
     // HC-side init stage errors
     OutOfAddresses,
-    HostControllerCommandFailed(u8),
+    HostControllerCommandFailed(u8, u32),
     PortResetFailed,
     PortInitFailed,
     // Setup stage errors
     InvalidConfiguration,
     InvalidDescriptorField,
     // Runtime errors
+    TimedOut,
     DeviceBusy,
     DeviceDisconnected,
-    TransferFailed,
+    TransferFailed(TransferError),
+    // Driver errors
+    DriverError,
+}
+
+impl From<TransferError> for UsbError {
+    fn from(value: TransferError) -> Self {
+        Self::TransferFailed(value)
+    }
 }
 
 impl From<UsbError> for Error {
     fn from(value: UsbError) -> Self {
         match value {
             UsbError::MemoryError(e) => e,
-            _ => Error::InvalidOperation,
+            e => {
+                log::warn!("usb: {e:?}");
+                Error::InvalidOperation
+            }
         }
     }
 }

kernel/driver/bus/usb/src/info.rs

@@ -1,9 +1,11 @@
+use core::fmt;
+
 use alloc::{string::String, vec::Vec};
 use yggdrasil_abi::primitive_enum;
 
-use crate::UsbDirection;
+use crate::communication::UsbDirection;
 
-#[derive(Debug, Clone, Copy)]
+#[derive(Debug, Clone, Copy, PartialEq)]
 pub enum UsbEndpointType {
     Control,
     Isochronous,
@@ -27,10 +29,21 @@ pub enum UsbUsageType {
     Reserved,
 }
 
+#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)]
+pub enum UsbVersion {
+    Usb11,
+    Usb20,
+    Usb21,
+    Usb30,
+    Usb31,
+    Usb32,
+}
+
 primitive_enum! {
     pub enum UsbDeviceClass: u8 {
         FromInterface = 0x00,
         Hid = 0x03,
+        MassStorage = 0x08,
         Unknown = 0xFF,
     }
 }
@@ -50,6 +63,7 @@ pub struct UsbInterfaceInfo {
     pub interface_class: UsbDeviceClass,
     pub interface_subclass: u8,
     pub interface_protocol: UsbDeviceProtocol,
+    pub interface_protocol_number: u8,
 }
 
 #[derive(Debug, Clone)]
@@ -73,12 +87,67 @@ pub struct UsbDeviceInfo {
     pub manufacturer: String,
     pub product: String,
 
+    pub usb_version: UsbVersion,
+
     pub id_vendor: u16,
     pub id_product: u16,
 
     pub device_class: UsbDeviceClass,
     pub device_subclass: u8,
     pub device_protocol: UsbDeviceProtocol,
+    pub device_protocol_number: u8,
 
+    /// Max packet size for endpoint zero
     pub max_packet_size: usize,
+
+    pub num_configurations: u8,
+}
+
+impl UsbVersion {
+    pub fn is_version_3(&self) -> bool {
+        matches!(self, Self::Usb30 | Self::Usb31 | Self::Usb32)
+    }
+
+    pub fn from_bcd_usb(value: u16) -> Option<Self> {
+        match value {
+            0x110 => Some(UsbVersion::Usb11),
+            0x200..=0x20F => Some(UsbVersion::Usb20),
+            0x210..=0x21F => Some(UsbVersion::Usb21),
+            0x300 => Some(UsbVersion::Usb30),
+            0x310 => Some(UsbVersion::Usb31),
+            0x320 => Some(UsbVersion::Usb32),
+            _ => None,
+        }
+    }
+}
+
+impl fmt::Display for UsbVersion {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let string = match self {
+            Self::Usb11 => "USB1.1",
+            Self::Usb20 => "USB2.0",
+            Self::Usb21 => "USB2.1",
+            Self::Usb30 => "USB3.0",
+            Self::Usb31 => "USB3.1",
+            Self::Usb32 => "USB3.2",
+        };
+        f.write_str(string)
+    }
+}
+
+impl UsbEndpointInfo {
+    pub fn is(&self, ty: UsbEndpointType, dir: UsbDirection) -> bool {
+        self.ty == ty && self.direction == dir
+    }
+}
+
+impl UsbConfigurationInfo {
+    pub fn find_endpoint<P: Fn(&UsbEndpointInfo) -> bool>(
+        &self,
+        predicate: P,
+    ) -> Option<(u8, &UsbEndpointInfo)> {
+        let index = self.endpoints.iter().position(predicate)?;
+        let info = &self.endpoints[index];
+        Some((index as u8 + 1, info))
+    }
 }

kernel/driver/bus/usb/src/lib.rs

@@ -1,6 +1,13 @@
 #![no_std]
 #![allow(clippy::new_without_default)]
-#![feature(iter_array_chunks, maybe_uninit_slice)]
+#![feature(
+    iter_array_chunks,
+    maybe_uninit_slice,
+    maybe_uninit_as_bytes,
+    maybe_uninit_uninit_array,
+    maybe_uninit_write_slice,
+    maybe_uninit_fill
+)]
 
 extern crate alloc;
 
@@ -15,7 +22,7 @@ pub mod util;
 
 pub mod class_driver;
 
-pub use communication::{UsbControlTransfer, UsbDirection, UsbTransferStatus, UsbTransferToken};
+// pub use communication::{UsbControlTransfer, UsbDirection, UsbTransferStatus, UsbTransferToken};
 
 pub trait UsbEndpoint: Sync {}
 

kernel/driver/bus/usb/src/pipe/control.rs

@@ -1,15 +1,12 @@
 use core::{
-    cmp::Ordering,
     mem::{size_of, MaybeUninit},
     ops::Deref,
 };
 
 use alloc::{boxed::Box, string::String};
+use async_trait::async_trait;
 use bytemuck::{Pod, Zeroable};
-use libk_mm::{
-    address::{AsPhysicalAddress, PhysicalAddress},
-    PageBox,
-};
+use libk_mm::PageBox;
 
 use crate::{
     descriptor::{
@@ -17,11 +14,8 @@ use crate::{
         UsbInterfaceDescriptor, UsbOtherSpeedConfiguration,
     },
     error::UsbError,
-    UsbControlTransfer, UsbDirection,
 };
 
-use super::UsbGenericPipe;
-
 #[derive(Debug)]
 pub struct ControlTransferSetup {
     pub bm_request_type: u8,
@@ -40,6 +34,11 @@ pub trait UsbDeviceRequest: Sized + Pod {
     const B_REQUEST: u8;
 }
 
+pub trait UsbClassSpecificRequest: Sized + Pod {
+    const BM_REQUEST_TYPE: u8;
+    const B_REQUEST: u8;
+}
+
 pub trait UsbDescriptorRequest: UsbDeviceRequest {
     const DESCRIPTOR_TYPE: u8;
 }
@@ -83,28 +82,23 @@ fn decode_usb_string(bytes: &[u8]) -> Result<String, UsbError> {
 
 // Pipe impl
 
-pub trait UsbControlPipe: UsbGenericPipe + Send + Sync {
-    fn start_transfer(
+#[async_trait]
+pub trait UsbControlPipe: Send + Sync {
+    async fn control_transfer(&self, setup: ControlTransferSetup) -> Result<(), UsbError>;
+    async fn control_transfer_in(
         &self,
         setup: ControlTransferSetup,
-        data: Option<(PhysicalAddress, usize, UsbDirection)>,
-    ) -> Result<UsbControlTransfer, UsbError>;
-
-    fn complete_transfer(&self, transfer: UsbControlTransfer);
+        buffer: &mut [MaybeUninit<u8>],
+    ) -> Result<usize, UsbError>;
+    async fn control_transfer_out(
+        &self,
+        setup: ControlTransferSetup,
+        buffer: &[u8],
+    ) -> Result<usize, UsbError>;
 }
 
 pub struct UsbControlPipeAccess(pub Box<dyn UsbControlPipe>);
 
-fn input_buffer<T: Pod>(
-    data: &mut PageBox<MaybeUninit<T>>,
-) -> (PhysicalAddress, usize, UsbDirection) {
-    (
-        unsafe { data.as_physical_address() },
-        size_of::<T>(),
-        UsbDirection::In,
-    )
-}
-
 #[derive(Debug)]
 pub enum ConfigurationDescriptorEntry<'a> {
     Configuration(&'a UsbConfigurationDescriptor),
@@ -183,80 +177,10 @@ impl ConfigurationDescriptorQuery {
 }
 
 impl UsbControlPipeAccess {
-    pub async fn perform_value_control(
-        &self,
-        setup: ControlTransferSetup,
-        buffer: Option<(PhysicalAddress, usize, UsbDirection)>,
-    ) -> Result<(), UsbError> {
-        let transfer = self.start_transfer(setup, buffer)?;
-        transfer.status.wait().await?;
-        self.complete_transfer(transfer);
-        Ok(())
-    }
+    pub async fn query_device_descriptor(&self) -> Result<UsbDeviceDescriptor, UsbError> {
+        let mut buffer = MaybeUninit::uninit();
 
-    async fn fill_configuation_descriptor(
-        &self,
-        index: u8,
-        buffer: &mut PageBox<[MaybeUninit<u8>]>,
-    ) -> Result<(), UsbError> {
-        self.perform_value_control(
-            ControlTransferSetup {
-                bm_request_type: 0b10000000,
-                b_request: 0x06,
-                w_value: 0x200 | (index as u16),
-                w_index: 0,
-                w_length: buffer.len().try_into().unwrap(),
-            },
-            Some((
-                unsafe { buffer.as_physical_address() },
-                buffer.len(),
-                UsbDirection::In,
-            )),
-        )
-        .await
-    }
-
-    pub async fn query_configuration_descriptor(
-        &self,
-        index: u8,
-    ) -> Result<ConfigurationDescriptorQuery, UsbError> {
-        // First, query the real length of the descriptor
-        let mut buffer = PageBox::new_uninit_slice(size_of::<UsbConfigurationDescriptor>())
-            .map_err(UsbError::MemoryError)?;
-        self.fill_configuation_descriptor(index, &mut buffer)
-            .await?;
-        let buffer = unsafe { PageBox::assume_init_slice(buffer) };
-
-        let desc: &UsbConfigurationDescriptor = bytemuck::from_bytes(&buffer);
-        let total_len = desc.total_length as usize;
-
-        // Return if everything's ready at this point
-        match total_len.cmp(&size_of::<UsbConfigurationDescriptor>()) {
-            Ordering::Less => todo!(),
-            Ordering::Equal => return Ok(ConfigurationDescriptorQuery { buffer }),
-            _ => (),
-        }
-
-        // Otherwise, query the rest of the data
-        let mut buffer = PageBox::new_uninit_slice(total_len).map_err(UsbError::MemoryError)?;
-        self.fill_configuation_descriptor(index, &mut buffer)
-            .await?;
-        let buffer = unsafe { PageBox::assume_init_slice(buffer) };
-
-        let desc: &UsbConfigurationDescriptor =
-            bytemuck::from_bytes(&buffer[..size_of::<UsbConfigurationDescriptor>()]);
-        let total_len = desc.total_length as usize;
-
-        if total_len != buffer.len() {
-            todo!();
-        }
-
-        Ok(ConfigurationDescriptorQuery { buffer })
-    }
-
-    pub async fn query_device_descriptor(&self) -> Result<PageBox<UsbDeviceDescriptor>, UsbError> {
-        let mut output = PageBox::new_uninit().map_err(UsbError::MemoryError)?;
-        self.perform_value_control(
+        self.control_transfer_in(
             ControlTransferSetup {
                 bm_request_type: 0b10000000,
                 b_request: 0x06,
@@ -264,54 +188,106 @@ impl UsbControlPipeAccess {
                 w_index: 0,
                 w_length: size_of::<UsbDeviceDescriptor>() as _,
             },
-            Some(input_buffer(&mut output)),
+            buffer.as_bytes_mut(),
         )
         .await?;
 
-        Ok(unsafe { output.assume_init() })
+        Ok(unsafe { buffer.assume_init() })
     }
 
-    pub async fn query_string(
+    async fn fill_configuation_descriptor(
         &self,
         index: u8,
-        buffer: &mut PageBox<MaybeUninit<[u8; 4096]>>,
-    ) -> Result<String, UsbError> {
-        self.perform_value_control(
+        buffer: &mut [MaybeUninit<u8>],
+    ) -> Result<(), UsbError> {
+        self.control_transfer_in(
             ControlTransferSetup {
                 bm_request_type: 0b10000000,
                 b_request: 0x06,
-                w_value: 0x300 | (index as u16),
+                w_value: 0x200 | (index as u16),
                 w_index: 0,
-                w_length: 4096,
+                w_length: buffer.len().try_into().unwrap(),
             },
-            Some(input_buffer(buffer)),
+            buffer,
         )
         .await?;
-        let data = unsafe { buffer.assume_init_ref() };
+        Ok(())
+    }
 
+    pub async fn query_string(&self, index: u8) -> Result<String, UsbError> {
+        let mut buffer = MaybeUninit::uninit_array::<256>();
+
+        let len = self
+            .control_transfer_in(
+                ControlTransferSetup {
+                    bm_request_type: 0b10000000,
+                    b_request: 0x06,
+                    w_value: 0x300 | (index as u16),
+                    w_index: 0,
+                    w_length: 4096,
+                },
+                &mut buffer[..],
+            )
+            .await?;
+        let data = unsafe { MaybeUninit::slice_assume_init_ref(&buffer[..len]) };
         let len = data[0] as usize;
 
         decode_usb_string(&data[2..len])
     }
 
+    pub async fn query_configuration_descriptor(
+        &self,
+        index: u8,
+    ) -> Result<ConfigurationDescriptorQuery, UsbError> {
+        // 4K should be enough for a configuration descriptor
+        let mut buffer = PageBox::new_uninit_slice(4096).map_err(UsbError::MemoryError)?;
+
+        self.fill_configuation_descriptor(index, &mut buffer[..])
+            .await?;
+
+        let buffer = unsafe { PageBox::assume_init_slice(buffer) };
+
+        let desc: &UsbConfigurationDescriptor =
+            bytemuck::from_bytes(&buffer[..size_of::<UsbConfigurationDescriptor>()]);
+        let total_len = desc.total_length as usize;
+
+        if total_len > 4096 {
+            unimplemented!("4KiB wasn't enough");
+        }
+
+        Ok(ConfigurationDescriptorQuery { buffer })
+    }
+
     pub async fn perform_action<D: UsbDeviceRequest>(
         &self,
         w_value: u16,
         w_index: u16,
     ) -> Result<(), UsbError> {
-        self.perform_value_control(
-            ControlTransferSetup {
-                bm_request_type: D::BM_REQUEST_TYPE,
-                b_request: D::B_REQUEST,
-                w_value,
-                w_index,
-                w_length: 0,
-            },
-            None,
-        )
+        self.control_transfer(ControlTransferSetup {
+            bm_request_type: D::BM_REQUEST_TYPE,
+            b_request: D::B_REQUEST,
+            w_value,
+            w_index,
+            w_length: 0,
+        })
         .await
     }
 
+    // pub async fn class_specific_request<D: UsbClassSpecificRequest>(
+    //     &self,
+    //     w_value: u16,
+    //     w_index: u16,
+    // ) -> Result<usize, UsbError> {
+    //     self.control_transfer(ControlTransferSetup {
+    //         bm_request_type: D::BM_REQUEST_TYPE,
+    //         b_request: D::B_REQUEST,
+    //         w_value,
+    //         w_index,
+    //         w_length: 0,
+    //     })
+    //     .await
+    // }
+
     pub async fn set_configuration(&self, value: u16) -> Result<(), UsbError> {
         self.perform_action::<SetConfiguration>(value, 0).await
     }

kernel/driver/bus/usb/src/pipe/interrupt.rs

@@ -1,32 +0,0 @@
-use core::ops::Deref;
-
-use alloc::boxed::Box;
-use libk_mm::PageBox;
-
-use crate::{communication::UsbInterruptTransfer, error::UsbError};
-
-use super::UsbGenericPipe;
-
-pub trait UsbInterruptInPipe: UsbGenericPipe + Send + Sync {
-    fn start_read(&self, buffer: &mut PageBox<[u8]>) -> Result<UsbInterruptTransfer, UsbError>;
-    fn complete_transfer(&self, transfer: UsbInterruptTransfer);
-}
-
-pub struct UsbInterruptInPipeAccess(pub Box<dyn UsbInterruptInPipe>);
-
-impl UsbInterruptInPipeAccess {
-    pub async fn read<'a>(&self, buffer: &'a mut PageBox<[u8]>) -> Result<&'a [u8], UsbError> {
-        let transfer = self.start_read(buffer)?;
-        let len = transfer.wait().await?;
-        self.complete_transfer(transfer);
-        Ok(&buffer[..len])
-    }
-}
-
-impl Deref for UsbInterruptInPipeAccess {
-    type Target = dyn UsbInterruptInPipe;
-
-    fn deref(&self) -> &Self::Target {
-        &*self.0
-    }
-}

kernel/driver/bus/usb/src/pipe/mod.rs

@@ -1,8 +1,12 @@
+use core::mem::MaybeUninit;
+
+use libk::dma::DmaBuffer;
+
+use crate::error::UsbError;
+
 pub mod control;
-pub mod interrupt;
+pub mod normal;
 
-pub trait UsbGenericPipe {}
-
-pub enum UsbPipe {
-    Control(control::UsbControlPipeAccess),
+pub trait UsbGenericPipe: Send + Sync {
+    fn allocate_dma_buffer(&self, len: usize) -> Result<DmaBuffer<[MaybeUninit<u8>]>, UsbError>;
 }

kernel/driver/bus/usb/src/pipe/normal.rs

@@ -0,0 +1,81 @@
+use core::{mem::MaybeUninit, ops::Deref};
+
+use alloc::boxed::Box;
+use async_trait::async_trait;
+use libk::dma::{DmaBuffer, DmaSlice, DmaSliceMut};
+
+use crate::error::{TransferError, UsbError};
+
+use super::UsbGenericPipe;
+
+#[async_trait]
+pub trait UsbNormalPipeIn: UsbGenericPipe {
+    async fn read_dma(&self, buffer: DmaSliceMut<'_, MaybeUninit<u8>>) -> Result<usize, UsbError>;
+
+    async fn read(&self, buffer: &mut [u8]) -> Result<usize, UsbError> {
+        let mut dma_buffer = self.allocate_dma_buffer(buffer.len())?;
+        let len = self.read_dma(dma_buffer.slice_mut(0..buffer.len())).await?;
+        let dma_slice = unsafe { MaybeUninit::slice_assume_init_ref(&dma_buffer[..len]) };
+        buffer[..len].copy_from_slice(dma_slice);
+        Ok(len)
+    }
+
+    async fn read_exact(&self, buffer: &mut [u8]) -> Result<(), UsbError> {
+        match self.read(buffer).await {
+            Ok(len) if len == buffer.len() => Ok(()),
+            Ok(len) => Err(UsbError::TransferFailed(TransferError::ShortPacket(len))),
+            Err(error) => Err(error),
+        }
+    }
+}
+
+#[async_trait]
+pub trait UsbNormalPipeOut: UsbGenericPipe {
+    async fn write_dma(&self, buffer: DmaSlice<'_, u8>) -> Result<usize, UsbError>;
+
+    async fn write(&self, buffer: &[u8]) -> Result<usize, UsbError> {
+        let mut dma_buffer = self.allocate_dma_buffer(buffer.len())?;
+        MaybeUninit::copy_from_slice(&mut dma_buffer, buffer);
+        let dma_buffer = unsafe { DmaBuffer::assume_init_slice(dma_buffer) };
+
+        self.write_dma(dma_buffer.slice(0..buffer.len())).await
+    }
+}
+
+pub struct UsbBulkInPipeAccess(pub Box<dyn UsbNormalPipeIn>);
+pub struct UsbBulkOutPipeAccess(pub Box<dyn UsbNormalPipeOut>);
+
+pub struct UsbInterruptInPipeAccess(pub Box<dyn UsbNormalPipeIn>);
+pub struct UsbInterruptOutPipeAccess(pub Box<dyn UsbNormalPipeOut>);
+
+impl Deref for UsbBulkInPipeAccess {
+    type Target = dyn UsbNormalPipeIn;
+
+    fn deref(&self) -> &Self::Target {
+        self.0.deref()
+    }
+}
+
+impl Deref for UsbBulkOutPipeAccess {
+    type Target = dyn UsbNormalPipeOut;
+
+    fn deref(&self) -> &Self::Target {
+        self.0.deref()
+    }
+}
+
+impl Deref for UsbInterruptInPipeAccess {
+    type Target = dyn UsbNormalPipeIn;
+
+    fn deref(&self) -> &Self::Target {
+        self.0.deref()
+    }
+}
+
+impl Deref for UsbInterruptOutPipeAccess {
+    type Target = dyn UsbNormalPipeOut;
+
+    fn deref(&self) -> &Self::Target {
+        self.0.deref()
+    }
+}

kernel/driver/fs/fat32/Cargo.toml

@@ -0,0 +1,14 @@
+[package]
+name = "ygg_driver_fat32"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+yggdrasil-abi.workspace = true
+device-api.workspace = true
+libk.workspace = true
+libk-util.workspace = true
+
+bytemuck.workspace = true
+log.workspace = true
+async-trait.workspace = true

kernel/driver/fs/fat32/src/data.rs

@@ -0,0 +1,172 @@
+use core::fmt;
+
+use libk_util::{get_le_u16, get_le_u32};
+
+use crate::FsLayout;
+
+#[derive(Debug, Clone, Copy)]
+#[repr(C)]
+pub struct Bpb<'a> {
+    bytes: &'a [u8],
+}
+
+#[derive(Debug, Clone, Copy)]
+#[repr(C)]
+pub struct Fat32Ebpb<'a> {
+    bytes: &'a [u8],
+}
+
+#[derive(Debug, Clone, Copy)]
+#[repr(C)]
+pub struct Fat32FsInfo {
+    bytes: [u8; 512],
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[repr(transparent)]
+pub struct ClusterNumber(pub u32);
+
+#[derive(PartialEq, Eq)]
+#[repr(transparent)]
+pub struct FatStr([u8]);
+
+impl<'a> Bpb<'a> {
+    pub fn from_bytes(bytes: &'a [u8]) -> Self {
+        assert_eq!(bytes.len(), 512);
+        Self { bytes }
+    }
+
+    pub fn bytes_per_sector(&self) -> usize {
+        get_le_u16(&self.bytes[0x0B..]) as usize
+    }
+
+    pub fn sectors_per_cluster(&self) -> usize {
+        self.bytes[0x0D] as usize
+    }
+
+    pub fn reserved_sectors(&self) -> usize {
+        get_le_u16(&self.bytes[0x0E..]) as usize
+    }
+
+    pub fn fat_count(&self) -> usize {
+        self.bytes[0x10] as usize
+    }
+
+    pub fn total_sectors_16(&self) -> u16 {
+        get_le_u16(&self.bytes[0x13..])
+    }
+
+    pub fn fat_size_16(&self) -> u16 {
+        get_le_u16(&self.bytes[0x16..])
+    }
+
+    pub fn total_sectors(&self) -> u64 {
+        let small = self.total_sectors_16() as u64;
+        if small == 0 {
+            get_le_u32(&self.bytes[0x20..]) as u64
+        } else {
+            small
+        }
+    }
+}
+
+impl<'a> Fat32Ebpb<'a> {
+    pub fn from_bytes(bytes: &'a [u8]) -> Self {
+        assert_eq!(bytes.len(), 512);
+        Self { bytes }
+    }
+
+    pub fn sectors_per_fat(&self) -> usize {
+        get_le_u32(&self.bytes[0x24..]) as usize
+    }
+
+    pub fn root_directory_cluster(&self) -> ClusterNumber {
+        ClusterNumber(get_le_u32(&self.bytes[0x2C..]))
+    }
+
+    pub fn fsinfo_sector(&self) -> u64 {
+        get_le_u16(&self.bytes[0x30..]) as u64
+    }
+
+    pub fn volume_label(&self) -> &FatStr {
+        FatStr::new(&self.bytes[0x47..0x52])
+    }
+}
+
+impl FatStr {
+    pub fn new(bytes: &[u8]) -> &Self {
+        let len = bytes
+            .iter()
+            .rposition(|&c| c != b' ' && c != 0)
+            .map(|len| len + 1)
+            .unwrap_or(0);
+        unsafe { core::mem::transmute(&bytes[..len]) }
+    }
+
+    #[inline]
+    pub fn from_str(str: &str) -> &Self {
+        unsafe { core::mem::transmute(str.trim_end_matches(' ').as_bytes()) }
+    }
+
+    #[inline]
+    pub fn len(&self) -> usize {
+        self.0.len()
+    }
+
+    #[inline]
+    pub fn is_empty(&self) -> bool {
+        self.0.is_empty()
+    }
+
+    #[inline]
+    pub fn as_bytes(&self) -> &[u8] {
+        unsafe { core::mem::transmute(self) }
+    }
+
+    #[inline]
+    pub fn eq_ignore_case(&self, other: &Self) -> bool {
+        self.0.eq_ignore_ascii_case(&other.0)
+    }
+}
+
+impl fmt::Display for FatStr {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        for &byte in self.0.iter() {
+            write!(f, "{}", byte.to_ascii_uppercase() as char)?;
+        }
+        Ok(())
+    }
+}
+
+impl fmt::Debug for FatStr {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "\"")?;
+        for &byte in self.0.iter() {
+            write!(f, "{}", byte.to_ascii_uppercase() as char)?;
+        }
+        write!(f, "\"")
+    }
+}
+
+impl Fat32FsInfo {
+    const SIGNATURE0: u32 = 0x41615252;
+    const SIGNATURE1: u32 = 0x61417272;
+
+    pub fn from_bytes(bytes: [u8; 512]) -> Self {
+        Self { bytes }
+    }
+
+    pub fn is_valid_fat32(&self) -> bool {
+        let signature0 = &self.bytes[0..4];
+        let signature1 = &self.bytes[484..488];
+
+        signature0 == &Self::SIGNATURE0.to_le_bytes()
+            && signature1 == &Self::SIGNATURE1.to_le_bytes()
+    }
+}
+
+impl ClusterNumber {
+    pub fn first_sector(&self, layout: &FsLayout) -> u64 {
+        layout.first_data_sector + (self.0 as u64 - 2) * layout.sectors_per_cluster as u64
+    }
+}

kernel/driver/fs/fat32/src/directory.rs

@@ -0,0 +1,443 @@
+use core::{any::Any, fmt, mem::MaybeUninit};
+
+use alloc::sync::Arc;
+use libk::{
+    block,
+    error::Error,
+    vfs::{
+        CommonImpl, CreateInfo, DirectoryImpl, DirectoryOpenPosition, Filename, Metadata, Node,
+        NodeFlags, NodeRef,
+    },
+};
+use libk_util::{
+    get_le_u16, get_le_u32,
+    string::{chars_equal_ignore_case, Utf16LeStr},
+};
+use yggdrasil_abi::{
+    io::{DirectoryEntry, FileMode, GroupId, UserId},
+    util::FixedString,
+};
+
+use crate::{
+    data::{ClusterNumber, FatStr},
+    file::FileNode,
+    Fat32Fs,
+};
+
+pub const DIRENT_SIZE: usize = 32;
+
+const ATTR_READ_ONLY: u8 = 0x01;
+const ATTR_HIDDEN: u8 = 0x02;
+const ATTR_SYSTEM: u8 = 0x04;
+const ATTR_VOLUME_ID: u8 = 0x08;
+const ATTR_DIRECTORY: u8 = 0x10;
+const ATTR_ARCHIVE: u8 = 0x20;
+
+const ATTR_LFN: u8 = ATTR_READ_ONLY | ATTR_HIDDEN | ATTR_SYSTEM | ATTR_VOLUME_ID;
+const ATTR_LFN_MASK: u8 =
+    ATTR_READ_ONLY | ATTR_HIDDEN | ATTR_SYSTEM | ATTR_VOLUME_ID | ATTR_DIRECTORY | ATTR_ARCHIVE;
+
+const LFN_LAST: u8 = 0x40;
+
+#[derive(Debug)]
+struct FatDirectoryEntry<'a> {
+    long_filename: Option<&'a Utf16LeStr>,
+    short_filename: ShortFilename<'a>,
+    is_directory: bool,
+    first_cluster: ClusterNumber,
+    size_bytes: u32,
+}
+
+pub struct DirectoryNode {
+    pub(crate) fs: Arc<Fat32Fs>,
+    pub(crate) cluster: ClusterNumber,
+    pub(crate) size_bytes: Option<u32>,
+    pub(crate) metadata: Metadata,
+    // Will be used when metadata needs to be updated
+    #[allow(unused)]
+    pub(crate) parent: Option<ClusterNumber>,
+}
+
+#[derive(PartialEq, Eq)]
+struct ShortFilename<'a> {
+    filename: &'a FatStr,
+    extension: &'a FatStr,
+}
+
+impl DirectoryNode {
+    async fn iterate<F: FnMut(usize, FatDirectoryEntry) -> Result<bool, Error>>(
+        &self,
+        mut predicate: F,
+        start_position: u64,
+        limit: usize,
+    ) -> Result<(usize, u64), Error> {
+        assert_eq!(start_position % DIRENT_SIZE as u64, 0);
+
+        let mut lfn_buffer = [0; 512];
+
+        let mut remaining = limit;
+        let mut offset = 0;
+        let mut position = start_position;
+        let entries_per_sector = self.fs.layout.bytes_per_sector / DIRENT_SIZE;
+        let mut end_of_dir = false;
+        let mut entry_count = 0;
+
+        while remaining != 0 && !end_of_dir {
+            let sector_in_cluster = position / self.fs.layout.bytes_per_sector as u64;
+            if sector_in_cluster >= self.fs.layout.sectors_per_cluster as u64 {
+                todo!("TODO: handle multi-cluster directories");
+            }
+            let offset_in_sector = (position % self.fs.layout.bytes_per_sector as u64) as usize;
+            let max_entries = core::cmp::min(
+                limit - offset,
+                entries_per_sector - offset_in_sector / DIRENT_SIZE,
+            );
+
+            self.fs
+                .with_cluster_sector(self.cluster, sector_in_cluster, |buffer| {
+                    for i in 0..max_entries {
+                        let offset = offset_in_sector + i * DIRENT_SIZE;
+                        let entry = &buffer[offset..offset + DIRENT_SIZE];
+
+                        if entry[0] == 0 {
+                            end_of_dir = true;
+                            break;
+                        }
+
+                        // If name[0] == 0xE5, the entry is free
+                        if entry[0] == 0xE5 {
+                            continue;
+                        }
+
+                        let attr = entry[11];
+
+                        if attr & ATTR_LFN_MASK == ATTR_LFN {
+                            // LFN entry
+                            let ord = ((entry[0] & !LFN_LAST) as usize - 1) * 2;
+
+                            lfn_buffer[ord * 13..ord * 13 + 10].copy_from_slice(&entry[1..11]);
+                            lfn_buffer[ord * 13 + 10..ord * 13 + 22]
+                                .copy_from_slice(&entry[14..26]);
+                            lfn_buffer[ord * 13 + 22..ord * 13 + 26]
+                                .copy_from_slice(&entry[28..32]);
+                        } else {
+                            let mut lfn_length = 0;
+                            for i in (0..lfn_buffer.len()).step_by(2) {
+                                let word = get_le_u16(&lfn_buffer[i..]);
+                                if word == 0 || word == 0xFFFF {
+                                    lfn_length = i;
+                                    break;
+                                }
+                            }
+
+                            let long_filename = if lfn_length > 0 {
+                                Utf16LeStr::from_utf16le(&lfn_buffer[..lfn_length]).ok()
+                            } else {
+                                None
+                            };
+
+                            let short_filename = ShortFilename {
+                                filename: FatStr::new(&entry[..8]),
+                                extension: FatStr::new(&entry[8..11]),
+                            };
+
+                            let is_directory = match attr & (ATTR_DIRECTORY | ATTR_VOLUME_ID) {
+                                0x00 => false,
+                                ATTR_DIRECTORY => true,
+                                ATTR_VOLUME_ID => continue,
+                                _ => continue,
+                            };
+
+                            let first_cluster_hi = get_le_u16(&entry[20..]);
+                            let first_cluster_lo = get_le_u16(&entry[26..]);
+                            let size_bytes = get_le_u32(&entry[28..]);
+                            let first_cluster =
+                                ((first_cluster_hi as u32) << 16) | (first_cluster_lo as u32);
+                            let first_cluster = ClusterNumber(first_cluster);
+
+                            let entry = FatDirectoryEntry {
+                                long_filename,
+                                short_filename,
+                                is_directory,
+                                first_cluster,
+                                size_bytes,
+                            };
+
+                            if !predicate(entry_count, entry)? {
+                                end_of_dir = true;
+                                break;
+                            }
+
+                            entry_count += 1;
+                            lfn_buffer.fill(0);
+                        }
+                    }
+                    Ok(())
+                })
+                .await?;
+
+            position += (max_entries * DIRENT_SIZE) as u64;
+            offset += max_entries;
+            remaining -= max_entries;
+        }
+
+        Ok((entry_count, position))
+    }
+
+    async fn read_entries_inner(
+        &self,
+        position: u64,
+        buffer: &mut [MaybeUninit<DirectoryEntry>],
+    ) -> Result<(usize, u64), Error> {
+        let size = self.size_bytes.unwrap_or(u32::MAX);
+        if position >= size as u64 {
+            return Ok((0, position));
+        }
+
+        let limit = core::cmp::min(
+            ((size - position as u32) / DIRENT_SIZE as u32) as usize,
+            buffer.len(),
+        );
+        let (count, position) = self
+            .iterate(
+                |index, entry| {
+                    let mut name = FixedString::empty();
+                    if let Some(lfn) = entry.long_filename {
+                        name.append_from_chars(lfn.chars()).unwrap();
+                    } else {
+                        name.append_from_bytes(entry.short_filename.filename.as_bytes());
+                        if !entry.short_filename.extension.is_empty() {
+                            name.append_from_bytes(b".");
+                            name.append_from_bytes(entry.short_filename.extension.as_bytes());
+                        }
+                    }
+                    let dirent = DirectoryEntry { ty: None, name };
+                    buffer[index].write(dirent);
+                    Ok(true)
+                },
+                position,
+                limit,
+            )
+            .await?;
+
+        assert_eq!(position % DIRENT_SIZE as u64, 0);
+
+        Ok((count, position))
+    }
+
+    async fn lookup_inner(&self, name: &Filename) -> Result<NodeRef, Error> {
+        let mut found = None;
+        let size = self.size_bytes.unwrap_or(u32::MAX);
+
+        self.iterate(
+            |_, entry| {
+                if entry.name_equals(name) {
+                    found = Some((entry.is_directory, entry.first_cluster, entry.size_bytes));
+                    Ok(false)
+                } else {
+                    Ok(true)
+                }
+            },
+            0,
+            (size as u64 / DIRENT_SIZE as u64) as usize,
+        )
+        .await?;
+
+        let (is_directory, cluster, size) = found.ok_or(Error::DoesNotExist)?;
+
+        let metadata = Metadata {
+            uid: UserId::root(),
+            gid: GroupId::root(),
+            mode: if is_directory {
+                FileMode::default_dir()
+            } else {
+                FileMode::default_file()
+            },
+            inode: Some(cluster.0),
+            ctime: 0,
+            mtime: 0,
+            block_count: (size.div_ceil(self.fs.layout.bytes_per_sector as u32)) as u64,
+            block_size: self.fs.layout.bytes_per_sector as u64,
+        };
+
+        if is_directory {
+            let directory = DirectoryNode {
+                fs: self.fs.clone(),
+                cluster,
+                size_bytes: Some(size),
+                parent: Some(self.cluster),
+                metadata,
+            };
+
+            Ok(Node::directory(
+                directory,
+                NodeFlags::empty(),
+                Some(metadata),
+                Some(self.fs.clone()),
+            ))
+        } else {
+            let file = FileNode {
+                fs: self.fs.clone(),
+                cluster,
+                size_bytes: size,
+                parent: Some(self.cluster),
+                metadata,
+            };
+
+            Ok(Node::regular(
+                file,
+                NodeFlags::empty(),
+                Some(metadata),
+                Some(self.fs.clone()),
+            ))
+        }
+    }
+}
+
+impl CommonImpl for DirectoryNode {
+    fn size(&self, _node: &NodeRef) -> Result<u64, Error> {
+        Ok(self.size_bytes.unwrap_or(0) as u64)
+    }
+
+    fn metadata(&self, _node: &NodeRef) -> Result<Metadata, Error> {
+        Ok(self.metadata.clone())
+    }
+
+    fn set_metadata(&self, _node: &NodeRef, _metadata: &Metadata) -> Result<(), Error> {
+        Err(Error::ReadOnly)
+    }
+
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}
+
+impl DirectoryImpl for DirectoryNode {
+    fn create_node(&self, _parent: &NodeRef, _info: &CreateInfo) -> Result<NodeRef, Error> {
+        Err(Error::ReadOnly)
+    }
+
+    fn attach_node(
+        &self,
+        _parent: &NodeRef,
+        _child: &NodeRef,
+        _name: &Filename,
+    ) -> Result<(), Error> {
+        Err(Error::ReadOnly)
+    }
+
+    fn unlink_node(
+        &self,
+        _parent: &NodeRef,
+        _child: &NodeRef,
+        _name: &Filename,
+    ) -> Result<(), Error> {
+        Err(Error::ReadOnly)
+    }
+
+    fn open(&self, _node: &NodeRef) -> Result<DirectoryOpenPosition, Error> {
+        Ok(DirectoryOpenPosition::FromPhysical(0))
+    }
+
+    fn lookup(&self, _node: &NodeRef, name: &Filename) -> Result<NodeRef, Error> {
+        // TODO validate FAT32 filename
+        block!(self.lookup_inner(name).await)?
+    }
+
+    fn read_entries(
+        &self,
+        _node: &NodeRef,
+        pos: u64,
+        entries: &mut [MaybeUninit<DirectoryEntry>],
+    ) -> Result<(usize, u64), Error> {
+        block!(self.read_entries_inner(pos, entries).await)?
+    }
+
+    fn len(&self, _node: &NodeRef) -> Result<usize, Error> {
+        Ok((self.size_bytes.unwrap_or(0) as u32 / DIRENT_SIZE as u32) as usize)
+    }
+}
+
+impl FatDirectoryEntry<'_> {
+    pub fn name_equals(&self, predicate: &str) -> bool {
+        if let Some(lfn) = self.long_filename {
+            // Match by LFN
+            chars_equal_ignore_case(lfn.chars(), predicate.chars())
+        } else {
+            // Match by SFN
+            let Some(predicate) = ShortFilename::from_str(predicate) else {
+                return false;
+            };
+            predicate.eq_ignore_case(&self.short_filename)
+        }
+    }
+}
+
+impl<'a> ShortFilename<'a> {
+    pub fn from_str(s: &'a str) -> Option<Self> {
+        let (filename, extension) = match s.split_once('.') {
+            Some(xs) => xs,
+            None => (s, ""),
+        };
+        let filename = FatStr::from_str(filename);
+        let extension = FatStr::from_str(extension);
+        if filename.is_empty() || filename.len() > 8 || extension.len() > 3 {
+            return None;
+        }
+        Some(Self {
+            filename,
+            extension,
+        })
+    }
+
+    pub fn eq_ignore_case(&self, other: &Self) -> bool {
+        self.filename.eq_ignore_case(other.filename)
+            && self.extension.eq_ignore_case(other.extension)
+    }
+}
+
+impl fmt::Debug for ShortFilename<'_> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "\"{}", self.filename)?;
+        if !self.extension.is_empty() {
+            write!(f, ".{}", self.extension)?;
+        }
+        write!(f, "\"")
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::{data::FatStr, directory::ShortFilename};
+
+    #[test]
+    fn test_short_filename_from_str() {
+        assert_eq!(
+            ShortFilename::from_str("a1b2c3d4.txt"),
+            Some(ShortFilename {
+                filename: FatStr::new(b"a1b2c3d4"),
+                extension: FatStr::new(b"txt")
+            })
+        );
+        assert_eq!(
+            ShortFilename::from_str("a1b2c3d4"),
+            Some(ShortFilename {
+                filename: FatStr::new(b"a1b2c3d4"),
+                extension: FatStr::new(b"")
+            })
+        );
+        assert_eq!(ShortFilename::from_str("a1b2c3d4e5.txt"), None);
+        assert_eq!(ShortFilename::from_str("a1b2c3d4.long"), None);
+        assert_eq!(ShortFilename::from_str("."), None);
+        assert_eq!(ShortFilename::from_str(""), None);
+        assert_eq!(ShortFilename::from_str(".ext"), None)
+    }
+
+    #[test]
+    fn test_short_filename_ignore_case_eq() {
+        assert!(ShortFilename::eq_ignore_case(
+            &ShortFilename::from_str("abcdefgh.txt").unwrap(),
+            &ShortFilename::from_str("AbCdEfGh.txT").unwrap()
+        ));
+    }
+}

kernel/driver/fs/fat32/src/file.rs

@@ -0,0 +1,156 @@
+use core::any::Any;
+
+use alloc::{sync::Arc, vec, vec::Vec};
+use libk::{
+    block,
+    error::Error,
+    task::sync::AsyncMutex,
+    vfs::{CommonImpl, InstanceData, Metadata, NodeRef, RegularImpl},
+};
+use yggdrasil_abi::io::OpenOptions;
+
+use crate::{data::ClusterNumber, Fat32Fs};
+
+pub struct FileNode {
+    pub(crate) fs: Arc<Fat32Fs>,
+    pub(crate) cluster: ClusterNumber,
+    pub(crate) size_bytes: u32,
+    pub(crate) metadata: Metadata,
+    // Will be used when metadata needs to be updated
+    #[allow(unused)]
+    pub(crate) parent: Option<ClusterNumber>,
+}
+
+// TODO use a "sliding window" to minimize memory usage when working with large files?
+struct OpenedFile {
+    cluster_chain: AsyncMutex<Vec<ClusterNumber>>,
+}
+
+impl OpenedFile {
+    fn new(first_cluster: ClusterNumber) -> Self {
+        Self {
+            cluster_chain: AsyncMutex::new(vec![first_cluster]),
+        }
+    }
+
+    async fn seek(&self, file: &FileNode, cluster_index: usize) -> Result<ClusterNumber, Error> {
+        let mut chain = self.cluster_chain.lock().await;
+        if cluster_index >= chain.len() {
+            let last = *chain.last().unwrap();
+            file.fs
+                .iterate_clusters(last, cluster_index + 1 - chain.len(), |cluster| {
+                    chain.push(cluster);
+                })
+                .await?;
+            assert_eq!(chain.len(), cluster_index + 1);
+        }
+
+        Ok(chain[cluster_index])
+    }
+}
+
+impl FileNode {
+    async fn read_inner(
+        &self,
+        instance: &OpenedFile,
+        mut pos: u64,
+        buffer: &mut [u8],
+    ) -> Result<usize, Error> {
+        if pos >= self.size_bytes as u64 {
+            return Ok(0);
+        }
+
+        let len = buffer.len().min((self.size_bytes as u64 - pos) as usize);
+        let mut rem = len;
+        let mut offset = 0;
+
+        let bps = self.fs.layout.bytes_per_sector as u64;
+        let spc = self.fs.layout.sectors_per_cluster as u64;
+
+        while rem != 0 {
+            let cluster_index = pos / (bps * spc);
+            let sector_in_cluster = (pos / bps) % spc;
+            let offset_in_sector = (pos % bps) as usize;
+            let amount = rem.min(bps as usize - offset_in_sector);
+
+            let cluster = instance.seek(self, cluster_index as usize).await?;
+
+            self.fs
+                .with_cluster_sector(cluster, sector_in_cluster, |data| {
+                    buffer[offset..offset + amount]
+                        .copy_from_slice(&data[offset_in_sector..offset_in_sector + amount]);
+                    Ok(())
+                })
+                .await?;
+
+            rem -= amount;
+            offset += amount;
+            pos += amount as u64;
+        }
+
+        Ok(offset)
+    }
+}
+
+impl CommonImpl for FileNode {
+    fn metadata(&self, _node: &NodeRef) -> Result<Metadata, Error> {
+        Ok(self.metadata)
+    }
+
+    fn size(&self, _node: &NodeRef) -> Result<u64, Error> {
+        Ok(self.size_bytes as u64)
+    }
+
+    fn set_metadata(&self, _node: &NodeRef, _metadata: &Metadata) -> Result<(), Error> {
+        Err(Error::ReadOnly)
+    }
+
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}
+
+impl RegularImpl for FileNode {
+    fn open(
+        &self,
+        _node: &NodeRef,
+        opts: OpenOptions,
+    ) -> Result<(u64, Option<InstanceData>), Error> {
+        if opts.contains_any(OpenOptions::TRUNCATE | OpenOptions::APPEND | OpenOptions::WRITE) {
+            return Err(Error::ReadOnly);
+        }
+        let instance = Arc::new(OpenedFile::new(self.cluster));
+        Ok((0, Some(instance)))
+    }
+
+    fn close(&self, _node: &NodeRef, _instance: Option<&InstanceData>) -> Result<(), Error> {
+        Ok(())
+    }
+
+    fn read(
+        &self,
+        _node: &NodeRef,
+        instance: Option<&InstanceData>,
+        pos: u64,
+        buf: &mut [u8],
+    ) -> Result<usize, Error> {
+        let instance = instance
+            .and_then(|p| p.downcast_ref::<OpenedFile>())
+            .ok_or(Error::InvalidFile)?;
+        block!(self.read_inner(instance, pos, buf).await)?
+    }
+
+    fn write(
+        &self,
+        _node: &NodeRef,
+        _instance: Option<&InstanceData>,
+        _pos: u64,
+        _buf: &[u8],
+    ) -> Result<usize, Error> {
+        Err(Error::ReadOnly)
+    }
+
+    fn truncate(&self, _node: &NodeRef, _new_size: u64) -> Result<(), Error> {
+        Err(Error::ReadOnly)
+    }
+}

kernel/driver/fs/fat32/src/lib.rs

@@ -0,0 +1,222 @@
+#![no_std]
+
+use alloc::sync::Arc;
+use async_trait::async_trait;
+use data::{Bpb, ClusterNumber, Fat32Ebpb, Fat32FsInfo};
+use directory::DirectoryNode;
+use libk::{
+    device::block::{cache::DeviceMapper, BlockDevice},
+    error::Error,
+    vfs::{Filesystem, FilesystemMountOption, Metadata, Node, NodeFlags, NodeRef},
+};
+use libk_util::get_le_u32;
+use yggdrasil_abi::io::{FileMode, GroupId, UserId};
+
+extern crate alloc;
+
+mod data;
+
+mod directory;
+mod file;
+
+pub struct FsLayout {
+    // From BPB/EBPB
+    bytes_per_sector: usize,
+    total_sectors: u64,
+    sectors_per_fat: usize,
+    sectors_per_cluster: usize,
+    root_directory_cluster: ClusterNumber,
+    first_fat_sector: u64,
+
+    // Computed
+    first_data_sector: u64,
+}
+
+pub struct Fat32Fs {
+    mapper: DeviceMapper,
+    pub layout: FsLayout,
+}
+
+#[async_trait]
+impl Filesystem for Fat32Fs {
+    fn display_name(&self) -> &'static str {
+        "fat32"
+    }
+}
+
+impl Fat32Fs {
+    pub async fn create<'a, I: IntoIterator<Item = FilesystemMountOption<'a>>>(
+        device: Arc<dyn BlockDevice>,
+        options: I,
+    ) -> Result<NodeRef, Error> {
+        let mut cached = true;
+        for option in options {
+            match option {
+                FilesystemMountOption::Sync => cached = false,
+                _ => (),
+            }
+        }
+
+        let fs = Arc::new(Self::create_fs(device, cached).await?);
+
+        // Setup root node
+        let root_metadata = Metadata {
+            uid: UserId::root(),
+            gid: GroupId::root(),
+            mode: FileMode::default_dir(),
+            ctime: 0,
+            mtime: 0,
+            inode: Some(fs.layout.root_directory_cluster.0),
+            block_size: fs.layout.bytes_per_sector as u64,
+            block_count: 0,
+        };
+        let root_directory = DirectoryNode {
+            fs: fs.clone(),
+            cluster: fs.layout.root_directory_cluster,
+            parent: None,
+            size_bytes: None,
+            metadata: root_metadata,
+        };
+        let root_node = Node::directory(root_directory, NodeFlags::empty(), None, Some(fs.clone()));
+
+        Ok(root_node)
+    }
+
+    async fn create_fs(device: Arc<dyn BlockDevice>, cached: bool) -> Result<Fat32Fs, Error> {
+        let mut boot_sector = [0; 512];
+        let mut fsinfo = [0; 512];
+        device.read_exact(0, &mut boot_sector).await?;
+
+        let bpb = Bpb::from_bytes(&boot_sector);
+        let ebpb = Fat32Ebpb::from_bytes(&boot_sector);
+        let fsinfo_sector = ebpb.fsinfo_sector();
+        let bytes_per_sector = bpb.bytes_per_sector();
+        let total_sectors = bpb.total_sectors();
+
+        if bpb.fat_size_16() != 0 {
+            log::warn!("fat32: not a FAT32 filesystem");
+            return Err(Error::InvalidArgument);
+        }
+
+        if fsinfo_sector >= total_sectors {
+            log::warn!("fat32: FSInfo sector is beyond filesystem end");
+            return Err(Error::InvalidArgument);
+        }
+
+        device
+            .read_exact(bytes_per_sector as u64 * fsinfo_sector, &mut fsinfo)
+            .await?;
+
+        let fsinfo = Fat32FsInfo::from_bytes(fsinfo);
+        if !fsinfo.is_valid_fat32() {
+            log::warn!("fat32: not a FAT32 filesystem");
+            return Err(Error::InvalidArgument);
+        }
+
+        let sectors_per_fat = ebpb.sectors_per_fat();
+        let sectors_per_cluster = bpb.sectors_per_cluster();
+        let root_directory_cluster = ebpb.root_directory_cluster();
+
+        // + RootDirSectors, but RootDirSectors = 0 on FAT32
+        let first_data_sector = bpb.reserved_sectors() + sectors_per_fat * bpb.fat_count();
+        let first_fat_sector = bpb.reserved_sectors();
+
+        log::info!("fat32: mounted {:?}", ebpb.volume_label());
+        log::info!(
+            "fat32: sector {}B, cluster {}B",
+            bytes_per_sector,
+            sectors_per_cluster * bytes_per_sector
+        );
+
+        let layout = FsLayout {
+            bytes_per_sector,
+            total_sectors,
+            sectors_per_fat,
+            sectors_per_cluster,
+            root_directory_cluster,
+
+            first_fat_sector: first_fat_sector as u64,
+            first_data_sector: first_data_sector as u64,
+        };
+
+        let mapper = if cached {
+            DeviceMapper::cached_with_capacity(
+                device,
+                // Most often 512
+                layout.bytes_per_sector,
+                // 512 * 16 * 4 = 32768
+                (layout.bytes_per_sector * layout.sectors_per_cluster) * 4,
+                128,
+                64,
+                "fat32",
+            )?
+        } else {
+            DeviceMapper::uncached(device, layout.bytes_per_sector, "fat32")?
+        };
+
+        Ok(Self { layout, mapper })
+    }
+
+    async fn with_cluster_sector<T, F: FnOnce(&[u8]) -> Result<T, Error>>(
+        &self,
+        cluster: ClusterNumber,
+        sector_in_cluster: u64,
+        mapper: F,
+    ) -> Result<T, Error> {
+        let sector = cluster.first_sector(&self.layout) + sector_in_cluster;
+        if sector >= self.layout.total_sectors {
+            log::warn!("fat32: sector {sector} beyond filesystem end");
+            return Err(Error::InvalidArgument);
+        }
+        self.mapper
+            .try_with(sector * self.layout.bytes_per_sector as u64, mapper)
+            .await
+    }
+
+    async fn with_fat_sector<T, F: FnOnce(&[u8]) -> Result<T, Error>>(
+        &self,
+        fat_sector: u64,
+        mapper: F,
+    ) -> Result<T, Error> {
+        if fat_sector >= self.layout.sectors_per_fat as u64 {
+            log::warn!("fat32: FAT sector {fat_sector} outside of FAT size");
+            return Err(Error::InvalidArgument);
+        }
+        let sector = self.layout.first_fat_sector + fat_sector;
+        self.mapper
+            .try_with(sector * self.layout.bytes_per_sector as u64, mapper)
+            .await
+    }
+
+    async fn iterate_clusters<F: FnMut(ClusterNumber)>(
+        &self,
+        start: ClusterNumber,
+        count: usize,
+        mut mapper: F,
+    ) -> Result<(), Error> {
+        let mut current = start;
+
+        for _ in 0..count {
+            let offset_in_fat = current.0 as usize * size_of::<u32>();
+            let sector_in_fat = (offset_in_fat / self.layout.bytes_per_sector) as u64;
+            let offset_in_sector = offset_in_fat % self.layout.bytes_per_sector;
+
+            let cluster = self
+                .with_fat_sector(sector_in_fat, |buffer| {
+                    let number = get_le_u32(&buffer[offset_in_sector..]) & 0x0FFFFFFF;
+                    Ok(number)
+                })
+                .await?;
+
+            if cluster >= 0x0FFFFFF7 {
+                return Err(Error::InvalidFile);
+            }
+
+            let cluster = ClusterNumber(cluster);
+            mapper(cluster);
+            current = cluster;
+        }
+
+        Ok(())
+    }
+}

kernel/driver/net/core/src/ephy.rs

@@ -0,0 +1,181 @@
+//! Ethernet PHY utilities
+
+use core::time::Duration;
+
+use libk::{
+    error::Error,
+    task::runtime::{psleep, pwait_try},
+};
+use yggdrasil_abi::bitflags;
+
+bitflags! {
+    pub struct BMCR: u16 {
+        const RESET: bit 15;
+        const LOOPBACK: bit 14;
+        const SPEED0: bit 13;
+        const ANE: bit 12;
+        const PWD: bit 11;
+        const ISOLATE: bit 10;
+        const RESTART_AN: bit 9;
+        const DUPLEX: bit 8;
+        const SPEED1: bit 6;
+    }
+}
+
+bitflags! {
+    pub struct BMSR: u16 {
+        const HAVE_100BASET4: bit 15;
+        const HAVE_100BASETX_FULL: bit 14;
+        const HAVE_100BASETX_HALF: bit 13;
+        const HAVE_10BASET_FULL: bit 12;
+        const HAVE_10BASET_HALF: bit 11;
+        const HAVE_100BASET2_FULL: bit 10;
+        const HAVE_100BASET2_HALF: bit 9;
+        const EXT_STATUS_1000BASET: bit 8;
+        const PREAMBLE_SUPP: bit 6;
+        const AN_COMPLETE: bit 5;
+        const REMOTE_FAULT: bit 4;
+        const AN_AVAILABLE: bit 3;
+        const LINK_STATUS: bit 2;
+        const JABBER: bit 1;
+        const EXTENDED_CAP: bit 0;
+    }
+}
+
+bitflags! {
+    pub struct ANAR: u16 {
+        const HAVE_10BASET: bit 5;
+        const HAVE_10BASET_FULL: bit 6;
+        const HAVE_100BASETX: bit 7;
+        const HAVE_100BASETX_FULL: bit 8;
+        const HAVE_100BASET4: bit 9;
+        const HAVE_PAUSE: bit 10;
+        const ASM_DIR: bit 11;
+        const REMOTE_FAULT: bit 13;
+        const NEXT_PAGE: bit 15;
+    }
+}
+
+bitflags! {
+    pub struct GBCR: u16 {
+        const HAVE_1000BASET_HALF: bit 8;
+        const HAVE_1000BASET_FULL: bit 9;
+    }
+}
+
+bitflags! {
+    pub struct GBESR: u16 {
+        const HAVE_1000BASET_HALF: bit 12;
+        const HAVE_1000BASET_FULL: bit 13;
+    }
+}
+
+pub trait MdioBus {
+    fn mii_read(&self, phyaddr: u8, regaddr: u8) -> Result<u16, Error>;
+    fn mii_write(&self, phyaddr: u8, regaddr: u8, value: u16) -> Result<(), Error>;
+}
+
+pub struct PhyAccess<'a, M: MdioBus> {
+    address: u8,
+    mii: &'a M,
+}
+
+pub const REG_BMCR: u8 = 0x00;
+pub const REG_BMSR: u8 = 0x01;
+pub const REG_PHYAD1: u8 = 0x02;
+pub const REG_PHYAD2: u8 = 0x03;
+pub const REG_ANAR: u8 = 0x04;
+pub const REG_GBCR: u8 = 0x09;
+pub const REG_GBESR: u8 = 0x0F;
+
+pub const ANAR_SELECTOR_802_3: u16 = 0b00001;
+pub const ANAR_SELECTOR_MASK: u16 = 0b11111;
+
+impl<'a, M: MdioBus> PhyAccess<'a, M> {
+    pub fn new(mii: &'a M, address: u8) -> Self {
+        Self { address, mii }
+    }
+
+    pub fn probe(_mii: &'a M, _address: u8) -> Result<Self, Error> {
+        todo!()
+    }
+
+    pub fn read_reg(&self, reg: u8) -> Result<u16, Error> {
+        self.mii.mii_read(self.address, reg)
+    }
+
+    pub fn write_reg(&self, reg: u8, value: u16) -> Result<(), Error> {
+        self.mii.mii_write(self.address, reg, value)
+    }
+
+    pub fn status(&self) -> Result<BMSR, Error> {
+        self.read_reg(REG_BMSR).map(BMSR::from)
+    }
+
+    pub fn id(&self) -> Result<(u16, u16), Error> {
+        let id0 = self.read_reg(REG_PHYAD1)?;
+        let id1 = self.read_reg(REG_PHYAD2)?;
+        Ok((id0, id1))
+    }
+
+    pub fn reset(&self, timeout: Duration) -> Result<(), Error> {
+        self.write_reg(REG_BMCR, BMCR::RESET.bits())?;
+        pwait_try(timeout, Duration::from_millis(10), || {
+            Ok(self.read_reg(REG_BMCR)? & BMCR::RESET.bits() == 0)
+        })
+    }
+
+    pub fn setup_link(&self, have_pause: bool, force_gbesr: GBESR) -> Result<(), Error> {
+        let bmsr = BMSR::from(self.read_reg(REG_BMSR)?);
+        let mut gbesr = if bmsr.contains(BMSR::EXT_STATUS_1000BASET) {
+            GBESR::from(self.read_reg(REG_GBESR)?)
+        } else {
+            GBESR::empty()
+        };
+        gbesr |= force_gbesr;
+        let mut anar = ANAR::from_capabilities(bmsr);
+        if have_pause {
+            anar |= ANAR::HAVE_PAUSE | ANAR::ASM_DIR;
+        }
+        let mut gbcr = GBCR::empty();
+        if gbesr.contains(GBESR::HAVE_1000BASET_HALF) {
+            gbcr |= GBCR::HAVE_1000BASET_HALF;
+        }
+        if gbesr.contains(GBESR::HAVE_1000BASET_FULL) {
+            gbcr |= GBCR::HAVE_1000BASET_FULL;
+        }
+
+        self.write_reg(REG_ANAR, anar.bits())?;
+        psleep(Duration::from_millis(10));
+
+        self.write_reg(REG_GBCR, gbcr.bits())?;
+        psleep(Duration::from_millis(10));
+
+        self.write_reg(REG_BMCR, (BMCR::ANE | BMCR::RESTART_AN).bits())?;
+        psleep(Duration::from_millis(10));
+
+        Ok(())
+    }
+}
+
+impl ANAR {
+    pub fn from_capabilities(bmsr: BMSR) -> ANAR {
+        let mut anar = ANAR::from(ANAR_SELECTOR_802_3);
+        if bmsr.contains(BMSR::HAVE_10BASET_HALF) {
+            anar |= ANAR::HAVE_10BASET;
+        }
+        if bmsr.contains(BMSR::HAVE_10BASET_FULL) {
+            anar |= ANAR::HAVE_10BASET_FULL;
+        }
+        if bmsr.contains(BMSR::HAVE_100BASETX_HALF) {
+            anar |= ANAR::HAVE_100BASETX;
+        }
+        if bmsr.contains(BMSR::HAVE_100BASETX_FULL) {
+            anar |= ANAR::HAVE_100BASETX_FULL;
+        }
+        if bmsr.contains(BMSR::HAVE_100BASET4) {
+            anar |= ANAR::HAVE_100BASET4;
+        }
+        anar
+    }
+}

kernel/driver/net/core/src/ethernet.rs

@@ -2,7 +2,7 @@ use core::mem::size_of;
 
 use alloc::sync::Arc;
 use bytemuck::Pod;
-use libk_mm::PageBox;
+use libk::dma::DmaBuffer;
 use yggdrasil_abi::{
     error::Error,
     net::{
@@ -24,7 +24,7 @@ pub struct L2Packet {
     pub l2_offset: usize,
     pub l3_offset: usize,
 
-    pub data: Arc<PageBox<[u8]>>,
+    pub data: Arc<DmaBuffer<[u8]>>,
 }
 
 impl L2Packet {
@@ -65,11 +65,12 @@ pub fn handle(packet: L2Packet) {
     let frame = packet.ethernet_frame();
     let ty = EtherType::from_network_order(frame.ethertype);
 
-    RawSocket::packet_received(packet.clone());
+    RawSocket::handle_rx(packet.clone());
 
     match ty {
         EtherType::ARP => l3::arp::handle_packet(packet),
         EtherType::IPV4 => l3::ip::handle_v4_packet(packet),
+        EtherType(0x0027) => (),
         p => {
             log::debug!(
                 "Unrecognized L2 protocol: {:#06x}",

kernel/driver/net/core/src/interface.rs

@@ -1,28 +1,36 @@
 use core::{
-    mem::size_of,
+    mem::{size_of, MaybeUninit},
     net::IpAddr,
     sync::atomic::{AtomicU32, AtomicUsize, Ordering},
 };
 
 use alloc::{boxed::Box, collections::BTreeMap, format, sync::Arc};
+use libk::dma::DmaBuffer;
 // TODO: link state management?
-use libk_mm::PageBox;
 use libk_util::{
     sync::spin_rwlock::{IrqSafeRwLock, IrqSafeRwLockReadGuard},
     OneTimeInit,
 };
 use yggdrasil_abi::{
     error::Error,
-    net::{protocols::EthernetFrame, MacAddress},
+    net::{link::LinkState, protocols::EthernetFrame, MacAddress},
 };
 
-use crate::l3::{arp::ArpTable, Route};
+use crate::{
+    l3::{arp::ArpTable, Route},
+    TxPacketBuilder,
+};
 
 pub trait NetworkDevice: Sync + Send {
-    fn transmit(&self, packet: PageBox<[u8]>) -> Result<(), Error>;
+    fn allocate_transmit_buffer(&self, len: usize) -> Result<DmaBuffer<[MaybeUninit<u8>]>, Error>;
+    fn transmit_buffer(&self, buffer: DmaBuffer<[u8]>) -> Result<(), Error>;
+
     fn packet_prefix_size(&self) -> usize;
 
     fn read_hardware_address(&self) -> MacAddress;
+    fn link_state(&self) -> LinkState {
+        LinkState::Other { up: true }
+    }
 }
 
 pub struct NetworkInterface {
@@ -99,16 +107,15 @@ impl NetworkInterface {
         ArpTable::insert_address(self.id, self.mac, address, true);
     }
 
+    pub fn link_state(&self) -> LinkState {
+        self.device.link_state()
+    }
+
     pub fn send_l2(&self, l2_frame: &EthernetFrame, l2_data: &[u8]) -> Result<(), Error> {
-        let l2_offset = self.device.packet_prefix_size();
-        let l2_data_offset = l2_offset + size_of::<EthernetFrame>();
-
-        let mut packet = PageBox::new_slice(0, l2_data_offset + l2_data.len())?;
-
-        packet[l2_offset..l2_data_offset].copy_from_slice(bytemuck::bytes_of(l2_frame));
-        packet[l2_data_offset..].copy_from_slice(l2_data);
-
-        self.device.transmit(packet)
+        let mut builder = TxPacketBuilder::new(self, size_of::<EthernetFrame>() + l2_data.len())?;
+        builder.push_bytes(bytemuck::bytes_of(l2_frame))?;
+        builder.push_bytes(l2_data)?;
+        builder.transmit(None)
     }
 }
 

kernel/driver/net/core/src/l3/mod.rs

@@ -6,7 +6,7 @@ use core::{
 
 use alloc::{sync::Arc, vec::Vec};
 use bytemuck::{Pod, Zeroable};
-use libk_mm::PageBox;
+use libk::dma::DmaBuffer;
 use libk_util::sync::spin_rwlock::{
     IrqSafeRwLock, IrqSafeRwLockReadGuard, IrqSafeRwLockWriteGuard,
 };
@@ -19,7 +19,7 @@ use yggdrasil_abi::{
     },
 };
 
-use crate::{interface::NetworkInterface, l4, PacketBuilder};
+use crate::{interface::NetworkInterface, l4, TxPacketBuilder};
 
 pub mod arp;
 pub mod ip;
@@ -40,7 +40,7 @@ pub struct L3Packet {
     pub l4_offset: usize,
     pub data_length: usize,
 
-    pub data: Arc<PageBox<[u8]>>,
+    pub data: Arc<DmaBuffer<[u8]>>,
 }
 
 pub trait IpFrame: Pod {
@@ -54,6 +54,7 @@ pub struct Route {
     pub subnet: SubnetAddr,
     pub interface: u32,
     pub gateway: Option<IpAddr>,
+    pub priority: u32,
 }
 
 pub struct L4ResolvedPacket<'a, 'i> {
@@ -114,12 +115,20 @@ impl Route {
         }
 
         let routes = ROUTES.read();
+        let mut best: Option<(&Route, IpAddr)> = None;
         for route in routes.iter() {
-            if route.subnet.contains(&address) {
-                return Some((route.interface, route.gateway, address));
+            if route.subnet.contains(&address) || route.subnet.base() == address {
+                if let Some((prev_best, _)) = best.as_ref() {
+                    if route.priority < prev_best.priority {
+                        best = Some((route, address));
+                    }
+                } else {
+                    best = Some((route, address));
+                }
             }
         }
-        None
+        let (best, address) = best?;
+        Some((best.interface, best.gateway, address))
     }
 
     pub fn insert(route: Self) -> Result<(), Error> {
@@ -220,8 +229,8 @@ pub async fn send_l4_ip_resolved(packet: &L4ResolvedPacket<'_, '_>) -> Result<()
 
     let l3_frame = packet.make_l3_frame()?;
 
-    let mut builder = PacketBuilder::new(
-        packet.interface.device.packet_prefix_size(),
+    let mut builder = TxPacketBuilder::new(
+        packet.interface,
         size_of::<EthernetFrame>() + size_of::<Ipv4Frame>() + packet.total_l4_len(),
     )?;
     builder.push(&EthernetFrame {
@@ -233,9 +242,7 @@ pub async fn send_l4_ip_resolved(packet: &L4ResolvedPacket<'_, '_>) -> Result<()
     builder.push_bytes(packet.l4_frame)?;
     builder.push_bytes(packet.l4_options)?;
     builder.push_bytes(packet.l4_data)?;
-
-    let (sent_packet, _len) = builder.finish();
-    packet.interface.device.transmit(sent_packet)
+    builder.transmit(None)
 }
 
 pub async fn send_l4_ip(packet: &L4UnresolvedPacket<'_>) -> Result<(), Error> {

kernel/driver/net/core/src/l4/icmp.rs

@@ -8,10 +8,14 @@ use yggdrasil_abi::{
     net::{
         protocols::{IcmpV4Frame, InetChecksum, IpProtocol},
         types::NetValueImpl,
+        SubnetAddr, SubnetV4Addr,
     },
 };
 
-use crate::{l3, L3Packet};
+use crate::{
+    l3::{self, Route},
+    L3Packet,
+};
 
 async fn send_v4_reply(
     destination_ip: Ipv4Addr,
@@ -64,6 +68,20 @@ async fn handle_v4(source_address: Ipv4Addr, l3_packet: L3Packet) -> Result<(),
 
     match (icmp_frame.ty, icmp_frame.code) {
         (8, 0) => send_v4_reply(source_address, icmp_frame, icmp_data).await,
+        // ICMP redirect to host
+        (5, 1) => {
+            // TODO don't just accept whatever gateway is supplied, it's a MitM asking to happen
+            let gateway = Ipv4Addr::from(u32::from_network_order(icmp_frame.rest));
+            log::warn!("ICMP redirect to host: {gateway}");
+            Route::insert(Route {
+                priority: 5,
+                interface: l3_packet.interface_id,
+                gateway: Some(IpAddr::V4(gateway)),
+                subnet: SubnetAddr::V4(SubnetV4Addr::from_address_mask(gateway, 32)),
+            })
+            .ok();
+            Ok(())
+        }
         (0, 0) => Ok(()),
         _ => {
             log::debug!(

kernel/driver/net/core/src/lib.rs

@@ -9,10 +9,11 @@ use core::mem::size_of;
 use alloc::sync::Arc;
 use bytemuck::Pod;
 use ethernet::L2Packet;
+use interface::NetworkInterface;
 use l3::L3Packet;
-use libk::task::runtime;
-use libk_mm::PageBox;
+use libk::{dma::DmaBuffer, task::runtime};
 use libk_util::queue::UnboundedMpmcQueue;
+use socket::RawSocket;
 use yggdrasil_abi::{error::Error, net::protocols::EthernetFrame};
 
 pub mod ethernet;
@@ -24,28 +25,34 @@ pub mod socket;
 pub mod interface;
 pub mod util;
 
+pub mod ephy;
+
 pub use interface::register_interface;
 
-pub struct Packet {
-    // TODO info about "received" interface
-    buffer: PageBox<[u8]>,
+pub struct RxPacket {
+    buffer: DmaBuffer<[u8]>,
     offset: usize,
     iface: u32,
 }
 
-pub struct PacketBuilder {
-    data: PageBox<[u8]>,
+pub struct TxPacketBuilder<'a> {
+    nic: &'a NetworkInterface,
+    data: DmaBuffer<[u8]>,
     pos: usize,
-    len: usize,
+    #[allow(unused)]
+    l2_offset: usize,
 }
 
-impl PacketBuilder {
-    pub fn new(l2_offset: usize, l2_size: usize) -> Result<Self, Error> {
-        let data = PageBox::new_slice(0, l2_offset + l2_size)?;
+impl<'a> TxPacketBuilder<'a> {
+    pub fn new(nic: &'a NetworkInterface, l2_size: usize) -> Result<Self, Error> {
+        let l2_offset = nic.device.packet_prefix_size();
+        let data = nic.device.allocate_transmit_buffer(l2_offset + l2_size)?;
+        let data = unsafe { DmaBuffer::assume_init_slice(data) };
         Ok(Self {
+            nic,
             data,
             pos: l2_offset,
-            len: l2_offset,
+            l2_offset,
         })
     }
 
@@ -63,14 +70,22 @@ impl PacketBuilder {
         Ok(())
     }
 
-    pub fn finish(self) -> (PageBox<[u8]>, usize) {
-        (self.data, self.len)
+    fn finish(self) -> (DmaBuffer<[u8]>, usize) {
+        (self.data, self.pos)
+    }
+
+    pub fn transmit(self, mute_raw: Option<u32>) -> Result<(), Error> {
+        let nic = self.nic;
+        let (data, len) = self.finish();
+        // Also transmit to raw
+        RawSocket::handle_tx(nic.id, &data, len, mute_raw);
+        nic.device.transmit_buffer(data)
     }
 }
 
-impl Packet {
+impl RxPacket {
     #[inline]
-    pub fn new(buffer: PageBox<[u8]>, offset: usize, iface: u32) -> Self {
+    pub fn new(buffer: DmaBuffer<[u8]>, offset: usize, iface: u32) -> Self {
         Self {
             buffer,
             offset,
@@ -79,11 +94,11 @@ impl Packet {
     }
 }
 
-static PACKET_QUEUE: UnboundedMpmcQueue<Packet> = UnboundedMpmcQueue::new();
+static PACKET_QUEUE: UnboundedMpmcQueue<RxPacket> = UnboundedMpmcQueue::new();
 static ACCEPT_QUEUE: UnboundedMpmcQueue<L3Packet> = UnboundedMpmcQueue::new();
 
 #[inline]
-pub fn receive_packet(packet: Packet) -> Result<(), Error> {
+pub fn receive_packet(packet: RxPacket) -> Result<(), Error> {
     PACKET_QUEUE.push_back(packet);
     Ok(())
 }

kernel/driver/net/core/src/socket/config.rs

@@ -250,6 +250,7 @@ fn describe_interface(interface: &NetworkInterface) -> InterfaceInfo {
         interface_name: interface.name.clone(),
         address: interface.address.read().map(Into::into),
         mac: interface.mac,
+        link: interface.link_state(),
     }
 }
 
@@ -275,6 +276,7 @@ fn add_route(
         interface: interface.id,
         gateway,
         subnet,
+        priority: 10,
     };
     Route::insert(route).map_err(|_| "Could not insert route")?;
     Ok(())

kernel/driver/net/core/src/socket/raw.rs

@@ -8,6 +8,7 @@ use core::{
 use alloc::{boxed::Box, collections::btree_map::BTreeMap, sync::Arc, vec::Vec};
 use async_trait::async_trait;
 use libk::{
+    dma::DmaBuffer,
     error::Error,
     task::runtime::maybe_timeout,
     vfs::{FileReadiness, Socket},
@@ -22,12 +23,17 @@ use yggdrasil_abi::{
     option::OptionValue,
 };
 
-use crate::{ethernet::L2Packet, interface::NetworkInterface};
+use crate::{ethernet::L2Packet, interface::NetworkInterface, TxPacketBuilder};
+
+enum RawPacket {
+    Ingress(L2Packet),
+    Egress(Arc<PageBox<[u8]>>),
+}
 
 pub struct RawSocket {
     id: u32,
     bound: IrqSafeRwLock<Option<u32>>,
-    receive_queue: BoundedMpmcQueue<L2Packet>,
+    receive_queue: BoundedMpmcQueue<RawPacket>,
 }
 
 static RAW_SOCKET_ID: AtomicU32 = AtomicU32::new(0);
@@ -48,32 +54,64 @@ impl RawSocket {
         socket
     }
 
-    fn bound_packet_received(&self, packet: L2Packet) {
+    fn bound_packet_received(&self, packet: RawPacket) {
         // TODO do something with the dropped packet?
         self.receive_queue.try_push_back(packet).ok();
     }
 
-    pub fn packet_received(packet: L2Packet) {
+    pub fn handle_rx(packet: L2Packet) {
         let bound_sockets = BOUND_RAW_SOCKETS.read();
         let raw_sockets = RAW_SOCKETS.read();
 
         if let Some(ids) = bound_sockets.get(&packet.interface_id) {
             for id in ids {
                 let socket = raw_sockets.get(id).unwrap();
-                log::info!("Packet -> {id}");
-                socket.bound_packet_received(packet.clone());
+                socket.bound_packet_received(RawPacket::Ingress(packet.clone()));
             }
         }
     }
 
-    fn packet_to_user(packet: L2Packet, buffer: &mut [u8]) -> Result<usize, Error> {
-        let full_len = packet.data.len();
-        let len = full_len - packet.l2_offset;
-        if buffer.len() < len {
-            return Err(Error::BufferTooSmall);
+    pub fn handle_tx(iface: u32, packet: &DmaBuffer<[u8]>, len: usize, except: Option<u32>) {
+        let bound_sockets = BOUND_RAW_SOCKETS.read();
+        let raw_sockets = RAW_SOCKETS.read();
+
+        if let Some(ids) = bound_sockets.get(&iface) {
+            let Ok(egress) = PageBox::from_slice(&packet[..len]).map(Arc::new) else {
+                return;
+            };
+
+            for id in ids {
+                if except.map_or(false, |i| i == *id) {
+                    continue;
+                }
+
+                let socket = raw_sockets.get(id).unwrap();
+                socket.bound_packet_received(RawPacket::Egress(egress.clone()));
+            }
+        }
+    }
+
+    fn packet_to_user(packet: RawPacket, buffer: &mut [u8]) -> Result<usize, Error> {
+        match packet {
+            RawPacket::Egress(egress) => {
+                // TODO l2offset in egress?
+                let len = egress.len();
+                if buffer.len() < len {
+                    return Err(Error::BufferTooSmall);
+                }
+                buffer[..len].copy_from_slice(&egress[..]);
+                Ok(len)
+            }
+            RawPacket::Ingress(ingress) => {
+                let full_len = ingress.data.len();
+                let len = full_len - ingress.l2_offset;
+                if buffer.len() < len {
+                    return Err(Error::BufferTooSmall);
+                }
+                buffer[..len].copy_from_slice(&ingress.data[ingress.l2_offset..full_len]);
+                Ok(len)
+            }
         }
-        buffer[..len].copy_from_slice(&packet.data[packet.l2_offset..full_len]);
-        Ok(len)
     }
 }
 
@@ -168,13 +206,14 @@ impl Socket for RawSocket {
         // TODO cap by MTU?
         let bound = self.bound.read().ok_or(Error::InvalidOperation)?;
         let interface = NetworkInterface::get(bound)?;
-        let l2_offset = interface.device.packet_prefix_size();
-        if message.payload.len() > 4096 - l2_offset {
+        // let l2_offset = interface.device.packet_prefix_size();
+        if message.payload.len() > 1024 {
             return Err(Error::InvalidArgument);
         }
-        let mut packet = PageBox::new_slice(0, l2_offset + message.payload.len())?;
-        packet[l2_offset..l2_offset + message.payload.len()].copy_from_slice(message.payload);
-        interface.device.transmit(packet)?;
+        let mut builder = TxPacketBuilder::new(&*interface, message.payload.len())?;
+        builder.push_bytes(message.payload)?;
+        // false to prevent loopback
+        builder.transmit(Some(self.id))?;
         Ok(message.payload.len())
     }
     async fn send_message(

kernel/driver/net/igbe/Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+name = "ygg_driver_net_igbe"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+device-api.workspace = true
+yggdrasil-abi.workspace = true
+libk-mm.workspace = true
+libk-util.workspace = true
+libk.workspace = true
+
+ygg_driver_pci.path = "../../bus/pci"
+ygg_driver_net_core.path = "../core"
+
+log.workspace = true
+tock-registers.workspace = true

kernel/driver/net/igbe/src/lib.rs

@@ -0,0 +1,212 @@
+#![no_std]
+
+use core::{mem::MaybeUninit, time::Duration};
+
+use alloc::sync::Arc;
+use device_api::{
+    device::{Device, DeviceInitContext},
+    dma::DmaAllocator,
+    interrupt::{InterruptAffinity, InterruptHandler, IrqVector},
+};
+use libk::{dma::DmaBuffer, error::Error};
+use libk_util::{
+    sync::{IrqSafeSpinlock, Spinlock},
+    OneTimeInit,
+};
+use regs::{Regs, ICR};
+use ring::{RxRing, TxRing};
+use ygg_driver_net_core::{
+    interface::{NetworkDevice, NetworkInterfaceType},
+    register_interface, RxPacket,
+};
+use ygg_driver_pci::{
+    device::{PciDeviceInfo, PreferredInterruptMode},
+    macros::pci_driver,
+    PciBaseAddress, PciConfigurationSpace,
+};
+use yggdrasil_abi::net::{link::LinkState, MacAddress};
+
+extern crate alloc;
+
+mod regs;
+mod ring;
+
+struct Igbe {
+    regs: IrqSafeSpinlock<Regs>,
+    dma: Arc<dyn DmaAllocator>,
+    pci: PciDeviceInfo,
+
+    mac: OneTimeInit<MacAddress>,
+    rx_ring: OneTimeInit<Spinlock<RxRing>>,
+    tx_ring: OneTimeInit<IrqSafeSpinlock<TxRing>>,
+    nic: OneTimeInit<u32>,
+}
+
+impl Igbe {
+    pub fn new(dma: Arc<dyn DmaAllocator>, regs: Regs, pci: PciDeviceInfo) -> Self {
+        Self {
+            dma,
+            pci,
+            mac: OneTimeInit::new(),
+            regs: IrqSafeSpinlock::new(regs),
+
+            rx_ring: OneTimeInit::new(),
+            tx_ring: OneTimeInit::new(),
+            nic: OneTimeInit::new(),
+        }
+    }
+}
+
+impl Device for Igbe {
+    unsafe fn init(self: Arc<Self>, _cx: DeviceInitContext) -> Result<(), Error> {
+        let msi_info = self
+            .pci
+            .map_interrupt(InterruptAffinity::Any, self.clone())?;
+
+        let rx_ring = RxRing::with_capacity(&*self.dma, 128, 2048 + 16)?;
+        let tx_ring = TxRing::with_capacity(&*self.dma, 128)?;
+
+        let mut regs = self.regs.lock();
+
+        regs.disable_interrupts();
+        let mac = regs.read_mac()?;
+        self.mac.init(mac);
+        regs.reset(Duration::from_millis(200))?;
+        // Intel 8257x manuals say an additional interrupt disable is needed after a global reset
+        regs.disable_interrupts();
+        regs.set_link_up()?;
+
+        // Initialize Rx
+        regs.initialize_receiver(&rx_ring);
+        regs.initialize_transmitter(&tx_ring);
+        // If MSI(-x) was initialized, notify the NIC about it
+        if let Some(msi_info) = msi_info {
+            regs.initialize_ivar(msi_info.vector);
+        }
+
+        self.rx_ring.init(Spinlock::new(rx_ring));
+        self.tx_ring.init(IrqSafeSpinlock::new(tx_ring));
+        let nic = register_interface(NetworkInterfaceType::Ethernet, self.clone());
+        self.nic.init(nic.id());
+
+        regs.enable_interrupts();
+
+        Ok(())
+    }
+
+    fn display_name(&self) -> &str {
+        "Intel Gigabit Ethernet"
+    }
+}
+
+impl InterruptHandler for Igbe {
+    fn handle_irq(self: Arc<Self>, _vector: IrqVector) -> bool {
+        let mut regs = self.regs.lock();
+        let cause = regs.interrupt_cause();
+        if cause.get() == 0 {
+            return false;
+        }
+        regs.clear_interrupts(cause.get());
+        let mut any = false;
+
+        if cause.matches_all(ICR::LSC::SET) {
+            let status = regs.read_link();
+            log::info!("igbe: link is {status}");
+            any = true;
+        }
+
+        if cause.matches_all(ICR::RXT0::SET) {
+            let mut rx = self.rx_ring.get().lock();
+            let nic = *self.nic.get();
+            let head = regs.rx_queue_head();
+            let tail = rx.handle_rx(&*self.dma, head, |packet, _| {
+                let packet = RxPacket::new(packet, 0, nic);
+                ygg_driver_net_core::receive_packet(packet).ok();
+            });
+            regs.set_rx_queue_tail(tail);
+            any = true;
+        }
+
+        if cause.matches_any(&[ICR::TXQE::SET, ICR::TXDW::SET]) {
+            let mut tx = self.tx_ring.get().lock();
+            let head = regs.tx_queue_head();
+            tx.handle_tx(head);
+            any = true;
+        }
+
+        if !any {
+            log::info!("igbe: unhandled irq {:#x}", cause.get());
+        }
+
+        any
+    }
+}
+
+impl NetworkDevice for Igbe {
+    fn read_hardware_address(&self) -> MacAddress {
+        *self.mac.get()
+    }
+
+    fn packet_prefix_size(&self) -> usize {
+        0
+    }
+
+    fn allocate_transmit_buffer(&self, len: usize) -> Result<DmaBuffer<[MaybeUninit<u8>]>, Error> {
+        DmaBuffer::new_uninit_slice(&*self.dma, len + 4)
+    }
+
+    fn transmit_buffer(&self, buffer: DmaBuffer<[u8]>) -> Result<(), Error> {
+        let mut tx = self.tx_ring.get().lock();
+
+        let Ok(head) = tx.tx_now(buffer) else {
+            return Err(Error::WouldBlock);
+        };
+
+        let mut regs = self.regs.lock();
+        regs.set_tx_queue_tail(head);
+
+        Ok(())
+    }
+
+    fn link_state(&self) -> LinkState {
+        LinkState::Ethernet(self.regs.lock().read_link())
+    }
+}
+
+pci_driver! {
+    matches: [
+        device (0x8086:0x10C9), // 82576 GbE
+        device (0x8086:0x1502), // 82579LM GbE (Lewisville)
+    ],
+    driver: {
+        fn probe(
+            &self,
+            info: &PciDeviceInfo,
+            dma: &Arc<dyn DmaAllocator>,
+        ) -> Result<Arc<dyn Device>, Error> {
+            let base = info
+                .config_space
+                .bar(0).ok_or(Error::InvalidArgument)?;
+
+            let use_mmio = match base {
+                PciBaseAddress::Memory32(_) | PciBaseAddress::Memory64(_) => {
+                    true
+                }
+                PciBaseAddress::Io(_) => {
+                    false
+                }
+            };
+
+            info.init_interrupts(PreferredInterruptMode::Msi(true))?;
+            info.set_command(true, use_mmio, !use_mmio, true);
+
+            let regs = unsafe { Regs::map(base) }?;
+            let device = Igbe::new(dma.clone(), regs, info.clone());
+            Ok(Arc::new(device))
+        }
+
+        fn driver_name(&self) -> &str {
+            "igbe"
+        }
+    }
+}

kernel/driver/net/igbe/src/regs.rs

@@ -0,0 +1,526 @@
+#![allow(non_snake_case)]
+use core::time::Duration;
+
+use libk::{
+    error::Error,
+    task::runtime::{psleep, pwait},
+};
+use libk_mm::{address::PhysicalAddress, device::RawDeviceMemoryMapping};
+use tock_registers::{fields::FieldValue, register_bitfields, LocalRegisterCopy, RegisterLongName};
+use ygg_driver_net_core::ephy::{MdioBus, PhyAccess, GBESR};
+use ygg_driver_pci::PciBaseAddress;
+use yggdrasil_abi::net::{
+    link::{Duplex, EthernetLinkState, EthernetSpeed},
+    MacAddress,
+};
+
+use crate::{RxRing, TxRing};
+
+enum Inner {
+    Memory(RawDeviceMemoryMapping),
+    // TODO I/O register access not yet implemented
+    #[allow(unused)]
+    Io(u16),
+}
+
+enum Eeprom {
+    // TODO not yet handled
+    #[allow(unused)]
+    Present,
+    MappedMemory(RawDeviceMemoryMapping),
+    // TODO I/O register access not yet implemented
+    #[allow(unused)]
+    MappedIo(u16),
+}
+
+pub struct Regs {
+    inner: Inner,
+    eeprom: Eeprom,
+}
+
+pub trait Reg {
+    const OFFSET: u16;
+}
+
+register_bitfields! {
+    u32,
+    pub CTRL [
+        SLU     OFFSET(6)   NUMBITS(1)  [],
+        RST     OFFSET(26)  NUMBITS(1)  [],
+        RFCE    OFFSET(27)  NUMBITS(1)  [],
+        TFCE    OFFSET(28)  NUMBITS(1)  [],
+    ],
+    pub STATUS [
+        FD      OFFSET(0)   NUMBITS(1)  [],
+        LU      OFFSET(1)   NUMBITS(1)  [],
+        SPEED   OFFSET(6)   NUMBITS(2)  [],
+    ],
+    pub MDIC [
+        DATA    OFFSET(0)   NUMBITS(16) [],
+        REGADD  OFFSET(16)  NUMBITS(5)  [],
+        PHYADD  OFFSET(21)  NUMBITS(5)  [],
+        OP      OFFSET(26)  NUMBITS(2)  [
+            Write = 0b01,
+            Read = 0b10
+        ],
+        R       OFFSET(28)  NUMBITS(1)  [],
+        I       OFFSET(29)  NUMBITS(1)  [],
+        E       OFFSET(30)  NUMBITS(1)  [],
+    ],
+    pub ICR [
+        TXDW    OFFSET(0)   NUMBITS(1)  [],
+        TXQE    OFFSET(1)   NUMBITS(1)  [],
+        LSC     OFFSET(2)   NUMBITS(1)  [],
+        RXT0    OFFSET(7)   NUMBITS(1)  [],
+    ],
+    pub IMS [
+        TXDW    OFFSET(0)   NUMBITS(1)  [],
+        TXQE    OFFSET(1)   NUMBITS(1)  [],
+        LSC     OFFSET(2)   NUMBITS(1)  [],
+        RXT0    OFFSET(7)   NUMBITS(1)  [],
+    ],
+    pub RCTL [
+        EN      OFFSET(1)   NUMBITS(1)  [],
+        UPE     OFFSET(3)   NUMBITS(1)  [],
+        MPE     OFFSET(4)   NUMBITS(1)  [],
+        BAM     OFFSET(15)  NUMBITS(1)  [],
+        BSIZE   OFFSET(16)  NUMBITS(2)  [
+            SIZE_2048 = 0b00,
+            SIZE_1024_16384 = 0b01,
+            SIZE_512_8192 = 0b10,
+            SIZE_256_4096 = 0b11
+        ],
+        BSEX    OFFSET(25)  NUMBITS(1)  [],
+    ],
+    pub RDLEN0 [
+        LEN0    OFFSET(7)   NUMBITS(13) [],
+    ],
+    pub CPUVEC [
+        CPUVEC  OFFSET(0)   NUMBITS(32) [],
+    ],
+    pub TCTL [
+        EN      OFFSET(1)   NUMBITS(1)  [],
+        PSP     OFFSET(3)   NUMBITS(1)  [],
+        CT      OFFSET(4)   NUMBITS(8)  [],
+        COLD    OFFSET(12)  NUMBITS(10) [],
+        MULR    OFFSET(28)  NUMBITS(1)  [],
+    ],
+    pub TDLEN [
+        LEN     OFFSET(7)   NUMBITS(13) [],
+    ],
+    pub TIDV [
+        IDV     OFFSET(0)   NUMBITS(16) [],
+    ],
+    pub TXDCTL [
+        PTHRESH     OFFSET(0)   NUMBITS(6)  [],
+        HTHRESH     OFFSET(8)   NUMBITS(8)  [],
+        WTHRESH     OFFSET(16)  NUMBITS(6)  [],
+        GRAN        OFFSET(24)  NUMBITS(1)  [],
+        LWTHRESH    OFFSET(25)  NUMBITS(7)  [],
+    ],
+    pub TADV [
+        IDV     OFFSET(0)   NUMBITS(16) [],
+    ],
+}
+pub mod IMC {
+    pub struct Register;
+}
+pub mod RDBAL0 {
+    pub struct Register;
+}
+pub mod RDBAH0 {
+    pub struct Register;
+}
+pub mod RDH0 {
+    pub struct Register;
+}
+pub mod RDT0 {
+    pub struct Register;
+}
+pub mod TIPG {
+    pub struct Register;
+}
+pub mod TDBAL {
+    pub struct Register;
+}
+pub mod TDBAH {
+    pub struct Register;
+}
+pub mod TDH {
+    pub struct Register;
+}
+pub mod TDT {
+    pub struct Register;
+}
+pub mod IVAR {
+    pub struct Register;
+}
+
+impl Reg for CTRL::Register {
+    const OFFSET: u16 = 0x0000;
+}
+impl Reg for STATUS::Register {
+    const OFFSET: u16 = 0x0008;
+}
+impl Reg for MDIC::Register {
+    const OFFSET: u16 = 0x0020;
+}
+impl Reg for ICR::Register {
+    const OFFSET: u16 = 0x00C0;
+}
+impl Reg for IMS::Register {
+    const OFFSET: u16 = 0x00D0;
+}
+impl Reg for IMC::Register {
+    const OFFSET: u16 = 0x00D8;
+}
+impl Reg for RCTL::Register {
+    const OFFSET: u16 = 0x0100;
+}
+impl Reg for RDBAL0::Register {
+    const OFFSET: u16 = 0x2800;
+}
+impl Reg for RDBAH0::Register {
+    const OFFSET: u16 = 0x2804;
+}
+impl Reg for RDLEN0::Register {
+    const OFFSET: u16 = 0x2808;
+}
+impl Reg for RDH0::Register {
+    const OFFSET: u16 = 0x2810;
+}
+impl Reg for RDT0::Register {
+    const OFFSET: u16 = 0x2818;
+}
+impl Reg for CPUVEC::Register {
+    const OFFSET: u16 = 0x2C10;
+}
+impl Reg for TCTL::Register {
+    const OFFSET: u16 = 0x0400;
+}
+impl Reg for TIPG::Register {
+    const OFFSET: u16 = 0x0410;
+}
+impl Reg for TDBAL::Register {
+    const OFFSET: u16 = 0x3800;
+}
+impl Reg for TDBAH::Register {
+    const OFFSET: u16 = 0x3804;
+}
+impl Reg for TDLEN::Register {
+    const OFFSET: u16 = 0x3808;
+}
+impl Reg for TDH::Register {
+    const OFFSET: u16 = 0x3810;
+}
+impl Reg for TDT::Register {
+    const OFFSET: u16 = 0x3818;
+}
+impl Reg for TIDV::Register {
+    const OFFSET: u16 = 0x3820;
+}
+impl Reg for TXDCTL::Register {
+    const OFFSET: u16 = 0x3828;
+}
+impl Reg for TADV::Register {
+    const OFFSET: u16 = 0x382C;
+}
+impl Reg for IVAR::Register {
+    const OFFSET: u16 = 0x1700;
+}
+
+impl Inner {
+    unsafe fn map(bar: PciBaseAddress) -> Result<Self, Error> {
+        let memory_base = match bar {
+            PciBaseAddress::Memory64(base) => PhysicalAddress::from_u64(base),
+            PciBaseAddress::Memory32(base) => PhysicalAddress::from_u32(base),
+            PciBaseAddress::Io(io_base) => {
+                return Ok(Self::Io(io_base));
+            }
+        };
+
+        unsafe {
+            RawDeviceMemoryMapping::map(memory_base.into_u64(), 0x4000, Default::default())
+                .map(Self::Memory)
+        }
+    }
+
+    fn get<R: Reg>(&self) -> u32 {
+        match self {
+            Self::Memory(mapping) => {
+                let ptr: *const u32 =
+                    core::ptr::with_exposed_provenance(mapping.address + R::OFFSET as usize);
+                unsafe { ptr.read_volatile() }
+            }
+            Self::Io(_io_base) => todo!(),
+        }
+    }
+
+    fn extract<R: Reg + RegisterLongName>(&self) -> LocalRegisterCopy<u32, R> {
+        LocalRegisterCopy::new(self.get::<R>())
+    }
+
+    fn write<R: Reg + RegisterLongName>(&self, value: FieldValue<u32, R>) {
+        self.set::<R>(value.value);
+    }
+
+    fn set<R: Reg>(&self, value: u32) {
+        match self {
+            Self::Memory(mapping) => {
+                let ptr: *mut u32 =
+                    core::ptr::with_exposed_provenance_mut(mapping.address + R::OFFSET as usize);
+                unsafe { ptr.write_volatile(value) };
+            }
+            Self::Io(_io_base) => todo!(),
+        }
+    }
+
+    fn modify<R: Reg + RegisterLongName>(&self, modify: FieldValue<u32, R>) {
+        let mut value = self.get::<R>();
+        value &= !modify.mask();
+        value |= modify.value;
+        self.set::<R>(value);
+    }
+
+    fn matches_all<R: Reg + RegisterLongName>(&self, pattern: FieldValue<u32, R>) -> bool {
+        pattern.matches_all(self.get::<R>())
+    }
+
+    fn matches_any<R: Reg + RegisterLongName>(&self, pattern: &[FieldValue<u32, R>]) -> bool {
+        let value = self.get::<R>();
+        for pattern in pattern {
+            if pattern.matches_all(value) {
+                return true;
+            }
+        }
+        false
+    }
+}
+
+impl Eeprom {
+    fn read(&self, _inner: &mut Inner, addr: u8) -> u16 {
+        match self {
+            Self::Present => todo!(),
+            Self::MappedMemory(mapping) => {
+                // EEPROM registers are read in words, not bytes/halves
+                let shift = (addr as usize & 1) << 4;
+                let reg = (addr as usize & !1) << 1;
+                let ptr: *const u32 = core::ptr::with_exposed_provenance(mapping.address + reg);
+                // This is safe, region is mapped as device memory
+                let word = unsafe { ptr.read_volatile() };
+                (word >> shift) as u16
+            }
+            &Self::MappedIo(_io_base) => todo!(),
+        }
+    }
+}
+
+impl MdioBus for Regs {
+    fn mii_read(&self, phyaddr: u8, regaddr: u8) -> Result<u16, Error> {
+        self.inner.write(
+            MDIC::PHYADD.val(phyaddr as u32) + MDIC::REGADD.val(regaddr as u32) + MDIC::OP::Read,
+        );
+
+        pwait(
+            Duration::from_millis(100),
+            Duration::from_millis(10),
+            || self.inner.matches_any(&[MDIC::R::SET, MDIC::E::SET]),
+        )?;
+
+        let mdic = self.inner.extract();
+        if mdic.matches_all(MDIC::E::SET) {
+            return Err(Error::InvalidOperation);
+        }
+
+        Ok(mdic.read(MDIC::DATA) as u16)
+    }
+
+    fn mii_write(&self, phyaddr: u8, regaddr: u8, value: u16) -> Result<(), Error> {
+        self.inner.write(
+            MDIC::PHYADD.val(phyaddr as u32)
+                + MDIC::REGADD.val(regaddr as u32)
+                + MDIC::DATA.val(value as u32)
+                + MDIC::OP::Write,
+        );
+
+        pwait(
+            Duration::from_millis(100),
+            Duration::from_millis(10),
+            || self.inner.matches_any(&[MDIC::R::SET, MDIC::E::SET]),
+        )?;
+
+        if self.inner.matches_all(MDIC::E::SET) {
+            return Err(Error::InvalidOperation);
+        }
+
+        Ok(())
+    }
+}
+
+impl Regs {
+    pub unsafe fn map(bar: PciBaseAddress) -> Result<Self, Error> {
+        let inner = unsafe { Inner::map(bar) }?;
+
+        // Map EEPROM registers
+        let eeprom = match &inner {
+            Inner::Memory(mapping) => {
+                let base = mapping.physical_base + ((mapping.address & 0xFFF) + 0x5400) as u64;
+                let mapping =
+                    unsafe { RawDeviceMemoryMapping::map(base, 0x10, Default::default()) }?;
+                Eeprom::MappedMemory(mapping)
+            }
+            &Inner::Io(_io_base) => todo!(),
+        };
+
+        Ok(Self { inner, eeprom })
+    }
+
+    fn read_eeprom(&mut self, addr: u8) -> u16 {
+        self.eeprom.read(&mut self.inner, addr)
+    }
+
+    pub fn read_mac(&mut self) -> Result<MacAddress, Error> {
+        let w0 = self.read_eeprom(0);
+        let w1 = self.read_eeprom(1);
+        let w2 = self.read_eeprom(2);
+        if w0 == 0 && w1 == 0 && w2 == 0 {
+            log::error!("Could not read EEPROM MAC address");
+            return Err(Error::InvalidArgument);
+        }
+        let w0 = w0.to_le_bytes();
+        let w1 = w1.to_le_bytes();
+        let w2 = w2.to_le_bytes();
+
+        let octets = [w0[0], w0[1], w1[0], w1[1], w2[0], w2[1]];
+        Ok(MacAddress::from(octets))
+    }
+
+    pub fn reset(&mut self, timeout: Duration) -> Result<(), Error> {
+        self.inner.write(CTRL::RST::SET);
+        psleep(Duration::from_millis(10));
+        pwait(timeout, Duration::from_millis(10), || {
+            self.inner.matches_all(CTRL::RST::CLEAR)
+        })
+    }
+
+    pub fn set_link_up(&mut self) -> Result<(), Error> {
+        self.inner
+            .modify(CTRL::SLU::SET + CTRL::RFCE::SET + CTRL::TFCE::SET);
+
+        let phy = PhyAccess::new(self, 0x01);
+
+        let (id0, id1) = phy.id()?;
+        log::info!("PHY {:04x}:{:04x}", id0, id1);
+
+        phy.reset(Duration::from_millis(200))?;
+        phy.setup_link(true, GBESR::empty())?;
+
+        Ok(())
+    }
+
+    pub fn read_link(&mut self) -> EthernetLinkState {
+        let status = self.inner.extract::<STATUS::Register>();
+
+        if status.matches_all(STATUS::LU::SET) {
+            let duplex = if status.matches_all(STATUS::FD::SET) {
+                Duplex::Full
+            } else {
+                Duplex::Half
+            };
+
+            let speed = match status.read(STATUS::SPEED) {
+                0b00 => EthernetSpeed::Speed10,
+                0b01 => EthernetSpeed::Speed100,
+                0b10 | 0b11 => EthernetSpeed::Speed1000,
+                _ => unreachable!(),
+            };
+
+            EthernetLinkState::Up(speed, duplex)
+        } else {
+            EthernetLinkState::Down
+        }
+    }
+
+    pub fn disable_interrupts(&mut self) {
+        self.inner.set::<IMC::Register>(0xFFFFFFFF);
+    }
+
+    pub fn clear_interrupts(&mut self, cause: u32) {
+        let _ = self.inner.get::<CPUVEC::Register>();
+        self.inner.set::<ICR::Register>(cause);
+    }
+
+    pub fn enable_interrupts(&mut self) {
+        self.inner.set::<IMS::Register>(0xFFFFFFFF);
+        // self.inner
+        //     .modify(IMS::LSC::SET + IMS::RXT0::SET + IMS::TXDW::SET + IMS::TXQE::SET);
+    }
+
+    pub fn interrupt_cause(&mut self) -> LocalRegisterCopy<u32, ICR::Register> {
+        self.inner.extract()
+    }
+
+    pub fn initialize_receiver(&mut self, rx_ring: &RxRing) {
+        let rx_queue_base = rx_ring.descriptors.bus_address().into_u64();
+        self.inner.set::<RDBAL0::Register>(rx_queue_base as u32);
+        self.inner
+            .set::<RDBAH0::Register>((rx_queue_base >> 32) as u32);
+
+        self.inner
+            .write(RDLEN0::LEN0.val((rx_ring.descriptors.len() / 8) as u32));
+        self.inner.set::<RDH0::Register>(0);
+        self.inner
+            .set::<RDT0::Register>((rx_ring.descriptors.len() - 1) as u32);
+
+        self.inner.write(
+            RCTL::EN::SET
+                + RCTL::BSIZE::SIZE_2048
+                + RCTL::BAM::SET
+                + RCTL::UPE::SET
+                + RCTL::MPE::SET,
+        );
+    }
+
+    pub fn initialize_transmitter(&mut self, tx_ring: &TxRing) {
+        let tx_queue_base = tx_ring.descriptors.bus_address().into_u64();
+        self.inner.set::<TDBAL::Register>(tx_queue_base as u32);
+        self.inner
+            .set::<TDBAH::Register>((tx_queue_base >> 32) as u32);
+
+        self.inner
+            .write(TDLEN::LEN.val((tx_ring.descriptors.len() / 8) as u32));
+        self.inner.set::<TDT::Register>(0);
+        self.inner.set::<TDH::Register>(0);
+
+        self.inner.set::<TIPG::Register>(0x0060200A);
+        self.inner.write(TIDV::IDV.val(100));
+        self.inner.write(TADV::IDV.val(100));
+        self.inner
+            .write(TCTL::EN::SET + TCTL::CT.val(15) + TCTL::COLD.val(63) + TCTL::PSP::SET);
+    }
+
+    pub fn initialize_ivar(&mut self, vector: usize) {
+        // Setup vector info for Tx/Rx queues 0
+        let mut val = 0;
+        for i in 0..4 {
+            val |= (0x80 | (vector as u32 & 0xFF)) << (8 * i);
+        }
+        self.inner.set::<IVAR::Register>(val);
+    }
+
+    pub fn rx_queue_head(&mut self) -> u16 {
+        self.inner.get::<RDH0::Register>() as u16
+    }
+
+    pub fn set_rx_queue_tail(&mut self, tail: u16) {
+        self.inner.set::<RDT0::Register>(tail as u32);
+    }
+
+    pub fn tx_queue_head(&mut self) -> u16 {
+        self.inner.get::<TDH::Register>() as u16
+    }
+
+    pub fn set_tx_queue_tail(&mut self, tail: u16) {
+        self.inner.set::<TDT::Register>(tail as u32);
+    }
+}

kernel/driver/net/igbe/src/ring.rs

@@ -0,0 +1,200 @@
+use core::mem::{self, MaybeUninit};
+
+use alloc::vec::Vec;
+use device_api::dma::DmaAllocator;
+use libk::{
+    dma::{BusAddress, DmaBuffer},
+    error::Error,
+};
+
+pub(crate) struct RxRing {
+    pub(crate) descriptors: DmaBuffer<[RxDescriptor]>,
+    buffers: Vec<DmaBuffer<[MaybeUninit<u8>]>>,
+    buffer_size: usize,
+    tail: u16,
+}
+
+pub(crate) struct TxRing {
+    pub(crate) descriptors: DmaBuffer<[TxDescriptor]>,
+    buffers: Vec<Option<DmaBuffer<[u8]>>>,
+    // Consumer end
+    tail: u16,
+    // Producer end
+    head: u16,
+}
+
+#[repr(C)]
+pub(crate) struct RxDescriptor {
+    address: BusAddress,
+    length: u16,
+    checksum: u16,
+    status: u8,
+    errors: u8,
+    special: u16,
+}
+
+#[repr(C)]
+pub(crate) struct TxDescriptor {
+    address: BusAddress,
+    length: u16,
+    cso: u8,
+    cmd: u8,
+    sta: u8,
+    _0: u8,
+    css: u8,
+    special: u8,
+}
+
+impl RxRing {
+    pub fn with_capacity(
+        dma: &dyn DmaAllocator,
+        capacity: usize,
+        buffer_size: usize,
+    ) -> Result<Self, Error> {
+        let buffers = (0..capacity)
+            .map(|_| DmaBuffer::new_uninit_slice(dma, buffer_size))
+            .collect::<Result<Vec<_>, _>>()?;
+        let descriptors = DmaBuffer::new_slice_with(
+            dma,
+            |i| RxDescriptor::new(buffers[i].bus_address(), buffer_size as u16),
+            capacity,
+        )?;
+
+        Ok(Self {
+            descriptors,
+            buffers,
+            tail: 0,
+            buffer_size,
+        })
+    }
+
+    // TODO move to background task/softirq to reduce amount of code run by the irq handler
+    pub fn handle_rx<F: FnMut(DmaBuffer<[u8]>, usize)>(
+        &mut self,
+        dma: &dyn DmaAllocator,
+        head: u16,
+        mut handler: F,
+    ) -> u16 {
+        let capacity = self.descriptors.len();
+        while self.tail != head {
+            let index = self.tail as usize;
+            // Replace the buffer
+
+            let new_buffer = DmaBuffer::new_uninit_slice(dma, self.buffer_size).unwrap();
+            let new_buffer_address = new_buffer.bus_address();
+            let buffer = mem::replace(&mut self.buffers[index], new_buffer);
+            let buffer = unsafe { DmaBuffer::assume_init_slice(buffer) };
+
+            let descriptor = &mut self.descriptors[index];
+
+            if descriptor.errors & !1 != 0 {
+                log::warn!("igbe: drop erroneous packet {:#x}", descriptor.errors);
+            } else {
+                let len = descriptor.length as usize;
+                handler(buffer, len);
+            }
+
+            // Replace the descriptor
+            *descriptor = RxDescriptor::new(new_buffer_address, self.buffer_size as u16);
+
+            self.tail = (self.tail + 1) & (capacity as u16 - 1);
+        }
+
+        (self.tail + capacity as u16 - 1) & (capacity as u16 - 1)
+    }
+}
+
+impl TxRing {
+    pub fn with_capacity(dma: &dyn DmaAllocator, capacity: usize) -> Result<Self, Error> {
+        let buffers = (0..capacity).map(|_| None).collect::<Vec<_>>();
+        let descriptors = DmaBuffer::new_slice_with(dma, |_| TxDescriptor::empty(), capacity)?;
+        Ok(Self {
+            descriptors,
+            buffers,
+            head: 0,
+            tail: 0,
+        })
+    }
+
+    pub fn handle_tx(&mut self, head: u16) {
+        self.tail = head;
+    }
+
+    pub fn tx_now(&mut self, buffer: DmaBuffer<[u8]>) -> Result<u16, DmaBuffer<[u8]>> {
+        // Queue full
+        let capacity = self.descriptors.len();
+        if (self.head + 1) & (capacity as u16 - 1) == self.tail {
+            log::warn!("igbe: tx queue full");
+            return Err(buffer);
+        }
+        let index = self.head as usize;
+
+        let descriptor = &mut self.descriptors[index];
+        // Only generate interrupts for every 1/4th of the buffer
+        let quarter = capacity / 4;
+        descriptor.setup_tx(
+            buffer.bus_address(),
+            buffer.len() as u16 - 4,
+            index % quarter == quarter - 1,
+        );
+        self.descriptors.cache_flush_element(index, true);
+
+        self.buffers[index] = Some(buffer);
+
+        self.head = (self.head + 1) & (capacity as u16 - 1);
+        Ok(self.head)
+    }
+}
+
+impl RxDescriptor {
+    pub fn new(address: BusAddress, length: u16) -> Self {
+        Self {
+            address,
+            length,
+            checksum: 0,
+            status: 0,
+            errors: 0,
+            special: 0,
+        }
+    }
+}
+
+impl TxDescriptor {
+    // Descriptor done
+    const STA_DD: u8 = 1 << 0;
+    // End of packet
+    const CMD_EOP: u8 = 1 << 0;
+    // Insert MAC FCS
+    const CMD_IFCS: u8 = 1 << 1;
+    // // Insert checksum
+    // const CMD_IC: u8 = 1 << 2;
+    // Report status
+    const CMD_RS: u8 = 1 << 3;
+
+    pub const fn empty() -> Self {
+        Self {
+            address: BusAddress::ZERO,
+            length: 0,
+            cso: 0,
+            cmd: 0,
+            sta: Self::STA_DD,
+            _0: 0,
+            css: 0,
+            special: 0,
+        }
+    }
+
+    pub fn setup_tx(&mut self, address: BusAddress, length: u16, ioc: bool) {
+        let mut cmd = Self::CMD_EOP | Self::CMD_IFCS;
+        if ioc {
+            cmd |= Self::CMD_RS;
+        }
+        self.address = address;
+        self.length = length;
+        self.css = 0;
+        self.cso = 0;
+        self.sta = 0;
+        self.cmd = cmd;
+        self.special = 0;
+    }
+}

kernel/driver/net/loopback/Cargo.toml

@@ -5,8 +5,10 @@ edition = "2021"
 
 [dependencies]
 yggdrasil-abi.workspace = true
+device-api.workspace = true
 libk-util.workspace = true
 libk-mm.workspace = true
+libk.workspace = true
 
 ygg_driver_net_core = { path = "../../net/core" }
 

kernel/driver/net/loopback/src/lib.rs

@@ -2,22 +2,33 @@
 
 extern crate alloc;
 
-use core::net::{IpAddr, Ipv4Addr};
+use core::{
+    mem::MaybeUninit,
+    net::{IpAddr, Ipv4Addr},
+};
 
 use alloc::sync::Arc;
-use libk_mm::PageBox;
+use device_api::dma::DmaAllocator;
+use libk::dma::{DmaBuffer, DummyDmaAllocator};
 use libk_util::OneTimeInit;
 use ygg_driver_net_core::{
     interface::{NetworkDevice, NetworkInterfaceType},
-    Packet,
+    RxPacket,
 };
 use yggdrasil_abi::{error::Error, net::MacAddress};
 
-struct LoopbackDevice;
+struct LoopbackDevice {
+    allocator: Arc<dyn DmaAllocator>,
+}
 
 impl NetworkDevice for LoopbackDevice {
-    fn transmit(&self, packet: PageBox<[u8]>) -> Result<(), Error> {
-        ygg_driver_net_core::receive_packet(Packet::new(packet, 0, *LOOPBACK_ID.get()))
+    fn allocate_transmit_buffer(&self, len: usize) -> Result<DmaBuffer<[MaybeUninit<u8>]>, Error> {
+        DmaBuffer::new_uninit_slice(&*self.allocator, len)
+    }
+
+    fn transmit_buffer(&self, buffer: DmaBuffer<[u8]>) -> Result<(), Error> {
+        let packet = RxPacket::new(buffer, 0, *LOOPBACK_ID.get());
+        ygg_driver_net_core::receive_packet(packet)
     }
 
     fn packet_prefix_size(&self) -> usize {
@@ -32,7 +43,9 @@ impl NetworkDevice for LoopbackDevice {
 static LOOPBACK_ID: OneTimeInit<u32> = OneTimeInit::new();
 
 pub fn init() {
-    let loopback = Arc::new(LoopbackDevice);
+    let loopback = Arc::new(LoopbackDevice {
+        allocator: Arc::new(DummyDmaAllocator),
+    });
     let interface =
         ygg_driver_net_core::register_interface(NetworkInterfaceType::Loopback, loopback);
 

kernel/driver/net/rtl81xx/Cargo.toml

@@ -0,0 +1,18 @@
+[package]
+name = "ygg_driver_net_rtl81xx"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+device-api.workspace = true
+yggdrasil-abi.workspace = true
+libk-mm.workspace = true
+libk-util.workspace = true
+libk.workspace = true
+
+ygg_driver_pci.path = "../../bus/pci"
+ygg_driver_net_core.path = "../core"
+
+tock-registers.workspace = true
+log.workspace = true
+futures-util.workspace = true

kernel/driver/net/rtl81xx/src/lib.rs

@@ -0,0 +1,88 @@
+#![no_std]
+
+use alloc::sync::Arc;
+use device_api::{device::Device, dma::DmaAllocator};
+use libk::error::Error;
+use rtl8139::Rtl8139;
+use rtl8168::Rtl8168;
+use ygg_driver_pci::{
+    capability::{
+        DevicePowerState, PciExpressCapability, PcieLinkControl, PowerManagementCapability,
+    },
+    device::{PciDeviceInfo, PreferredInterruptMode},
+    macros::pci_driver,
+    PciBaseAddress, PciCommandRegister, PciConfigurationSpace,
+};
+
+extern crate alloc;
+
+pub mod rtl8139;
+pub mod rtl8168;
+
+pci_driver! {
+    matches: [device (0x10EC:0x8139)],
+    driver: {
+        fn driver_name(&self) -> &str {
+            "rtl8139"
+        }
+
+        fn probe(&self, info: &PciDeviceInfo, dma: &Arc<dyn DmaAllocator>) -> Result<Arc<dyn Device>, Error> {
+            info.init_interrupts(PreferredInterruptMode::Msi(false))?;
+
+            if let Some(mut pcie) = info.config_space.capability::<PciExpressCapability>() {
+                let mut lcr = pcie.link_control();
+                lcr.remove(PcieLinkControl::ASPM_MASK | PcieLinkControl::ECPM);
+                pcie.set_link_control(lcr);
+            }
+
+            // Enable MMIO + interrupts + bus mastering
+            let mut command = info.config_space.command();
+            command |= (PciCommandRegister::BUS_MASTER | PciCommandRegister::ENABLE_MEMORY).bits();
+            command &= !(PciCommandRegister::ENABLE_IO | PciCommandRegister::DISABLE_INTERRUPTS).bits();
+            info.config_space.set_command(command);
+
+            let base = info
+                .config_space
+                .bar(1)
+                .and_then(PciBaseAddress::as_memory)
+                .ok_or(Error::InvalidArgument)?;
+
+            let device = Rtl8139::new(dma.clone(), base, info.clone())?;
+            Ok(Arc::new(device))
+        }
+    }
+}
+
+pci_driver! {
+    matches: [device (0x10EC:0x8168)],
+    driver: {
+        fn driver_name(&self) -> &str {
+            "rtl816x"
+        }
+
+        fn probe(&self, info: &PciDeviceInfo, dma: &Arc<dyn DmaAllocator>) -> Result<Arc<dyn Device>, Error> {
+            // Enable MMIO + interrupts + bus mastering
+            info.set_command(false, true, true, true);
+
+            let base = info
+                .config_space
+                .bar(2)
+                .and_then(PciBaseAddress::as_memory)
+                .ok_or(Error::InvalidArgument)?;
+
+            if let Some(power) = info.config_space.capability::<PowerManagementCapability>() {
+                power.set_device_power_state(DevicePowerState::D0);
+                power.set_pme_en(true);
+            }
+
+            if let Some(mut pcie) = info.config_space.capability::<PciExpressCapability>() {
+                let mut lcr = pcie.link_control();
+                lcr.remove(PcieLinkControl::ASPM_MASK | PcieLinkControl::ECPM);
+                pcie.set_link_control(lcr);
+            }
+
+            let device = Rtl8168::new(dma.clone(), base, info.clone())?;
+            Ok(Arc::new(device))
+        }
+    }
+}

kernel/driver/net/rtl81xx/src/rtl8139.rs

@@ -0,0 +1,434 @@
+use core::mem::MaybeUninit;
+
+use alloc::sync::Arc;
+use device_api::{
+    device::{Device, DeviceInitContext},
+    dma::DmaAllocator,
+    interrupt::{InterruptHandler, IrqVector},
+};
+use libk::{dma::DmaBuffer, error::Error};
+use libk_mm::{address::PhysicalAddress, device::DeviceMemoryIo};
+use libk_util::{queue::BoundedQueue, sync::IrqSafeSpinlock, OneTimeInit};
+use tock_registers::{
+    interfaces::{ReadWriteable, Readable, Writeable},
+    register_bitfields, register_structs,
+    registers::{ReadOnly, ReadWrite},
+};
+use ygg_driver_net_core::{
+    interface::{NetworkDevice, NetworkInterfaceType},
+    RxPacket,
+};
+use ygg_driver_pci::device::PciDeviceInfo;
+use yggdrasil_abi::net::MacAddress;
+
+register_bitfields! {
+    u8,
+
+    pub CR [
+        /// Reset. Setting this bit forces the RTL8139D(L) to perform a software reset. This bit is
+        /// cleared by the RTL8139D(L) after the reset completes.
+        RST     OFFSET(4)   NUMBITS(1)  [],
+        /// Receiver enable
+        RE      OFFSET(3)   NUMBITS(1)  [],
+        /// Transmitter enable
+        TE      OFFSET(2)   NUMBITS(1)  [],
+        /// Rx buffer empty
+        BUFE    OFFSET(0)   NUMBITS(1)  [],
+    ],
+
+    pub CONFIG1 [
+        /// LED pin control
+        LEDS1   OFFSET(7)   NUMBITS(1)  [],
+        /// LED pin control
+        LEDS0   OFFSET(6)   NUMBITS(1)  [],
+        /// Driver load. Software may use this bit to make sure that the driver has been loaded.
+        DVRLOAD OFFSET(5)   NUMBITS(1)  [],
+        /// LWAKE active mode. The LWACT bit and LWPTN bit in CONFIG4 are used to program the LWAKE
+        /// pin's output signal.
+        LWACT   OFFSET(4)   NUMBITS(1)  [],
+        /// The operational registers are mapped into PCI memory space
+        MEMMAP  OFFSET(3)   NUMBITS(1)  [],
+        /// The operational registers are mapped into PCI I/O space
+        IOMAP   OFFSET(2)   NUMBITS(1)  [],
+        /// Set to enable Vital Product Data
+        VPD     OFFSET(1)   NUMBITS(1)  [],
+        /// Power management enable
+        PMEn    OFFSET(0)   NUMBITS(1)  [],
+    ],
+}
+
+register_bitfields! {
+    u16,
+
+    pub IMR_ISR [
+        /// System error interrupt
+        SERR        OFFSET(15)  NUMBITS(1)  [],
+        /// Time out interrupt
+        TIMEOUT     OFFSET(14)  NUMBITS(1)  [],
+        /// Cable length change interrupt
+        LENCHG      OFFSET(13)  NUMBITS(1)  [],
+        /// Rx FIFO overflow interrupt
+        FOVW        OFFSET(6)   NUMBITS(1)  [],
+        /// Packet underrun/Link change interrupt
+        PUN_LINKCHG OFFSET(5)   NUMBITS(1)  [],
+        /// Rx buffer overflow
+        RXOVW       OFFSET(4)   NUMBITS(1)  [],
+        /// Tx error interrupt
+        TER         OFFSET(3)   NUMBITS(1)  [],
+        /// Tx OK interrupt
+        TOK         OFFSET(2)   NUMBITS(1)  [],
+        /// Rx error interrupt
+        RER         OFFSET(1)   NUMBITS(1)  [],
+        /// Rx OK interrupt
+        ROK         OFFSET(0)   NUMBITS(1)  [],
+    ],
+}
+
+register_bitfields! {
+    u32,
+
+    pub TSDn [
+        /// Carrier sense lost
+        CRS         OFFSET(31)  NUMBITS(1)  [],
+        /// Tx aborted
+        TABT        OFFSET(30)  NUMBITS(1)  [],
+        /// Out of window collision
+        OWC         OFFSET(29)  NUMBITS(1)  [],
+        /// CD heart beat
+        CDH         OFFSET(28)  NUMBITS(1)  [],
+        /// Number of collision count
+        NCC         OFFSET(24)  NUMBITS(4)  [],
+        /// Early Tx threshold
+        ERTXTH      OFFSET(16)  NUMBITS(6)  [],
+        /// Tx OK
+        TOK         OFFSET(15)  NUMBITS(1)  [],
+        /// Tx FIFO underrun
+        TUN         OFFSET(14)  NUMBITS(1)  [],
+        /// Tx descriptor is owned by the DMA. Set to 0 by the driver to initiate a send
+        OWN         OFFSET(13)  NUMBITS(1)  [],
+        /// Tx size
+        SIZE        OFFSET(0)   NUMBITS(13) [],
+    ],
+
+    pub RCR [
+        /// Early Rx threshold bits
+        ERTH        OFFSET(24)  NUMBITS(4)  [],
+        /// Multiple early interrupt select
+        MULERINT    OFFSET(17)  NUMBITS(1)  [],
+        /// TBD, didn't really understand the docs, lmao
+        RER8        OFFSET(16)  NUMBITS(1)  [],
+        /// Rx FIFO threshold
+        RXFTH       OFFSET(13)  NUMBITS(3)  [],
+        /// Rx buffer length (+16 bytes)
+        RBLEN       OFFSET(11)  NUMBITS(2)  [
+            Len8K = 0,
+            Len16K = 1,
+            Len32K = 2,
+            Len64K = 3,
+        ],
+        /// Max DMA burst size per Rx DMA burst
+        MXDMA       OFFSET(8)   NUMBITS(3)  [],
+        /// When 0: the packet will get split if crossing the Rx buffer end
+        /// When 1: the packet will get placed contiguously
+        WRAP        OFFSET(7)   NUMBITS(1)  [],
+        /// Accept error packet
+        AER         OFFSET(5)   NUMBITS(1)  [],
+        /// Accept runt (packets smaller than 64 bytes)
+        AR          OFFSET(4)   NUMBITS(1)  [],
+        /// Accept broadcast packets
+        AB          OFFSET(3)   NUMBITS(1)  [],
+        /// Accept multicast packets
+        AM          OFFSET(2)   NUMBITS(1)  [],
+        /// Accept physical match
+        APM         OFFSET(1)   NUMBITS(1)  [],
+        /// Accept all packets
+        AAP         OFFSET(0)   NUMBITS(1)  [],
+    ],
+}
+
+register_structs! {
+    #[allow(non_snake_case)]
+    pub Regs {
+        (0x0000 => pub IDRn: [ReadWrite<u32>; 2]),
+        (0x0008 => pub MARn: [ReadWrite<u8>; 8]),
+        (0x0010 => pub TSDn: [ReadWrite<u32, TSDn::Register>; 4]),
+        (0x0020 => pub TSADn: [ReadWrite<u32>; 4]),
+        (0x0030 => pub RBSTART: ReadWrite<u32>),
+        (0x0034 => pub ERBCR: ReadOnly<u16>),
+        (0x0036 => pub ERSR: ReadOnly<u8>),
+        (0x0037 => pub CR: ReadWrite<u8, CR::Register>),
+        (0x0038 => pub CAPR: ReadWrite<u16>),
+        (0x003A => pub CBR: ReadOnly<u16>),
+        (0x003C => pub IMR: ReadWrite<u16, IMR_ISR::Register>),
+        (0x003E => pub ISR: ReadWrite<u16, IMR_ISR::Register>),
+        (0x0040 => pub TCR: ReadWrite<u32>),
+        (0x0044 => pub RCR: ReadWrite<u32, RCR::Register>),
+        (0x0048 => pub TCTR: ReadWrite<u32>),
+        (0x004C => pub MPC: ReadWrite<u32>),
+        (0x0050 => pub r9346CR: ReadWrite<u8>),
+        (0x0051 => pub CONFIG0: ReadOnly<u8>),
+        (0x0052 => pub CONFIG1: ReadWrite<u8, CONFIG1::Register>),
+        (0x0053 => _1),
+        (0x0054 => pub TIMERINT: ReadWrite<u32>),
+        (0x0058 => pub MSR: ReadWrite<u8>),
+        (0x0059 => pub CONFIG3: ReadWrite<u8>),
+        (0x005A => pub CONFIG4: ReadWrite<u8>),
+        (0x005B => _2),
+        (0x005C => pub MULINT: ReadWrite<u16>),
+        (0x005E => pub RERID: ReadOnly<u8>),
+        (0x005F => _3),
+        (0x0060 => pub TSAD: ReadOnly<u16>),
+        (0x0062 => pub BMCR: ReadWrite<u16>),
+        (0x0064 => pub BMSR: ReadOnly<u16>),
+        (0x0066 => pub ANAR: ReadWrite<u16>),
+        (0x0068 => pub ANLPAR: ReadOnly<u16>),
+        (0x006A => pub ANER: ReadOnly<u16>),
+        (0x006C => pub DIS: ReadOnly<u16>),
+        (0x006E => pub FCSC: ReadOnly<u16>),
+        (0x0070 => pub NWAYTR: ReadWrite<u16>),
+        (0x0072 => pub REC: ReadOnly<u16>),
+        (0x0074 => pub CSCR: ReadWrite<u16>),
+        (0x0076 => _4),
+        (0x0078 => pub PHY1_PARM: ReadWrite<u32>),
+        (0x007C => pub TW_PARM: ReadWrite<u32>),
+        (0x0080 => pub PHY2_PARM: ReadWrite<u8>),
+        (0x0081 => _5),
+        (0x0084 => pub CRCn: [ReadWrite<u8>; 8]),
+        (0x008C => pub WAKEUPn: [ReadWrite<u32>; 16]),
+        (0x00CC => pub LSBCRCn: [ReadWrite<u8>; 8]),
+        (0x00D4 => _6),
+        (0x00D8 => pub CONFIG5: ReadWrite<u8>),
+        (0x00D9 => _7),
+        (0x0100 => @END),
+    }
+}
+
+struct Rx {
+    buffer: DmaBuffer<[MaybeUninit<u8>]>,
+    rd: usize,
+}
+
+// TODO place a secondary Tx queue here, to send the queued packets when more slots become
+// available
+struct Tx {
+    buffers: [Option<DmaBuffer<[u8]>>; 4],
+    wr: usize,
+    rd: usize,
+    queue: BoundedQueue<DmaBuffer<[u8]>>,
+}
+
+pub struct Rtl8139 {
+    regs: IrqSafeSpinlock<DeviceMemoryIo<'static, Regs>>,
+    mac: MacAddress,
+    pci: PciDeviceInfo,
+
+    nic: OneTimeInit<u32>,
+    rx: OneTimeInit<IrqSafeSpinlock<Rx>>,
+    tx: IrqSafeSpinlock<Tx>,
+
+    dma: Arc<dyn DmaAllocator>,
+}
+
+impl Tx {
+    pub fn tx_now(&mut self, regs: &Regs, packet: DmaBuffer<[u8]>) -> Result<(), Error> {
+        let packet_address = packet.bus_address().try_into_u32()?;
+        let packet_len = packet.len();
+        if packet_len > 1500 {
+            return Err(Error::InvalidArgument);
+        }
+
+        self.buffers[self.wr] = Some(packet);
+        regs.TSADn[self.wr].set(packet_address);
+        regs.TSDn[self.wr].write(TSDn::SIZE.val(packet_len as u32));
+
+        self.wr = (self.wr + 1) % 4;
+
+        Ok(())
+    }
+
+    pub fn is_full(&self) -> bool {
+        (self.wr + 1) % 4 == self.rd
+    }
+}
+
+impl Rtl8139 {
+    // 8K + overflow space
+    const RX_BUFFER_LEN: usize = 8192;
+    const RX_BUFFER_OVERFLOW: usize = 4096;
+
+    pub fn new(
+        dma: Arc<dyn DmaAllocator>,
+        base: PhysicalAddress,
+        info: PciDeviceInfo,
+    ) -> Result<Self, Error> {
+        let regs = unsafe { DeviceMemoryIo::<Regs>::map(base, Default::default()) }?;
+        let mac0 = regs.IDRn[0].get().to_le_bytes();
+        let mac1 = regs.IDRn[1].get().to_le_bytes();
+        let mac = MacAddress::from([mac0[0], mac0[1], mac0[2], mac0[3], mac1[0], mac1[1]]);
+
+        Ok(Self {
+            mac,
+            regs: IrqSafeSpinlock::new(regs),
+            pci: info,
+
+            nic: OneTimeInit::new(),
+            rx: OneTimeInit::new(),
+            tx: IrqSafeSpinlock::new(Tx {
+                buffers: [const { None }; 4],
+                wr: 0,
+                rd: 0,
+                queue: BoundedQueue::new(64),
+            }),
+
+            dma,
+        })
+    }
+}
+
+impl InterruptHandler for Rtl8139 {
+    fn handle_irq(self: Arc<Self>, _vector: IrqVector) -> bool {
+        let regs = self.regs.lock();
+        let status = regs.ISR.extract();
+        // Clear ISR bits
+        regs.ISR.write(IMR_ISR::TOK::SET + IMR_ISR::ROK::SET);
+
+        let mut any = false;
+        if status.matches_all(IMR_ISR::TOK::SET) {
+            let mut tx = self.tx.lock();
+
+            tx.rd = (tx.rd + 1) % 4;
+            if let Some(packet) = tx.queue.pop() {
+                // Should not fail, we just freed a single descriptor
+                tx.tx_now(&regs, packet).unwrap();
+            }
+
+            any = true;
+        }
+        if status.matches_all(IMR_ISR::ROK::SET) {
+            let nic = *self.nic.get();
+            let mut rx = self.rx.get().lock();
+            let cbr = regs.CBR.get();
+
+            loop {
+                let rx_pos = rx.rd % Self::RX_BUFFER_LEN;
+                if rx_pos == cbr as usize {
+                    break;
+                }
+
+                // 4-byte preamble
+                let rx_len_0 = unsafe { rx.buffer[rx_pos + 2].assume_init() };
+                let rx_len_1 = unsafe { rx.buffer[rx_pos + 3].assume_init() };
+                let rx_len = u16::from_le_bytes([rx_len_0, rx_len_1]) as usize;
+
+                if rx_len >= 16 {
+                    if let Ok(mut packet_buf) = DmaBuffer::new_uninit_slice(&*self.dma, rx_len) {
+                        packet_buf.copy_from_slice(&rx.buffer[rx_pos + 4..rx_pos + rx_len + 4]);
+                        let packet_buf = unsafe { DmaBuffer::assume_init_slice(packet_buf) };
+                        // let packet_buf = unsafe { packet_buf.assume_init_slice() };
+                        let packet = RxPacket::new(packet_buf, 0, nic);
+                        ygg_driver_net_core::receive_packet(packet).ok();
+                    }
+                }
+
+                // rx_len + 4, aligned to 4 bytes
+                let total_len = (rx_len + 7) & !3;
+                rx.rd = rx.rd.wrapping_add(total_len);
+            }
+
+            regs.CAPR.set((rx.rd as u16).max(cbr).wrapping_sub(0x10));
+            rx.rd %= Self::RX_BUFFER_LEN;
+
+            any = true;
+        }
+
+        any
+    }
+}
+
+impl Device for Rtl8139 {
+    unsafe fn init(self: Arc<Self>, _cx: DeviceInitContext) -> Result<(), Error> {
+        log::info!("Initialize rtl8139 driver");
+        log::info!("MAC: {}", self.mac);
+
+        // Setup the initial Rx buffer
+        let rx_buffer = DmaBuffer::new_uninit_slice(
+            &*self.dma,
+            Self::RX_BUFFER_LEN + Self::RX_BUFFER_OVERFLOW,
+        )?;
+        let rx_buffer_address = rx_buffer.bus_address().try_into_u32()?;
+
+        self.pci.map_interrupt(Default::default(), self.clone())?;
+
+        let regs = self.regs.lock();
+
+        // Power up the device
+        regs.CONFIG1.set(0);
+
+        // Reset the device
+        let mut timeout = 1000000;
+        regs.CR.write(CR::RST::SET);
+        while timeout > 0 && regs.CR.matches_all(CR::RST::SET) {
+            core::hint::spin_loop();
+            timeout -= 1;
+        }
+        if timeout == 0 {
+            log::error!("rtl8139: reset timed out");
+            return Err(Error::TimedOut);
+        }
+
+        // Configure Rx
+        regs.RBSTART.set(rx_buffer_address);
+        regs.RCR
+            .write(RCR::WRAP::SET + RCR::AB::SET + RCR::APM::SET + RCR::AR::SET + RCR::AM::SET);
+
+        // Enable Rx/Tx interrupts
+        regs.IMR.write(IMR_ISR::ROK::SET + IMR_ISR::TOK::SET);
+
+        // Start Rx/Tx
+        regs.CR.modify(CR::TE::SET + CR::RE::SET);
+
+        self.rx.init(IrqSafeSpinlock::new(Rx {
+            buffer: rx_buffer,
+            rd: 0,
+        }));
+        let nic =
+            ygg_driver_net_core::register_interface(NetworkInterfaceType::Ethernet, self.clone());
+        self.nic.init(nic.id());
+
+        Ok(())
+    }
+
+    fn display_name(&self) -> &str {
+        "Realtek RTL8139 10/100Mbps Ethernet Controller"
+    }
+}
+
+impl NetworkDevice for Rtl8139 {
+    fn allocate_transmit_buffer(&self, len: usize) -> Result<DmaBuffer<[MaybeUninit<u8>]>, Error> {
+        DmaBuffer::new_uninit_slice(&*self.dma, len)
+    }
+
+    fn transmit_buffer(&self, packet: DmaBuffer<[u8]>) -> Result<(), Error> {
+        let mut tx = self.tx.lock();
+
+        // Buffer still in Tx, cannot send
+        if tx.is_full() {
+            if tx.queue.push(packet).is_err() {
+                log::warn!("rtl8139: out of tx buffers + queue full");
+                return Err(Error::WouldBlock);
+            } else {
+                return Ok(());
+            }
+        }
+        let regs = self.regs.lock();
+        tx.tx_now(&regs, packet)
+    }
+
+    fn packet_prefix_size(&self) -> usize {
+        0
+    }
+
+    fn read_hardware_address(&self) -> MacAddress {
+        self.mac
+    }
+}

kernel/driver/net/rtl81xx/src/rtl8168.rs

@@ -0,0 +1,790 @@
+use core::{
+    mem::{self, MaybeUninit},
+    sync::atomic::{self, Ordering},
+    time::Duration,
+};
+
+use alloc::{sync::Arc, vec::Vec};
+use device_api::{
+    device::{Device, DeviceInitContext},
+    dma::DmaAllocator,
+    interrupt::{InterruptHandler, IrqVector},
+};
+use futures_util::task::AtomicWaker;
+use libk::{
+    dma::{BusAddress, DmaBuffer},
+    error::Error,
+    task::runtime::{self, psleep, pwait},
+};
+use libk_mm::{address::PhysicalAddress, device::DeviceMemoryIo, L3_PAGE_SIZE};
+use libk_util::{event::BitmapEvent, sync::IrqSafeSpinlock, OneTimeInit};
+use tock_registers::{
+    interfaces::{ReadWriteable, Readable, Writeable},
+    register_bitfields, register_structs,
+    registers::{ReadOnly, ReadWrite, WriteOnly},
+    LocalRegisterCopy,
+};
+use ygg_driver_net_core::{
+    ephy::{MdioBus, PhyAccess, GBESR},
+    interface::{NetworkDevice, NetworkInterfaceType},
+    RxPacket,
+};
+use ygg_driver_pci::device::{PciDeviceInfo, PreferredInterruptMode};
+use yggdrasil_abi::{
+    bitflags,
+    net::{
+        link::{Duplex, EthernetLinkState, EthernetSpeed, LinkState},
+        MacAddress,
+    },
+};
+
+register_bitfields! {
+    u8,
+    CR [
+        /// Request the MAC to stop its Tx/Rx activity
+        STOP    OFFSET(7)   NUMBITS(1)  [],
+        /// Software reset bit. Set by driver, cleared by device
+        RST     OFFSET(4)   NUMBITS(1)  [],
+        /// Rx enable
+        RE      OFFSET(3)   NUMBITS(1)  [],
+        /// Tx enable
+        TE      OFFSET(2)   NUMBITS(1)  [],
+    ],
+    TPPOLL [
+        HPQ     OFFSET(7)   NUMBITS(1)  [],
+        NPQ     OFFSET(6)   NUMBITS(1)  [],
+        FSWInt  OFFSET(0)   NUMBITS(1)  [],
+    ],
+    R9346CR [
+        EEM     OFFSET(6)   NUMBITS(2)  [
+            Normal = 0b00,
+            ConfigWriteEnable = 0b11
+        ],
+    ],
+    CONFIG2 [
+        MSI     OFFSET(5)   NUMBITS(1)  [],
+    ],
+    PHYSTATUS [
+        TXFLOW  OFFSET(6)   NUMBITS(1)  [],
+        RXFLOW  OFFSET(5)   NUMBITS(1)  [],
+        MF1000  OFFSET(4)   NUMBITS(1)  [],
+        M100    OFFSET(3)   NUMBITS(1)  [],
+        M10     OFFSET(2)   NUMBITS(1)  [],
+        LINKSTS OFFSET(1)   NUMBITS(1)  [],
+        FULLDUP OFFSET(0)   NUMBITS(1)  [],
+    ],
+}
+
+register_bitfields! {
+    u16,
+    IMR_ISR [
+        TIMEOUT OFFSET(14)  NUMBITS(1)  [],
+        FEMP    OFFSET(9)   NUMBITS(1)  [],
+        SWINT   OFFSET(8)   NUMBITS(1)  [],
+        TDU     OFFSET(7)   NUMBITS(1)  [],
+        FOVW    OFFSET(6)   NUMBITS(1)  [],
+        LINKCHG OFFSET(5)   NUMBITS(1)  [],
+        RDU     OFFSET(4)   NUMBITS(1)  [],
+        TER     OFFSET(3)   NUMBITS(1)  [],
+        TOK     OFFSET(2)   NUMBITS(1)  [],
+        RER     OFFSET(1)   NUMBITS(1)  [],
+        ROK     OFFSET(0)   NUMBITS(1)  [],
+    ],
+    CPCR [
+        /// (rtl8169sc) PCI endianness mode
+        ENDIAN      OFFSET(9)   NUMBITS(1)  [
+            Little = 0,
+            Big = 1,
+        ],
+        /// (no docs found) TBD
+        MACSTATDIS  OFFSET(7)   NUMBITS(1)  [],
+        /// Rx VLAN de-tagging enable
+        RXVLAN      OFFSET(6)   NUMBITS(1)  [],
+        /// Rx checksum offload enable
+        RXCHKSUM    OFFSET(5)   NUMBITS(1)  [],
+        /// PCI dual address cycle enable
+        DAC         OFFSET(4)   NUMBITS(1)  [],
+        /// PCI multiple read/write enable
+        MULRW       OFFSET(3)   NUMBITS(1)  [],
+
+        // Not documented, found in OpenBSD's re driver
+        /// C+ Rx mode enable
+        RXENB       OFFSET(1)   NUMBITS(1)  [],
+        /// C+ Tx mode enable
+        TXENB       OFFSET(0)   NUMBITS(1)  [],
+    ],
+}
+
+register_bitfields! {
+    u32,
+    TCR [
+        HWVERID0    OFFSET(28)  NUMBITS(2)  [],
+        HWVERID1    OFFSET(26)  NUMBITS(1)  [],
+        IFG         OFFSET(24)  NUMBITS(2)  [],
+        HWVERID2    OFFSET(23)  NUMBITS(1)  [],
+        TX_NOCRC    OFFSET(16)  NUMBITS(1)  [],
+        QUEUE_EMPTY OFFSET(11)  NUMBITS(1)  [],
+        MXDMA       OFFSET(8)   NUMBITS(3)  [
+            Burst16 = 0b000,
+            Burst32 = 0b001,
+            Burst64 = 0b010,
+            Burst128 = 0b011,
+            Burst256 = 0b100,
+            Burst512 = 0b101,
+            Unlimited = 0b111,
+        ],
+    ],
+    RCR [
+        RXFTH       OFFSET(13)  NUMBITS(3)  [
+            Threshold64 = 0b010,
+            Threshold128 = 0b011,
+            Threshold256 = 0b100,
+            Threshold512 = 0b101,
+            Threshold1024 = 0b110,
+            NoThreshold = 0b111,
+        ],
+        EARLYOFFV2  OFFSET(11)  NUMBITS(1)  [],
+        MXDMA       OFFSET(8)   NUMBITS(3)  [
+            Burst64 = 0b010,
+            Burst128 = 0b011,
+            Burst256 = 0b100,
+            Burst512 = 0b101,
+            Burst1024 = 0b110,
+            Unlimited = 0b111
+        ],
+        R9356SEL    OFFSET(6)   NUMBITS(1)  [],
+        AER         OFFSET(5)   NUMBITS(1)  [],
+        AR          OFFSET(4)   NUMBITS(1)  [],
+        AB          OFFSET(3)   NUMBITS(1)  [],
+        AM          OFFSET(2)   NUMBITS(1)  [],
+        APM         OFFSET(1)   NUMBITS(1)  [],
+        AAP         OFFSET(0)   NUMBITS(1)  [],
+    ],
+    PHYAR [
+        FLAG        OFFSET(31)  NUMBITS(1)  [
+            Read = 0,
+            Write = 1,
+        ],
+        REGADDR     OFFSET(16)  NUMBITS(5)  [],
+        DATA        OFFSET(0)   NUMBITS(16) [],
+    ],
+}
+
+register_structs! {
+    #[allow(non_snake_case)]
+    Regs {
+        (0x0000 => IDRn: [ReadWrite<u32>; 2]),
+        (0x0008 => MARn: [ReadWrite<u32>; 2]),
+        (0x0010 => DTCCR: ReadWrite<u64>),
+        (0x0018 => _0),
+        (0x0020 => TNPDS: [ReadWrite<u32>; 2]),
+        (0x0028 => THPDS: [ReadWrite<u32>; 2]),
+        (0x0030 => _1),
+        (0x0037 => CR: ReadWrite<u8, CR::Register>),
+        (0x0038 => TPPOLL: WriteOnly<u8, TPPOLL::Register>),
+        (0x0039 => _2),
+        (0x003C => IMR: ReadWrite<u16, IMR_ISR::Register>),
+        (0x003E => ISR: ReadWrite<u16, IMR_ISR::Register>),
+        (0x0040 => TCR: ReadWrite<u32, TCR::Register>),
+        (0x0044 => RCR: ReadWrite<u32, RCR::Register>),
+        (0x0048 => TCTR: ReadWrite<u32>),
+        (0x004C => MPKT: ReadWrite<u32>),
+        (0x0050 => R9346CR: ReadWrite<u8, R9346CR::Register>),
+        (0x0051 => CONFIG0: ReadWrite<u8>),
+        (0x0052 => CONFIG1: ReadWrite<u8>),
+        (0x0053 => CONFIG2: ReadWrite<u8, CONFIG2::Register>),
+        (0x0054 => CONFIG3: ReadWrite<u8>),
+        (0x0055 => CONFIG4: ReadWrite<u8>),
+        (0x0056 => CONFIG5: ReadWrite<u8>),
+        (0x0057 => _4),
+        (0x0058 => TIMERINT: ReadWrite<u32>),
+        (0x005C => _5),
+        (0x0060 => PHYAR: ReadWrite<u32, PHYAR::Register>),
+        (0x0064 => _6),
+        (0x006C => PHYSTATUS: ReadOnly<u8, PHYSTATUS::Register>),
+        (0x006D => _7),
+        (0x0082 => LDPS: ReadWrite<u8>),
+        (0x0083 => _13),
+        (0x0084 => WAKEUPn: [ReadWrite<u32>; 16]),
+        (0x00C4 => CRCn: [ReadWrite<u16>; 5]),
+        (0x00CE => _8),
+        (0x00DA => RMS: ReadWrite<u16>),
+        (0x00DC => _9),
+        (0x00E0 => CPCR: ReadWrite<u16, CPCR::Register>),
+        (0x00E2 => _10),
+        (0x00E4 => RDSAR: [ReadWrite<u32>; 2]),
+        (0x00EC => MTPS: ReadWrite<u8>),
+        (0x00ED => _11),
+        (0x00F0 => MISC: ReadWrite<u32>),
+        (0x00F4 => _12),
+        (0x0100 => @END),
+    }
+}
+
+bitflags! {
+    struct ChipFlags: u32 {
+        const MACSTAT: bit 0;
+        const NEED_PHY_RESET: bit 1;
+        const RXDV_GATED: bit 2;
+        const EARLYOFFV2: bit 3;
+    }
+}
+
+#[derive(Clone, Copy, PartialEq, Debug)]
+#[allow(non_camel_case_types)]
+enum Revision {
+    rtl8168gu,
+    rtl8168h,
+    other(u32),
+}
+
+#[repr(C)]
+struct Descriptor {
+    cmd: u32,
+    vlan_cmd: u32,
+    address: BusAddress,
+}
+
+struct RxRing {
+    entries: DmaBuffer<[Descriptor]>,
+    buffers: Vec<DmaBuffer<[MaybeUninit<u8>]>>,
+    rd: usize,
+}
+
+struct TxRing {
+    entries: DmaBuffer<[Descriptor]>,
+    buffers: Vec<Option<DmaBuffer<[u8]>>>,
+    wr: usize,
+    rd: usize,
+}
+
+pub struct Rtl8168 {
+    regs: IrqSafeSpinlock<DeviceMemoryIo<'static, Regs>>,
+    mac: MacAddress,
+    pci: PciDeviceInfo,
+
+    nic: OneTimeInit<u32>,
+    rx: OneTimeInit<IrqSafeSpinlock<RxRing>>,
+    tx: OneTimeInit<IrqSafeSpinlock<TxRing>>,
+
+    events: BitmapEvent<AtomicWaker>,
+
+    dma: Arc<dyn DmaAllocator>,
+}
+
+impl RxRing {
+    pub fn with_capacity(dma: &dyn DmaAllocator, capacity: usize) -> Result<Self, Error> {
+        let buffers = (0..capacity)
+            .map(|_| DmaBuffer::new_uninit_slice(dma, L3_PAGE_SIZE))
+            .collect::<Result<Vec<_>, Error>>()?;
+        let entries = DmaBuffer::new_slice_with(
+            dma,
+            |i| Descriptor::new_rx(buffers[i].bus_address(), i == capacity - 1),
+            capacity,
+        )?;
+        Ok(Self {
+            entries,
+            buffers,
+            rd: 0,
+        })
+    }
+
+    pub fn consume<F: Fn(DmaBuffer<[u8]>)>(
+        &mut self,
+        dma: &dyn DmaAllocator,
+        handler: F,
+    ) -> Result<usize, Error> {
+        let mut count = 0;
+        loop {
+            let index = self.rd % self.entries.len();
+            let entry = &self.entries[index];
+
+            if entry.is_host_owned() {
+                let new_buffer = DmaBuffer::new_uninit_slice(dma, 4096)?;
+                let new_buffer_address = new_buffer.bus_address();
+                let buffer = mem::replace(&mut self.buffers[index], new_buffer);
+                let buffer = unsafe { DmaBuffer::assume_init_slice(buffer) };
+                handler(buffer);
+
+                self.entries[index].setup_rx(new_buffer_address);
+
+                self.rd = self.rd.wrapping_add(1);
+                count += 1;
+            } else {
+                break;
+            }
+        }
+        Ok(count)
+    }
+
+    pub fn base_address(&self) -> BusAddress {
+        self.entries.bus_address()
+    }
+}
+
+impl TxRing {
+    pub fn with_capacity(dma: &dyn DmaAllocator, capacity: usize) -> Result<Self, Error> {
+        let entries =
+            DmaBuffer::new_slice_with(dma, |i| Descriptor::empty_tx(i == capacity - 1), capacity)?;
+        let buffers = (0..capacity).map(|_| None).collect();
+
+        Ok(Self {
+            entries,
+            buffers,
+            wr: 0,
+            rd: 0,
+        })
+    }
+
+    pub fn is_full(&self) -> bool {
+        self.wr.wrapping_add(1) % self.entries.len() == self.rd % self.entries.len()
+    }
+
+    pub fn push(&mut self, packet: DmaBuffer<[u8]>) -> Result<(), Error> {
+        let packet_base = packet.bus_address();
+        let packet_size = packet.len();
+        let capacity = self.entries.len();
+
+        // TODO packet size checks
+
+        if self.is_full() {
+            log::warn!("rtl81xx: tx ring full: wr={}, rd={}", self.wr, self.rd);
+            return Err(Error::WouldBlock);
+        }
+
+        let index = self.wr % capacity;
+        packet.cache_flush_all(true);
+        self.buffers[index] = Some(packet);
+        self.entries[index].setup_tx(packet_base, packet_size, index == capacity - 1);
+        self.entries.cache_flush_element(index, true);
+        self.entries[index].cmd |= Descriptor::CMD_OWN;
+        self.entries.cache_flush_element(index, true);
+
+        self.wr = self.wr.wrapping_add(1);
+
+        Ok(())
+    }
+
+    pub fn consume(&mut self) -> Result<usize, Error> {
+        let mut count = 0;
+        loop {
+            let index = self.rd % self.entries.len();
+            let entry = &mut self.entries[index];
+
+            if self.rd != self.wr && entry.is_host_owned() {
+                self.rd = self.rd.wrapping_add(1);
+                count += 1;
+            } else {
+                break;
+            }
+        }
+        Ok(count)
+    }
+
+    pub fn base_address(&self) -> BusAddress {
+        self.entries.bus_address()
+    }
+}
+
+impl Descriptor {
+    // Descriptor owned by DMA
+    const CMD_OWN: u32 = 1 << 31;
+    // Last descriptor in the ring
+    const CMD_END: u32 = 1 << 30;
+    // Last segment of a packet
+    const CMD_LS: u32 = 1 << 28;
+    // First segment of a packet
+    const CMD_FS: u32 = 1 << 29;
+
+    pub fn new_rx(buffer: BusAddress, last: bool) -> Self {
+        let mut cmd = Self::CMD_OWN | 0x1000;
+        if last {
+            cmd |= Self::CMD_END;
+        }
+        Self {
+            cmd,
+            vlan_cmd: 0,
+            address: buffer,
+        }
+    }
+
+    pub fn empty_tx(last: bool) -> Self {
+        let cmd = if last { Self::CMD_END } else { 0 };
+        Self {
+            cmd,
+            vlan_cmd: 0,
+            address: BusAddress::ZERO,
+        }
+    }
+
+    pub fn setup_rx(&mut self, buffer: BusAddress) {
+        let cmd = self.cmd;
+        self.address = buffer;
+        self.vlan_cmd = 0;
+        unsafe {
+            atomic::fence(Ordering::Release);
+            core::ptr::write_volatile(&mut self.cmd, cmd | Self::CMD_OWN);
+        }
+    }
+
+    pub fn setup_tx(&mut self, buffer: BusAddress, size: usize, last: bool) {
+        let mut cmd = Self::CMD_FS | Self::CMD_LS | (size as u32);
+        if last {
+            cmd |= Self::CMD_END;
+        }
+        self.address = buffer;
+        self.vlan_cmd = 0;
+        self.cmd = cmd;
+    }
+
+    pub fn is_host_owned(&self) -> bool {
+        self.cmd & Self::CMD_OWN == 0
+    }
+}
+
+impl Regs {
+    const TCR_REVISION_MASK: u32 = 0x7C800000;
+
+    pub fn get_link_state(&self) -> EthernetLinkState {
+        let phystatus = self.PHYSTATUS.extract();
+        if phystatus.matches_all(PHYSTATUS::LINKSTS::SET) {
+            let duplex = if phystatus.matches_all(PHYSTATUS::FULLDUP::SET) {
+                Duplex::Full
+            } else {
+                Duplex::Half
+            };
+            let (speed, duplex) = if phystatus.matches_all(PHYSTATUS::MF1000::SET) {
+                // 1000MF is always Full-Duplex
+                (EthernetSpeed::Speed1000, Duplex::Full)
+            } else if phystatus.matches_all(PHYSTATUS::M100::SET) {
+                (EthernetSpeed::Speed100, duplex)
+            } else if phystatus.matches_all(PHYSTATUS::M10::SET) {
+                (EthernetSpeed::Speed10, duplex)
+            } else {
+                (EthernetSpeed::Unknown, duplex)
+            };
+
+            EthernetLinkState::Up(speed, duplex)
+        } else {
+            EthernetLinkState::Down
+        }
+    }
+
+    pub fn get_chip_revision(&self) -> Revision {
+        let rev = self.TCR.get() & Self::TCR_REVISION_MASK;
+        match rev {
+            0x50800000 => Revision::rtl8168gu,
+            0x54000000 => Revision::rtl8168h,
+            _ => Revision::other(rev),
+        }
+    }
+
+    pub fn reset_device(&self, timeout: Duration) -> Result<(), Error> {
+        self.CR.write(CR::RST::SET);
+
+        psleep(Duration::from_millis(10));
+        pwait(timeout, Duration::from_millis(10), || {
+            self.CR.matches_all(CR::RST::CLEAR)
+        })
+    }
+
+    pub fn reset_phy(&self, timeout: Duration) -> Result<(), Error> {
+        let phy = PhyAccess::new(self, 0x00);
+
+        let (id0, id1) = phy.id()?;
+        log::info!("PHY {id0:04x}:{id1:04x}");
+
+        phy.write_reg(0x1F, 0x00)?;
+        phy.write_reg(0x0E, 0x00)?;
+
+        phy.reset(timeout)?;
+        phy.setup_link(true, GBESR::empty())?;
+
+        psleep(Duration::from_millis(100));
+
+        phy.write_reg(0x1F, 0x00)?;
+        phy.write_reg(0x0E, 0x00)?;
+
+        Ok(())
+    }
+
+    pub fn stop(&self, timeout: Duration) -> Result<(), Error> {
+        self.CR.write(CR::STOP::SET);
+
+        pwait(timeout, Duration::from_millis(10), || {
+            self.TCR.matches_all(TCR::QUEUE_EMPTY::SET)
+        })?;
+
+        psleep(Duration::from_millis(100));
+
+        self.IMR.set(0x0000);
+        self.ISR.set(0xFFFF);
+
+        Ok(())
+    }
+}
+
+impl MdioBus for Regs {
+    fn mii_read(&self, _phyaddr: u8, reg: u8) -> Result<u16, Error> {
+        self.PHYAR
+            .write(PHYAR::REGADDR.val(reg as u32) + PHYAR::FLAG::Read);
+        psleep(Duration::from_millis(10));
+        pwait(
+            Duration::from_millis(100),
+            Duration::from_millis(10),
+            || self.PHYAR.matches_all(PHYAR::FLAG::SET),
+        )?;
+        Ok(self.PHYAR.read(PHYAR::DATA) as u16)
+    }
+
+    fn mii_write(&self, _phyaddr: u8, reg: u8, value: u16) -> Result<(), Error> {
+        self.PHYAR.write(
+            PHYAR::REGADDR.val(reg as u32) + PHYAR::DATA.val(value as u32) + PHYAR::FLAG::Write,
+        );
+        psleep(Duration::from_millis(10));
+        pwait(
+            Duration::from_millis(100),
+            Duration::from_millis(10),
+            || self.PHYAR.matches_all(PHYAR::FLAG::CLEAR),
+        )
+    }
+}
+
+impl Rtl8168 {
+    pub fn new(
+        dma: Arc<dyn DmaAllocator>,
+        base: PhysicalAddress,
+        info: PciDeviceInfo,
+    ) -> Result<Self, Error> {
+        let regs = unsafe { DeviceMemoryIo::<Regs>::map(base, Default::default()) }?;
+        let mac0 = regs.IDRn[0].get().to_le_bytes();
+        let mac1 = regs.IDRn[1].get().to_le_bytes();
+        let mac = MacAddress::from([mac0[0], mac0[1], mac0[2], mac0[3], mac1[0], mac1[1]]);
+        Ok(Self {
+            regs: IrqSafeSpinlock::new(regs),
+            mac,
+            pci: info,
+
+            nic: OneTimeInit::new(),
+            rx: OneTimeInit::new(),
+            tx: OneTimeInit::new(),
+
+            events: BitmapEvent::new(AtomicWaker::new()),
+
+            dma,
+        })
+    }
+}
+
+impl InterruptHandler for Rtl8168 {
+    fn handle_irq(self: Arc<Self>, _vector: IrqVector) -> bool {
+        let regs = self.regs.lock();
+
+        let status = regs.ISR.extract();
+
+        if status.get() == 0 || status.get() == 0xFFFF {
+            return false;
+        }
+
+        regs.IMR.set(0x0000);
+        // Acknowledge interrupt
+        regs.ISR.set(status.get());
+
+        // Forward to softirq
+        self.events.signal(status.get() as u64);
+
+        // Restore interrupt mask
+        regs.IMR.set(0xFFFF);
+
+        true
+    }
+}
+
+impl Rtl8168 {
+    // "softirq" handler for rtl8168
+    async fn packet_handler(&self) {
+        let nic = *self.nic.get();
+        let tx = self.tx.get();
+        let rx = self.rx.get();
+
+        loop {
+            let events = LocalRegisterCopy::new(self.events.wait().await as u16);
+
+            if events.matches_all(IMR_ISR::LINKCHG::SET) {
+                let status = self.link_state();
+                log::info!("rtl8168: link is {status}");
+            }
+
+            if events.matches_all(IMR_ISR::ROK::SET) {
+                let mut rx = rx.lock();
+                rx.consume(&*self.dma, |buffer| {
+                    // TODO add packet len hint to packets
+                    let packet = RxPacket::new(buffer, 0, nic);
+                    ygg_driver_net_core::receive_packet(packet).ok();
+                })
+                .ok();
+            }
+
+            if events.matches_all(IMR_ISR::TOK::SET) {
+                let mut tx = tx.lock();
+                tx.consume().ok();
+            }
+        }
+    }
+}
+
+impl Device for Rtl8168 {
+    unsafe fn init(self: Arc<Self>, _cx: DeviceInitContext) -> Result<(), Error> {
+        log::info!("Initialize rtl8168");
+        log::info!("MAC: {}", self.mac);
+
+        // Bind IRQ
+        self.pci
+            .init_interrupts(PreferredInterruptMode::Msi(true))?;
+        let msi_info = self.pci.map_interrupt(Default::default(), self.clone())?;
+        if msi_info.is_none() {
+            log::error!("rtl8168 in non-MSI(-x) mode is not supported yet");
+            return Err(Error::InvalidArgument);
+        }
+
+        let regs = self.regs.lock();
+
+        let hwrev = regs.get_chip_revision();
+
+        if let Revision::other(id) = hwrev {
+            log::error!("rtl8168: unsupported chip ID {:#08x}, aborting", id);
+            return Err(Error::InvalidArgument);
+        }
+
+        regs.R9346CR.write(R9346CR::EEM::ConfigWriteEnable);
+        regs.CONFIG2.modify(CONFIG2::MSI::SET);
+        regs.R9346CR.write(R9346CR::EEM::Normal);
+
+        let rx_ring = RxRing::with_capacity(&*self.dma, 1024)?;
+        let tx_ring = TxRing::with_capacity(&*self.dma, 1024)?;
+        let rx_ring_base = rx_ring.base_address().into_u64();
+        let tx_ring_base = tx_ring.base_address().into_u64();
+
+        log::info!("Revision: {:?}", hwrev);
+        let flags = hwrev.flags();
+
+        // If the MAC was performing any I/O prior to driver init, stop it
+        regs.stop(Duration::from_secs(1))?;
+
+        // Perform software reset
+        regs.reset_device(Duration::from_millis(100))?;
+
+        let mut cpcr = CPCR::TXENB::SET + CPCR::MULRW::SET + CPCR::RXCHKSUM::SET;
+        if flags.contains(ChipFlags::MACSTAT) {
+            cpcr += CPCR::MACSTATDIS::SET;
+        }
+        regs.CPCR.write(cpcr);
+
+        if flags.contains(ChipFlags::NEED_PHY_RESET) {
+            log::info!("Reset PHY and start AN");
+            regs.reset_phy(Duration::from_millis(1000))?;
+        }
+
+        // TODO initialize MAC address
+
+        //regs.R9346CR.write(R9346CR::EEM::Normal);
+
+        regs.RDSAR[1].set((rx_ring_base >> 32) as u32);
+        regs.RDSAR[0].set(rx_ring_base as u32);
+        regs.TNPDS[1].set((tx_ring_base >> 32) as u32);
+        regs.TNPDS[0].set(tx_ring_base as u32);
+
+        if flags.contains(ChipFlags::RXDV_GATED) {
+            regs.MISC.set(regs.MISC.get() & !0x80000);
+        }
+
+        // Setup Tx
+        regs.TCR.modify(TCR::MXDMA::Unlimited + TCR::IFG.val(3));
+        // Setup Rx
+        regs.MTPS.set(16);
+        let mut rcr = RCR::MXDMA::Unlimited
+            + RCR::RXFTH::NoThreshold
+            + RCR::AAP::SET
+            + RCR::AB::SET
+            + RCR::AM::SET
+            + RCR::APM::SET;
+        if flags.contains(ChipFlags::EARLYOFFV2) {
+            rcr += RCR::EARLYOFFV2::SET;
+        }
+        regs.RCR.set(rcr.value | 0x1800);
+
+        // Enable Rx/Tx
+        regs.CR.write(CR::RE::SET + CR::TE::SET);
+
+        regs.IMR.set(0xFFFF);
+        regs.ISR.set(0xFFFF);
+
+        regs.MPKT.set(0);
+
+        regs.RMS.set(4096);
+
+        self.rx.init(IrqSafeSpinlock::new(rx_ring));
+        self.tx.init(IrqSafeSpinlock::new(tx_ring));
+
+        let interface =
+            ygg_driver_net_core::register_interface(NetworkInterfaceType::Ethernet, self.clone());
+        self.nic.init(interface.id());
+
+        drop(regs);
+
+        let this = self.clone();
+        runtime::spawn(async move { this.packet_handler().await })?;
+
+        Ok(())
+    }
+
+    fn display_name(&self) -> &str {
+        "Realtek RTL8111/8168 Gigabit Ethernet Controller"
+    }
+}
+
+impl NetworkDevice for Rtl8168 {
+    fn allocate_transmit_buffer(&self, len: usize) -> Result<DmaBuffer<[MaybeUninit<u8>]>, Error> {
+        DmaBuffer::new_uninit_slice(&*self.dma, len)
+    }
+
+    fn transmit_buffer(&self, buffer: DmaBuffer<[u8]>) -> Result<(), Error> {
+        let mut tx = self.tx.get().lock();
+        let regs = self.regs.lock();
+
+        if !matches!(regs.get_link_state(), EthernetLinkState::Up(_, _)) {
+            return Err(Error::NetworkUnreachable);
+        }
+
+        tx.push(buffer)?;
+        regs.TPPOLL.write(TPPOLL::NPQ::SET);
+
+        Ok(())
+    }
+
+    fn packet_prefix_size(&self) -> usize {
+        0
+    }
+
+    fn read_hardware_address(&self) -> MacAddress {
+        self.mac
+    }
+
+    fn link_state(&self) -> LinkState {
+        LinkState::Ethernet(self.regs.lock().get_link_state())
+    }
+}
+
+impl Revision {
+    fn flags(&self) -> ChipFlags {
+        match self {
+            Self::rtl8168h | Self::rtl8168gu => {
+                ChipFlags::NEED_PHY_RESET
+                    | ChipFlags::MACSTAT
+                    | ChipFlags::RXDV_GATED
+                    | ChipFlags::EARLYOFFV2
+            }
+            _ => unreachable!(),
+        }
+    }
+}

kernel/driver/net/stmmac/Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+name = "ygg_driver_net_stmmac"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+device-tree.workspace = true
+device-api.workspace = true
+yggdrasil-abi.workspace = true
+libk-mm.workspace = true
+libk-util.workspace = true
+libk.workspace = true
+ygg_driver_net_core.path = "../core"
+
+tock-registers.workspace = true
+log.workspace = true
+bytemuck.workspace = true

kernel/driver/net/stmmac/src/lib.rs

@@ -0,0 +1,343 @@
+#![no_std]
+
+use core::mem::MaybeUninit;
+
+use alloc::{sync::Arc, vec::Vec};
+use device_api::{
+    clock::{ClockHandle, ResetHandle},
+    device::{Device, DeviceInitContext},
+    dma::DmaAllocator,
+    interrupt::{FullIrq, InterruptHandler, IrqVector},
+};
+use device_tree::driver::{device_tree_driver, util::read_mac_address, Node, ProbeContext};
+use libk::{device::external_interrupt_controller, dma::DmaBuffer, error::Error};
+use libk_mm::{address::PhysicalAddress, device::DeviceMemoryIo};
+use libk_util::{sync::IrqSafeSpinlock, OneTimeInit};
+use regs::{
+    dma::{DMACiCR, DMACiIER, DMACiSR, DMACiTXCR, DMAC0RXCR, DMAMR, DMASBMR},
+    mac::{
+        MACAiHR, MACAiLR, MACCR, MACIER, MACISR, MACPFR, MACPHYCSR, MACQ0TXFCR, MACRXFCR, MACRXQC0R,
+    },
+    mtl::{MTLRXQiOMR, MTLTXQiOMR, MTLOMR},
+    Regs,
+};
+use ring::{RxDescriptor, RxRing, TxDescriptor, TxRing};
+use tock_registers::interfaces::{ReadWriteable, Readable, Writeable};
+use ygg_driver_net_core::{
+    interface::{NetworkDevice, NetworkInterfaceType},
+    RxPacket,
+};
+use yggdrasil_abi::net::{
+    link::{EthernetLinkState, LinkState},
+    MacAddress,
+};
+
+extern crate alloc;
+
+pub mod regs;
+pub mod ring;
+
+struct Inner {
+    regs: IrqSafeSpinlock<DeviceMemoryIo<'static, Regs>>,
+
+    tx_ring: IrqSafeSpinlock<TxRing>,
+    rx_ring: IrqSafeSpinlock<RxRing>,
+}
+
+struct Stmmac {
+    base: PhysicalAddress,
+    dma: OneTimeInit<Arc<dyn DmaAllocator>>,
+    mac: MacAddress,
+    resets: Vec<ResetHandle>,
+    clocks: Vec<ClockHandle>,
+    irq: FullIrq,
+
+    inner: OneTimeInit<Inner>,
+    iface_id: OneTimeInit<u32>,
+}
+
+impl Stmmac {
+    pub fn start_xmit(&self, frame: DmaBuffer<[u8]>) -> Result<(), Error> {
+        let inner = self.inner.get();
+        let regs = inner.regs.lock();
+
+        let mut tx_ring = inner.tx_ring.lock();
+        let index = tx_ring.push_xmit(frame)?;
+        let ring_pos = tx_ring
+            .buffer_base()
+            .add(size_of::<TxDescriptor>() * index)
+            .try_into_u32()
+            .unwrap();
+
+        regs.DMA.DMAC0TXDTPR.set(ring_pos);
+
+        Ok(())
+    }
+}
+
+impl InterruptHandler for Stmmac {
+    fn handle_irq(self: Arc<Self>, _vector: IrqVector) -> bool {
+        let inner = self.inner.get();
+        let regs = inner.regs.lock();
+        let mac_isr = regs.MAC.MACISR.extract();
+        let dma0_sr = regs.DMA.DMAC0SR.extract();
+
+        if mac_isr.matches_all(MACISR::RGSMIIIS::SET) {
+            let macphycsr = regs.MAC.MACPHYCSR.extract();
+            let state = if macphycsr.matches_all(MACPHYCSR::LNKSTS::Up) {
+                "up"
+            } else {
+                "down"
+            };
+            let mode = if macphycsr.matches_all(MACPHYCSR::LNKMOD::FullDuplex) {
+                "full-duplex"
+            } else {
+                "half-duplex"
+            };
+            log::info!("RGMII link status update");
+            log::info!("Link state: link={state}, mode={mode}");
+        }
+
+        if dma0_sr.matches_all(DMACiSR::TI::SET) {
+            regs.DMA.DMAC0SR.modify(DMACiSR::TI::SET);
+            inner.tx_ring.lock().consume().ok();
+        }
+        if dma0_sr.matches_all(DMACiSR::RI::SET) {
+            let dma = self.dma.get();
+            regs.DMA.DMAC0SR.modify(DMACiSR::RI::SET);
+            let iface = *self.iface_id.get();
+            let mut rx_ring = inner.rx_ring.lock();
+            rx_ring
+                .consume(dma.as_ref(), |packet| {
+                    let packet = RxPacket::new(packet, 0, iface);
+                    ygg_driver_net_core::receive_packet(packet).ok();
+                })
+                .ok();
+        }
+
+        true
+    }
+}
+
+impl Device for Stmmac {
+    unsafe fn init(self: Arc<Self>, cx: DeviceInitContext) -> Result<(), Error> {
+        // TODO this is a dirty fix for the device tree double initialization problem
+        if self.dma.is_initialized() {
+            return Err(Error::AlreadyExists);
+        }
+        let dma = self.dma.init(cx.dma_allocator.clone());
+
+        let intc = external_interrupt_controller()?;
+
+        intc.register_irq(self.irq.irq, self.irq.options, self.clone())?;
+
+        let tx_ring_capacity = 32;
+        let rx_ring_capacity = 32;
+
+        log::info!("dwmac: setup {:#x}, mac={}", self.base, self.mac);
+
+        for clock in self.clocks.iter() {
+            clock.enable()?;
+        }
+        for reset in self.resets.iter() {
+            reset.deassert()?;
+        }
+
+        let regs = unsafe { DeviceMemoryIo::<Regs>::map(self.base, Default::default()) }?;
+
+        // DMA initialization
+
+        // Perform a software reset
+        regs.DMA.DMAMR.modify(DMAMR::SWR::SET);
+
+        let mut timeout = 100000;
+        while timeout > 0 && regs.DMA.DMAMR.matches_all(DMAMR::SWR::SET) {
+            core::hint::spin_loop();
+            timeout -= 1;
+        }
+        if timeout == 0 {
+            log::warn!("dwmac: reset timeout");
+            return Err(Error::TimedOut);
+        }
+
+        // Setup DMA parameters
+        // TODO get these params from device tree
+        regs.DMA.DMASBMR.write(
+            DMASBMR::FB::SET
+                + DMASBMR::BLEN256::SET
+                + DMASBMR::BLEN128::SET
+                + DMASBMR::BLEN64::SET
+                + DMASBMR::BLEN32::SET
+                + DMASBMR::EN_LPI::SET
+                + DMASBMR::WR_OSR_LMT.val(3)
+                + DMASBMR::RD_OSR_LMT.val(3),
+        );
+
+        // Setup DMA Tx/Rx rings
+        let tx_ring = TxRing::with_capacity(dma.as_ref(), tx_ring_capacity)?;
+        let rx_ring = RxRing::with_capacity(dma.as_ref(), rx_ring_capacity)?;
+        let tx_ring_base = tx_ring.buffer_base().try_into_u32().unwrap();
+        let rx_ring_base = rx_ring.buffer_base().try_into_u32().unwrap();
+
+        regs.DMA.DMAC0TXRLR.set(tx_ring_capacity as u32 - 1);
+        regs.DMA.DMAC0TXDLAR.set(tx_ring_base);
+        regs.DMA.DMAC0TXDTPR.set(tx_ring_base);
+
+        regs.DMA.DMAC0RXRLR.set(rx_ring_capacity as u32 - 1);
+        regs.DMA.DMAC0RXDLAR.set(rx_ring_base);
+        regs.DMA
+            .DMAC0RXDTPR
+            .set(rx_ring_base + (rx_ring_capacity * size_of::<RxDescriptor>()) as u32);
+
+        // Setup DMA maximum segmen size, Rx buffer size + max burst len
+        regs.DMA.DMAC0CR.write(DMACiCR::PBLX8::CLEAR);
+        regs.DMA
+            .DMAC0TXCR
+            .write(DMACiTXCR::TXPBL.val(32) + DMACiTXCR::OSF::SET);
+        regs.DMA
+            .DMAC0RXCR
+            .write(DMAC0RXCR::RBSZ.val(4096) + DMAC0RXCR::RXPBL.val(32));
+
+        // Enable DMA interrupts
+        // TODO enable abnormal interrupts to handle errors properly
+        regs.DMA.DMAC0IER.write(
+            DMACiIER::NIE::SET
+                + DMACiIER::RIE::SET
+                + DMACiIER::TIE::SET
+                + DMACiIER::RBUE::SET
+                + DMACiIER::RSE::SET,
+        );
+
+        // Start Tx/Rx DMAs
+        regs.DMA.DMAC0TXCR.modify(DMACiTXCR::ST::SET);
+        regs.DMA.DMAC0RXCR.modify(DMAC0RXCR::SR::SET);
+
+        // MTL initialization
+
+        regs.MTL
+            .MTLOMR
+            .write(MTLOMR::RAA::AlgStrictPrio + MTLOMR::SCHALG::AlgStrictPrio);
+
+        // TODO get TQS, RQS from device tree
+        regs.MTL
+            .MTLTXQ0OMR
+            .write(MTLTXQiOMR::TSF::SET + MTLTXQiOMR::TQS.val(7) + MTLTXQiOMR::TXQEN::Enable);
+        regs.MTL.MTLRXQ0OMR.write(
+            MTLRXQiOMR::RQS.val(7)
+                + MTLRXQiOMR::DIS_TCP_EF::SET
+                + MTLRXQiOMR::RSF::SET
+                + MTLRXQiOMR::FEP::SET
+                + MTLRXQiOMR::FUP::SET,
+        );
+
+        // MAC initialization
+
+        // Setup the MAC address
+        let mac_bytes: [u8; 6] = self.mac.into();
+        regs.MAC.MACA0HR.write(
+            MACAiHR::AE::SET
+                + MACAiHR::ADDR4.val(mac_bytes[4] as u32)
+                + MACAiHR::ADDR5.val(mac_bytes[5] as u32),
+        );
+        regs.MAC.MACA0LR.write(
+            MACAiLR::ADDR0.val(mac_bytes[0] as u32)
+                + MACAiLR::ADDR1.val(mac_bytes[1] as u32)
+                + MACAiLR::ADDR2.val(mac_bytes[2] as u32)
+                + MACAiLR::ADDR3.val(mac_bytes[3] as u32),
+        );
+
+        // TODO proper filtering, don't use promiscuous mode all the time
+        regs.MAC.MACPFR.write(MACPFR::PR::SET + MACPFR::RA::SET);
+
+        // Setup Tx/Rx flow control, enable Rx queue 0
+        regs.MAC.MACQ0TXFCR.write(MACQ0TXFCR::TFE::SET);
+        regs.MAC.MACRXFCR.write(MACRXFCR::RFE::SET);
+        regs.MAC.MACRXQC0R.write(MACRXQC0R::RXQ0EN::Enable);
+
+        // Setup MAC-level interrupts
+        regs.MAC
+            .MACIER
+            .write(MACIER::TXSTSIE::SET + MACIER::RXSTSIE::SET + MACIER::RGSMIIIE::SET);
+
+        // Setup link information
+        regs.MAC
+            .MACCR
+            .modify(MACCR::PS::Ps1000Mbps + MACCR::DM::FullDuplex);
+
+        // Start Tx/Rx
+        regs.MAC.MACCR.modify(MACCR::TE::SET + MACCR::RE::SET);
+
+        // Apply PHY settings
+        regs.MAC
+            .MACPHYCSR
+            .modify(MACPHYCSR::LUD::SET + MACPHYCSR::TC::SET);
+
+        self.inner.init(Inner {
+            regs: IrqSafeSpinlock::new(regs),
+
+            tx_ring: IrqSafeSpinlock::new(tx_ring),
+            rx_ring: IrqSafeSpinlock::new(rx_ring),
+        });
+
+        let iface =
+            ygg_driver_net_core::register_interface(NetworkInterfaceType::Ethernet, self.clone());
+        self.iface_id.init(iface.id());
+
+        intc.enable_irq(self.irq.irq)?;
+
+        Ok(())
+    }
+
+    fn display_name(&self) -> &str {
+        "Synopsys DesignWare Ethernet MAC"
+    }
+}
+
+impl NetworkDevice for Stmmac {
+    fn allocate_transmit_buffer(&self, len: usize) -> Result<DmaBuffer<[MaybeUninit<u8>]>, Error> {
+        DmaBuffer::new_uninit_slice(&**self.dma.get(), len)
+    }
+
+    fn transmit_buffer(&self, buffer: DmaBuffer<[u8]>) -> Result<(), Error> {
+        self.start_xmit(buffer)
+    }
+
+    fn packet_prefix_size(&self) -> usize {
+        0
+    }
+
+    fn read_hardware_address(&self) -> MacAddress {
+        self.mac
+    }
+
+    fn link_state(&self) -> LinkState {
+        // TODO
+        LinkState::Ethernet(EthernetLinkState::Down)
+    }
+}
+
+device_tree_driver! {
+    compatible: ["starfive,jh7110-dwmac"],
+    driver: {
+        fn probe(&self, node: &Arc<Node>, context: &ProbeContext) -> Option<Arc<dyn Device>> {
+            let base = node.map_base(context, 0)?;
+            let clocks = node.clocks()?.collect();
+            let resets = node.resets()?.collect();
+            let mac = read_mac_address(node)?;
+            // TODO named_interrupt()
+            let irq = node.interrupt(0)?;
+
+            Some(Arc::new(Stmmac {
+                base,
+                clocks,
+                resets,
+                mac,
+                irq,
+
+                dma: OneTimeInit::new(),
+                inner: OneTimeInit::new(),
+                iface_id: OneTimeInit::new(),
+            }))
+        }
+    }
+}

kernel/driver/net/stmmac/src/regs/dma.rs

@@ -0,0 +1,243 @@
+use tock_registers::{register_bitfields, register_structs, registers::ReadWrite};
+
+register_bitfields! {
+    u32,
+    pub DMAMR [
+        /// Interrupt mode
+        INTM    OFFSET(16)  NUMBITS(2)  [],
+        /// Transmit priority
+        TXPR    OFFSET(11)  NUMBITS(1)  [],
+        /// Descriptor posted write
+        DSPW    OFFSET(8)   NUMBITS(1)  [],
+        /// Transmit arbitration algorithm
+        TAA     OFFSET(2)   NUMBITS(3)  [],
+        /// Software reset
+        ///
+        /// When this bit is set, the MAC and the DMA controllers are reset. This bit is cleared
+        /// automatically when the MAC/DMA reset completes.
+        SWR     OFFSET(0)   NUMBITS(1)  [],
+    ],
+
+    pub DMASBMR [
+        /// Enable Low Power interface
+        EN_LPI      OFFSET(31)  NUMBITS(1)  [],
+        /// Unlock on Magic packet or Remote Wakeup packet
+        LPI_XIT_PKT OFFSET(30)  NUMBITS(1)  [],
+        /// AXI maximum write outstanding request limit
+        ///
+        /// Maximum outstanding requests = WR_OSR_LMT + 1
+        WR_OSR_LMT  OFFSET(24)  NUMBITS(2)  [],
+        /// AXI maximum reaad outstanding request limit
+        ///
+        /// Maximum outstanding requests = RD_OSR_LMT + 1
+        RD_OSR_LMT  OFFSET(16)  NUMBITS(2)  [],
+        /// 1 Kbyte boundary crossing enable for the AXI master
+        ///
+        /// If set, the burst transfers performed by the AXI master do not cross 1 Kbyte boundary
+        /// If not, the transfers do not cross 4 Kbyte boundary
+        ONEKBBE     OFFSET(13)  NUMBITS(1)  [],
+        /// Address aligned bursts
+        ///
+        /// If set, the master performs addres-aligned burst transfers on read and write channels
+        AAL         OFFSET(12)  NUMBITS(1)  [],
+        /// Automatic AXI LPI enable
+        ///
+        /// When set, enables AXI master to enter into LPI state when there is no activity on the
+        /// Ethernet peripheral for a number of clock cycles programmed in ETH_DMALPIEI register.
+        AALE        OFFSET(10)  NUMBITS(1)  [],
+        /// AXI burst length 256
+        BLEN256     OFFSET(7)   NUMBITS(1)  [],
+        /// AXI burst length 128
+        BLEN128     OFFSET(6)   NUMBITS(1)  [],
+        /// AXI burst length 64
+        BLEN64      OFFSET(5)   NUMBITS(1)  [],
+        /// AXI burst length 32
+        BLEN32      OFFSET(4)   NUMBITS(1)  [],
+        /// AXI burst length 16
+        BLEN16      OFFSET(3)   NUMBITS(1)  [],
+        /// AXI burst length 8
+        BLEN8       OFFSET(2)   NUMBITS(1)  [],
+        /// AXI burst length 4
+        BLEN4       OFFSET(1)   NUMBITS(1)  [],
+        /// Fixed burst length
+        FB          OFFSET(0)   NUMBITS(1)  [],
+    ],
+
+    pub DMACiCR [
+        /// Descriptor skip length
+        DSL     OFFSET(18)  NUMBITS(1)  [],
+        /// 8xPBL mode
+        PBLX8   OFFSET(16)  NUMBITS(1)  [],
+        /// Maximum segment size
+        MSS     OFFSET(0)   NUMBITS(14)  [],
+    ],
+
+    pub DMACiTXCR [
+        /// Tx QOS
+        TQOS    OFFSET(24)  NUMBITS(4)  [],
+        /// Transmit programmable burst length
+        TXPBL   OFFSET(16)  NUMBITS(6)  [],
+        /// Ignore PBL requirement
+        IPBL    OFFSET(15)  NUMBITS(1)  [],
+        /// TCP segmentation enabled
+        TSE     OFFSET(12)  NUMBITS(1)  [],
+        /// Operate on second packet
+        OSF     OFFSET(4)   NUMBITS(1)  [],
+        /// Tx channel weight
+        TCW     OFFSET(1)   NUMBITS(3)  [],
+        /// Start or stop Tx command
+        ///
+        /// When set, the DMA checks the transmit list at current position for a packet to be
+        /// transmitted. If the DMA does not own the current descriptor, the transmission enters
+        /// the Suspended state with TBU bit in DMACiSR set.
+        ST      OFFSET(0)   NUMBITS(1)  [],
+    ],
+
+    pub DMAC0RXCR [
+        /// DMA Rx channel 0 packet flush
+        ///
+        /// When set, the DMA automatically flushes the packet from the Rx queues destined to DMA
+        /// Rx channel 0 when the DMA Rx 0 is stopped after a system bus error.
+        RPF     OFFSET(31)  NUMBITS(1)  [],
+        /// Rx QOS
+        RQOS    OFFSET(24)  NUMBITS(4)  [],
+        /// Rx programmable burst length
+        RXPBL   OFFSET(16)  NUMBITS(6)  [],
+        /// Receive buffer size
+        RBSZ    OFFSET(1)   NUMBITS(14) [],
+        /// Start or stop Rx
+        ///
+        /// When set, the DMA tries to acquire the descriptor from the receive list and process the
+        /// incoming packets.
+        SR      OFFSET(0)   NUMBITS(1)  [],
+    ],
+
+    pub DMACiIER [
+        /// Normal interrupt summary enable
+        ///
+        /// When this bit is set, the following interrupts are enabled in the Channel i status
+        /// register:
+        ///
+        /// * Tx interrupt
+        /// * Tx buffer unavailable
+        /// * Rx interrupt
+        /// * Early Rx interrupt
+        NIE     OFFSET(15)  NUMBITS(1)  [],
+        /// Abnormal interrupt summary enable
+        ///
+        /// When this bit is set, the following interrupt summary is enabled:
+        ///
+        /// * Tx process stopped
+        /// * Rx buffer unavailable
+        /// * Rx process stopped
+        /// * Rx watchdog timeout
+        /// * Early Tx interrupt
+        /// * Fatal bus error
+        AIE     OFFSET(14)  NUMBITS(1)  [],
+        /// Context descriptor error enable (+AIE)
+        CDEE    OFFSET(13)  NUMBITS(1)  [],
+        /// Fatal bus error enable (+AIE)
+        FBEE    OFFSET(12)  NUMBITS(1)  [],
+        /// Early Rx interrupt enable
+        ///
+        /// When this bit is set along with the NIE bit, the Early Rx interrupt is enabled
+        ERIE    OFFSET(11)  NUMBITS(1)  [],
+        /// Early Tx interrupt enable
+        ///
+        /// When this bit is set along with the AIE bit, the Early Tx interrupt is enabled
+        ETIE    OFFSET(10)  NUMBITS(1)  [],
+        /// Receive watchdog timeout enable (+AIE)
+        RWTE    OFFSET(9)   NUMBITS(1)  [],
+        /// Rx stopped enable (+AIE)
+        RSE     OFFSET(8)   NUMBITS(1)  [],
+        /// Rx buffer unavailable enable (+AIE)
+        RBUE    OFFSET(7)   NUMBITS(1)  [],
+        /// Rx interrupt enable (+NIE)
+        RIE     OFFSET(6)   NUMBITS(1)  [],
+        /// Tx buffer unavailable enable (+NIE)
+        TBUE    OFFSET(2)   NUMBITS(1)  [],
+        /// Tx stopped enable (+AIE)
+        TXSE    OFFSET(1)   NUMBITS(1)  [],
+        /// Tx interrupt enable (+NIE)
+        TIE     OFFSET(0)   NUMBITS(1)  [],
+    ],
+
+    pub DMACiSR [
+        /// Rx DMA error bits
+        REB     OFFSET(19)  NUMBITS(3)  [],
+        /// Tx DMA error bits
+        TEB     OFFSET(16)  NUMBITS(3)  [],
+        /// Normal interrupt summary
+        NIS     OFFSET(15)  NUMBITS(1)  [],
+        /// Abnormal interrupt summary
+        AIS     OFFSET(14)  NUMBITS(1)  [],
+        /// Context descriptor error
+        CDE     OFFSET(13)  NUMBITS(1)  [],
+        /// Fatal bus error
+        FBE     OFFSET(12)  NUMBITS(1)  [],
+        /// Early Rx interrupt
+        ERI     OFFSET(11)  NUMBITS(1)  [],
+        /// Early Tx interrupt
+        ETI     OFFSET(10)  NUMBITS(1)  [],
+        /// Rx watchdog timeout
+        RWT     OFFSET(9)   NUMBITS(1)  [],
+        /// Rx process stopped
+        RPS     OFFSET(8)   NUMBITS(1)  [],
+        /// Rx buffer unavailable. To resume processing Rx descriptors, the driver should change
+        /// the ownership of the descriptors and issue a Rx Poll Demand command. In ring mode, the
+        /// driver should advance the Rx descriptor tail pointer of a channel.
+        RBU     OFFSET(7)   NUMBITS(1)  [],
+        /// Rx interrupt
+        RI      OFFSET(6)   NUMBITS(1)  [],
+        /// Tx buffer unavailable (see RBU, but for Tx)
+        TBU     OFFSET(2)   NUMBITS(1)  [],
+        /// Tx process stopped
+        TPS     OFFSET(1)   NUMBITS(1)  [],
+        /// Tx interrupt
+        TI      OFFSET(0)   NUMBITS(1)  [],
+    ],
+}
+
+register_structs! {
+    pub DmaRegs {
+        (0x000 => pub DMAMR: ReadWrite<u32, DMAMR::Register>),
+        (0x004 => pub DMASBMR: ReadWrite<u32, DMASBMR::Register>),
+        (0x008 => pub DMAISR: ReadWrite<u32>),
+        (0x00C => pub DMADSR: ReadWrite<u32>),
+        (0x010 => _0),
+        (0x020 => pub DMAA4TXACR: ReadWrite<u32>),
+        (0x024 => pub DMAA4RXACR: ReadWrite<u32>),
+        (0x028 => pub DMAA4DACR: ReadWrite<u32>),
+        (0x02C => _1),
+        (0x040 => pub DMALPIEI: ReadWrite<u32>),
+        (0x044 => _2),
+        (0x100 => pub DMAC0CR: ReadWrite<u32, DMACiCR::Register>),
+        (0x104 => pub DMAC0TXCR: ReadWrite<u32, DMACiTXCR::Register>),
+        (0x108 => pub DMAC0RXCR: ReadWrite<u32, DMAC0RXCR::Register>),
+        (0x10C => _3),
+        (0x114 => pub DMAC0TXDLAR: ReadWrite<u32>),
+        (0x118 => _4),
+        (0x11C => pub DMAC0RXDLAR: ReadWrite<u32>),
+        (0x120 => pub DMAC0TXDTPR: ReadWrite<u32>),
+        (0x124 => _5),
+        (0x128 => pub DMAC0RXDTPR: ReadWrite<u32>),
+        (0x12C => pub DMAC0TXRLR: ReadWrite<u32>),
+        (0x130 => pub DMAC0RXRLR: ReadWrite<u32>),
+        (0x134 => pub DMAC0IER: ReadWrite<u32, DMACiIER::Register>),
+        (0x138 => pub DMAC0RXIWTR: ReadWrite<u32>),
+        (0x13C => pub DMAC0SFCSR: ReadWrite<u32>),
+        (0x140 => _6),
+        (0x144 => pub DMAC0CATXDR: ReadWrite<u32>),
+        (0x148 => _7),
+        (0x14C => pub DMAC0CARXDR: ReadWrite<u32>),
+        (0x150 => _8),
+        (0x154 => pub DMAC0CATXBR: ReadWrite<u32>),
+        (0x158 => _9),
+        (0x15C => pub DMAC0CARXBR: ReadWrite<u32>),
+        (0x160 => pub DMAC0SR: ReadWrite<u32, DMACiSR::Register>),
+        (0x164 => _10),
+        (0x16C => pub DMAC0MFCR: ReadWrite<u32>),
+        (0x170 => _11),
+        (0x200 => @END),
+    }
+}

kernel/driver/net/stmmac/src/regs/mac.rs

@@ -0,0 +1,500 @@
+use libk::error::Error;
+use tock_registers::{
+    interfaces::{Readable, Writeable},
+    register_bitfields, register_structs,
+    registers::{ReadOnly, ReadWrite},
+};
+
+register_bitfields! {
+    u32,
+    pub MACCR [
+        /// * When set: MAC recognizes ARP requests, responds with ARP responses.
+        /// * When clear: no additional ARP logic, frames are indicates as Type
+        ///               frame in the RxStatus.
+        ARPEN   OFFSET(31)  NUMBITS(1)  [],
+        /// Source address insertion or replacement setting.
+        SARC    OFFSET(28)  NUMBITS(3)  [],
+        /// IP checksum offload.
+        ///
+        /// When set, enables IPv4 header checksum checking, IPv4, IPv6 TCP+UDP and ICMP
+        /// payload checksum checking.
+        IPC     OFFSET(27)  NUMBITS(1)  [],
+        /// Inter-packet gap setting.
+        IPG     OFFSET(24)  NUMBITS(1)  [],
+        /// Giant packet size limit control enable.
+        GPSCLE  OFFSET(23)  NUMBITS(1)  [],
+        /// IEEE 802.3as support for 2K packets.
+        S2KP    OFFSET(22)  NUMBITS(1)  [],
+        /// CRC stripping for Type packets.
+        CST     OFFSET(21)  NUMBITS(1)  [],
+        /// Automatic pad or CRC stripping.
+        ACS     OFFSET(20)  NUMBITS(1)  [],
+        /// Watchdog disable.
+        ///
+        /// When set, disables the watchdog timer on the receiver. The MAC can receive packets
+        /// of up to 16383 bytes.
+        /// When clear, the MAC does not allow packets of more than 2048 bytes (or 10240 with
+        /// JE).
+        WD      OFFSET(19)  NUMBITS(1)  [],
+        /// Packet burst enable.
+        BE      OFFSET(18)  NUMBITS(1)  [],
+        /// Jabber disable.
+        ///
+        /// When set, disables the jabber timer on the transmitter. The MAC can send packets of
+        /// up to 16383 bytes.
+        JD      OFFSET(17)  NUMBITS(1)  [],
+        /// Jumbo packet enable.
+        JE      OFFSET(16)  NUMBITS(1)  [],
+        /// Port select
+        PS      OFFSET(15)  NUMBITS(1)  [
+            Ps1000Mbps = 0,
+            Ps10Or100Mbps = 1,
+        ],
+        /// MAC speed
+        FES     OFFSET(14)  NUMBITS(1)  [
+            Fes10Mbps = 0,
+            Fes100Mbps = 1,
+        ],
+        /// Duplex mode
+        DM      OFFSET(13)  NUMBITS(1)  [
+            HalfDuplex = 0,
+            FullDuplex = 1,
+        ],
+        /// Loopback mode. When set, GMII output is looped back into input. The GMII RX clock
+        /// input is required for the loopback to work, because the GMII TX clock is not
+        /// looped back internally.
+        LM      OFFSET(12)  NUMBITS(1)  [],
+        /// Enable carrier-sense before transmission in Full-Duplex mode
+        ECRSFD  OFFSET(11)  NUMBITS(1)  [],
+        /// Disable receive own
+        DO      OFFSET(10)  NUMBITS(1)  [],
+        /// Disable carrier-sense during transmission
+        DCRS    OFFSET(9)   NUMBITS(1)  [],
+        /// Disable retry
+        DR      OFFSET(8)   NUMBITS(1)  [],
+        /// Backoff limit
+        BL      OFFSET(5)   NUMBITS(2)  [
+            /// k = min(n, 10)
+            BackoffMin10    = 0,
+            /// k = min(n, 8)
+            BackoffMin8     = 1,
+            /// k = min(n, 4)
+            BackoffMin4     = 2,
+            /// k = min(n, 1)
+            BackoffMin1     = 3,
+        ],
+        /// Deferral check
+        DC      OFFSET(4)   NUMBITS(1)  [],
+        /// Preamble length
+        PRELEN  OFFSET(2)   NUMBITS(2)  [
+            Pre7b = 0,
+            Pre5b = 1,
+            Pre3b = 2,
+        ],
+        /// Transmitter enable
+        TE      OFFSET(1)   NUMBITS(1)  [],
+        /// Receiver enable
+        RE      OFFSET(0)   NUMBITS(1)  [],
+    ],
+
+    pub MACPFR [
+        /// Receive all. When set, the MAC receiver passes all received packets to the application,
+        /// whether they pass the address filter or not.
+        RA      OFFSET(31)  NUMBITS(1)  [],
+        /// Drop Non-TCP/UDP over IP packets
+        DNTU    OFFSET(21)  NUMBITS(1)  [],
+        /// Layer 3 and 4 filter enable
+        IPFE    OFFSET(20)  NUMBITS(1)  [],
+        /// VLAN filter enable
+        VTFE    OFFSET(16)  NUMBITS(1)  [],
+        /// Hash or perfect filter
+        HPF     OFFSET(10)  NUMBITS(1)  [],
+        /// Source address filter
+        SAF     OFFSET(9)   NUMBITS(1)  [],
+        /// SA inverse filtering
+        SAIF    OFFSET(8)   NUMBITS(1)  [],
+        /// Pass control checks
+        PCF     OFFSET(6)   NUMBITS(2)  [],
+        /// Disable broadcast packets
+        DBF     OFFSET(5)   NUMBITS(1)  [],
+        /// Pass all multicast
+        PM      OFFSET(4)   NUMBITS(1)  [],
+        /// DA inverse filtering
+        DAIF    OFFSET(3)   NUMBITS(1)  [],
+        /// Hash multicast
+        HMC     OFFSET(2)   NUMBITS(1)  [],
+        /// Hash unicast
+        HUC     OFFSET(1)   NUMBITS(1)  [],
+        /// Promiscuous mode
+        PR      OFFSET(0)   NUMBITS(1)  [],
+    ],
+
+    pub MACQ0TXFCR [
+        /// Pause time
+        PT      OFFSET(16)  NUMBITS(16) [],
+        /// Disable zero-quanta pause
+        DZPQ    OFFSET(7)   NUMBITS(1)  [],
+        /// Pause low threshold
+        PLT     OFFSET(4)   NUMBITS(3)  [],
+        /// Transmit flow control enable
+        TFE     OFFSET(1)   NUMBITS(1)  [],
+        /// Flow control busy or backpressure activate
+        FCB_BPA OFFSET(0)   NUMBITS(1)  [],
+    ],
+
+    pub MACRXFCR [
+        /// Unicast packet detect
+        UP      OFFSET(1)   NUMBITS(1)  [],
+        /// Rx flow control enable
+        RFE     OFFSET(0)   NUMBITS(1)  [],
+    ],
+
+    pub MACRXQC0R [
+        /// Rx queue 1 enable
+        RXQ1EN  OFFSET(2)   NUMBITS(2)  [
+            Disable = 0b00,
+            EnableAV = 0b01,
+            Enable = 0b10,
+        ],
+        /// Rx queue 0 enable
+        RXQ0EN  OFFSET(0)   NUMBITS(2)  [
+            Disable = 0b00,
+            EnableAV = 0b01,
+            Enable = 0b10,
+        ],
+    ],
+
+    pub MACISR [
+        /// Rx status interrupt
+        RXSTSIS     OFFSET(14)  NUMBITS(1)  [],
+        /// Tx status interrupt
+        TXSTSIS     OFFSET(13)  NUMBITS(1)  [],
+        /// Timestamp interrupt
+        TSIS        OFFSET(12)  NUMBITS(1)  [],
+        /// MMC Tx interrupt
+        MMCTXIS     OFFSET(10)  NUMBITS(1)  [],
+        /// MMC Rx interrupt
+        MMCRXIS     OFFSET(9)   NUMBITS(1)  [],
+        /// MMC interrupt
+        MMCIS       OFFSET(8)   NUMBITS(1)  [],
+        /// LPI interrupt
+        LPIIS       OFFSET(5)   NUMBITS(1)  [],
+        /// PMT interrupt
+        PMTIS       OFFSET(4)   NUMBITS(1)  [],
+        /// PHY interrupt
+        PHYIS       OFFSET(3)   NUMBITS(1)  [],
+        /// RGMII interrupt
+        RGSMIIIS    OFFSET(0)   NUMBITS(1)  [],
+    ],
+
+    pub MACIER [
+        /// Rx status interrupt enable
+        RXSTSIE     OFFSET(14)  NUMBITS(1)  [],
+        /// Tx status interrupt enable
+        TXSTSIE     OFFSET(13)  NUMBITS(1)  [],
+        /// Timestamp interrupt enable
+        TSIE        OFFSET(12)  NUMBITS(1)  [],
+        /// LPI interrupt enable
+        LPIIE       OFFSET(5)   NUMBITS(1)  [],
+        /// PMT interrupt enable
+        PMTIE       OFFSET(4)   NUMBITS(1)  [],
+        /// PHY interrupt enable
+        PHYIE       OFFSET(3)   NUMBITS(1)  [],
+        /// RGMII interrupt enable
+        RGSMIIIE    OFFSET(0)   NUMBITS(1)  [],
+    ],
+
+    pub MACPHYCSR [
+        /// Link status
+        LNKSTS      OFFSET(19)  NUMBITS(1)  [
+            Down = 0,
+            Up = 1,
+        ],
+        /// Link speed
+        LNKSPEED    OFFSET(17)  NUMBITS(2)  [
+            Speed2_5MHz = 0b00,
+            Speed25MHz = 0b01,
+            Speed125MHz = 0b10,
+        ],
+        /// Link mode
+        LNKMOD      OFFSET(16)  NUMBITS(1)  [
+            HalfDuplex = 0,
+            FullDuplex = 1,
+        ],
+        /// Link up or down
+        ///
+        /// This bit indicates whether the link is up or down during transmission of configuration
+        /// in the RGMII interface
+        LUD         OFFSET(1)   NUMBITS(1)  [
+            Down = 0,
+            Up = 1,
+        ],
+        /// Transmit configuration in RGMII
+        TC          OFFSET(0)   NUMBITS(1)  [],
+    ],
+
+    pub MACMDIOAR [
+        /// Preamble suppress enable
+        ///
+        /// If set, SMA suppresses the 32-bit preamble and transmits MDIO frames with only 1
+        /// preamble bit
+        PSE     OFFSET(27)  NUMBITS(1)  [],
+        /// Back to back transactions
+        ///
+        /// If set and the NTC has value greater than 0, the MAC informs the completion of a read
+        /// or write command at the end of frame transfer. Must not be set with NTC=0.
+        BTB     OFFSET(26)  NUMBITS(1)  [],
+        /// Physical layer address
+        ///
+        /// In Clause 22, indicates a PHY device the MAC is addressing.
+        PA      OFFSET(21)  NUMBITS(5)  [],
+        /// Register/Device address
+        ///
+        /// In Clause 22, selects the PHY register
+        RDA     OFFSET(16)  NUMBITS(5)  [],
+        /// Number of trailing clocks
+        NTC     OFFSET(12)  NUMBITS(3)  [],
+        /// CSR clock range
+        CR      OFFSET(8)   NUMBITS(4)  [
+            CSR_20_TO_35_MHZ = 0b0010,
+        ],
+        /// Skip address packet
+        ///
+        /// If set, the SMA does not send the address packets before read, write or post-read
+        /// increment address packets. Only valid with C45E set.
+        SKAP    OFFSET(4)   NUMBITS(1)  [],
+        /// GMII operation command
+        GOC     OFFSET(2)   NUMBITS(2)  [
+            Write = 0b01,
+            C45EPostReadIncrement = 0b10,
+            Read = 0b11
+        ],
+        /// Clause 45 PHY enable
+        C45E    OFFSET(1)   NUMBITS(1)  [],
+        /// GMII busy
+        GB      OFFSET(0)   NUMBITS(1)  [],
+    ],
+
+    pub MACMDIODR [
+        /// Register address
+        ///
+        /// Specifies the PHY register when used with Clause 45
+        RA      OFFSET(16)  NUMBITS(16) [],
+        /// GMII data
+        GD      OFFSET(0)   NUMBITS(16) [],
+    ],
+
+    pub MACAiHR [
+        /// MAC address enable bit. Should always be set to 1 for MAC address 0.
+        AE      OFFSET(31)  NUMBITS(1)  [],
+        ADDR5   OFFSET(8)   NUMBITS(8)  [],
+        ADDR4   OFFSET(0)   NUMBITS(8)  [],
+    ],
+    pub MACAiLR [
+        ADDR3   OFFSET(24)  NUMBITS(8)  [],
+        ADDR2   OFFSET(16)  NUMBITS(8)  [],
+        ADDR1   OFFSET(8)   NUMBITS(8)  [],
+        ADDR0   OFFSET(0)   NUMBITS(8)  [],
+    ],
+}
+
+register_structs! {
+    pub MacRegs {
+        /// Operating mode configuration register
+        ///
+        /// Reset value: 0x0000 8000
+        ///
+        /// Established the operating mode of the MAC.
+        (0x000 => pub MACCR: ReadWrite<u32, MACCR::Register>),
+        /// Extended operating mode configuration register
+        ///
+        /// XXX
+        (0x004 => pub MACECR: ReadWrite<u32>),
+        /// Packet filtering control register
+        (0x008 => pub MACPFR: ReadWrite<u32, MACPFR::Register>),
+        /// Watchdog timeout register
+        (0x00C => pub MACWTR: ReadWrite<u32>),
+        /// Hash table 0 register
+        (0x010 => pub MACHT0R: ReadWrite<u32>),
+        /// Hash table 1 register
+        (0x014 => pub MACHT1R: ReadWrite<u32>),
+        (0x018 => _0),
+        /// VLAN tag register
+        (0x050 => pub MACVTR: ReadWrite<u32>),
+        (0x054 => _1),
+        /// VLAN hash table register
+        (0x058 => pub MACVHTR: ReadWrite<u32>),
+        (0x05C => _2),
+        /// VLAN inclusion register
+        (0x060 => pub MACVIR: ReadWrite<u32>),
+        /// Inner VLAN inclusion register
+        (0x064 => pub MACIVIR: ReadWrite<u32>),
+        (0x068 => _3),
+        /// Tx queue 0 flow control register
+        (0x070 => pub MACQ0TXFCR: ReadWrite<u32, MACQ0TXFCR::Register>),
+        (0x074 => _4),
+        /// Rx flow control register
+        (0x090 => pub MACRXFCR: ReadWrite<u32, MACRXFCR::Register>),
+        /// Rx queue control register
+        (0x094 => pub MACRXQCR: ReadWrite<u32>),
+        (0x098 => _5),
+        /// Rx queue control 0 register
+        (0x0A0 => pub MACRXQC0R: ReadWrite<u32, MACRXQC0R::Register>),
+        /// Rx queue control 1 register
+        (0x0A4 => pub MACRXQC1R: ReadWrite<u32>),
+        /// Rx queue control 2 register
+        (0x0A8 => pub MACRXQC2R: ReadWrite<u32>),
+        (0x0AC => _6),
+        /// Interrupt status register
+        (0x0B0 => pub MACISR: ReadWrite<u32, MACISR::Register>),
+        /// Interrupt enable register
+        (0x0B4 => pub MACIER: ReadWrite<u32, MACIER::Register>),
+        /// Rx/Tx status register
+        (0x0B8 => pub MACRXTXSR: ReadWrite<u32>),
+        (0x0BC => _7),
+        /// PMT control status register
+        (0x0C0 => pub MACPCSR: ReadWrite<u32>),
+        /// Remote wakeup packet filter register
+        (0x0C4 => pub MACRWKPFR: ReadWrite<u32>),
+        (0x0C8 => _8),
+        /// LPI control and status register
+        (0x0D0 => pub MACLCSR: ReadWrite<u32>),
+        /// LPI timers control register
+        (0x0D4 => pub MACLTCR: ReadWrite<u32>),
+        /// Tx LPI entry timer
+        (0x0D8 => pub MACLETR: ReadWrite<u32>),
+        /// One-microsecond-tick counter register
+        (0x0DC => pub MAC1USTCR: ReadWrite<u32>),
+        (0x0E0 => _9),
+        /// PHYIF control and status register
+        (0x0F8 => pub MACPHYCSR: ReadWrite<u32, MACPHYCSR::Register>),
+        (0x0FC => _10),
+        /// Version register
+        (0x110 => pub MACVR: ReadOnly<u32>),
+        /// Debug register
+        (0x114 => pub MACDR: ReadOnly<u32>),
+        (0x118 => _11),
+        /// HW feature 0 register
+        (0x11C => pub MACHWF0R: ReadWrite<u32>),
+        /// HW feature 1 register
+        (0x120 => pub MACHWF1R: ReadWrite<u32>),
+        /// HW feature 2 register
+        (0x124 => pub MACHWF2R: ReadWrite<u32>),
+        /// HW feature 3 register
+        (0x128 => pub MACHWF3R: ReadWrite<u32>),
+        (0x12C => _12),
+        /// MDIO address register
+        (0x200 => pub MACMDIOAR: ReadWrite<u32, MACMDIOAR::Register>),
+        /// MDIO data register
+        (0x204 => pub MACMDIODR: ReadWrite<u32, MACMDIODR::Register>),
+        (0x208 => _13),
+        /// ARP address register
+        (0x210 => pub MACARPAR: ReadWrite<u32>),
+        (0x214 => _14),
+        /// CSR software control register
+        (0x230 => pub MACCSRSWCR: ReadWrite<u32>),
+        (0x234 => _15),
+        /// MAC address 0 high register
+        (0x300 => pub MACA0HR: ReadWrite<u32, MACAiHR::Register>),
+        /// MAC address 0 low register
+        (0x304 => pub MACA0LR: ReadWrite<u32, MACAiLR::Register>),
+        /// MAC address 1 high register
+        (0x308 => pub MACA1HR: ReadWrite<u32>),
+        /// MAC address 1 low register
+        (0x30C => pub MACA1LR: ReadWrite<u32>),
+        /// MAC address 2 high register
+        (0x310 => pub MACA2HR: ReadWrite<u32>),
+        /// MAC address 2 low register
+        (0x314 => pub MACA2LR: ReadWrite<u32>),
+        /// MAC address 3 high register
+        (0x318 => pub MACA3HR: ReadWrite<u32>),
+        /// MAC address 3 low register
+        (0x31C => pub MACA3LR: ReadWrite<u32>),
+        (0x320 => _16),
+        /// MMC control register
+        (0x700 => pub MMC_CONTROL: ReadWrite<u32>),
+        /// MMC Rx interrupt register
+        (0x704 => pub MMC_RX_INTERRUPT: ReadWrite<u32>),
+        /// MMC Tx interrupt register
+        (0x708 => pub MMC_TX_INTERRUPT: ReadWrite<u32>),
+        /// MMC Rx interrupt mask register
+        (0x70C => pub MMC_RX_INTERRUPT_MASK: ReadWrite<u32>),
+        /// MMC Tx interrupt mask register
+        (0x710 => pub MMC_TX_INTERRUPT_MASK: ReadWrite<u32>),
+        (0x714 => _17),
+        (0xC00 => @END),
+    }
+}
+
+impl MacRegs {
+    fn mdio_wait_busy(&self) -> Result<(), Error> {
+        let mut timeout = 10000;
+        while timeout > 0 && self.MACMDIOAR.matches_all(MACMDIOAR::GB::SET) {
+            core::hint::spin_loop();
+            timeout -= 1;
+        }
+        if timeout == 0 {
+            Err(Error::TimedOut)
+        } else {
+            Ok(())
+        }
+    }
+
+    pub fn mdio_read_c45(
+        &self,
+        phy_addr: u8,
+        mmd_addr: u8,
+        mii_reg: u8,
+        csr_clk_range: u8,
+    ) -> Result<u16, Error> {
+        self.mdio_wait_busy()?;
+        self.MACMDIODR.write(MACMDIODR::RA.val(mii_reg as u32));
+        self.MACMDIOAR.write(
+            MACMDIOAR::CR.val(csr_clk_range as u32)
+                + MACMDIOAR::GOC::Read
+                + MACMDIOAR::PA.val(phy_addr as u32)
+                + MACMDIOAR::RDA.val(mmd_addr as u32)
+                + MACMDIOAR::C45E::SET
+                + MACMDIOAR::GB::SET,
+        );
+        self.mdio_wait_busy()?;
+        Ok(self.MACMDIODR.read(MACMDIODR::GD) as u16)
+    }
+
+    pub fn mdio_read_c22(
+        &self,
+        mii_addr: u8,
+        mii_reg: u8,
+        csr_clk_range: u8,
+    ) -> Result<u16, Error> {
+        self.mdio_wait_busy()?;
+        self.MACMDIOAR.write(
+            MACMDIOAR::GOC::Read
+                + MACMDIOAR::CR.val(csr_clk_range as u32)
+                + MACMDIOAR::PA.val(mii_addr as u32)
+                + MACMDIOAR::RDA.val(mii_reg as u32)
+                + MACMDIOAR::GB::SET,
+        );
+        self.mdio_wait_busy()?;
+        Ok(self.MACMDIODR.read(MACMDIODR::GD) as u16)
+    }
+
+    pub fn mdio_write_c22(
+        &self,
+        mii_addr: u8,
+        mii_reg: u8,
+        value: u16,
+        csr_clk_range: u8,
+    ) -> Result<(), Error> {
+        self.mdio_wait_busy()?;
+        self.MACMDIODR.write(MACMDIODR::GD.val(value as u32));
+        self.MACMDIOAR.write(
+            MACMDIOAR::GOC::Write
+                + MACMDIOAR::CR.val(csr_clk_range as u32)
+                + MACMDIOAR::PA.val(mii_addr as u32)
+                + MACMDIOAR::RDA.val(mii_reg as u32)
+                + MACMDIOAR::GB::SET,
+        );
+        self.mdio_wait_busy()
+    }
+}

kernel/driver/net/stmmac/src/regs/mod.rs

@@ -0,0 +1,16 @@
+#![allow(non_snake_case)]
+
+use tock_registers::register_structs;
+
+pub mod dma;
+pub mod mac;
+pub mod mtl;
+
+register_structs! {
+    pub Regs {
+        (0x0000 => pub MAC: mac::MacRegs),
+        (0x0C00 => pub MTL: mtl::MtlRegs),
+        (0x1000 => pub DMA: dma::DmaRegs),
+        (0x1200 => @END),
+    }
+}

kernel/driver/net/stmmac/src/regs/mtl.rs

@@ -0,0 +1,132 @@
+use tock_registers::{register_bitfields, register_structs, registers::ReadWrite};
+
+register_bitfields! {
+    u32,
+    pub MTLOMR [
+        CNTCLR  OFFSET(9)   NUMBITS(1)  [],
+        CNTPRST OFFSET(8)   NUMBITS(1)  [],
+        SCHALG  OFFSET(5)   NUMBITS(2)  [
+            AlgWeightedRR = 0b00,
+            AlgStrictPrio = 0b11,
+        ],
+        RAA     OFFSET(2)   NUMBITS(1)  [
+            AlgStrictPrio   = 0,
+            AlgWeightedPrio = 1
+        ],
+        DTXSTS  OFFSET(1)   NUMBITS(1)  [],
+    ],
+
+    pub MTLTXQiOMR [
+        TQS     OFFSET(16)  NUMBITS(4)  [],
+        TTC     OFFSET(4)   NUMBITS(3)  [],
+        TXQEN   OFFSET(2)   NUMBITS(2)  [
+            Disable     = 0b00,
+            EnableAv    = 0b01,
+            Enable      = 0b11,
+        ],
+        TSF     OFFSET(1)   NUMBITS(1)  [],
+        FTQ     OFFSET(0)   NUMBITS(1)  [],
+    ],
+
+    pub MTLRXQiOMR [
+        RQS         OFFSET(20)  NUMBITS(4)  [],
+        /// Threshold for deactivating flow control
+        RFD         OFFSET(14)  NUMBITS(3)  [
+            FullMinus1K     = 0,
+            FullMinus1_5K   = 1,
+            FullMinus2K     = 2,
+            FullMinus2_5K   = 3,
+            FullMinus3K     = 4,
+            FullMinus3_5K   = 5
+        ],
+        /// Threshold for activating flow control. See [RFD].
+        RFA         OFFSET(8)   NUMBITS(3)  [],
+        /// Enable hardware flow control. When set, the flow control signal operation, based on the
+        /// fill-level of the Rx queue, is enabled.
+        EHFC        OFFSET(7)   NUMBITS(1)  [],
+        /// Disable dropping of TCP/IP checksum error packets
+        DIS_TCP_EF  OFFSET(6)   NUMBITS(1)  [],
+        /// Receive queue store and forward. When set, the Ethernet peripheral reads a packet from
+        /// the Rx queue only after the complete packet has been written to it, ignoring the RTC
+        /// field.
+        RSF         OFFSET(5)   NUMBITS(1)  [],
+        /// Forward error packets. When set, the Rx queue does not drop the packets with CRC
+        /// errors, watchdog timeouts, overflows and receive errors.
+        FEP         OFFSET(4)   NUMBITS(1)  [],
+        /// Forward undersized good packets. When set, the Rx queue does not drop packets with no
+        /// errors, but with a length of less than 64 bytes.
+        FUP         OFFSET(3)   NUMBITS(1)  [],
+        /// Rx threshold level
+        RTC         OFFSET(0)   NUMBITS(2)  [
+            Threshold64     = 0b00,
+            Threshold32     = 0b01,
+            Threshold96     = 0b10,
+            Threshold128    = 0b11,
+        ],
+    ],
+}
+
+register_structs! {
+    pub MtlRegs {
+        /// MTL operating mode register
+        (0x000 => pub MTLOMR: ReadWrite<u32, MTLOMR::Register>),
+        (0x004 => _0),
+        /// Interrupt status register
+        (0x020 => pub MTLISR: ReadWrite<u32>),
+        (0x024 => _1),
+        /// Tx queue 0 operating mode register
+        (0x100 => pub MTLTXQ0OMR: ReadWrite<u32, MTLTXQiOMR::Register>),
+        /// Tx queue 0 underflow register
+        (0x104 => pub MTLTXQ0UR: ReadWrite<u32>),
+        /// Tx queue 0 debug register
+        (0x108 => pub MTLTXQ0DR: ReadWrite<u32>),
+        (0x10C => _2),
+        /// Tx queue 0 ETS status register
+        (0x114 => pub MTLTXQ0ESR: ReadWrite<u32>),
+        /// Tx queue 0 quantum weight register
+        (0x118 => pub MTLTXQ0QWR: ReadWrite<u32>),
+        (0x11C => _3),
+        /// Queue 0 interrupt control status register
+        (0x12C => pub MTLQ0ICSR: ReadWrite<u32>),
+        /// Rx queue 0 operating mode register
+        (0x130 => pub MTLRXQ0OMR: ReadWrite<u32, MTLRXQiOMR::Register>),
+        /// Rx queue 0 missed packet and overflow counter register
+        (0x134 => pub MTLRXQ0MPOCR: ReadWrite<u32>),
+        /// Rx queue 0 debug register
+        (0x138 => pub MTLRXQ0DR: ReadWrite<u32>),
+        /// Rx queue 0 control register
+        (0x13C => pub MTLRXQ0CR: ReadWrite<u32>),
+        /// Tx queue 1 operating mode register
+        (0x140 => pub MTLTXQ1OMR: ReadWrite<u32>),
+        /// Tx queue 1 underflow register
+        (0x144 => pub MTLTXQ1UR: ReadWrite<u32>),
+        /// Tx queue 1 debug register
+        (0x148 => pub MTLTXQ1DR: ReadWrite<u32>),
+        (0x14C => _4),
+        /// Tx queue 1 ETS control register
+        (0x150 => pub MTLTXQ1ECR: ReadWrite<u32>),
+        /// Tx queue 1 ETS status register
+        (0x154 => pub MTLTXQ1ESR: ReadWrite<u32>),
+        /// Tx queue 1 quantum weight register
+        (0x158 => pub MTLTXQ1QWR: ReadWrite<u32>),
+        /// Tx queue 1 send slope credit register
+        (0x15C => pub MTLTXQ1SSCR: ReadWrite<u32>),
+        /// Tx queue 1 hiCredit register
+        (0x160 => pub MTLTXQ1HCR: ReadWrite<u32>),
+        /// Tx queue 1 loCredit register
+        (0x164 => pub MTLTXQ1LCR: ReadWrite<u32>),
+        (0x168 => _5),
+        /// Queue 1 interrupt control status register
+        (0x16C => pub MTLQ1ICSR: ReadWrite<u32>),
+        /// Rx queue 1 operating mode register
+        (0x170 => pub MTLRXQ1OMR: ReadWrite<u32>),
+        /// Rx queue 1 missed packet and overflow counter register
+        (0x174 => pub MTLRXQ1MPOCR: ReadWrite<u32>),
+        /// Rx queue 1 debug register
+        (0x178 => pub MTLRXQ1DR: ReadWrite<u32>),
+        /// Rx queue 1 control register
+        (0x17C => pub MTLRXQ1CR: ReadWrite<u32>),
+        (0x180 => _6),
+        (0x400 => @END),
+    }
+}

kernel/driver/net/stmmac/src/ring.rs

@@ -0,0 +1,254 @@
+use core::mem::{self, MaybeUninit};
+
+use alloc::vec::Vec;
+use device_api::dma::DmaAllocator;
+use libk::{
+    dma::{BusAddress, DmaBuffer},
+    error::Error,
+};
+
+pub struct TxRing {
+    entries: DmaBuffer<[TxDescriptor]>,
+    buffers: Vec<Option<DmaBuffer<[u8]>>>,
+
+    wr: usize,
+    rd: usize,
+}
+
+pub struct RxRing {
+    entries: DmaBuffer<[RxDescriptor]>,
+    buffers: Vec<DmaBuffer<[MaybeUninit<u8>]>>,
+
+    rd: usize,
+}
+
+#[derive(Clone, Copy, Debug)]
+#[repr(C)]
+pub struct TxDescriptor {
+    tdes0: u32,
+    tdes1: u32,
+    tdes2: u32,
+    tdes3: u32,
+}
+
+#[derive(Clone, Copy, Debug)]
+#[repr(C)]
+pub struct RxDescriptor {
+    rdes0: u32,
+    rdes1: u32,
+    rdes2: u32,
+    rdes3: u32,
+}
+
+impl TxRing {
+    pub fn with_capacity(dma: &dyn DmaAllocator, capacity: usize) -> Result<Self, Error> {
+        let entries = DmaBuffer::new_slice(dma, TxDescriptor::empty(), capacity)?;
+        let buffers = (0..capacity).map(|_| None).collect();
+        Ok(Self {
+            entries,
+            buffers,
+
+            wr: 0,
+            rd: 0,
+        })
+    }
+
+    pub fn buffer_base(&self) -> BusAddress {
+        self.entries.bus_address()
+    }
+
+    pub fn capacity(&self) -> usize {
+        self.entries.len()
+    }
+
+    pub fn can_xmit(&self) -> bool {
+        self.wr.wrapping_add(1) != self.rd
+    }
+
+    pub fn outstanding_tx(&self) -> bool {
+        self.wr != self.rd
+    }
+
+    pub fn push_xmit(&mut self, frame: DmaBuffer<[u8]>) -> Result<usize, Error> {
+        if !self.can_xmit() {
+            return Err(Error::WouldBlock);
+        }
+
+        let address = frame.bus_address();
+        let frame_len = frame.len();
+
+        let capacity = self.entries.len();
+        let index = self.wr % capacity;
+        assert!(self.buffers[index].is_none());
+
+        frame.cache_flush_all(true);
+        self.entries[index].setup_tx(address, frame_len, true)?;
+        self.entries.cache_flush_element(index, true);
+
+        self.buffers[index] = Some(frame);
+        self.wr = self.wr.wrapping_add(1);
+
+        Ok(self.wr % self.capacity())
+    }
+
+    pub fn consume(&mut self) -> Result<usize, Error> {
+        let mut count = 0;
+
+        loop {
+            let index = self.rd % self.entries.len();
+            let entry = &self.entries[index];
+
+            if self.rd == self.wr {
+                break;
+            }
+
+            if let Some(status) = entry.tx_status() {
+                if status != 0 {
+                    log::warn!("tx_ring[{index}] error: {status:#x}");
+                }
+                let _ = self.buffers[index].take().unwrap();
+                self.rd = self.rd.wrapping_add(1);
+                count += 1;
+            } else {
+                break;
+            }
+        }
+
+        Ok(count)
+    }
+}
+
+impl TxDescriptor {
+    const TDES3_OWN: u32 = 1 << 31;
+    const TDES3_FD: u32 = 1 << 29;
+    const TDES3_LD: u32 = 1 << 28;
+    const TDES2_IOC: u32 = 1 << 31;
+
+    pub const fn empty() -> Self {
+        Self {
+            tdes0: 0,
+            tdes1: 0,
+            tdes2: 0,
+            tdes3: 0,
+        }
+    }
+
+    pub fn tx_status(&self) -> Option<u32> {
+        if self.tdes3 & Self::TDES3_OWN == 0 {
+            Some(self.tdes3 & !(0xFFFF << 16))
+        } else {
+            None
+        }
+    }
+
+    pub fn setup_tx(
+        &mut self,
+        frame: BusAddress,
+        frame_len: usize,
+        ioc: bool,
+    ) -> Result<(), Error> {
+        let tdes0 = frame.try_into_u32().map_err(|_| Error::InvalidArgument)?;
+        if frame_len & !0x3FFF != 0 {
+            return Err(Error::InvalidArgument);
+        }
+        let mut tdes2 = frame_len as u32;
+        if ioc {
+            tdes2 |= Self::TDES2_IOC;
+        }
+        let tdes3 = Self::TDES3_OWN | Self::TDES3_FD | Self::TDES3_LD;
+
+        self.tdes0 = tdes0;
+        self.tdes1 = 0;
+        self.tdes2 = tdes2;
+        self.tdes3 = tdes3;
+        Ok(())
+    }
+}
+
+impl RxRing {
+    pub fn with_capacity(dma: &dyn DmaAllocator, capacity: usize) -> Result<Self, Error> {
+        let mut entries = DmaBuffer::new_slice(dma, RxDescriptor::empty(), capacity)?;
+        let buffers = (0..capacity)
+            .map(|_| DmaBuffer::new_uninit_slice(dma, 4096))
+            .collect::<Result<Vec<_>, _>>()?;
+        for i in 0..capacity {
+            entries[i].setup_rx(buffers[i].bus_address(), true)?;
+        }
+        Ok(Self {
+            buffers,
+            entries,
+
+            rd: 0,
+        })
+    }
+
+    pub fn buffer_base(&self) -> BusAddress {
+        self.entries.bus_address()
+    }
+
+    pub fn consume<F: Fn(DmaBuffer<[u8]>)>(
+        &mut self,
+        dma: &dyn DmaAllocator,
+        packet_handler: F,
+    ) -> Result<usize, Error> {
+        let mut count = 0;
+
+        loop {
+            let index = self.rd % self.entries.len();
+            let entry = &mut self.entries[index];
+
+            if let Some(_) = entry.rx_completed() {
+                // Grab the current buffer (the one just written to by the DMA), replace it
+                // with the newly allocated one, and mark the descriptor as DMA-owned again
+                let new_buffer = DmaBuffer::new_uninit_slice(dma, 4096)?;
+                let new_buffer_address = new_buffer.bus_address();
+                let buffer = mem::replace(&mut self.buffers[index], new_buffer);
+                let buffer = unsafe { DmaBuffer::assume_init_slice(buffer) };
+                // TODO packet size hint
+                packet_handler(buffer);
+                entry.setup_rx(new_buffer_address, true)?;
+                self.rd = self.rd.wrapping_add(1);
+                count += 1;
+            } else {
+                break;
+            }
+        }
+
+        Ok(count)
+    }
+}
+
+impl RxDescriptor {
+    const RDES3_OWN: u32 = 1 << 31;
+    const RDES3_IOC: u32 = 1 << 30;
+    const RDES3_BUF1V: u32 = 1 << 24;
+
+    pub const fn empty() -> Self {
+        Self {
+            rdes0: 0,
+            rdes1: 0,
+            rdes2: 0,
+            rdes3: 0,
+        }
+    }
+
+    pub fn rx_completed(&self) -> Option<usize> {
+        if self.rdes3 & Self::RDES3_OWN == 0 {
+            Some((self.rdes3 & 0x7FFF) as usize)
+        } else {
+            None
+        }
+    }
+
+    pub fn setup_rx(&mut self, buffer: BusAddress, ioc: bool) -> Result<(), Error> {
+        self.rdes0 = buffer.try_into_u32().map_err(|_| Error::InvalidArgument)?;
+        self.rdes1 = 0;
+        self.rdes2 = 0;
+        self.rdes3 = Self::RDES3_BUF1V;
+        if ioc {
+            self.rdes3 |= Self::RDES3_IOC;
+        }
+        self.rdes3 |= Self::RDES3_OWN;
+        Ok(())
+    }
+}

kernel/driver/usb/xhci/Cargo.toml

@@ -16,8 +16,9 @@ libk = { path = "../../../libk" }
 
 xhci_lib = { git = "https://github.com/rust-osdev/xhci.git", package = "xhci" }
 
-atomic_enum = "0.3.0"
-log = "0.4.22"
-tock-registers = "0.8.1"
-bytemuck = { version = "1.16.1", features = ["derive"] }
-futures-util = { version = "0.3.30", default-features = false, features = ["alloc", "async-await"] }
+async-trait.workspace = true
+log.workspace = true
+atomic_enum.workspace = true
+tock-registers.workspace = true
+bytemuck.workspace = true
+futures-util.workspace = true

kernel/driver/usb/xhci/src/context.rs

@@ -0,0 +1,136 @@
+use core::ops::{Deref, DerefMut};
+
+use device_api::dma::DmaAllocator;
+use libk::dma::{BusAddress, DmaBuffer};
+use libk_util::sync::spin_rwlock::IrqSafeRwLock;
+use xhci_lib::context::{self, DeviceHandler, InputHandler};
+use ygg_driver_usb::error::UsbError;
+
+use crate::regs::{ContextSize, PortNumber};
+
+pub enum XhciDeviceContext {
+    Context32(IrqSafeRwLock<DmaBuffer<context::Device32Byte>>),
+    Context64(IrqSafeRwLock<DmaBuffer<context::Device64Byte>>),
+}
+
+pub enum XhciInputContext {
+    Context32(DmaBuffer<context::Input32Byte>),
+    Context64(DmaBuffer<context::Input64Byte>),
+}
+
+impl XhciDeviceContext {
+    pub fn new(dma: &dyn DmaAllocator, size: ContextSize) -> Result<Self, UsbError> {
+        match size {
+            ContextSize::Context32 => DmaBuffer::new(dma, context::Device::new_32byte())
+                .map(IrqSafeRwLock::new)
+                .map(Self::Context32),
+            ContextSize::Context64 => DmaBuffer::new(dma, context::Device::new_64byte())
+                .map(IrqSafeRwLock::new)
+                .map(Self::Context64),
+        }
+        .map_err(UsbError::MemoryError)
+    }
+
+    pub fn map<T, F: FnOnce(&dyn DeviceHandler) -> T>(&self, mapper: F) -> T {
+        match self {
+            Self::Context32(cx) => mapper(&**cx.read()),
+            Self::Context64(cx) => mapper(&**cx.read()),
+        }
+    }
+
+    pub fn bus_address(&self) -> BusAddress {
+        match self {
+            Self::Context32(cx) => cx.read().bus_address(),
+            Self::Context64(cx) => cx.read().bus_address(),
+        }
+    }
+    // pub fn physical_address(&self) -> PhysicalAddress {
+    // }
+}
+
+impl XhciInputContext {
+    pub fn new(dma: &dyn DmaAllocator, size: ContextSize) -> Result<Self, UsbError> {
+        match size {
+            ContextSize::Context32 => {
+                DmaBuffer::new(dma, context::Input::new_32byte()).map(Self::Context32)
+            }
+            ContextSize::Context64 => {
+                DmaBuffer::new(dma, context::Input::new_64byte()).map(Self::Context64)
+            }
+        }
+        .map_err(UsbError::MemoryError)
+    }
+
+    pub fn new_address_device(
+        dma: &dyn DmaAllocator,
+        size: ContextSize,
+        root_hub_port_number: PortNumber,
+        max_packet_size: usize,
+        speed: u8,
+        dequeue_pointer: BusAddress,
+    ) -> Result<Self, UsbError> {
+        let mut cx = Self::new(dma, size)?;
+
+        {
+            let control = cx.control_mut();
+
+            control.set_add_context_flag(0); // Enable slot context
+            control.set_add_context_flag(1); // Enable endpoint 0 context
+        }
+
+        {
+            let slot = cx.device_mut().slot_mut();
+
+            slot.set_context_entries(1);
+            slot.set_interrupter_target(0);
+            slot.set_root_hub_port_number(root_hub_port_number.into());
+            slot.set_speed(speed);
+        }
+
+        {
+            let ep0 = cx.device_mut().endpoint_mut(1);
+
+            ep0.set_endpoint_type(context::EndpointType::Control);
+            ep0.set_error_count(3);
+            // Use the provided max_packet_size, or the default one for the given speed
+            ep0.set_max_packet_size(max_packet_size as u16);
+            ep0.set_tr_dequeue_pointer(dequeue_pointer.into_u64());
+            ep0.set_dequeue_cycle_state();
+        }
+
+        Ok(cx)
+    }
+
+    // pub fn physical_address(&self) -> PhysicalAddress {
+    //     match self {
+    //         Self::Context32(cx) => unsafe { cx.as_physical_address() },
+    //         Self::Context64(cx) => unsafe { cx.as_physical_address() },
+    //     }
+    // }
+    pub fn bus_address(&self) -> BusAddress {
+        match self {
+            Self::Context32(cx) => cx.bus_address(),
+            Self::Context64(cx) => cx.bus_address(),
+        }
+    }
+}
+
+impl Deref for XhciInputContext {
+    type Target = dyn InputHandler;
+
+    fn deref(&self) -> &Self::Target {
+        match self {
+            Self::Context32(cx) => &**cx,
+            Self::Context64(cx) => &**cx,
+        }
+    }
+}
+
+impl DerefMut for XhciInputContext {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        match self {
+            Self::Context32(cx) => &mut **cx,
+            Self::Context64(cx) => &mut **cx,
+        }
+    }
+}

kernel/driver/usb/xhci/src/controller.rs

@@ -1,312 +1,446 @@
-use core::{future::poll_fn, sync::atomic::Ordering, task::Poll, time::Duration};
-
-use alloc::{boxed::Box, collections::BTreeMap, sync::Arc, vec, vec::Vec};
-use atomic_enum::atomic_enum;
-use device_api::{device::Device, interrupt::InterruptHandler};
-use futures_util::task::AtomicWaker;
-use libk::task::runtime;
-use libk_mm::{
-    address::{AsPhysicalAddress, PhysicalAddress},
-    PageBox,
+use core::{
+    mem::MaybeUninit,
+    sync::atomic::{AtomicU8, Ordering},
 };
-use libk_util::{sync::spin_rwlock::IrqSafeRwLock, OneTimeInit};
+
+use alloc::{boxed::Box, collections::btree_map::BTreeMap, sync::Arc, vec::Vec};
+use async_trait::async_trait;
+use device_api::{
+    device::{Device, DeviceInitContext},
+    dma::DmaAllocator,
+    interrupt::{InterruptHandler, IrqVector},
+};
+use libk::{
+    dma::{BusAddress, DmaBuffer},
+    error::Error,
+    task::runtime,
+};
+use libk_util::{
+    sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock},
+    OneTimeInit,
+};
+use tock_registers::{
+    interfaces::{ReadWriteable, Readable, Writeable},
+    LocalRegisterCopy,
+};
+use ygg_driver_pci::{device::PciDeviceInfo, PciConfigurationSpace};
 use ygg_driver_usb::{
     bus::UsbBusManager,
-    device::{UsbBusAddress, UsbDevice, UsbDeviceAccess},
+    device::{UsbBusAddress, UsbDeviceAccess, UsbSpeed},
     error::UsbError,
+    info::UsbVersion,
     pipe::control::UsbControlPipeAccess,
-    util::UsbAddressAllocator,
     UsbHostController,
 };
-use yggdrasil_abi::error::Error;
+use yggdrasil_abi::bitflags;
 
 use crate::{
+    context::{XhciDeviceContext, XhciInputContext},
     device::XhciBusDevice,
     pipe::ControlPipe,
-    regs::{Mapper, PortSpeed, Regs},
-    ring::{
-        CommandExecutor, CommandRing, ControlTransferRing, Event, EventRing, EventRingSegmentTable,
-        GenericTransferRing,
+    regs::{
+        self,
+        extended::ExtendedCapability,
+        operational::{PortRegs, CONFIG, PORTSC, USBCMD, USBSTS},
+        PortNumber, Regs,
     },
-    XhciContext,
+    ring::{
+        command::CommandRing,
+        event::{Event, EventRing, EventRingSegmentTable},
+        transfer::TransferRing,
+        CommandExecutor, GenericRing,
+    },
+    util::EventBitmap,
 };
 
-#[atomic_enum]
-#[derive(PartialEq, Eq)]
-pub enum PortState {
-    Disconnected, // Default + set by "handle detach"
-    Init,         // Set by "port task"
-    Running,      // Set by "port task"
+bitflags! {
+    pub struct Quirks: u32 {
+        const INTEL: bit 0;
+        const INTEL_PANTHER_POINT: bit 1;
+    }
 }
 
-struct PortStruct {
-    state: AtomicPortState,
-    notify: AtomicWaker,
-    address: IrqSafeRwLock<Option<UsbBusAddress>>,
+#[allow(unused)]
+struct ScratchpadArray {
+    buffers: Vec<DmaBuffer<[MaybeUninit<u8>]>>,
+    array: DmaBuffer<[BusAddress]>,
+}
+
+struct RootHubPort {
+    version: UsbVersion,
+    slot_type: u8,
 }
 
 pub struct Xhci {
-    regs: Regs,
-
-    bus_address: OneTimeInit<u16>,
-    address_allocator: UsbAddressAllocator,
-
-    port_count: usize,
-    // TODO use to allocate proper contexts
+    pub(crate) regs: Regs,
     #[allow(unused)]
-    context_size: usize,
+    pci: PciDeviceInfo,
+    pub(crate) dma: Arc<dyn DmaAllocator>,
+    quirks: Quirks,
 
-    dcbaa: IrqSafeRwLock<PageBox<[PhysicalAddress]>>,
-    endpoints: IrqSafeRwLock<BTreeMap<(u8, u8), Arc<dyn GenericTransferRing>>>,
-    event_ring: EventRing,
+    dcbaa: IrqSafeRwLock<DmaBuffer<[BusAddress]>>,
+    #[allow(unused)]
+    scratchpads: Option<ScratchpadArray>,
     pub(crate) command_ring: CommandRing,
+    event_ring: EventRing,
+    erst: EventRingSegmentTable,
 
-    port_states: Vec<PortStruct>,
+    root_hub_ports: Vec<Option<RootHubPort>>,
+    pub(crate) endpoints: IrqSafeRwLock<BTreeMap<(u8, u8), Arc<TransferRing>>>,
+    pub(crate) slots: Vec<IrqSafeRwLock<Option<Arc<XhciBusDevice>>>>,
+    pub(crate) port_slot_map: Vec<AtomicU8>,
+    bus_index: OneTimeInit<u16>,
+    port_event_map: EventBitmap,
+}
+
+impl ScratchpadArray {
+    pub fn new(
+        dma: &dyn DmaAllocator,
+        capacity: usize,
+        element_size: usize,
+    ) -> Result<Self, UsbError> {
+        let buffers = (0..capacity)
+            .map(|_| DmaBuffer::new_zeroed_slice(dma, element_size))
+            .collect::<Result<Vec<_>, _>>()
+            .map_err(UsbError::MemoryError)?;
+        let array = DmaBuffer::new_slice_with(dma, |i| buffers[i].bus_address(), capacity)
+            .map_err(UsbError::MemoryError)?;
+        Ok(Self { buffers, array })
+    }
 }
 
 impl Xhci {
-    pub fn new(regs: xhci_lib::Registers<Mapper>) -> Result<Self, UsbError> {
-        let event_ring = EventRing::new(128)?;
-        let command_ring = CommandRing::new(128)?;
+    pub fn new(
+        dma: Arc<dyn DmaAllocator>,
+        pci: PciDeviceInfo,
+        regs: Regs,
+    ) -> Result<Self, UsbError> {
+        let mut quirks = Quirks::empty();
+        if pci.vendor_id == 0x8086 {
+            quirks |= Quirks::INTEL;
 
-        let regs = Regs::from(regs);
+            if pci.device_id == 0x1E31 {
+                quirks |= Quirks::INTEL_PANTHER_POINT;
+            }
+        }
 
-        let port_count = regs.port_count();
-        let slot_count = regs.max_slot_count();
-        let context_size = regs.context_size();
-
-        let dcbaa = PageBox::new_slice(PhysicalAddress::ZERO, slot_count + 1)
+        let mut dcbaa = DmaBuffer::new_slice(&*dma, BusAddress::ZERO, regs.slot_count + 1)
             .map_err(UsbError::MemoryError)?;
+        let command_ring = CommandRing::new(&*dma, 128)?;
+        let event_ring = EventRing::new(&*dma, 64)?;
+        let erst = EventRingSegmentTable::for_event_rings(&*dma, &[&event_ring])?;
+
+        // Setup scratch buffers
+        // TODO: Linux seems to just ignore the PAGESIZE, it's (1 << 0) everywhere
+        let scratchpads = if regs.scratch_count != 0 {
+            let array = ScratchpadArray::new(&*dma, regs.scratch_count, 0x1000)?;
+            dcbaa[0] = array.array.bus_address();
+            // dcbaa[0] = unsafe { array.array.as_physical_address() };
+            Some(array)
+        } else {
+            None
+        };
+
+        let mut root_hub_ports: Vec<Option<_>> = (0..regs.port_count).map(|_| None).collect();
+
+        for cap in regs.extended_capabilities.iter() {
+            match cap {
+                ExtendedCapability::ProtocolSupport(support) => {
+                    let Some(version) = support.usb_revision() else {
+                        continue;
+                    };
+
+                    for port in support.port_range() {
+                        log::info!("* Port {port}: {version}");
+                        root_hub_ports[port as usize - 1] = Some(RootHubPort {
+                            version,
+                            slot_type: support.slot_type(),
+                        });
+                    }
+                }
+                ExtendedCapability::LegacySupport(legsup) => {
+                    legsup.write().perform_bios_handoff(10000000)?;
+                }
+            }
+        }
+
+        // TODO this is only needed if there's an EHCI controller as well
+        if quirks.contains(Quirks::INTEL_PANTHER_POINT) {
+            log::info!("quirk: intel pantherpoint xhci/ehci port switchover");
+            const INTEL_USB3PSSEN: usize = 0xD8;
+            const INTEL_XUSB2PR: usize = 0xD0;
+
+            let space = &pci.config_space;
+
+            space.write_u32(INTEL_USB3PSSEN, 0xFFFFFFFF);
+            space.write_u32(INTEL_XUSB2PR, 0xFFFFFFFF);
+        }
+
+        let port_slot_map = (0..regs.port_count).map(|_| AtomicU8::new(0)).collect();
+        let slots = (0..regs.slot_count)
+            .map(|_| IrqSafeRwLock::new(None))
+            .collect();
 
         Ok(Self {
             regs,
-
-            bus_address: OneTimeInit::new(),
-            address_allocator: UsbAddressAllocator::new(),
-
-            port_count,
-            context_size,
-
-            event_ring,
-            command_ring,
+            pci,
+            dma,
+            quirks,
 
             dcbaa: IrqSafeRwLock::new(dcbaa),
-            endpoints: IrqSafeRwLock::new(BTreeMap::new()),
+            scratchpads,
+            command_ring,
+            event_ring,
+            erst,
 
-            port_states: Vec::from_iter((0..port_count).map(|_| PortStruct {
-                state: AtomicPortState::new(PortState::Disconnected),
-                notify: AtomicWaker::new(),
-                address: IrqSafeRwLock::new(None),
-            })),
+            root_hub_ports,
+            bus_index: OneTimeInit::new(),
+            endpoints: IrqSafeRwLock::new(BTreeMap::new()),
+            slots,
+            port_slot_map,
+            port_event_map: EventBitmap::new(),
         })
     }
 
-    pub fn register_device_context(&self, slot_id: u8, context: PhysicalAddress) {
-        self.dcbaa.write()[slot_id as usize] = context;
+    fn notify_endpoint(&self, slot_id: u8, endpoint_id: u8, address: BusAddress, status: u32) {
+        if let Some(endpoint) = self.endpoints.read().get(&(slot_id, endpoint_id)) {
+            endpoint.notify(address, status);
+        } else {
+            log::warn!("Endpoint {slot_id}:{endpoint_id} does not exist");
+        }
     }
 
-    pub fn register_endpoint(
-        &self,
-        slot_id: u8,
-        endpoint_id: u8,
-        ring: Arc<dyn GenericTransferRing>,
-    ) {
-        self.endpoints.write().insert((slot_id, endpoint_id), ring);
-    }
-
-    pub fn shutdown_endpoint(&self, slot_id: u8, endpoint_id: u8) {
-        if let Some(endpoint) = self.endpoints.write().remove(&(slot_id, endpoint_id)) {
+    fn kill_endpoint(&self, slot_id: u8, endpoint_id: u8) {
+        let endpoint = self.endpoints.write().remove(&(slot_id, endpoint_id));
+        if let Some(endpoint) = endpoint {
             endpoint.shutdown();
-        } else {
-            log::warn!(
-                "Endpoint {}:{} does not exist, maybe already shut down?",
-                slot_id,
-                endpoint_id
-            );
         }
     }
 
-    pub fn notify_transfer(
-        &self,
-        slot_id: u8,
-        endpoint_id: u8,
-        address: PhysicalAddress,
-        status: u32,
-    ) {
-        if let Some(ep) = self.endpoints.read().get(&(slot_id, endpoint_id)) {
-            ep.notify(address, status);
-        }
-    }
+    async fn allocate_device_slot(
+        self: &Arc<Self>,
+        port_id: PortNumber,
+        slot_type: u8,
+        speed: UsbSpeed,
+    ) -> Result<(u8, Arc<XhciBusDevice>, Arc<TransferRing>), UsbError> {
+        let device_context = XhciDeviceContext::new(&*self.dma, self.regs.context_size)?;
+        let slot_id = self.command_ring.enable_slot(&**self, slot_type).await?;
+        let (control_pipe, control_ring) = ControlPipe::new(self.clone(), slot_id, 1, 128)?;
+        let control_pipe = UsbControlPipeAccess(Box::new(control_pipe));
 
-    async fn assign_device(
-        self: Arc<Self>,
-        speed: PortSpeed,
-        slot_id: u8,
-        root_hub_port_number: u8,
-    ) -> Result<Box<dyn UsbDevice>, UsbError> {
-        let address = self.address_allocator.allocate().unwrap();
-        let ring = Arc::new(ControlTransferRing::new(slot_id, 1, 128)?);
+        self.dcbaa.write()[slot_id as usize] = device_context.bus_address();
+        self.endpoints
+            .write()
+            .insert((slot_id, 1), control_ring.clone());
 
-        let context =
-            XhciContext::new_32byte_address_device(&ring, speed, address, root_hub_port_number)?;
-        let mut input = context.input.write();
-
-        self.register_device_context(slot_id, unsafe { context.output.as_physical_address() });
-
-        self.command_ring
-            .address_device(&*self, slot_id, &mut input)
-            .await?;
-
-        self.register_endpoint(slot_id, 1, ring.clone());
-
-        let pipe = ControlPipe::new(self.clone(), slot_id, ring.clone());
-
-        // TODO: If the device is a Full-speed one, determine its max packet size for the control
-        // endpoint
-        if speed == PortSpeed::Full {
-            todo!()
-        }
-
-        drop(input);
-
-        let bus_address = UsbBusAddress {
-            bus: *self.bus_address.get(),
-            device: address,
-        };
-
-        let device = XhciBusDevice {
-            xhci: self.clone(),
+        let slot = Arc::new(XhciBusDevice {
+            port_id,
             slot_id,
-            port_id: root_hub_port_number,
-            bus_address,
+            device_context,
+            control_pipe,
             speed,
-            context: Arc::new(context),
-            rings: IrqSafeRwLock::new(vec![ring]),
-            control_pipe: UsbControlPipeAccess(Box::new(pipe)),
-        };
-
-        Ok(Box::new(device))
-    }
-
-    async fn port_task(self: Arc<Self>, index: usize) -> Result<(), UsbError> {
-        let state = &self.port_states[index];
-
-        self.reset_port(index).await?;
-
-        let regs = self.regs.ports.read(index);
-        let speed =
-            PortSpeed::try_from(regs.portsc.port_speed()).map_err(|_| UsbError::PortInitFailed)?;
-
-        let slot_id = self.command_ring.enable_slot(self.as_ref()).await?;
-
-        let device = self
-            .clone()
-            .assign_device(speed, slot_id, (index + 1) as _)
-            .await?;
-        let device = UsbDeviceAccess::setup(device).await?;
-
-        let old = state.address.write().replace(device.bus_address());
-        assert!(old.is_none());
-
-        UsbBusManager::register_device(Arc::new(device));
-
-        state.state.store(PortState::Running, Ordering::Release);
-
-        Ok(())
-    }
-
-    fn handle_device_attached(self: Arc<Self>, port: usize) -> Result<(), UsbError> {
-        log::info!("Port {}: device attached", port);
-
-        if let Err(err) = self.port_states[port].state.compare_exchange(
-            PortState::Disconnected,
-            PortState::Init,
-            Ordering::Acquire,
-            Ordering::Relaxed,
-        ) {
-            log::warn!("Could not start port init task: port state is {:?}", err);
-            return Err(UsbError::DeviceBusy);
-        }
-
-        runtime::spawn(async move { self.port_task(port).await }).map_err(UsbError::SystemError)?;
-        Ok(())
-    }
-
-    fn handle_device_detached(&self, port: usize) -> Result<(), UsbError> {
-        let state = &self.port_states[port];
-
-        match state.state.swap(PortState::Disconnected, Ordering::Release) {
-            PortState::Init => todo!(),
-            PortState::Running => {
-                log::info!("Port {}: device detached", port);
-                let address = state
-                    .address
-                    .write()
-                    .take()
-                    .expect("Port marked as Running, but has no address");
-
-                UsbBusManager::detach_device(address);
-                state.notify.wake();
-
-                Ok(())
-            }
-            // Already disconnected
-            PortState::Disconnected => Ok(()),
-        }
-    }
-
-    fn handle_port_event(self: Arc<Self>, port: usize) -> Result<(), UsbError> {
-        let state = &self.port_states[port];
-        let port_regs = self.regs.ports.read(port);
-
-        if port_regs.portsc.connect_status_change() {
-            if port_regs.portsc.current_connect_status() {
-                self.handle_device_attached(port)
-            } else {
-                self.handle_device_detached(port)
-            }
-        } else {
-            // Some other event
-            state.notify.wake();
-            Ok(())
-        }
-    }
-
-    async fn reset_port(&self, port: usize) -> Result<(), UsbError> {
-        log::debug!("Reset port {}", port);
-
-        self.regs.ports.update(port, |u| {
-            u.portsc.set_port_reset();
+            detach_handler: IrqSafeSpinlock::new(None),
+            endpoints: IrqSafeRwLock::new(BTreeMap::from_iter([(1, control_ring.clone())])),
+            xhci: self.clone(),
+            bus_address: OneTimeInit::new(),
         });
 
-        // Wait for port reset
-        // TODO handle disconnect during reset?
-        let result = runtime::with_timeout(
-            poll_fn(|cx| {
-                let state = &self.port_states[port];
+        *self.slots[slot_id as usize].write() = Some(slot.clone());
+        self.port_slot_map[port_id.index()].store(slot_id, Ordering::Release);
 
-                state.notify.register(cx.waker());
-                if !self.regs.ports.read(port).portsc.port_reset() {
-                    Poll::Ready(())
-                } else {
-                    Poll::Pending
+        Ok((slot_id, slot, control_ring))
+    }
+
+    // TODO proper timeout
+    async fn reset_port(&self, regs: &PortRegs) -> Result<(), UsbError> {
+        // Reset the port
+        regs.modify_portsc_preserving(PORTSC::PR::SET);
+
+        let mut timeout = 10000000;
+        while timeout > 0 {
+            let status = regs.PORTSC.extract();
+
+            if status.matches_all(PORTSC::PR::CLEAR) {
+                break;
+            }
+
+            timeout -= 1;
+        }
+
+        // Clear PRC
+        regs.modify_portsc_preserving(PORTSC::PRC::SET);
+        Ok(())
+    }
+
+    // TODO clean up resources if init fails
+    async fn setup_port(
+        self: &Arc<Self>,
+        regs: &PortRegs,
+        number: PortNumber,
+    ) -> Result<(), UsbError> {
+        let root_hub_port = self.root_hub_ports[number.index()]
+            .as_ref()
+            .ok_or(UsbError::PortInitFailed)?;
+
+        let need_reset = !root_hub_port.version.is_version_3();
+
+        if need_reset {
+            self.reset_port(regs).await?;
+        }
+
+        let status = regs.PORTSC.extract();
+
+        let speed = status.read(PORTSC::PS) as u8;
+        let pls = status.read(PORTSC::PLS);
+
+        let (usb_speed, max_packet_size) = match speed {
+            1 => (UsbSpeed::Full, 8),
+            2 => (UsbSpeed::Low, 8),
+            3 => (UsbSpeed::High, 64),
+            4 => (UsbSpeed::Super, 512),
+            _ => {
+                log::error!("Port {number} invalid speed value {speed}");
+                return Err(UsbError::DeviceDisconnected);
+            }
+        };
+
+        // PLS != 0 after reset
+        if pls != 0 {
+            log::error!("Port {number} pls!=0 after reset");
+            return Err(UsbError::DeviceDisconnected);
+        }
+
+        let (slot_id, slot, control_ring) = self
+            .allocate_device_slot(number, root_hub_port.slot_type, usb_speed)
+            .await?;
+
+        let input_cx = XhciInputContext::new_address_device(
+            &*self.dma,
+            self.regs.context_size,
+            number,
+            max_packet_size,
+            speed,
+            control_ring.bus_address(),
+        )?;
+
+        self.command_ring
+            .address_device(&**self, slot_id, &input_cx, false)
+            .await
+            .inspect_err(|error| {
+                log::error!("Port {number} Address Device TRB (BSR=0) failed: {error:?}")
+            })?;
+
+        // TODO update max_packet_size for default control endpoint after fetching the device
+        //      descriptor
+
+        let bus_address = slot
+            .device_context
+            .map(|device| device.slot().usb_device_address());
+        slot.bus_address.init(UsbBusAddress {
+            bus: *self.bus_index.get(),
+            device: bus_address,
+        });
+
+        let device = UsbDeviceAccess::setup(slot).await?;
+        UsbBusManager::register_device(device.into());
+
+        Ok(())
+    }
+
+    async fn handle_disconnect(&self, number: PortNumber) -> Result<(), UsbError> {
+        let slot_id = self.port_slot_map[number.index()].swap(0, Ordering::Acquire);
+        if slot_id == 0 {
+            return Ok(());
+        }
+        log::info!("Port {number} disconnected");
+
+        let slot = self.slots[slot_id as usize].write().take();
+        if let Some(slot) = slot {
+            UsbBusManager::detach_device(*slot.bus_address.get());
+
+            // Clean up xHC resources
+            for (&endpoint_id, endpoint) in slot.endpoints.read().iter() {
+                endpoint.shutdown();
+                self.kill_endpoint(slot_id, endpoint_id);
+            }
+        } else {
+            log::warn!("No device in slot {slot_id}");
+        }
+
+        // TODO stop endpoints on the xHC side?
+        self.command_ring.disable_slot(self, slot_id).await?;
+
+        Ok(())
+    }
+
+    async fn port_handler_task(self: Arc<Self>) -> Result<(), Error> {
+        // Inject notify for ports that are already connected
+        {
+            let ports = self.regs.ports.write();
+            for (index, port) in self.root_hub_ports.iter().enumerate() {
+                if port.is_none() {
+                    continue;
                 }
-            }),
-            Duration::from_secs(1),
-        )
-        .await;
 
-        match result {
-            Ok(()) => Ok(()),
-            Err(_) => Err(UsbError::PortResetFailed),
+                if ports[index].PORTSC.matches_all(PORTSC::CCS::SET) {
+                    self.port_event_map.signal(index);
+                }
+            }
+        }
+
+        loop {
+            let events = self.port_event_map.wait_any(self.regs.port_count).await;
+
+            for index in events {
+                let number = PortNumber::from_index(index);
+                let ports = self.regs.ports.write();
+                let regs = &ports[index];
+
+                let status = regs.PORTSC.extract();
+                let neutral = regs::portsc_to_neutral(status);
+                let mut clear = LocalRegisterCopy::<u32, PORTSC::Register>::new(0);
+
+                // Clear affected change bits
+                let connected = if status.matches_all(PORTSC::CSC::SET) {
+                    clear.modify(PORTSC::CSC::SET);
+                    Some(status.matches_all(PORTSC::CCS::SET))
+                } else {
+                    None
+                };
+                if status.matches_all(PORTSC::PEC::SET) {
+                    clear.modify(PORTSC::PEC::SET);
+                }
+                if status.matches_all(PORTSC::PLC::SET) {
+                    clear.modify(PORTSC::PLC::SET);
+                }
+
+                regs.PORTSC.set(neutral.get() | clear.get());
+
+                if let Some(connected) = connected {
+                    if connected {
+                        log::info!("Port {number} connected");
+                        if let Err(error) = self.setup_port(regs, number).await {
+                            log::error!("Port {number} setup error: {error:?}");
+                        }
+                    }
+
+                    if !connected {
+                        if let Err(error) = self.handle_disconnect(number).await {
+                            log::error!("Port {number} disconnect error: {error:?}");
+                        }
+                    }
+                }
+            }
         }
     }
 
-    fn handle_event(self: Arc<Self>) {
+    fn handle_event(&self) {
         while let Some(event) = self.event_ring.try_dequeue() {
             match event {
-                Event::PortChange(port) => {
-                    self.clone().handle_port_event(port - 1).ok();
+                Event::PortChange(number) => {
+                    if number > 0 {
+                        self.port_event_map.signal(number - 1);
+                    }
                 }
                 Event::CommandCompletion { address, reply } => {
                     self.command_ring.notify(address, reply);
@@ -317,65 +451,105 @@ impl Xhci {
                     endpoint_id,
                     status,
                 } => {
-                    self.notify_transfer(slot_id, endpoint_id, address, status);
+                    self.notify_endpoint(slot_id, endpoint_id, address, status);
                 }
             }
         }
 
         self.regs
-            .set_interrupter_0_dequeue_pointer(self.event_ring.dequeue_pointer());
+            .runtime
+            .write()
+            .set_interrupter_dequeue_pointer(0, self.event_ring.dequeue_pointer());
     }
 }
 
-impl UsbHostController for Xhci {}
-
+#[async_trait]
 impl CommandExecutor for Xhci {
     fn ring_doorbell(&self, index: usize, target: u8) {
-        self.regs.ring_doorbell(index, target);
+        self.regs.doorbells[index].store(target as u32, Ordering::Release);
+    }
+
+    async fn reset_endpoint(
+        &self,
+        slot_id: u8,
+        endpoint_id: u8,
+        dequeue_pointer: BusAddress,
+        dequeue_cycle: bool,
+    ) -> Result<(), UsbError> {
+        log::warn!("xhci: reset stalled endpoint {slot_id}:{endpoint_id}");
+
+        self.command_ring
+            .reset_endpoint(&*self, slot_id, endpoint_id, false)
+            .await?;
+        self.command_ring
+            .set_tr_dequeue_pointer(&*self, slot_id, endpoint_id, dequeue_pointer, dequeue_cycle)
+            .await?;
+
+        Ok(())
     }
 }
 
 impl Device for Xhci {
-    unsafe fn init(self: Arc<Self>) -> Result<(), Error> {
-        log::info!("Init USB xHCI");
+    unsafe fn init(self: Arc<Self>, _cx: DeviceInitContext) -> Result<(), Error> {
+        self.regs.hc_reset(10000000)?;
+        log::info!("xHC reset complete");
 
-        self.regs.reset();
-        self.regs.set_max_slot_count();
+        // Configure the HC
+        let dcbaap = self.dcbaa.read().bus_address();
+        let cr_base = self.command_ring.bus_base();
 
-        let erst = EventRingSegmentTable::for_event_rings(&[&self.event_ring])?;
-        let dcbaa = self.dcbaa.read();
+        let op = self.regs.operational.write();
+        let rt = self.regs.runtime.write();
 
-        self.regs
-            .configure(&dcbaa, &self.command_ring, &self.event_ring, &erst);
+        log::info!("xhci: configure HC");
+        op.CONFIG
+            .modify(CONFIG::MaxSlotsEn.val(self.regs.slot_count as u32));
+        op.set_dcbaap(dcbaap);
+        op.set_crcr(cr_base, true);
 
-        let bus = UsbBusManager::register_bus(self.clone());
-        self.bus_address.init(bus);
+        log::info!("xhci: configure interrupter");
+        rt.configure_interrupter(0, &self.event_ring, &self.erst);
 
-        for port in 0..self.port_count {
-            let p = self.regs.ports.read(port);
-            if p.portsc.current_connect_status() {
-                self.clone().handle_device_attached(port).ok();
+        // Enable interrupts and start the HC
+        log::info!("xhci: start HC");
+        op.USBCMD.modify(USBCMD::INTE::SET + USBCMD::HSEE::SET);
+        op.USBCMD.modify(USBCMD::RS::SET);
+
+        if self.quirks.contains(Quirks::INTEL) {
+            for _ in 0..100000000 {
+                core::hint::spin_loop();
             }
         }
 
+        op.wait_usbsts_bit(USBSTS::CNR::CLEAR, 100000000)?;
+
+        let bus = UsbBusManager::register_bus(self.clone());
+        self.bus_index.init(bus);
+
+        runtime::spawn(self.clone().port_handler_task()).ok();
+
         Ok(())
     }
 
     fn display_name(&self) -> &str {
-        "USB xHCI"
+        "xHCI"
     }
 }
 
 impl InterruptHandler for Xhci {
-    fn handle_irq(self: Arc<Self>, _vector: Option<usize>) -> bool {
-        if let Some(status) = self.regs.handle_interrupt() {
-            if status.event_interrupt() {
-                self.handle_event();
-            }
+    fn handle_irq(self: Arc<Self>, _vector: IrqVector) -> bool {
+        let status = self.regs.handle_interrupt(0);
 
-            true
-        } else {
-            false
+        if status.matches_all(USBSTS::HSE::SET) {
+            log::error!("xhci: Host System Error occurred");
         }
+
+        if status.matches_all(USBSTS::EINT::SET) {
+            self.handle_event();
+        }
+
+        true
     }
 }
+
+impl UsbHostController for Xhci {}

kernel/driver/usb/xhci/src/device.rs

@@ -1,38 +1,175 @@
-use alloc::{boxed::Box, sync::Arc, vec::Vec};
-use futures_util::{future::BoxFuture, FutureExt};
-use libk_util::sync::spin_rwlock::IrqSafeRwLock;
-use xhci_lib::context::{self, InputHandler};
+use alloc::{boxed::Box, collections::btree_map::BTreeMap, sync::Arc};
+use async_trait::async_trait;
+use libk_util::{
+    sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock},
+    OneTimeInit,
+};
+use xhci_lib::context;
 use ygg_driver_usb::{
-    device::{UsbBusAddress, UsbDevice, UsbSpeed},
+    communication::UsbDirection,
+    device::{UsbBusAddress, UsbDevice, UsbDeviceDetachHandler, UsbSpeed},
     error::UsbError,
     info::UsbEndpointType,
-    pipe::{control::UsbControlPipeAccess, interrupt::UsbInterruptInPipeAccess},
-    UsbDirection, UsbHostController,
+    pipe::{
+        control::UsbControlPipeAccess,
+        normal::{UsbNormalPipeIn, UsbNormalPipeOut},
+    },
+    UsbHostController,
 };
 
 use crate::{
-    pipe::InterruptInPipe,
-    regs::PortSpeed,
-    ring::{transfer::InterruptInTransferRing, GenericTransferRing},
-    Xhci, XhciContext,
+    context::{XhciDeviceContext, XhciInputContext},
+    pipe::{NormalInPipe, NormalOutPipe},
+    regs::PortNumber,
+    ring::transfer::TransferRing,
+    Xhci,
 };
 
-// TODO device context information
 pub struct XhciBusDevice {
-    pub(crate) port_id: u8,
+    pub(crate) port_id: PortNumber,
     pub(crate) slot_id: u8,
-    pub(crate) bus_address: UsbBusAddress,
-
-    pub(crate) speed: PortSpeed,
+    pub(crate) speed: UsbSpeed,
+    pub(crate) bus_address: OneTimeInit<UsbBusAddress>,
 
     pub(crate) xhci: Arc<Xhci>,
 
-    pub(crate) context: Arc<XhciContext<8>>,
-    pub(crate) rings: IrqSafeRwLock<Vec<Arc<dyn GenericTransferRing>>>,
+    pub(crate) device_context: XhciDeviceContext,
+    pub(crate) endpoints: IrqSafeRwLock<BTreeMap<u8, Arc<TransferRing>>>,
     pub(crate) control_pipe: UsbControlPipeAccess,
+    pub(crate) detach_handler: IrqSafeSpinlock<Option<Arc<dyn UsbDeviceDetachHandler>>>,
+}
+
+#[async_trait]
+impl UsbDevice for XhciBusDevice {
+    fn port_number(&self) -> u8 {
+        self.port_id.into()
+    }
+
+    fn bus_address(&self) -> UsbBusAddress {
+        *self.bus_address.get()
+    }
+
+    fn control_pipe(&self) -> &UsbControlPipeAccess {
+        &self.control_pipe
+    }
+
+    fn handle_detach(&self) {
+        if let Some(handler) = self.detach_handler.lock().as_ref() {
+            handler.handle_device_detach();
+        }
+    }
+
+    fn set_detach_handler(&self, handler: Arc<dyn UsbDeviceDetachHandler>) {
+        *self.detach_handler.lock() = Some(handler);
+    }
+
+    fn controller_ref(&self) -> &dyn UsbHostController {
+        self.xhci.as_ref()
+    }
+
+    fn debug(&self) {}
+
+    fn speed(&self) -> UsbSpeed {
+        self.speed
+    }
+
+    async fn open_normal_in_pipe(
+        &self,
+        number: u8,
+        max_packet_size: u16,
+        ty: UsbEndpointType,
+    ) -> Result<Box<dyn UsbNormalPipeIn>, UsbError> {
+        let dci = Self::dci(ty, UsbDirection::In, number) as u8;
+        let (pipe, ring) = NormalInPipe::new(self.xhci.clone(), self.slot_id, dci, 128)?;
+        self.add_endpoint(number, ring.clone(), ty, UsbDirection::In, max_packet_size)
+            .await?;
+        Ok(Box::new(pipe))
+    }
+
+    async fn open_normal_out_pipe(
+        &self,
+        number: u8,
+        max_packet_size: u16,
+        ty: UsbEndpointType,
+    ) -> Result<Box<dyn UsbNormalPipeOut>, UsbError> {
+        let dci = Self::dci(ty, UsbDirection::Out, number) as u8;
+        let (pipe, ring) = NormalOutPipe::new(self.xhci.clone(), self.slot_id, dci, 128)?;
+        self.add_endpoint(number, ring.clone(), ty, UsbDirection::Out, max_packet_size)
+            .await?;
+        Ok(Box::new(pipe))
+    }
 }
 
 impl XhciBusDevice {
+    pub async fn add_endpoint(
+        &self,
+        endpoint_number: u8,
+        ring: Arc<TransferRing>,
+        ty: UsbEndpointType,
+        direction: UsbDirection,
+        max_packet_size: u16,
+    ) -> Result<(), UsbError> {
+        let dci = Self::dci(ty, direction, endpoint_number) as u8;
+
+        log::debug!(
+            "Setup endpoint dci #{}: {} {:?} {:?}",
+            dci,
+            endpoint_number,
+            ty,
+            direction
+        );
+
+        let ep_type = match (ty, direction) {
+            (UsbEndpointType::Interrupt, UsbDirection::In) => context::EndpointType::InterruptIn,
+            (UsbEndpointType::Interrupt, UsbDirection::Out) => context::EndpointType::InterruptOut,
+            (UsbEndpointType::Bulk, UsbDirection::In) => context::EndpointType::BulkIn,
+            (UsbEndpointType::Bulk, UsbDirection::Out) => context::EndpointType::BulkOut,
+            (UsbEndpointType::Isochronous, UsbDirection::In) => context::EndpointType::IsochIn,
+            (UsbEndpointType::Isochronous, UsbDirection::Out) => context::EndpointType::IsochOut,
+            (UsbEndpointType::Control, _) => context::EndpointType::Control,
+        };
+
+        let mut input_cx = XhciInputContext::new(&*self.xhci.dma, self.xhci.regs.context_size)?;
+
+        {
+            let control = input_cx.control_mut();
+
+            control.set_add_context_flag(0);
+            control.set_add_context_flag(dci as usize);
+        }
+
+        {
+            let slot = input_cx.device_mut().slot_mut();
+
+            slot.set_context_entries(dci + 1);
+        }
+
+        {
+            let endpoint = input_cx.device_mut().endpoint_mut(dci as usize);
+
+            endpoint.set_endpoint_type(ep_type);
+            endpoint.set_error_count(3);
+            // TODO Pick a better value here
+            endpoint.set_interval(3);
+            endpoint.set_max_packet_size(max_packet_size);
+            endpoint.set_tr_dequeue_pointer(ring.bus_address().into_u64());
+            endpoint.set_dequeue_cycle_state();
+        }
+
+        self.xhci
+            .command_ring
+            .configure_endpoint(self.xhci.as_ref(), self.slot_id, &input_cx)
+            .await?;
+
+        self.endpoints.write().insert(dci, ring.clone());
+        self.xhci
+            .endpoints
+            .write()
+            .insert((self.slot_id, dci), ring);
+
+        Ok(())
+    }
+
     fn dci(ty: UsbEndpointType, dir: UsbDirection, number: u8) -> usize {
         match ty {
             UsbEndpointType::Control => (number as usize) * 2 + 1,
@@ -46,112 +183,4 @@ impl XhciBusDevice {
             }
         }
     }
-
-    async fn setup_endpoint_inner(
-        &self,
-        ring: Arc<dyn GenericTransferRing>,
-        dci: u8,
-        ty: UsbEndpointType,
-        direction: UsbDirection,
-    ) -> Result<(), UsbError> {
-        log::debug!("Setup endpoint dci #{}: {:?} {:?}", dci, ty, direction);
-
-        let mut input = self.context.input.write();
-
-        let ep_type = match (ty, direction) {
-            (UsbEndpointType::Interrupt, UsbDirection::In) => context::EndpointType::InterruptIn,
-            _ => todo!(),
-        };
-
-        {
-            let control = input.control_mut();
-
-            control.set_add_context_flag(0);
-            control.clear_add_context_flag(1);
-            control.set_add_context_flag(dci as _);
-        }
-
-        {
-            let slot = input.device_mut().slot_mut();
-
-            slot.set_context_entries(31);
-        }
-
-        {
-            let ep_cx = input.device_mut().endpoint_mut(dci as _);
-
-            ep_cx.set_tr_dequeue_pointer(ring.dequeue_pointer().into());
-            ep_cx.set_dequeue_cycle_state();
-            ep_cx.set_endpoint_type(ep_type);
-            ep_cx.set_error_count(3);
-
-            // TODO get from endpoint info
-            ep_cx.set_max_packet_size(8);
-        }
-
-        self.xhci
-            .command_ring
-            .configure_endpoint(self.xhci.as_ref(), self.slot_id, &mut input)
-            .await?;
-
-        self.rings.write().push(ring.clone());
-        self.xhci.register_endpoint(self.slot_id, dci, ring);
-
-        Ok(())
-    }
-}
-
-impl UsbDevice for XhciBusDevice {
-    fn control_pipe(&self) -> &UsbControlPipeAccess {
-        &self.control_pipe
-    }
-
-    fn port_number(&self) -> u8 {
-        self.port_id
-    }
-
-    fn bus_address(&self) -> UsbBusAddress {
-        self.bus_address
-    }
-
-    fn controller_ref(&self) -> &dyn UsbHostController {
-        self.xhci.as_ref()
-    }
-
-    fn open_interrupt_in_pipe(
-        &self,
-        number: u8,
-    ) -> BoxFuture<Result<UsbInterruptInPipeAccess, UsbError>> {
-        async move {
-            let dci = Self::dci(UsbEndpointType::Interrupt, UsbDirection::In, number) as u8;
-            let ring = Arc::new(InterruptInTransferRing::new(self.slot_id, dci as _, 128)?);
-
-            self.setup_endpoint_inner(
-                ring.clone(),
-                dci,
-                UsbEndpointType::Interrupt,
-                UsbDirection::In,
-            )
-            .await?;
-
-            let pipe = InterruptInPipe::new(self.xhci.clone(), self.slot_id, number, dci, ring);
-
-            Ok(UsbInterruptInPipeAccess(Box::new(pipe)))
-        }
-        .boxed()
-    }
-
-    fn handle_detach(&self) {
-        log::info!("Device detach handler");
-        for ring in self.rings.write().drain(..) {
-            self.xhci
-                .shutdown_endpoint(ring.slot_id(), ring.endpoint_id());
-        }
-    }
-
-    fn speed(&self) -> UsbSpeed {
-        self.speed.into()
-    }
-
-    fn debug(&self) {}
 }

kernel/driver/usb/xhci/src/lib.rs

@@ -1,110 +1,70 @@
 #![no_std]
 #![allow(clippy::new_without_default)]
-#![feature(iter_array_chunks)]
+#![feature(iter_array_chunks, let_chains, maybe_uninit_slice)]
 
 extern crate alloc;
 
+use alloc::sync::Arc;
+use controller::Xhci;
+use device_api::{device::Device, dma::DmaAllocator, interrupt::InterruptAffinity};
+use regs::Regs;
+use ygg_driver_pci::{
+    capability::{DevicePowerState, PowerManagementCapability},
+    device::{PciDeviceInfo, PreferredInterruptMode},
+    macros::pci_driver,
+    PciCommandRegister, PciConfigurationSpace,
+};
+use yggdrasil_abi::error::Error;
+
+mod context;
 mod controller;
 mod device;
 mod pipe;
 mod regs;
 mod ring;
+mod util;
 
-use alloc::sync::Arc;
-pub use controller::Xhci;
-use device_api::{device::Device, interrupt::InterruptAffinity};
-use libk_mm::PageBox;
-use libk_util::sync::spin_rwlock::IrqSafeRwLock;
-use regs::{Mapper, PortSpeed};
-use ring::{ControlTransferRing, GenericTransferRing};
-use xhci_lib::context::{self, InputHandler};
-use ygg_driver_pci::{
-    device::{PciDeviceInfo, PreferredInterruptMode},
-    PciCommandRegister, PciConfigurationSpace,
-};
-use ygg_driver_usb::error::UsbError;
-use yggdrasil_abi::error::Error;
-
-pub struct XhciContext<const N: usize> {
-    pub(crate) input: IrqSafeRwLock<PageBox<context::Input<N>>>,
-    pub(crate) output: PageBox<context::Device<N>>,
-}
-
-impl XhciContext<8> {
-    pub fn new_32byte() -> Result<Self, UsbError> {
-        let input = PageBox::new(context::Input::new_32byte()).map_err(UsbError::MemoryError)?;
-        let output = PageBox::new(context::Device::new_32byte()).map_err(UsbError::MemoryError)?;
-
-        Ok(Self {
-            input: IrqSafeRwLock::new(input),
-            output,
-        })
-    }
-
-    pub fn new_32byte_address_device(
-        default_control_ring: &ControlTransferRing,
-        speed: PortSpeed,
-        address: u8,
-        root_hub_port_number: u8,
-    ) -> Result<Self, UsbError> {
-        let mut input =
-            PageBox::new(context::Input::new_32byte()).map_err(UsbError::MemoryError)?;
-        let output = PageBox::new(context::Device::new_32byte()).map_err(UsbError::MemoryError)?;
-
-        // Setup input context
-        {
-            let control = input.control_mut();
-
-            control.set_add_context_flag(0);
-            control.set_add_context_flag(1);
+// Only x86-64 supports xHCI currently, because only it has MSI-X implemented
+pci_driver! {
+    matches: [class (0x0C:0x03:0x30)],
+    driver: {
+        fn driver_name(&self) -> &str {
+            "xhci"
         }
 
-        {
-            let slot = input.device_mut().slot_mut();
+        fn probe(&self, info: &PciDeviceInfo, dma: &Arc<dyn DmaAllocator>) -> Result<Arc<dyn Device>, Error> {
+            // TODO Chip Hardware Reset
+            let bar0 = info
+                .config_space
+                .bar(0)
+                .expect("xHCI doesn't have BAR0 configured")
+                .as_memory()
+                .expect("xHCI's BAR0 is not memory-type");
 
-            slot.set_context_entries(1);
-            slot.set_interrupter_target(0);
-            slot.set_usb_device_address(address);
-            slot.set_root_hub_port_number(root_hub_port_number);
-            slot.set_speed(speed.into());
+            if let Some(power) = info.config_space.capability::<PowerManagementCapability>() {
+                log::info!("xHC has power management capability");
+                let power_state = power.get_device_power_state();
+
+                if power_state != DevicePowerState::D0 {
+                    power.set_device_power_state(DevicePowerState::D0);
+                }
+                // Enable PME# signal generation
+                power.set_pme_en(true);
+            }
+
+            let mut cmd = PciCommandRegister::from_bits_retain(info.config_space.command());
+            cmd &= !(PciCommandRegister::DISABLE_INTERRUPTS | PciCommandRegister::ENABLE_IO);
+            cmd |= PciCommandRegister::ENABLE_MEMORY | PciCommandRegister::BUS_MASTER;
+            info.config_space.set_command(cmd.bits());
+
+            info.init_interrupts(PreferredInterruptMode::Msi(true))?;
+
+            let regs = Regs::map(bar0)?;
+            let xhci = Arc::new(Xhci::new(dma.clone(), info.clone(), regs)?);
+
+            info.map_interrupt(InterruptAffinity::Any, xhci.clone())?;
+
+            Ok(xhci)
         }
-
-        {
-            let ep0 = input.device_mut().endpoint_mut(1);
-
-            ep0.set_endpoint_type(context::EndpointType::Control);
-            ep0.set_tr_dequeue_pointer(default_control_ring.dequeue_pointer().into());
-            ep0.set_dequeue_cycle_state();
-            ep0.set_error_count(3);
-            ep0.set_max_packet_size(speed.default_max_packet_size() as _);
-        }
-
-        Ok(Self {
-            input: IrqSafeRwLock::new(input),
-            output,
-        })
     }
 }
-
-pub fn probe(info: &PciDeviceInfo) -> Result<Arc<dyn Device>, Error> {
-    // TODO Chip Hardware Reset
-    let bar0 = info
-        .config_space
-        .bar(0)
-        .expect("xHCI doesn't have BAR0 configured")
-        .as_memory()
-        .expect("xHCI's BAR0 is not memory-type");
-
-    let mut cmd = PciCommandRegister::from_bits_retain(info.config_space.command());
-    cmd &= !(PciCommandRegister::DISABLE_INTERRUPTS | PciCommandRegister::ENABLE_IO);
-    cmd |= PciCommandRegister::ENABLE_MEMORY | PciCommandRegister::BUS_MASTER;
-    info.config_space.set_command(cmd.bits());
-
-    let regs = unsafe { xhci_lib::Registers::new(bar0.try_into_usize().unwrap(), Mapper::new()) };
-    let xhci = Arc::new(Xhci::new(regs)?);
-
-    info.init_interrupts(PreferredInterruptMode::Msi)?;
-    info.map_interrupt(InterruptAffinity::Any, xhci.clone())?;
-
-    Ok(xhci)
-}

kernel/driver/usb/xhci/src/pipe.rs

@@ -1,85 +1,199 @@
-use alloc::sync::Arc;
-use libk_mm::{address::PhysicalAddress, PageBox};
+use core::mem::MaybeUninit;
+
+use alloc::{boxed::Box, sync::Arc};
+use async_trait::async_trait;
+use libk::dma::{DmaBuffer, DmaSlice, DmaSliceMut};
 use ygg_driver_usb::{
-    communication::UsbInterruptTransfer,
-    error::UsbError,
+    error::{TransferError, UsbError},
     pipe::{
         control::{ControlTransferSetup, UsbControlPipe},
-        interrupt::UsbInterruptInPipe,
+        normal::{UsbNormalPipeIn, UsbNormalPipeOut},
         UsbGenericPipe,
     },
-    UsbControlTransfer, UsbDirection,
 };
 
 use crate::{
-    ring::{transfer::InterruptInTransferRing, ControlTransferRing},
-    Xhci,
+    controller::Xhci,
+    ring::transfer::{ControlDataStage, TransferRing},
 };
 
 pub struct ControlPipe {
     xhci: Arc<Xhci>,
-    ring: Arc<ControlTransferRing>,
+    pub(crate) ring: Arc<TransferRing>,
 }
 
-#[allow(unused)]
-pub struct InterruptInPipe {
+pub struct NormalInPipe {
     xhci: Arc<Xhci>,
-
-    slot_id: u8,
-    endpoint_id: u8,
-    dci: u8,
-
-    ring: Arc<InterruptInTransferRing>,
+    pub(crate) ring: Arc<TransferRing>,
 }
 
-impl UsbGenericPipe for ControlPipe {}
-
-impl UsbControlPipe for ControlPipe {
-    fn start_transfer(
-        &self,
-        setup: ControlTransferSetup,
-        data: Option<(PhysicalAddress, usize, UsbDirection)>,
-    ) -> Result<UsbControlTransfer, UsbError> {
-        self.ring.start_transfer(self.xhci.as_ref(), setup, data)
-    }
-
-    fn complete_transfer(&self, transfer: UsbControlTransfer) {
-        self.ring.complete_transfer(transfer)
-    }
+pub struct NormalOutPipe {
+    xhci: Arc<Xhci>,
+    pub(crate) ring: Arc<TransferRing>,
 }
 
 impl ControlPipe {
-    pub fn new(xhci: Arc<Xhci>, _slot_id: u8, ring: Arc<ControlTransferRing>) -> Self {
-        Self { xhci, ring }
-    }
-}
-
-impl UsbGenericPipe for InterruptInPipe {}
-
-impl UsbInterruptInPipe for InterruptInPipe {
-    fn start_read(&self, buffer: &mut PageBox<[u8]>) -> Result<UsbInterruptTransfer, UsbError> {
-        self.ring.start_transfer(self.xhci.as_ref(), buffer)
-    }
-
-    fn complete_transfer(&self, transfer: UsbInterruptTransfer) {
-        self.ring.complete_transfer(transfer)
-    }
-}
-
-impl InterruptInPipe {
     pub fn new(
         xhci: Arc<Xhci>,
         slot_id: u8,
         endpoint_id: u8,
-        dci: u8,
-        ring: Arc<InterruptInTransferRing>,
-    ) -> Self {
-        Self {
-            xhci,
+        capacity: usize,
+    ) -> Result<(Self, Arc<TransferRing>), UsbError> {
+        let ring = Arc::new(TransferRing::new(
+            &*xhci.dma,
             slot_id,
             endpoint_id,
-            dci,
+            capacity,
+        )?);
+        Ok((
+            Self {
+                xhci,
+                ring: ring.clone(),
+            },
             ring,
-        }
+        ))
+    }
+}
+
+impl NormalInPipe {
+    pub fn new(
+        xhci: Arc<Xhci>,
+        slot_id: u8,
+        endpoint_id: u8,
+        capacity: usize,
+    ) -> Result<(Self, Arc<TransferRing>), UsbError> {
+        let ring = Arc::new(TransferRing::new(
+            &*xhci.dma,
+            slot_id,
+            endpoint_id,
+            capacity,
+        )?);
+        Ok((
+            Self {
+                xhci,
+                ring: ring.clone(),
+            },
+            ring,
+        ))
+    }
+}
+
+impl NormalOutPipe {
+    pub fn new(
+        xhci: Arc<Xhci>,
+        slot_id: u8,
+        endpoint_id: u8,
+        capacity: usize,
+    ) -> Result<(Self, Arc<TransferRing>), UsbError> {
+        let ring = Arc::new(TransferRing::new(
+            &*xhci.dma,
+            slot_id,
+            endpoint_id,
+            capacity,
+        )?);
+        Ok((
+            Self {
+                xhci,
+                ring: ring.clone(),
+            },
+            ring,
+        ))
+    }
+}
+
+#[async_trait]
+impl UsbControlPipe for ControlPipe {
+    async fn control_transfer(&self, setup: ControlTransferSetup) -> Result<(), UsbError> {
+        self.ring
+            .control_transfer(self.xhci.as_ref(), setup, ControlDataStage::None)
+            .await?;
+        Ok(())
+    }
+
+    async fn control_transfer_in(
+        &self,
+        setup: ControlTransferSetup,
+        buffer: &mut [MaybeUninit<u8>],
+    ) -> Result<usize, UsbError> {
+        let data_len = buffer.len();
+        let mut dma_buffer = DmaBuffer::new_uninit_slice(&*self.xhci.dma, data_len)
+            .map_err(UsbError::MemoryError)?;
+
+        let result = self
+            .ring
+            .control_transfer(
+                self.xhci.as_ref(),
+                setup,
+                ControlDataStage::In(&mut dma_buffer),
+            )
+            .await;
+        let result = allow_short_packet(data_len, result);
+
+        match result {
+            Ok(len) => {
+                buffer[..len].copy_from_slice(&dma_buffer[..len]);
+                Ok(len)
+            }
+            Err(error) => Err(error),
+        }
+    }
+
+    async fn control_transfer_out(
+        &self,
+        setup: ControlTransferSetup,
+        buffer: &[u8],
+    ) -> Result<usize, UsbError> {
+        let mut dma_buffer =
+            DmaBuffer::from_slice(&*self.xhci.dma, buffer).map_err(UsbError::MemoryError)?;
+
+        self.ring
+            .control_transfer(
+                self.xhci.as_ref(),
+                setup,
+                ControlDataStage::Out(&mut dma_buffer),
+            )
+            .await
+    }
+}
+
+impl UsbGenericPipe for NormalInPipe {
+    fn allocate_dma_buffer(&self, len: usize) -> Result<DmaBuffer<[MaybeUninit<u8>]>, UsbError> {
+        DmaBuffer::new_uninit_slice(&*self.xhci.dma, len).map_err(UsbError::MemoryError)
+    }
+}
+
+#[async_trait]
+impl UsbNormalPipeIn for NormalInPipe {
+    async fn read_dma(&self, buffer: DmaSliceMut<'_, MaybeUninit<u8>>) -> Result<usize, UsbError> {
+        let result = self
+            .ring
+            .normal_transfer(self.xhci.as_ref(), buffer.bus_address(), buffer.len())
+            .await;
+        allow_short_packet(buffer.len(), result)
+    }
+}
+
+impl UsbGenericPipe for NormalOutPipe {
+    fn allocate_dma_buffer(&self, len: usize) -> Result<DmaBuffer<[MaybeUninit<u8>]>, UsbError> {
+        DmaBuffer::new_uninit_slice(&*self.xhci.dma, len).map_err(UsbError::MemoryError)
+    }
+}
+
+#[async_trait]
+impl UsbNormalPipeOut for NormalOutPipe {
+    async fn write_dma(&self, buffer: DmaSlice<'_, u8>) -> Result<usize, UsbError> {
+        self.ring
+            .normal_transfer(self.xhci.as_ref(), buffer.bus_address(), buffer.len())
+            .await
+    }
+}
+
+fn allow_short_packet(data_len: usize, result: Result<usize, UsbError>) -> Result<usize, UsbError> {
+    match result {
+        // Short packets are okay for control transfers
+        Err(UsbError::TransferFailed(TransferError::ShortPacket(residual))) => {
+            Ok(data_len.saturating_sub(residual))
+        }
+        result => result,
     }
 }

kernel/driver/usb/xhci/src/regs.rs

@@ -1,283 +0,0 @@
-use core::{cell::UnsafeCell, num::NonZeroUsize};
-
-use alloc::{sync::Arc, vec::Vec};
-use libk_mm::{
-    address::{AsPhysicalAddress, PhysicalAddress},
-    device::RawDeviceMemoryMapping,
-    PageBox,
-};
-use libk_util::sync::spin_rwlock::IrqSafeRwLock;
-use xhci_lib::{
-    accessor::{array, marker},
-    registers::{
-        operational::UsbStatusRegister, Capability, Doorbell, InterrupterRegisterSet, Operational,
-        PortRegisterSet,
-    },
-};
-use ygg_driver_usb::device::UsbSpeed;
-use yggdrasil_abi::primitive_enum;
-
-use crate::ring::{CommandRing, EventRing, EventRingSegmentTable, GenericRing};
-
-#[derive(Clone)]
-pub struct Mapper {
-    mappings: Vec<Arc<RawDeviceMemoryMapping>>,
-}
-
-pub struct LockedArray<T> {
-    array: UnsafeCell<array::Generic<T, Mapper, marker::ReadWrite>>,
-    locks: Vec<IrqSafeRwLock<()>>,
-}
-
-unsafe impl<T> Sync for LockedArray<T> {}
-unsafe impl<T> Send for LockedArray<T> {}
-
-pub struct Regs {
-    operational: IrqSafeRwLock<Operational<Mapper>>,
-    interrupters: IrqSafeRwLock<InterrupterRegisterSet<Mapper>>,
-    capability: Capability<Mapper>,
-    doorbells: LockedArray<Doorbell>,
-    pub ports: LockedArray<PortRegisterSet>,
-}
-
-impl<T> LockedArray<T> {
-    #[inline]
-    #[allow(clippy::mut_from_ref)]
-    unsafe fn get_mut(&self) -> &mut array::Generic<T, Mapper, marker::ReadWrite> {
-        &mut *self.array.get()
-    }
-
-    pub fn update<U: FnOnce(&mut T)>(&self, index: usize, f: U) {
-        let _guard = self.locks[index].write();
-        unsafe { self.get_mut() }.update_volatile_at(index, f);
-    }
-
-    pub fn read(&self, index: usize) -> T {
-        let _guard = self.locks[index].read();
-        unsafe { self.get_mut() }.read_volatile_at(index)
-    }
-}
-
-impl<T> From<array::Generic<T, Mapper, marker::ReadWrite>> for LockedArray<T> {
-    fn from(value: array::Generic<T, Mapper, marker::ReadWrite>) -> Self {
-        let locks = Vec::from_iter((0..value.len()).map(|_| IrqSafeRwLock::new(())));
-
-        Self {
-            array: UnsafeCell::new(value),
-            locks,
-        }
-    }
-}
-
-impl From<xhci_lib::Registers<Mapper>> for Regs {
-    fn from(value: xhci_lib::Registers<Mapper>) -> Self {
-        Self {
-            operational: IrqSafeRwLock::new(value.operational),
-            capability: value.capability,
-            interrupters: IrqSafeRwLock::new(value.interrupter_register_set),
-            doorbells: LockedArray::from(value.doorbell),
-            ports: LockedArray::from(value.port_register_set),
-        }
-    }
-}
-
-impl Regs {
-    pub fn reset(&self) {
-        let mut o = self.operational.write();
-
-        // TODO Get ownership from firmware
-
-        // Stop the controller
-        o.usbcmd.update_volatile(|u| {
-            u.clear_run_stop();
-        });
-
-        while !o.usbsts.read_volatile().hc_halted() {
-            core::hint::spin_loop();
-        }
-
-        // Reset the controller
-        o.usbcmd.update_volatile(|u| {
-            u.set_host_controller_reset();
-        });
-        while o.usbcmd.read_volatile().host_controller_reset()
-            || o.usbsts.read_volatile().controller_not_ready()
-        {
-            core::hint::spin_loop();
-        }
-    }
-
-    pub fn max_slot_count(&self) -> usize {
-        self.capability
-            .hcsparams1
-            .read_volatile()
-            .number_of_device_slots() as _
-    }
-
-    pub fn set_max_slot_count(&self) -> usize {
-        let device_slot_count = self.max_slot_count();
-        let mut o = self.operational.write();
-        // Set max slots enabled
-        o.config.update_volatile(|u| {
-            u.set_max_device_slots_enabled(device_slot_count as _);
-        });
-
-        device_slot_count as _
-    }
-
-    pub fn context_size(&self) -> usize {
-        match self.capability.hccparams1.read_volatile().context_size() {
-            true => 64,
-            false => 32,
-        }
-    }
-
-    pub fn port_count(&self) -> usize {
-        self.capability.hcsparams1.read_volatile().number_of_ports() as _
-    }
-
-    pub fn configure(
-        &self,
-        dcbaa: &PageBox<[PhysicalAddress]>,
-        cmd_ring: &CommandRing,
-        evt_ring: &EventRing,
-        erst: &EventRingSegmentTable,
-    ) {
-        let mut o = self.operational.write();
-        let mut i = self.interrupters.write();
-
-        o.dcbaap.update_volatile(|u| unsafe {
-            u.set(dcbaa.as_physical_address().into());
-        });
-        o.crcr.update_volatile(|u| {
-            u.set_command_ring_pointer(cmd_ring.base().into());
-            u.set_ring_cycle_state();
-        });
-
-        let mut intr0 = i.interrupter_mut(0);
-        intr0.erstsz.update_volatile(|u| {
-            u.set(erst.capacity().try_into().unwrap());
-        });
-        intr0.erdp.update_volatile(|u| {
-            log::debug!("::: Dequeue Pointer: {:#x}", evt_ring.dequeue_pointer());
-            u.set_event_ring_dequeue_pointer(evt_ring.dequeue_pointer().into());
-        });
-        intr0.erstba.update_volatile(|u| {
-            u.set(erst.physical_address().into());
-        });
-        // intr0.imod.update_volatile(|u| {
-        //     u.set_interrupt_moderation_interval(0)
-        //         .set_interrupt_moderation_counter(0);
-        // });
-        intr0.iman.update_volatile(|u| {
-            u.set_interrupt_enable();
-        });
-
-        o.usbcmd.update_volatile(|u| {
-            u.set_interrupter_enable().set_run_stop();
-        });
-    }
-
-    pub fn handle_interrupt(&self) -> Option<UsbStatusRegister> {
-        let mut o = self.operational.write();
-        let mut i = self.interrupters.write();
-
-        let status = o.usbsts.read_volatile();
-
-        if !status.event_interrupt() {
-            return None;
-        }
-
-        o.usbsts.write_volatile(status);
-
-        if status.host_system_error() {
-            return Some(status);
-        }
-
-        // Acknowledge interrupts
-        let mut intr0 = i.interrupter_mut(0);
-        intr0.iman.update_volatile(|u| {
-            u.clear_interrupt_pending();
-        });
-
-        Some(status)
-    }
-
-    pub fn set_interrupter_0_dequeue_pointer(&self, pointer: PhysicalAddress) {
-        let mut i = self.interrupters.write();
-
-        i.interrupter_mut(0).erdp.update_volatile(|u| {
-            u.set_event_ring_dequeue_pointer(pointer.into());
-            u.clear_event_handler_busy();
-        });
-    }
-
-    pub fn ring_doorbell(&self, index: usize, target: u8) {
-        self.doorbells.update(index, |u| {
-            u.set_doorbell_target(target);
-        });
-    }
-}
-
-impl Mapper {
-    pub fn new() -> Self {
-        Self {
-            mappings: Vec::new(),
-        }
-    }
-}
-
-impl xhci_lib::accessor::Mapper for Mapper {
-    unsafe fn map(&mut self, phys_start: usize, bytes: usize) -> NonZeroUsize {
-        let mapping = RawDeviceMemoryMapping::map(phys_start as u64, bytes, Default::default())
-            .expect("Could not map an USB xHCI region");
-        let address = mapping.address;
-        self.mappings.push(Arc::new(mapping));
-        NonZeroUsize::new_unchecked(address)
-    }
-
-    fn unmap(&mut self, _virt_start: usize, _bytes: usize) {
-        // TODO
-    }
-}
-
-// Register value definitions
-
-primitive_enum! {
-    pub enum PortSpeed: u8 {
-        Full = 1,
-        Low = 2,
-        High = 3,
-        SuperGen1x1 = 4,
-        SuperGen2x1 = 5,
-        SuperGen1x2 = 6,
-        SuperGen2x2 = 7,
-    }
-}
-
-impl PortSpeed {
-    pub fn default_max_packet_size(&self) -> usize {
-        match self {
-            Self::Low => 8,
-            Self::High => 64,
-            Self::SuperGen1x1 | Self::SuperGen1x2 | Self::SuperGen2x1 | Self::SuperGen2x2 => 512,
-
-            // See Section 4.3., point 7. of the initialization list
-            Self::Full => 8,
-        }
-    }
-}
-
-impl From<PortSpeed> for UsbSpeed {
-    fn from(value: PortSpeed) -> Self {
-        match value {
-            PortSpeed::Low => UsbSpeed::Low,
-            PortSpeed::Full => UsbSpeed::Full,
-            PortSpeed::High => UsbSpeed::High,
-            PortSpeed::SuperGen1x1
-            | PortSpeed::SuperGen1x2
-            | PortSpeed::SuperGen2x1
-            | PortSpeed::SuperGen2x2 => UsbSpeed::Super,
-        }
-    }
-}

kernel/driver/usb/xhci/src/regs/capability.rs

@@ -0,0 +1,66 @@
+use tock_registers::{register_bitfields, register_structs, registers::ReadOnly};
+
+register_bitfields! {
+    u32,
+    pub HCSPARAMS1 [
+        MaxSlots            OFFSET(0)   NUMBITS(8)  [],
+        MaxIntrs            OFFSET(8)   NUMBITS(11) [],
+        MaxPorts            OFFSET(24)  NUMBITS(8)  [],
+    ],
+    pub HCSPARAMS2 [
+        IST                 OFFSET(0)   NUMBITS(4)  [],
+        ERSTMax             OFFSET(4)   NUMBITS(4)  [],
+        MaxScratchpadBufsHi OFFSET(21)  NUMBITS(5)  [],
+        ScratchpadRestore   OFFSET(26)  NUMBITS(1)  [],
+        MaxScratchpadBufsLo OFFSET(27)  NUMBITS(5)  [],
+    ],
+    pub HCSPARAMS3 [
+        U1ExitLatency       OFFSET(0)   NUMBITS(8)  [],
+        U2ExitLatency       OFFSET(16)  NUMBITS(16) [],
+    ],
+    pub HCCPARAMS1 [
+        AC64                OFFSET(0)   NUMBITS(1)  [],
+        BNC                 OFFSET(1)   NUMBITS(1)  [],
+        CSZ                 OFFSET(2)   NUMBITS(1)  [],
+        PPC                 OFFSET(3)   NUMBITS(1)  [],
+        PIND                OFFSET(4)   NUMBITS(1)  [],
+        LHRC                OFFSET(5)   NUMBITS(1)  [],
+        LTC                 OFFSET(6)   NUMBITS(1)  [],
+        NSS                 OFFSET(7)   NUMBITS(1)  [],
+        PAE                 OFFSET(8)   NUMBITS(1)  [],
+        SPC                 OFFSET(9)   NUMBITS(1)  [],
+        SEC                 OFFSET(10)  NUMBITS(1)  [],
+        CFC                 OFFSET(11)  NUMBITS(1)  [],
+        MaxPSASize          OFFSET(12)  NUMBITS(4)  [],
+        XECP                OFFSET(16)  NUMBITS(16) [],
+    ],
+    pub HCCPARAMS2 [
+        U3C                 OFFSET(0)   NUMBITS(1)  [],
+        CMC                 OFFSET(1)   NUMBITS(1)  [],
+        FSC                 OFFSET(2)   NUMBITS(1)  [],
+        CTC                 OFFSET(3)   NUMBITS(1)  [],
+        LEC                 OFFSET(4)   NUMBITS(1)  [],
+        CIC                 OFFSET(5)   NUMBITS(1)  [],
+        ETC                 OFFSET(6)   NUMBITS(1)  [],
+        ETC_TSC             OFFSET(7)   NUMBITS(1)  [],
+        GSC                 OFFSET(8)   NUMBITS(1)  [],
+        VTC                 OFFSET(9)   NUMBITS(1)  [],
+    ],
+}
+
+register_structs! {
+    #[allow(non_snake_case)]
+    pub CapabilityRegs {
+        (0x00 => pub CAPLENGTH: ReadOnly<u8>),
+        (0x01 => _0),
+        (0x02 => pub HCIVERSION: ReadOnly<u16>),
+        (0x04 => pub HCSPARAMS1: ReadOnly<u32, HCSPARAMS1::Register>),
+        (0x08 => pub HCSPARAMS2: ReadOnly<u32, HCSPARAMS2::Register>),
+        (0x0C => pub HCSPARAMS3: ReadOnly<u32, HCSPARAMS3::Register>),
+        (0x10 => pub HCCPARAMS1: ReadOnly<u32, HCCPARAMS1::Register>),
+        (0x14 => pub DBOFF: ReadOnly<u32>),
+        (0x18 => pub RTSOFF: ReadOnly<u32>),
+        (0x1C => pub HCCPARAMS2: ReadOnly<u32, HCCPARAMS2::Register>),
+        (0x20 => @END),
+    }
+}

kernel/driver/usb/xhci/src/regs/extended.rs

@@ -0,0 +1,122 @@
+use core::{
+    ops::Range,
+    sync::atomic::{AtomicU32, Ordering},
+};
+
+use alloc::vec::Vec;
+use libk::error::Error;
+use libk_mm::{address::PhysicalAddress, device::DeviceMemoryIo};
+use libk_util::sync::spin_rwlock::IrqSafeRwLock;
+use ygg_driver_usb::{error::UsbError, info::UsbVersion};
+
+pub struct ProtocolSupport {
+    words: [u32; 4],
+}
+
+pub struct LegacySupport(DeviceMemoryIo<'static, [AtomicU32; 4]>);
+
+pub enum ExtendedCapability {
+    LegacySupport(IrqSafeRwLock<LegacySupport>),
+    ProtocolSupport(ProtocolSupport),
+}
+
+impl ExtendedCapability {
+    pub fn parse_list(
+        base: PhysicalAddress,
+        extended_capability_offset: usize,
+    ) -> Result<Vec<Self>, Error> {
+        let mut extended_capabilities = Vec::new();
+
+        if extended_capability_offset != 0 {
+            let extended_capability_base = base.add(extended_capability_offset);
+            let mut current = extended_capability_base;
+
+            loop {
+                let header =
+                    unsafe { DeviceMemoryIo::<[AtomicU32; 4]>::map(current, Default::default()) }?;
+                let word0 = header[0].load(Ordering::Acquire);
+                let id = word0 as u8;
+                let next_pointer = ((word0 >> 8) & 0xFF) as usize;
+
+                match id {
+                    0x01 => extended_capabilities.push(ExtendedCapability::LegacySupport(
+                        IrqSafeRwLock::new(LegacySupport(header)),
+                    )),
+                    0x02 => {
+                        let words = [
+                            word0,
+                            header[1].load(Ordering::Acquire),
+                            header[2].load(Ordering::Acquire),
+                            header[3].load(Ordering::Acquire),
+                        ];
+                        extended_capabilities.push(ExtendedCapability::ProtocolSupport(
+                            ProtocolSupport { words },
+                        ))
+                    }
+                    _ => (),
+                }
+
+                if next_pointer == 0 {
+                    break;
+                } else {
+                    current = current.add(next_pointer << 2);
+                }
+            }
+        }
+
+        Ok(extended_capabilities)
+    }
+}
+
+impl ProtocolSupport {
+    pub fn usb_revision(&self) -> Option<UsbVersion> {
+        UsbVersion::from_bcd_usb((self.words[0] >> 16) as u16)
+    }
+
+    pub fn slot_type(&self) -> u8 {
+        (self.words[3] & 0x1F) as u8
+    }
+
+    pub fn port_range(&self) -> Range<u8> {
+        let port_offset = self.words[2] as u8;
+        let port_count = (self.words[2] >> 8) as u8;
+
+        port_offset.max(1)..port_offset.saturating_add(port_count)
+    }
+}
+
+impl LegacySupport {
+    fn is_bios_owned(&self) -> bool {
+        self.0[0].load(Ordering::Acquire) & (1 << 16) != 0
+    }
+
+    fn set_os_owned(&mut self) {
+        self.0[0].fetch_or(1 << 24, Ordering::Release);
+    }
+
+    pub fn perform_bios_handoff(&mut self, mut timeout_cycles: u64) -> Result<(), UsbError> {
+        if !self.is_bios_owned() {
+            return Ok(());
+        }
+
+        // Clear SMI events
+        self.0[1].fetch_and(
+            !((1 << 0) | (1 << 4) | (1 << 13) | (1 << 14) | (1 << 15)),
+            Ordering::Release,
+        );
+
+        // Set OS owned semaphore
+        self.set_os_owned();
+        while timeout_cycles > 0 && self.is_bios_owned() {
+            core::hint::spin_loop();
+            timeout_cycles -= 1;
+        }
+
+        if timeout_cycles > 0 {
+            Ok(())
+        } else {
+            log::error!("xhci: BIOS handoff timed out");
+            Err(UsbError::DeviceBusy)
+        }
+    }
+}

kernel/driver/usb/xhci/src/regs/mod.rs

@@ -0,0 +1,184 @@
+use core::{fmt, sync::atomic::AtomicU32};
+
+use alloc::vec::Vec;
+use capability::{CapabilityRegs, HCCPARAMS1, HCSPARAMS1, HCSPARAMS2};
+use extended::ExtendedCapability;
+use libk::error::Error;
+use libk_mm::{address::PhysicalAddress, device::DeviceMemoryIo};
+use libk_util::sync::spin_rwlock::IrqSafeRwLock;
+use operational::{OperationalRegs, PortRegs, PORTSC, USBCMD, USBSTS};
+use runtime::{RuntimeRegs, IMAN};
+use tock_registers::{
+    interfaces::{ReadWriteable, Readable, Writeable},
+    LocalRegisterCopy,
+};
+
+pub mod capability;
+pub mod extended;
+pub mod operational;
+pub mod runtime;
+
+pub struct Regs {
+    #[allow(unused)]
+    pub(crate) capability: DeviceMemoryIo<'static, CapabilityRegs>,
+    pub(crate) operational: IrqSafeRwLock<DeviceMemoryIo<'static, OperationalRegs>>,
+    pub(crate) runtime: IrqSafeRwLock<DeviceMemoryIo<'static, RuntimeRegs>>,
+    pub(crate) doorbells: DeviceMemoryIo<'static, [AtomicU32]>,
+    pub(crate) extended_capabilities: Vec<ExtendedCapability>,
+    // TODO per-port locks
+    pub(crate) ports: IrqSafeRwLock<DeviceMemoryIo<'static, [PortRegs]>>,
+    pub(crate) port_count: usize,
+    pub(crate) slot_count: usize,
+    pub(crate) scratch_count: usize,
+    pub(crate) context_size: ContextSize,
+}
+
+#[derive(Clone, Copy, PartialEq, Eq, Debug)]
+pub enum ContextSize {
+    Context32,
+    Context64,
+}
+
+#[derive(Clone, Copy, PartialEq, Eq, Debug)]
+#[repr(transparent)]
+pub struct PortNumber(u8);
+
+impl Regs {
+    const PORT_REGS_OFFSET: usize = 0x400;
+
+    pub fn map(base: PhysicalAddress) -> Result<Self, Error> {
+        let capability =
+            unsafe { DeviceMemoryIo::<CapabilityRegs>::map(base, Default::default()) }?;
+
+        let operational_offset = capability.CAPLENGTH.get() as usize;
+        let doorbell_offset = capability.DBOFF.get() & !0x3;
+        let runtime_offset = capability.RTSOFF.get() & !0x1F;
+        let extended_capability_offset =
+            (capability.HCCPARAMS1.read(HCCPARAMS1::XECP) as usize) << 2;
+
+        let port_count = capability.HCSPARAMS1.read(HCSPARAMS1::MaxPorts) as usize;
+        let slot_count = capability.HCSPARAMS1.read(HCSPARAMS1::MaxSlots) as usize;
+
+        let operational_base = base.add(operational_offset);
+        let doorbell_base = base.add(doorbell_offset as usize);
+        let runtime_base = base.add(runtime_offset as usize);
+
+        let scratch_lo = capability.HCSPARAMS2.read(HCSPARAMS2::MaxScratchpadBufsLo);
+        let scratch_hi = capability.HCSPARAMS2.read(HCSPARAMS2::MaxScratchpadBufsHi);
+        let scratch_count = ((scratch_hi << 5) | scratch_lo) as usize;
+
+        let context_size = if capability.HCCPARAMS1.matches_all(HCCPARAMS1::CSZ::SET) {
+            ContextSize::Context64
+        } else {
+            ContextSize::Context32
+        };
+
+        let operational = unsafe {
+            DeviceMemoryIo::<OperationalRegs>::map(operational_base, Default::default())
+        }?;
+        let runtime =
+            unsafe { DeviceMemoryIo::<RuntimeRegs>::map(runtime_base, Default::default()) }?;
+
+        let ports = unsafe {
+            DeviceMemoryIo::map_slice(
+                operational_base.add(Self::PORT_REGS_OFFSET),
+                port_count,
+                Default::default(),
+            )
+        }?;
+        let doorbells =
+            unsafe { DeviceMemoryIo::map_slice(doorbell_base, 256, Default::default()) }?;
+        let extended_capabilities =
+            ExtendedCapability::parse_list(base, extended_capability_offset)?;
+
+        Ok(Self {
+            capability,
+            operational: IrqSafeRwLock::new(operational),
+            runtime: IrqSafeRwLock::new(runtime),
+            doorbells,
+            ports: IrqSafeRwLock::new(ports),
+            extended_capabilities,
+
+            port_count,
+            slot_count,
+            scratch_count,
+            context_size,
+        })
+    }
+
+    pub fn hc_reset(&self, timeout_cycles: u64) -> Result<(), Error> {
+        let op = self.operational.write();
+
+        // Wait for CNR to get cleared
+        op.wait_usbsts_bit(USBSTS::CNR::CLEAR, timeout_cycles)?;
+
+        // Clear run/stop and wait for the HC to halt
+        op.USBCMD.modify(USBCMD::RS::CLEAR);
+        op.wait_usbsts_bit(USBSTS::HCH::SET, timeout_cycles)?;
+
+        // Reset the HC, wait for CNR to get cleared again
+        op.USBCMD.modify(USBCMD::HCRST::SET);
+        op.wait_usbsts_bit(USBSTS::CNR::CLEAR, timeout_cycles)?;
+
+        // Halt the HC again
+        op.USBCMD.modify(USBCMD::RS::CLEAR);
+        op.wait_usbsts_bit(USBSTS::HCH::SET, timeout_cycles)?;
+
+        Ok(())
+    }
+
+    pub fn handle_interrupt(&self, vector: usize) -> LocalRegisterCopy<u32, USBSTS::Register> {
+        let op = self.operational.write();
+        let rt = self.runtime.write();
+
+        let status = op.USBSTS.extract();
+
+        // Clear the RW1C bits
+        op.USBSTS.set(status.get());
+
+        // Acknowledge interrupts for given vector (vector == interrupter index)
+        let interrupter = &rt.IRn[vector];
+
+        interrupter.IMAN.modify(IMAN::IP::SET);
+
+        status
+    }
+}
+
+pub fn portsc_to_neutral(
+    portsc: LocalRegisterCopy<u32, PORTSC::Register>,
+) -> LocalRegisterCopy<u32, PORTSC::Register> {
+    const RO_MASK: u32 = (1 << 0) | (1 << 3) | (0xF << 10) | (1 << 30);
+    const RWS_MASK: u32 = (0xF << 5)
+        | (1 << 9)
+        | (0x3 << 14)
+        | (1 << 25)
+        | (1 << 26)
+        | (1 << 27)
+        | (1 << 30)
+        | (1 << 31);
+
+    portsc.bitand(RO_MASK | RWS_MASK)
+}
+
+impl Into<u8> for PortNumber {
+    fn into(self) -> u8 {
+        self.0
+    }
+}
+
+impl PortNumber {
+    pub fn from_index(index: usize) -> Self {
+        Self(index as u8 + 1)
+    }
+
+    pub fn index(&self) -> usize {
+        self.0 as usize - 1
+    }
+}
+
+impl fmt::Display for PortNumber {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        fmt::Display::fmt(&self.0, f)
+    }
+}