pci: move to .init_array-based driver registration

Cargo.toml

@@ -39,6 +39,7 @@ ahash = { version = "0.8.11", default-features = false, features = ["no-rng"] }
 
 # acpi
 acpi = { git = "https://github.com/alnyan/acpi.git", package = "acpi", branch = "acpi-system" }
+rsdp = { git = "https://github.com/alnyan/acpi.git", package = "rsdp", branch = "acpi-system" }
 aml = { git = "https://github.com/alnyan/acpi.git", branch = "acpi-system" }
 acpi-system = { git = "https://github.com/alnyan/acpi-system.git" }
 

kernel/Cargo.toml

@@ -45,6 +45,7 @@ bytemuck.workspace = true
 futures-util.workspace = true
 crossbeam-queue.workspace = true
 async-trait.workspace = true
+cfg-if.workspace = true
 
 git-version = "0.3.9"
 
@@ -61,12 +62,12 @@ kernel-arch-riscv64.workspace = true
 yboot-proto.workspace = true
 kernel-arch-x86_64.workspace = true
 kernel-arch-x86.workspace = true
+ygg_driver_acpi.path = "driver/acpi"
 
 ygg_driver_nvme = { path = "driver/block/nvme" }
+ygg_driver_net_rtl81xx.path = "driver/net/rtl81xx"
 
 acpi.workspace = true
-aml.workspace = true
-acpi-system.workspace = true
 
 [target.'cfg(target_arch = "x86")'.dependencies]
 kernel-arch-i686.workspace = true
@@ -87,6 +88,9 @@ kernel-arch-x86.workspace = true
 kernel-arch-aarch64.workspace = true
 kernel-arch-riscv64.workspace = true
 
+ygg_driver_acpi.path = "driver/acpi"
+ygg_driver_net_rtl81xx.path = "driver/net/rtl81xx"
+
 [features]
 default = ["fb_console"]
 fb_console = []

kernel/arch/interface/src/mem/mod.rs

@@ -35,6 +35,8 @@ pub struct DeviceMemoryAttributes {
 /// Describes a single device memory mapping
 #[derive(Debug)]
 pub struct RawDeviceMemoryMapping<A: KernelTableManager> {
+    /// Physical base address of the object
+    pub physical_base: u64,
     /// Virtual address of the mapped object
     pub address: usize,
     /// Base address of the mapping start
@@ -98,7 +100,8 @@ impl<A: KernelTableManager> RawDeviceMemoryMapping<A> {
         address
     }
 
-    pub fn into_raw_parts(self) -> (usize, usize, usize, usize) {
+    pub fn into_raw_parts(self) -> (u64, usize, usize, usize, usize) {
+        let physical_base = self.physical_base;
         let address = self.address;
         let base_address = self.base_address;
         let page_count = self.page_count;
@@ -106,7 +109,7 @@ impl<A: KernelTableManager> RawDeviceMemoryMapping<A> {
 
         core::mem::forget(self);
 
-        (address, base_address, page_count, page_size)
+        (physical_base, address, base_address, page_count, page_size)
     }
 
     /// # Safety
@@ -114,12 +117,14 @@ impl<A: KernelTableManager> RawDeviceMemoryMapping<A> {
     /// Preconditions: all the fields must come from a [RawDeviceMemoryMapping::into_raw_parts]
     /// call.
     pub unsafe fn from_raw_parts(
+        physical_base: u64,
         address: usize,
         base_address: usize,
         page_count: usize,
         page_size: usize,
     ) -> Self {
         Self {
+            physical_base,
             address,
             base_address,
             page_count,

kernel/arch/x86/src/lib.rs

@@ -6,4 +6,10 @@ extern crate alloc;
 
 pub mod cpuid;
 pub mod gdt;
+pub mod intrinsics;
 pub mod registers;
+
+#[cfg(any(target_arch = "x86_64", rust_analyzer))]
+pub const ISA_IRQ_OFFSET: u32 = 1024;
+#[cfg(any(target_arch = "x86", rust_analyzer))]
+pub const ISA_IRQ_OFFSET: u32 = 0;

kernel/arch/x86_64/src/mem/mod.rs

@@ -222,6 +222,7 @@ unsafe fn map_device_memory(
         let address = base_address + l2_offset;
 
         Ok(RawDeviceMemoryMapping::from_raw_parts(
+            l2_aligned.into_u64(),
             address,
             base_address,
             page_count,
@@ -233,6 +234,7 @@ unsafe fn map_device_memory(
         let address = base_address + l3_offset;
 
         Ok(RawDeviceMemoryMapping::from_raw_parts(
+            l3_aligned.into_u64(),
             address,
             base_address,
             page_count,

kernel/driver/acpi/Cargo.toml

@@ -0,0 +1,18 @@
+[package]
+name = "ygg_driver_acpi"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+libk-util.workspace = true
+libk-mm.workspace = true
+libk.workspace = true
+device-api.workspace = true
+kernel-arch-x86.path = "../../arch/x86"
+
+acpi.workspace = true
+rsdp.workspace = true
+aml.workspace = true
+acpi-system.workspace = true
+
+log.workspace = true

kernel/driver/acpi/src/aml_handler.rs

@@ -0,0 +1,131 @@
+use core::time::Duration;
+
+use crate::AcpiHandlerImpl;
+
+impl aml::Handler for AcpiHandlerImpl {
+    fn read_io_u8(&self, port: u16) -> u8 {
+        <Self as acpi_system::Handler>::io_read_u8(port)
+    }
+
+    fn read_io_u16(&self, port: u16) -> u16 {
+        <Self as acpi_system::Handler>::io_read_u16(port)
+    }
+
+    fn read_io_u32(&self, port: u16) -> u32 {
+        <Self as acpi_system::Handler>::io_read_u32(port)
+    }
+
+    fn write_io_u8(&self, port: u16, value: u8) {
+        <Self as acpi_system::Handler>::io_write_u8(port, value)
+    }
+
+    fn write_io_u16(&self, port: u16, value: u16) {
+        <Self as acpi_system::Handler>::io_write_u16(port, value)
+    }
+
+    fn write_io_u32(&self, port: u16, value: u32) {
+        <Self as acpi_system::Handler>::io_write_u32(port, value)
+    }
+
+    fn read_u8(&self, address: usize) -> u8 {
+        <Self as acpi_system::Handler>::mem_read_u8(address as u64)
+    }
+
+    fn read_u16(&self, address: usize) -> u16 {
+        <Self as acpi_system::Handler>::mem_read_u16(address as u64)
+    }
+
+    fn read_u32(&self, address: usize) -> u32 {
+        <Self as acpi_system::Handler>::mem_read_u32(address as u64)
+    }
+
+    fn read_u64(&self, address: usize) -> u64 {
+        <Self as acpi_system::Handler>::mem_read_u64(address as u64)
+    }
+
+    fn write_u8(&self, address: usize, value: u8) {
+        <Self as acpi_system::Handler>::mem_write_u8(address as u64, value)
+    }
+
+    fn write_u16(&self, address: usize, value: u16) {
+        <Self as acpi_system::Handler>::mem_write_u16(address as u64, value)
+    }
+
+    fn write_u32(&self, address: usize, value: u32) {
+        <Self as acpi_system::Handler>::mem_write_u32(address as u64, value)
+    }
+
+    fn write_u64(&self, address: usize, value: u64) {
+        <Self as acpi_system::Handler>::mem_write_u64(address as u64, value)
+    }
+
+    fn read_pci_u8(&self, _segment: u16, _bus: u8, _device: u8, _function: u8, _offset: u16) -> u8 {
+        0xFF
+    }
+
+    fn read_pci_u16(
+        &self,
+        _segment: u16,
+        _bus: u8,
+        _device: u8,
+        _function: u8,
+        _offset: u16,
+    ) -> u16 {
+        0xFFFF
+    }
+
+    fn read_pci_u32(
+        &self,
+        _segment: u16,
+        _bus: u8,
+        _device: u8,
+        _function: u8,
+        _offset: u16,
+    ) -> u32 {
+        0xFFFFFFFF
+    }
+
+    fn write_pci_u8(
+        &self,
+        _segment: u16,
+        _bus: u8,
+        _device: u8,
+        _function: u8,
+        _offset: u16,
+        _value: u8,
+    ) {
+    }
+
+    fn write_pci_u16(
+        &self,
+        _segment: u16,
+        _bus: u8,
+        _device: u8,
+        _function: u8,
+        _offset: u16,
+        _value: u16,
+    ) {
+    }
+
+    fn write_pci_u32(
+        &self,
+        _segment: u16,
+        _bus: u8,
+        _device: u8,
+        _function: u8,
+        _offset: u16,
+        _value: u32,
+    ) {
+    }
+
+    fn read_ec_u8(&self, _address: u64) -> u8 {
+        0x00
+    }
+
+    fn write_ec_u8(&self, _address: u64, _value: u8) {}
+
+    fn sleep(&self, _duration: Duration) {
+        todo!()
+        // util::polling_sleep(duration).unwrap();
+    }
+}

kernel/driver/acpi/src/handler.rs

@@ -0,0 +1,171 @@
+use core::{ptr::NonNull, time::Duration};
+
+use acpi::PhysicalMapping;
+use acpi_system::AcpiSystemError;
+use alloc::sync::Arc;
+use device_api::{
+    device::Device,
+    interrupt::{InterruptHandler, Irq},
+};
+use kernel_arch_x86::{intrinsics, ISA_IRQ_OFFSET};
+use libk::device::external_interrupt_controller;
+use libk_mm::{
+    address::{PhysicalAddress, Virtualize},
+    pointer::PhysicalRef,
+};
+
+use crate::{
+    mem::{read_memory, write_memory},
+    ACPI_SYSTEM,
+};
+
+#[derive(Clone, Copy)]
+#[doc(hidden)]
+pub struct AcpiHandlerImpl;
+
+struct SciHandler;
+
+impl acpi_system::Handler for AcpiHandlerImpl {
+    type MappedSlice = PhysicalRef<'static, [u8]>;
+
+    unsafe fn map_slice(address: u64, length: u64) -> Self::MappedSlice {
+        unsafe {
+            PhysicalRef::map_slice(
+                PhysicalAddress::from_u64(address),
+                length.try_into().unwrap(),
+            )
+        }
+    }
+
+    fn io_read_u8(port: u16) -> u8 {
+        let value = unsafe { intrinsics::inb(port) };
+        log::trace!("io_read_u8 {:#x} <- {:#x}", port, value);
+        value
+    }
+
+    fn io_read_u16(port: u16) -> u16 {
+        let value = unsafe { intrinsics::inw(port) };
+        log::trace!("io_read_u16 {:#x} <- {:#x}", port, value);
+        value
+    }
+
+    fn io_read_u32(port: u16) -> u32 {
+        let value = unsafe { intrinsics::inl(port) };
+        log::trace!("io_read_u32 {:#x} <- {:#x}", port, value);
+        value
+    }
+
+    fn io_write_u8(port: u16, value: u8) {
+        log::trace!("io_write_u8 {:#x}, {:#x}", port, value);
+        unsafe { intrinsics::outb(port, value) }
+    }
+
+    fn io_write_u16(port: u16, value: u16) {
+        log::trace!("io_write_u16 {:#x}, {:#x}", port, value);
+        unsafe { intrinsics::outw(port, value) }
+    }
+
+    fn io_write_u32(port: u16, value: u32) {
+        log::trace!("io_write_u32 {:#x}, {:#x}", port, value);
+        unsafe { intrinsics::outl(port, value) }
+    }
+
+    fn mem_read_u8(address: u64) -> u8 {
+        let value = unsafe { read_memory(PhysicalAddress::from_u64(address)) };
+        log::trace!("mem_read_u8 {:#x} -> {:#x}", address, value);
+        value
+    }
+
+    fn mem_read_u16(address: u64) -> u16 {
+        let value = unsafe { read_memory(PhysicalAddress::from_u64(address)) };
+        log::trace!("mem_read_u16 {:#x} -> {:#x}", address, value);
+        value
+    }
+
+    fn mem_read_u32(address: u64) -> u32 {
+        let value = unsafe { read_memory(PhysicalAddress::from_u64(address)) };
+        log::trace!("mem_read_u32 {:#x} -> {:#x}", address, value);
+        value
+    }
+
+    fn mem_read_u64(address: u64) -> u64 {
+        let value = unsafe { read_memory(PhysicalAddress::from_u64(address)) };
+        log::trace!("mem_read_u64 {:#x} -> {:#x}", address, value);
+        value
+    }
+
+    fn mem_write_u8(address: u64, value: u8) {
+        log::trace!("mem_write_u8 {:#x}, {:#x}", address, value);
+        unsafe { write_memory(PhysicalAddress::from_u64(address), value) }
+    }
+
+    fn mem_write_u16(address: u64, value: u16) {
+        log::trace!("mem_write_u16 {:#x}, {:#x}", address, value);
+        unsafe { write_memory(PhysicalAddress::from_u64(address), value) }
+    }
+
+    fn mem_write_u32(address: u64, value: u32) {
+        log::trace!("mem_write_u32 {:#x}, {:#x}", address, value);
+        unsafe { write_memory(PhysicalAddress::from_u64(address), value) }
+    }
+
+    fn mem_write_u64(address: u64, value: u64) {
+        log::trace!("mem_write_u64 {:#x}, {:#x}", address, value);
+        unsafe { write_memory(PhysicalAddress::from_u64(address), value) }
+    }
+
+    fn install_interrupt_handler(irq: u32) -> Result<(), AcpiSystemError> {
+        log::info!("Installing ACPI SCI handler at IRQ #{}", irq);
+
+        let intc = external_interrupt_controller().expect("No external intc");
+        let handler = Arc::new(SciHandler);
+        let irq = Irq::External(irq + ISA_IRQ_OFFSET);
+
+        intc.register_irq(irq, Default::default(), handler).unwrap();
+        intc.enable_irq(irq).unwrap();
+
+        Ok(())
+    }
+
+    fn stall(_duration: Duration) {
+        // TODO polling_sleep is not yet implemented properly
+        todo!()
+        // util::polling_sleep(duration).ok();
+    }
+}
+
+impl rsdp::handler::AcpiHandler for AcpiHandlerImpl {
+    unsafe fn map_physical_region<T>(
+        &self,
+        physical_address: usize,
+        size: usize,
+    ) -> PhysicalMapping<Self, T> {
+        unsafe {
+            PhysicalMapping::new(
+                physical_address,
+                NonNull::new_unchecked(
+                    PhysicalAddress::from_usize(physical_address).virtualize() as *mut T
+                ),
+                size,
+                size,
+                *self,
+            )
+        }
+    }
+
+    fn unmap_physical_region<T>(_region: &acpi::PhysicalMapping<Self, T>) {}
+}
+
+impl InterruptHandler for SciHandler {
+    fn handle_irq(self: Arc<Self>, _vector: Option<usize>) -> bool {
+        log::trace!("ACPI SCI received");
+        ACPI_SYSTEM.get().lock().handle_sci();
+        true
+    }
+}
+
+impl Device for SciHandler {
+    fn display_name(&self) -> &str {
+        "ACPI SCI handler"
+    }
+}

kernel/driver/acpi/src/lib.rs

@@ -0,0 +1,89 @@
+#![feature(allocator_api)]
+#![no_std]
+
+use acpi::AcpiTables;
+use acpi_system::{AcpiInterruptMethod, AcpiSystem};
+use alloc::boxed::Box;
+use libk::error::Error;
+use libk_util::{sync::IrqSafeSpinlock, OneTimeInit};
+
+extern crate alloc;
+
+pub mod mem;
+pub use mem::AcpiAllocator;
+pub mod handler;
+pub use handler::AcpiHandlerImpl;
+pub mod aml_handler;
+
+pub use acpi_system::{
+    EventAction, FixedEvent, InterruptPolarity, InterruptTrigger, IrqDescriptor, PciPin,
+};
+
+static ACPI_SYSTEM: OneTimeInit<IrqSafeSpinlock<AcpiSystem<AcpiHandlerImpl>>> = OneTimeInit::new();
+
+pub fn add_event_handler<F: Fn(&AcpiSystem<AcpiHandlerImpl>) -> EventAction + 'static>(
+    event: &FixedEvent,
+    handler: F,
+) -> Result<(), Error> {
+    ACPI_SYSTEM
+        .get()
+        .lock()
+        .enable_fixed_event(event, Box::new(handler))
+        .map_err(|_| Error::InvalidArgument)
+}
+
+pub fn get_pci_route(
+    aml_path: &str,
+    device: u16,
+    function: u16,
+    pin: PciPin,
+) -> Option<IrqDescriptor> {
+    ACPI_SYSTEM
+        .get()
+        .lock()
+        .pci_route(aml_path, device, function, pin)
+        .ok()
+}
+
+/// Initializes ACPI management
+pub fn switch_to_acpi(tables: &'static AcpiTables<AcpiHandlerImpl>) -> Result<(), Error> {
+    // NOTE mostly broken for real HW
+    let mut system = AcpiSystem::new(tables, Box::new(AcpiHandlerImpl)).unwrap();
+
+    system.initialize(AcpiInterruptMethod::Apic).unwrap();
+
+    // system
+    //     .enable_fixed_event(
+    //         &FixedEvent::POWER_BUTTON,
+    //         Box::new(|_| {
+    //             log::info!("Power button was pressed");
+
+    //             // TODO the correct way would be to
+    //             // 1. Nicely ask all the processes to quit
+    //             // 2. Wait for some time
+    //             // 3. Kill the remaining ones
+    //             // 4. Halt other cores
+    //             // 5. Sync filesystem
+    //             // 6. Do something with the devices
+    //             // 7. Actually enter the S5 state
+
+    //             unsafe {
+    //                 PLATFORM
+    //                     .send_ipi(IpiDeliveryTarget::OtherCpus, IpiMessage::Shutdown)
+    //                     .unwrap();
+    //             }
+
+    //             SHUTDOWN_FENCE.signal();
+    //             SHUTDOWN_FENCE.wait_all(CPU_COUNT.load(Ordering::Acquire));
+
+    //             log::info!("CPUs are parked, can shutdown now");
+
+    //             EventAction::EnterSleepState(AcpiSleepState::S5)
+    //         }),
+    //     )
+    //     .unwrap();
+
+    ACPI_SYSTEM.init(IrqSafeSpinlock::new(system));
+
+    Ok(())
+}

kernel/driver/acpi/src/mem.rs

@@ -0,0 +1,64 @@
+//! ACPI memory IO and management functions
+
+use core::{
+    alloc::{AllocError, Allocator, GlobalAlloc, Layout},
+    ptr::NonNull,
+};
+
+use libk_mm::{address::PhysicalAddress, device::DeviceMemoryMapping, heap::GLOBAL_HEAP};
+
+#[derive(Clone, Copy)]
+#[doc(hidden)]
+pub struct AcpiAllocator;
+
+unsafe impl Allocator for AcpiAllocator {
+    fn allocate(&self, layout: Layout) -> Result<NonNull<[u8]>, AllocError> {
+        let ptr = unsafe { GLOBAL_HEAP.alloc(layout) };
+        log::trace!("ACPI alloc: {:?} -> {:p}", layout, ptr);
+
+        if ptr.is_null() {
+            Err(AllocError)
+        } else {
+            unsafe {
+                Ok(NonNull::slice_from_raw_parts(
+                    NonNull::new_unchecked(ptr),
+                    layout.size(),
+                ))
+            }
+        }
+    }
+
+    unsafe fn deallocate(&self, ptr: NonNull<u8>, layout: Layout) {
+        log::trace!("ACPI dealloc: {:?}, {:?}", ptr, layout);
+        unsafe { GLOBAL_HEAP.dealloc(ptr.as_ptr(), layout) };
+    }
+}
+
+// TODO don't map memory as device if not necessary
+pub unsafe fn read_memory<T>(address: PhysicalAddress) -> T {
+    let io =
+        unsafe { DeviceMemoryMapping::map(address, size_of::<T>(), Default::default()).unwrap() };
+    let address = io.address();
+
+    unsafe {
+        if address % align_of::<T>() == 0 {
+            (address as *const T).read_volatile()
+        } else {
+            (address as *const T).read_unaligned()
+        }
+    }
+}
+
+pub unsafe fn write_memory<T>(address: PhysicalAddress, value: T) {
+    let io =
+        unsafe { DeviceMemoryMapping::map(address, size_of::<T>(), Default::default()).unwrap() };
+    let address = io.address();
+
+    unsafe {
+        if address % align_of::<T>() == 0 {
+            (address as *mut T).write_volatile(value)
+        } else {
+            (address as *mut T).write_unaligned(value)
+        }
+    }
+}

kernel/driver/block/ahci/src/lib.rs

@@ -20,6 +20,7 @@ use regs::{PortRegs, Regs};
 use tock_registers::interfaces::{ReadWriteable, Readable, Writeable};
 use ygg_driver_pci::{
     device::{PciDeviceInfo, PreferredInterruptMode},
+    macros::pci_driver,
     PciCommandRegister, PciConfigurationSpace,
 };
 use yggdrasil_abi::{error::Error, io::FileMode};
@@ -191,51 +192,6 @@ impl Device for AhciController {
     }
 }
 
-static SATA_DRIVES: IrqSafeSpinlock<Vec<Arc<AhciPort>>> = IrqSafeSpinlock::new(Vec::new());
-
-pub fn probe(info: &PciDeviceInfo) -> Result<Arc<dyn Device>, Error> {
-    let bar5 = info.config_space.bar(5).ok_or(Error::InvalidOperation)?;
-    let bar5 = bar5.as_memory().ok_or(Error::InvalidOperation)?;
-
-    let mut cmd = PciCommandRegister::from_bits_retain(info.config_space.command());
-    cmd &= !(PciCommandRegister::DISABLE_INTERRUPTS | PciCommandRegister::ENABLE_IO);
-    cmd |= PciCommandRegister::ENABLE_MEMORY | PciCommandRegister::BUS_MASTER;
-    info.config_space.set_command(cmd.bits());
-
-    info.init_interrupts(PreferredInterruptMode::Msi)?;
-
-    // // TODO support regular PCI interrupts (ACPI dependency)
-    // let Some(mut msi) = info.config_space.capability::<MsiCapability>() else {
-    //     log::warn!("Ignoring AHCI: does not support MSI (and the OS doesn't yet support PCI IRQ)");
-    //     return Err(Error::InvalidOperation);
-    // };
-
-    // Map the registers
-    let regs = unsafe { DeviceMemoryIo::<Regs>::map(bar5, Default::default()) }?;
-
-    let version = Version::try_from(regs.VS.get())?;
-    let ahci_only = regs.CAP.matches_all(CAP::SAM::SET);
-    let max_port_count = regs.CAP.read(CAP::NP) as usize;
-    let has_64_bit = regs.CAP.matches_all(CAP::S64A::SET);
-
-    // TODO extract Number of Command Slots
-
-    let ahci = Arc::new(AhciController {
-        regs: IrqSafeSpinlock::new(regs),
-        ports: OneTimeInit::new(),
-        received_fis_buffers: OneTimeInit::new(),
-        version,
-        max_port_count,
-        ahci_only,
-        has_64_bit,
-    });
-
-    // TODO use multiple vectors if capable
-    info.map_interrupt(InterruptAffinity::Any, ahci.clone())?;
-
-    Ok(ahci)
-}
-
 pub fn register_sata_drive(drive: Arc<AhciPort>, probe: bool) {
     let index = {
         let mut drives = SATA_DRIVES.lock();
@@ -273,3 +229,57 @@ pub fn register_sata_drive(drive: Arc<AhciPort>, probe: bool) {
         .ok();
     }
 }
+
+static SATA_DRIVES: IrqSafeSpinlock<Vec<Arc<AhciPort>>> = IrqSafeSpinlock::new(Vec::new());
+
+pci_driver! {
+    matches: [class (0x01:0x06:0x01)],
+    driver: {
+        fn driver_name(&self) -> &str {
+            "ahci"
+        }
+
+        fn probe(&self, info: &PciDeviceInfo) -> Result<Arc<dyn Device>, Error> {
+            let bar5 = info.config_space.bar(5).ok_or(Error::InvalidOperation)?;
+            let bar5 = bar5.as_memory().ok_or(Error::InvalidOperation)?;
+
+            let mut cmd = PciCommandRegister::from_bits_retain(info.config_space.command());
+            cmd &= !(PciCommandRegister::DISABLE_INTERRUPTS | PciCommandRegister::ENABLE_IO);
+            cmd |= PciCommandRegister::ENABLE_MEMORY | PciCommandRegister::BUS_MASTER;
+            info.config_space.set_command(cmd.bits());
+
+            info.init_interrupts(PreferredInterruptMode::Msi(true))?;
+
+            // // TODO support regular PCI interrupts (ACPI dependency)
+            // let Some(mut msi) = info.config_space.capability::<MsiCapability>() else {
+            //     log::warn!("Ignoring AHCI: does not support MSI (and the OS doesn't yet support PCI IRQ)");
+            //     return Err(Error::InvalidOperation);
+            // };
+
+            // Map the registers
+            let regs = unsafe { DeviceMemoryIo::<Regs>::map(bar5, Default::default()) }?;
+
+            let version = Version::try_from(regs.VS.get())?;
+            let ahci_only = regs.CAP.matches_all(CAP::SAM::SET);
+            let max_port_count = regs.CAP.read(CAP::NP) as usize;
+            let has_64_bit = regs.CAP.matches_all(CAP::S64A::SET);
+
+            // TODO extract Number of Command Slots
+
+            let ahci = Arc::new(AhciController {
+                regs: IrqSafeSpinlock::new(regs),
+                ports: OneTimeInit::new(),
+                received_fis_buffers: OneTimeInit::new(),
+                version,
+                max_port_count,
+                ahci_only,
+                has_64_bit,
+            });
+
+            // TODO use multiple vectors if capable
+            info.map_interrupt(InterruptAffinity::Any, ahci.clone())?;
+
+            Ok(ahci)
+        }
+    }
+}

kernel/driver/block/nvme/src/lib.rs

@@ -1,6 +1,8 @@
 #![feature(const_trait_impl, let_chains, if_let_guard, maybe_uninit_slice)]
 #![allow(missing_docs)]
 #![no_std]
+// TODO
+#![allow(unused)]
 
 extern crate alloc;
 
@@ -35,6 +37,7 @@ use tock_registers::{
 };
 use ygg_driver_pci::{
     device::{PciDeviceInfo, PreferredInterruptMode},
+    macros::pci_driver,
     PciCommandRegister, PciConfigurationSpace,
 };
 use yggdrasil_abi::{error::Error, io::FileMode};
@@ -417,54 +420,6 @@ impl Device for NvmeController {
 // TODO
 unsafe impl Sync for NvmeController {}
 
-static NVME_CONTROLLERS: IrqSafeSpinlock<Vec<Arc<NvmeController>>> =
-    IrqSafeSpinlock::new(Vec::new());
-
-pub fn probe(info: &PciDeviceInfo) -> Result<Arc<dyn Device>, Error> {
-    let bar0 = info
-        .config_space
-        .bar(0)
-        .unwrap()
-        .as_memory()
-        .expect("Expected a memory BAR0");
-
-    info.init_interrupts(PreferredInterruptMode::Msi)?;
-
-    let mut cmd = PciCommandRegister::from_bits_retain(info.config_space.command());
-    cmd &= !(PciCommandRegister::DISABLE_INTERRUPTS | PciCommandRegister::ENABLE_IO);
-    cmd |= PciCommandRegister::ENABLE_MEMORY | PciCommandRegister::BUS_MASTER;
-    info.config_space.set_command(cmd.bits());
-
-    let regs = unsafe { DeviceMemoryIo::<Regs>::map(bar0, Default::default()) }?;
-
-    // Disable the controller
-    regs.CC.modify(CC::ENABLE::CLEAR);
-
-    let doorbell_shift = regs.CAP.read(CAP::DSTRD) as usize + 1;
-    let min_page_size = 1 << (regs.CAP.read(CAP::MPSMIN) + 12);
-
-    if min_page_size > PAGE_SIZE {
-        log::error!("Cannot support NVMe HC: min page size ({min_page_size}) > host page size ({PAGE_SIZE})");
-        return Err(Error::InvalidArgument);
-    }
-
-    let device = NvmeController {
-        regs: IrqSafeSpinlock::new(regs),
-        admin_q: OneTimeInit::new(),
-        ioqs: OneTimeInit::new(),
-        drive_table: IrqSafeSpinlock::new(BTreeMap::new()),
-        controller_id: OneTimeInit::new(),
-
-        pci: info.clone(),
-
-        io_queue_count: AtomicUsize::new(1),
-        doorbell_shift,
-        min_page_size,
-    };
-
-    Ok(Arc::new(device))
-}
-
 pub fn register_nvme_controller(controller: Arc<NvmeController>) {
     let mut list = NVME_CONTROLLERS.lock();
     let id = list.len();
@@ -496,3 +451,60 @@ pub fn register_nvme_namespace(namespace: Arc<NvmeNamespace>, probe: bool) {
         .ok();
     }
 }
+
+static NVME_CONTROLLERS: IrqSafeSpinlock<Vec<Arc<NvmeController>>> =
+    IrqSafeSpinlock::new(Vec::new());
+
+pci_driver! {
+    matches: [class (0x01:0x08:0x02)],
+    driver: {
+        fn driver_name(&self) -> &str {
+            "nvme"
+        }
+
+        fn probe(&self, info: &PciDeviceInfo) -> Result<Arc<dyn Device>, Error> {
+            let bar0 = info
+                .config_space
+                .bar(0)
+                .unwrap()
+                .as_memory()
+                .expect("Expected a memory BAR0");
+
+            info.init_interrupts(PreferredInterruptMode::Msi(true))?;
+
+            let mut cmd = PciCommandRegister::from_bits_retain(info.config_space.command());
+            cmd &= !(PciCommandRegister::DISABLE_INTERRUPTS | PciCommandRegister::ENABLE_IO);
+            cmd |= PciCommandRegister::ENABLE_MEMORY | PciCommandRegister::BUS_MASTER;
+            info.config_space.set_command(cmd.bits());
+
+            let regs = unsafe { DeviceMemoryIo::<Regs>::map(bar0, Default::default()) }?;
+
+            // Disable the controller
+            regs.CC.modify(CC::ENABLE::CLEAR);
+
+            let doorbell_shift = regs.CAP.read(CAP::DSTRD) as usize + 1;
+            let min_page_size = 1 << (regs.CAP.read(CAP::MPSMIN) + 12);
+
+            if min_page_size > PAGE_SIZE {
+                log::error!("Cannot support NVMe HC: min page size ({min_page_size}) > host page size ({PAGE_SIZE})");
+                return Err(Error::InvalidArgument);
+            }
+
+            let device = NvmeController {
+                regs: IrqSafeSpinlock::new(regs),
+                admin_q: OneTimeInit::new(),
+                ioqs: OneTimeInit::new(),
+                drive_table: IrqSafeSpinlock::new(BTreeMap::new()),
+                controller_id: OneTimeInit::new(),
+
+                pci: info.clone(),
+
+                io_queue_count: AtomicUsize::new(1),
+                doorbell_shift,
+                min_page_size,
+            };
+
+            Ok(Arc::new(device))
+        }
+    }
+}

kernel/driver/block/scsi/Cargo.toml

@@ -0,0 +1,14 @@
+[package]
+name = "ygg_driver_scsi"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+yggdrasil-abi.workspace = true
+device-api.workspace = true
+libk-util.workspace = true
+libk-mm.workspace = true
+libk.workspace = true
+
+async-trait.workspace = true
+log.workspace = true

kernel/driver/block/scsi/src/command.rs

@@ -0,0 +1,102 @@
+use libk::error::Error;
+
+use crate::device::ScsiDeviceType;
+
+pub trait ScsiCommand {
+    type Response;
+    const REQUEST_LEN: usize;
+    const RESPONSE_LEN: usize;
+
+    fn into_bytes(self) -> [u8; Self::REQUEST_LEN];
+    fn parse_response(bytes: &[u8]) -> Result<Self::Response, Error>;
+}
+
+// Add more info when needed
+pub struct ScsiInquiry;
+#[derive(Debug)]
+pub struct ScsiInquiryResponse {
+    pub device_type: ScsiDeviceType,
+}
+
+impl ScsiCommand for ScsiInquiry {
+    type Response = ScsiInquiryResponse;
+    const REQUEST_LEN: usize = 6;
+    const RESPONSE_LEN: usize = 36;
+
+    fn into_bytes(self) -> [u8; Self::REQUEST_LEN] {
+        [0x12, 0x00, 0x00, 0x00, 0x00, 0x00]
+    }
+
+    fn parse_response(bytes: &[u8]) -> Result<Self::Response, Error> {
+        if bytes.len() != 36 {
+            return Err(Error::InvalidArgument);
+        }
+        let device_type = ScsiDeviceType::try_from(bytes[0] & 0x1F).unwrap_or_default();
+        Ok(ScsiInquiryResponse { device_type })
+    }
+}
+
+pub struct ScsiTestUnitReady;
+#[derive(Debug)]
+pub struct ScsiTestUnitReadyResponse;
+
+impl ScsiCommand for ScsiTestUnitReady {
+    type Response = ScsiTestUnitReadyResponse;
+    const RESPONSE_LEN: usize = 0;
+    const REQUEST_LEN: usize = 6;
+
+    fn into_bytes(self) -> [u8; Self::REQUEST_LEN] {
+        [0x00, 0x00, 0x00, 0x00, 0x00, 0x00]
+    }
+
+    fn parse_response(_bytes: &[u8]) -> Result<Self::Response, Error> {
+        Ok(ScsiTestUnitReadyResponse)
+    }
+}
+
+pub struct ScsiRequestSense;
+#[derive(Debug)]
+pub struct ScsiRequestSenseResponse;
+
+impl ScsiCommand for ScsiRequestSense {
+    type Response = ScsiRequestSenseResponse;
+    const RESPONSE_LEN: usize = 0;
+    const REQUEST_LEN: usize = 6;
+
+    fn into_bytes(self) -> [u8; Self::REQUEST_LEN] {
+        [0x03, 0x00, 0x00, 0x00, 0x00, 0x00]
+    }
+
+    fn parse_response(_bytes: &[u8]) -> Result<Self::Response, Error> {
+        Ok(ScsiRequestSenseResponse)
+    }
+}
+
+pub struct ScsiReadCapacity;
+#[derive(Debug)]
+pub struct ScsiReadCapacityResponse {
+    pub block_size: u32,
+    pub block_count: u32,
+}
+
+impl ScsiCommand for ScsiReadCapacity {
+    type Response = ScsiReadCapacityResponse;
+    const REQUEST_LEN: usize = 10;
+    const RESPONSE_LEN: usize = 8;
+
+    fn into_bytes(self) -> [u8; Self::REQUEST_LEN] {
+        [0x25, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]
+    }
+
+    fn parse_response(bytes: &[u8]) -> Result<Self::Response, Error> {
+        if bytes.len() != 8 {
+            return Err(Error::InvalidArgument);
+        }
+        let block_count = u32::from_be_bytes(bytes[0..4].try_into().unwrap());
+        let block_size = u32::from_be_bytes(bytes[4..8].try_into().unwrap());
+        Ok(ScsiReadCapacityResponse {
+            block_size,
+            block_count,
+        })
+    }
+}

kernel/driver/block/scsi/src/device.rs

@@ -0,0 +1,24 @@
+use yggdrasil_abi::primitive_enum;
+
+primitive_enum! {
+    #[derive(Default)]
+    pub enum ScsiDeviceType: u8 {
+        DirectAccessBlock = 0x00,
+        SequentialAccess = 0x01,
+        Printer = 0x02,
+        Processor = 0x03,
+        WriteOnce = 0x04,
+        CdDvd = 0x05,
+        OpticalMemory = 0x07,
+        MediumChanger = 0x08,
+        StorageArrayController = 0x0C,
+        EnclosureServices = 0x0D,
+        SimplifiedDirectAccess = 0x0E,
+        OpticalCard = 0x0F,
+        BridgeController = 0x10,
+        ObjectBasedStorage = 0x11,
+        AutomationDriveInterface = 0x12,
+        #[default]
+        Other = 0x1F,
+    }
+}

kernel/driver/block/scsi/src/lib.rs

@@ -0,0 +1,248 @@
+#![feature(generic_const_exprs, maybe_uninit_slice)]
+#![allow(incomplete_features)]
+#![no_std]
+
+use core::{mem::MaybeUninit, time::Duration};
+
+use alloc::{
+    boxed::Box, collections::btree_map::BTreeMap, format, string::String, sync::Arc, vec::Vec,
+};
+use async_trait::async_trait;
+use command::{ScsiReadCapacity, ScsiRequestSense, ScsiTestUnitReady};
+use device_api::device::Device;
+use libk::{
+    device::{block::BlockDevice, manager::probe_partitions},
+    error::Error,
+    fs::devfs,
+    task::runtime,
+};
+use libk_mm::{address::PhysicalAddress, table::MapAttributes, PageProvider, PageSlice};
+use libk_util::{
+    sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock},
+    OneTimeInit,
+};
+use transport::{ScsiTransport, ScsiTransportWrapper};
+use yggdrasil_abi::io::FileMode;
+
+extern crate alloc;
+
+pub mod command;
+pub mod device;
+pub mod transport;
+
+// TODO SCSI detach
+pub struct ScsiDevice {
+    transport: IrqSafeSpinlock<ScsiTransportWrapper>,
+    lba_count: u64,
+    lba_size: usize,
+    index: OneTimeInit<u32>,
+    names: IrqSafeRwLock<Vec<String>>,
+}
+
+impl ScsiDevice {
+    // TODO support LUNs other than 0
+    pub async fn setup<T: ScsiTransport + 'static>(transport: T) -> Result<Arc<Self>, Error> {
+        let mut transport = ScsiTransportWrapper::new(transport);
+
+        let mut attempts = 5;
+        let mut timeout = 100;
+        while attempts > 0 {
+            // TEST UNIT READY (6)
+            if transport
+                .perform_command(0, ScsiTestUnitReady)
+                .await
+                .is_ok()
+            {
+                break;
+            }
+
+            // If not, send a REQUEST SENSE (6)
+            if transport.perform_command(0, ScsiRequestSense).await.is_ok() {
+                break;
+            }
+
+            log::warn!("scsi: unit not ready [{attempts}/5]");
+
+            runtime::sleep(Duration::from_millis(timeout)).await;
+            timeout *= 2;
+            attempts -= 1;
+        }
+        if attempts == 0 {
+            log::error!("scsi: unit not ready");
+            return Err(Error::DoesNotExist);
+        }
+
+        // TODO INQUIRY fails for real USB flash drives
+        // transport.perform_command(0, ScsiInquiry).await?;
+
+        let capacity_info = transport.perform_command(0, ScsiReadCapacity).await?;
+        log::info!(
+            "scsi: lba_size={}, lba_count={}",
+            capacity_info.block_size,
+            capacity_info.block_count
+        );
+
+        Ok(Arc::new(Self {
+            transport: IrqSafeSpinlock::new(transport),
+            lba_count: capacity_info.block_count.into(),
+            lba_size: capacity_info.block_size as usize,
+            index: OneTimeInit::new(),
+            names: IrqSafeRwLock::new(Vec::new()),
+        }))
+    }
+
+    pub fn detach(&self) {
+        if let Some(&index) = self.index.try_get() {
+            detach(index);
+        }
+    }
+}
+
+#[async_trait]
+impl BlockDevice for ScsiDevice {
+    async fn read_aligned(
+        &self,
+        position: u64,
+        buffer: &mut PageSlice<MaybeUninit<u8>>,
+    ) -> Result<(), Error> {
+        if buffer.len() % self.lba_size != 0 {
+            return Err(Error::InvalidArgument);
+        }
+        let lba_start = position / self.lba_size as u64;
+        let lba_count = buffer.len() / self.lba_size;
+        if lba_start.saturating_add(lba_count as u64) >= self.lba_count {
+            return Err(Error::InvalidArgument);
+        }
+        log::info!("scsi: read lba={lba_start}, count={lba_count}");
+        let mut transport = self.transport.lock();
+        for i in 0..lba_count {
+            let lba = lba_start + i as u64;
+            let offset = self.lba_size * i;
+            let slice = unsafe {
+                MaybeUninit::slice_assume_init_mut(&mut buffer[offset..offset + self.lba_size])
+            };
+
+            let len = transport.read(0, lba, slice).await?;
+            if len != self.lba_size {
+                log::warn!("scsi: truncated read received at lba {lba}");
+                return Err(Error::InvalidOperation);
+            }
+        }
+        Ok(())
+    }
+
+    async fn write_aligned(&self, _position: u64, _buffer: &PageSlice<u8>) -> Result<(), Error> {
+        // TODO AtaWriteDmaEx
+        Err(Error::NotImplemented)
+    }
+
+    fn block_size(&self) -> usize {
+        self.lba_size
+    }
+
+    fn block_count(&self) -> u64 {
+        self.lba_count
+    }
+
+    fn max_blocks_per_request(&self) -> usize {
+        65536 / self.lba_size
+    }
+}
+
+impl PageProvider for ScsiDevice {
+    fn get_page(&self, _offset: u64) -> Result<PhysicalAddress, Error> {
+        Err(Error::NotImplemented)
+    }
+
+    fn release_page(&self, _offset: u64, _phys: PhysicalAddress) -> Result<(), Error> {
+        Err(Error::NotImplemented)
+    }
+
+    fn clone_page(
+        &self,
+        _offset: u64,
+        _src_phys: PhysicalAddress,
+        _src_attrs: MapAttributes,
+    ) -> Result<PhysicalAddress, Error> {
+        Err(Error::NotImplemented)
+    }
+}
+
+impl Device for ScsiDevice {
+    fn display_name(&self) -> &str {
+        "SCSI Storage Device"
+    }
+}
+
+impl Drop for ScsiDevice {
+    fn drop(&mut self) {
+        if let Some(index) = self.index.try_get() {
+            log::info!("scsi{index} dropped");
+        }
+    }
+}
+
+// TODO this is crap
+static SCSI_DEVICES: IrqSafeSpinlock<BTreeMap<u32, Arc<ScsiDevice>>> =
+    IrqSafeSpinlock::new(BTreeMap::new());
+static SCSI_BITMAP: IrqSafeSpinlock<u32> = IrqSafeSpinlock::new(0);
+
+pub fn attach(device: Arc<ScsiDevice>) -> Result<(), Error> {
+    let index = {
+        let mut bitmap = SCSI_BITMAP.lock();
+        let index = (0..8)
+            .position(|p| *bitmap & (1 << p) == 0)
+            .ok_or(Error::InvalidOperation)
+            .inspect_err(|_| log::warn!("Cannot attach SCSI device: too many of them"))?
+            as u32;
+        let mut devices = SCSI_DEVICES.lock();
+        *bitmap |= 1 << index;
+        assert!(!devices.contains_key(&index));
+        devices.insert(index, device.clone());
+        index
+    };
+
+    let name = format!("scsi{index}");
+    device.index.init(index);
+    device.names.write().push(name.clone());
+    devfs::add_named_block_device(device.clone(), name.clone(), FileMode::new(0o600)).ok();
+    log::info!("{name} attached");
+
+    // TODO this code is repeated everywhere
+    runtime::spawn(async move {
+        let name = name;
+        probe_partitions(device.clone(), |index, partition| {
+            let partition_name = format!("{name}p{}", index + 1);
+            log::info!("{name}: partition {partition_name}");
+            device.names.write().push(partition_name.clone());
+            devfs::add_named_block_device(
+                Arc::new(partition),
+                partition_name,
+                FileMode::new(0o600),
+            )
+            .ok();
+        })
+        .await
+        .ok();
+    })
+    .ok();
+
+    Ok(())
+}
+
+pub fn detach(index: u32) {
+    let mut devices = SCSI_DEVICES.lock();
+    let mut bitmap = SCSI_BITMAP.lock();
+
+    if let Some(device) = devices.remove(&index) {
+        {
+            let names = device.names.read();
+            for name in names.iter() {
+                devfs::remove_node(name).ok();
+            }
+        }
+
+        *bitmap &= !(1 << index);
+        log::info!("scsi{index} detached");
+    }
+}

kernel/driver/block/scsi/src/transport.rs

@@ -0,0 +1,72 @@
+use alloc::boxed::Box;
+use async_trait::async_trait;
+use libk::error::Error;
+
+use crate::command::ScsiCommand;
+
+#[async_trait]
+pub trait ScsiTransport: Send + Sync {
+    /// Perform a no-data request
+    async fn perform_request_raw(
+        &mut self,
+        lun: u8,
+        request_data: &[u8],
+        response_buffer: &mut [u8],
+    ) -> Result<usize, Error>;
+}
+
+pub struct ScsiTransportWrapper {
+    inner: Box<dyn ScsiTransport>,
+}
+
+impl ScsiTransportWrapper {
+    pub fn new<T: ScsiTransport + 'static>(inner: T) -> Self {
+        Self {
+            inner: Box::new(inner),
+        }
+    }
+
+    pub async fn read(&mut self, lun: u8, lba: u64, buffer: &mut [u8]) -> Result<usize, Error> {
+        if lba >= u32::MAX as u64 || buffer.len() > u16::MAX as usize {
+            return Err(Error::InvalidArgument);
+        }
+        let lba_bytes = (lba as u32).to_be_bytes();
+        // Issue a READ (10) command
+        let request_buffer = [
+            0x28,
+            0x00,
+            lba_bytes[0],
+            lba_bytes[1],
+            lba_bytes[2],
+            lba_bytes[3],
+            0x00,
+            0x00,
+            0x01,
+            0x00,
+        ];
+
+        self.inner
+            .perform_request_raw(lun, &request_buffer, buffer)
+            .await
+    }
+
+    pub async fn perform_command<R: ScsiCommand>(
+        &mut self,
+        lun: u8,
+        request: R,
+    ) -> Result<R::Response, Error>
+    where
+        [u8; R::RESPONSE_LEN]: Sized,
+        [u8; R::REQUEST_LEN]: Sized,
+    {
+        let mut response_buffer = [0; R::RESPONSE_LEN];
+        let request_buffer = request.into_bytes();
+
+        let response_len = self
+            .inner
+            .perform_request_raw(lun, &request_buffer, &mut response_buffer)
+            .await?;
+
+        R::parse_response(&response_buffer[..response_len])
+    }
+}

kernel/driver/bus/pci/Cargo.toml

@@ -16,7 +16,9 @@ bitflags.workspace = true
 tock-registers.workspace = true
 
 [target.'cfg(target_arch = "x86_64")'.dependencies]
+ygg_driver_acpi.path = "../../acpi"
 acpi.workspace = true
+kernel-arch-x86.workspace = true
 
 [lints]
 workspace = true

kernel/driver/bus/pci/src/capability.rs

@@ -1,9 +1,12 @@
 //! PCI capability structures and queries
 
+use core::mem::offset_of;
+
 use alloc::{sync::Arc, vec, vec::Vec};
 use device_api::interrupt::{
     InterruptAffinity, InterruptHandler, MessageInterruptController, MsiInfo,
 };
+use kernel_arch_x86::intrinsics;
 use libk_mm::{address::PhysicalAddress, device::DeviceMemoryIoMut};
 use tock_registers::{
     interfaces::{Readable, Writeable},
@@ -11,6 +14,8 @@ use tock_registers::{
 };
 use yggdrasil_abi::error::Error;
 
+use crate::PciBaseAddress;
+
 use super::{PciCapability, PciCapabilityId, PciConfigurationSpace};
 
 pub trait VirtioCapabilityData<'s, S: PciConfigurationSpace + ?Sized + 's>: Sized {
@@ -41,6 +46,9 @@ pub trait VirtioCapability {
     type Output<'a, S: PciConfigurationSpace + ?Sized + 'a>: VirtioCapabilityData<'a, S>;
 }
 
+/// Power management capability entry
+pub struct PowerManagementCapability;
+
 /// MSI-X capability query
 pub struct MsiXCapability;
 
@@ -57,6 +65,15 @@ pub struct VirtioNotifyConfigCapability;
 /// VirtIO interrupt status
 pub struct VirtioInterruptStatusCapability;
 
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum DevicePowerState {
+    D0,
+    D1,
+    D2,
+    D3Cold,
+    D3Hot,
+}
+
 /// Represents an entry in MSI-X vector table
 #[repr(C)]
 pub struct MsiXEntry {
@@ -68,8 +85,20 @@ pub struct MsiXEntry {
     pub control: ReadWrite<u32>,
 }
 
+enum MsiXVectorTableAccess<'a> {
+    Memory(DeviceMemoryIoMut<'a, [MsiXEntry]>),
+    Io(u16),
+}
+
 pub struct MsiXVectorTable<'a> {
-    vectors: DeviceMemoryIoMut<'a, [MsiXEntry]>,
+    access: MsiXVectorTableAccess<'a>,
+    len: usize,
+}
+
+/// PCI Power Management capability data structure
+pub struct PowerManagementData<'s, S: PciConfigurationSpace + ?Sized + 's> {
+    space: &'s S,
+    offset: usize,
 }
 
 /// MSI-X capability data structure
@@ -122,6 +151,19 @@ impl<T: VirtioCapability> PciCapability for T {
     }
 }
 
+impl PciCapability for PowerManagementCapability {
+    const ID: PciCapabilityId = PciCapabilityId::PowerManagement;
+    type CapabilityData<'a, S: PciConfigurationSpace + ?Sized + 'a> = PowerManagementData<'a, S>;
+
+    fn data<'s, S: PciConfigurationSpace + ?Sized + 's>(
+        space: &'s S,
+        offset: usize,
+        _len: usize,
+    ) -> Self::CapabilityData<'s, S> {
+        PowerManagementData { space, offset }
+    }
+}
+
 impl PciCapability for MsiXCapability {
     const ID: PciCapabilityId = PciCapabilityId::MsiX;
     type CapabilityData<'a, S: PciConfigurationSpace + ?Sized + 'a> = MsiXData<'a, S>;
@@ -246,6 +288,56 @@ impl<'s, S: PciConfigurationSpace + ?Sized + 's> VirtioCapabilityData<'s, S>
     }
 }
 
+impl<'s, S: PciConfigurationSpace + ?Sized + 's> PowerManagementData<'s, S> {
+    pub fn set_device_power_state(&self, state: DevicePowerState) {
+        let pmcsr = self.space.read_u16(self.offset + 4) & !0x3;
+        let current = self.get_device_power_state();
+
+        if state == current {
+            return;
+        }
+
+        log::info!("Set device power state: {state:?}");
+
+        match state {
+            DevicePowerState::D0 => {
+                // power = 0b00 | PME_EN
+                self.space.write_u16(self.offset + 4, pmcsr);
+            }
+            _ => {
+                log::warn!("TODO: {state:?} power state");
+            }
+        }
+    }
+
+    pub fn set_pme_en(&self, state: bool) {
+        let pmcsr = self.space.read_u16(self.offset + 4);
+        let new = if state {
+            pmcsr | (1 << 8)
+        } else {
+            pmcsr & !(1 << 8)
+        };
+        if pmcsr == new {
+            return;
+        }
+
+        log::info!("Set PMCSR.PME_En = {state}");
+
+        self.space.write_u16(self.offset + 4, new);
+    }
+
+    pub fn get_device_power_state(&self) -> DevicePowerState {
+        let pmcsr = self.space.read_u16(self.offset + 4);
+        match pmcsr & 0x3 {
+            0b00 => DevicePowerState::D0,
+            0b01 => DevicePowerState::D1,
+            0b10 => DevicePowerState::D2,
+            0b11 => DevicePowerState::D3Hot,
+            _ => unreachable!(),
+        }
+    }
+}
+
 impl<'s, S: PciConfigurationSpace + ?Sized + 's> MsiXData<'s, S> {
     // TODO use pending bits as well
     /// Maps and returns the vector table associated with the device's MSI-X capability
@@ -260,13 +352,27 @@ impl<'s, S: PciConfigurationSpace + ?Sized + 's> MsiXData<'s, S> {
         let Some(base) = self.space.bar(bir) else {
             return Err(Error::DoesNotExist);
         };
-        let Some(base) = base.as_memory() else {
-            return Err(Error::InvalidOperation);
-        };
 
-        log::debug!("MSI-X table address: {:#x}", base.add(table_offset));
+        match base {
-
+            PciBaseAddress::Memory32(mem32) => unsafe {
-        unsafe { MsiXVectorTable::from_raw_parts(base.add(table_offset), table_size) }
+                log::info!("MSI-X table address: {:#x}", mem32 + table_offset as u32);
+                MsiXVectorTable::memory_from_raw_parts(
+                    PhysicalAddress::from_u32(mem32).add(table_offset),
+                    table_size,
+                )
+            },
+            PciBaseAddress::Memory64(mem64) => unsafe {
+                log::info!("MSI-X table address: {:#x}", mem64 + table_offset as u64);
+                MsiXVectorTable::memory_from_raw_parts(
+                    PhysicalAddress::from_u64(mem64).add(table_offset),
+                    table_size,
+                )
+            },
+            PciBaseAddress::Io(io) => unsafe {
+                log::info!("MSI-X table I/O: {:#x}", io + table_offset as u16);
+                MsiXVectorTable::io_from_raw_parts(io + table_offset as u16, table_size)
+            },
+        }
     }
 
     /// Changes the global enable status for the device's MSI-X capability. If set, regular IRQs
@@ -292,15 +398,79 @@ impl<'s, S: PciConfigurationSpace + ?Sized + 's> MsiXData<'s, S> {
     }
 }
 
+impl MsiXVectorTableAccess<'_> {
+    fn set_vector_masked(&mut self, vector: usize, masked: bool) {
+        let old = self.read_control(vector);
+        let new = if masked { old | 1 } else { old & !1 };
+        if old != new {
+            self.write_control(vector, new);
+        }
+    }
+
+    fn read_control(&mut self, vector: usize) -> u32 {
+        match self {
+            &mut Self::Io(base) => unsafe {
+                let a = base
+                    + (vector * size_of::<MsiXEntry>() + offset_of!(MsiXEntry, control)) as u16;
+                intrinsics::inl(a)
+            },
+            Self::Memory(vectors) => vectors[vector].control.get(),
+        }
+    }
+
+    fn write_address(&mut self, vector: usize, value: u64) {
+        match self {
+            &mut Self::Io(base) => unsafe {
+                let a = base + (vector * size_of::<MsiXEntry>()) as u16;
+                intrinsics::outl(a, value as u32);
+                intrinsics::outl(a + 4, (value >> 32) as u32);
+            },
+            Self::Memory(vectors) => vectors[vector].address.set(value),
+        }
+    }
+
+    fn write_data(&mut self, vector: usize, value: u32) {
+        match self {
+            &mut Self::Io(base) => unsafe {
+                let a =
+                    base + (vector * size_of::<MsiXEntry>() + offset_of!(MsiXEntry, data)) as u16;
+                intrinsics::outl(a, value)
+            },
+            Self::Memory(vectors) => vectors[vector].data.set(value),
+        }
+    }
+
+    fn write_control(&mut self, vector: usize, value: u32) {
+        match self {
+            &mut Self::Io(base) => unsafe {
+                let a = base
+                    + (vector * size_of::<MsiXEntry>() + offset_of!(MsiXEntry, control)) as u16;
+                intrinsics::outl(a, value)
+            },
+            Self::Memory(vectors) => vectors[vector].control.set(value),
+        }
+    }
+}
+
 impl MsiXVectorTable<'_> {
-    unsafe fn from_raw_parts(base: PhysicalAddress, len: usize) -> Result<Self, Error> {
+    unsafe fn memory_from_raw_parts(base: PhysicalAddress, len: usize) -> Result<Self, Error> {
         let vectors = DeviceMemoryIoMut::map_slice(base, len, Default::default())?;
-        Ok(Self { vectors })
+        Ok(Self {
+            access: MsiXVectorTableAccess::Memory(vectors),
+            len,
+        })
+    }
+
+    unsafe fn io_from_raw_parts(base: u16, len: usize) -> Result<Self, Error> {
+        Ok(Self {
+            access: MsiXVectorTableAccess::Io(base),
+            len,
+        })
     }
 
     pub fn mask_all(&mut self) {
-        for vector in self.vectors.iter_mut() {
+        for i in 0..self.len {
-            vector.set_masked(true);
+            self.access.set_vector_masked(i, true);
         }
     }
 
@@ -324,26 +494,15 @@ impl MsiXVectorTable<'_> {
 
         for (i, info) in range.iter().enumerate() {
             let index = i + start;
-            self.vectors[index].address.set(info.address as _);
+            self.access.write_address(index, info.address as _);
-            self.vectors[index].data.set(info.value);
+            self.access.write_data(index, info.value);
-            self.vectors[index].set_masked(false);
+            self.access.set_vector_masked(index, false);
         }
 
         Ok(range)
     }
 }
 
-impl MsiXEntry {
-    /// If set, prevents the MSI-X interrupt from being delivered
-    fn set_masked(&mut self, masked: bool) {
-        if masked {
-            self.control.set(self.control.get() | 1);
-        } else {
-            self.control.set(self.control.get() & !1);
-        }
-    }
-}
-
 impl<'s, S: PciConfigurationSpace + ?Sized + 's> MsiData<'s, S> {
     pub fn register<C: MessageInterruptController + ?Sized>(
         &mut self,

kernel/driver/bus/pci/src/device.rs

@@ -11,7 +11,7 @@ use yggdrasil_abi::error::Error;
 
 use crate::{
     capability::{MsiCapability, MsiXCapability, MsiXVectorTable},
-    PciAddress, PciConfigSpace, PciConfigurationSpace, PciSegmentInfo,
+    PciAddress, PciCommandRegister, PciConfigSpace, PciConfigurationSpace, PciSegmentInfo,
 };
 
 /// Describes a PCI device
@@ -53,7 +53,7 @@ pub enum PciInterruptPin {
 
 #[derive(Clone, Copy, PartialEq, Eq, Debug)]
 pub enum PreferredInterruptMode {
-    Msi,
+    Msi(bool),
     Legacy,
 }
 
@@ -78,18 +78,6 @@ pub struct PciInterruptRoute {
     pub options: IrqOptions,
 }
 
-pub enum PciMatch {
-    Generic(fn(&PciDeviceInfo) -> bool),
-    Vendor(u16, u16),
-    Class(u8, Option<u8>, Option<u8>),
-}
-
-pub struct PciDriver {
-    pub(crate) name: &'static str,
-    pub(crate) check: PciMatch,
-    pub(crate) probe: fn(&PciDeviceInfo) -> Result<Arc<dyn Device>, Error>,
-}
-
 /// Used to store PCI bus devices which were enumerated by the kernel
 pub struct PciBusDevice {
     pub(crate) info: PciDeviceInfo,
@@ -98,30 +86,77 @@ pub struct PciBusDevice {
 }
 
 impl PciDeviceInfo {
+    pub fn set_command(
+        &self,
+        enable_irq: bool,
+        enable_mem: bool,
+        enable_io: bool,
+        enable_bus_master: bool,
+    ) {
+        let command = PciCommandRegister::from_bits_retain(self.config_space.command());
+        let mut new = command;
+        if enable_irq {
+            new &= !PciCommandRegister::DISABLE_INTERRUPTS;
+        } else {
+            new |= PciCommandRegister::DISABLE_INTERRUPTS;
+        }
+        if enable_mem {
+            new |= PciCommandRegister::ENABLE_MEMORY;
+        } else {
+            new &= !PciCommandRegister::ENABLE_MEMORY;
+        }
+        if enable_io {
+            new |= PciCommandRegister::ENABLE_IO;
+        } else {
+            new &= !PciCommandRegister::ENABLE_IO;
+        }
+        if enable_bus_master {
+            new |= PciCommandRegister::BUS_MASTER;
+        } else {
+            new &= !PciCommandRegister::BUS_MASTER;
+        }
+        if new != command {
+            self.config_space.set_command(new.bits());
+        }
+    }
+
     pub fn init_interrupts(&self, preferred_mode: PreferredInterruptMode) -> Result<(), Error> {
         self.interrupt_config
             .try_init_with(|| {
-                let configured_mode =
+                let configured_mode = if self.segment.has_msi
-                    if self.segment.has_msi && preferred_mode == PreferredInterruptMode::Msi {
+                    && let PreferredInterruptMode::Msi(want_msix) = preferred_mode
-                        if let Some(mut msix) = self.config_space.capability::<MsiXCapability>() {
+                {
-                            let mut vt = msix.vector_table().unwrap();
+                    // Try MSI-X first
-
+                    let mut result = None;
+                    if want_msix
+                        && let Some(mut msix) = self.config_space.capability::<MsiXCapability>()
+                    {
+                        if let Ok(mut vt) = msix.vector_table() {
                             vt.mask_all();
 
                             msix.set_function_mask(false);
                             msix.set_enabled(true);
 
-                            ConfiguredInterruptMode::MsiX(vt)
+                            result = Some(ConfiguredInterruptMode::MsiX(vt));
-                        } else if self.config_space.capability::<MsiCapability>().is_some() {
-                            ConfiguredInterruptMode::Msi
-                        } else {
-                            self.legacy_interrupt_mode()
                         }
-                    } else {
+                    }
-                        // Ignore preferred_mode, the only supported is Legacy
-                        self.legacy_interrupt_mode()
-                    };
 
+                    // Then try MSI
+                    if result.is_none() && self.config_space.capability::<MsiCapability>().is_some()
+                    {
+                        result = Some(ConfiguredInterruptMode::Msi)
+                    }
+
+                    // And then fall back to legacy
+                    if let Some(result) = result {
+                        result
+                    } else {
+                        self.legacy_interrupt_mode()
+                    }
+                } else {
+                    // Ignore preferred_mode, the only supported is Legacy
+                    self.legacy_interrupt_mode()
+                };
                 IrqSafeRwLock::new(InterruptConfig {
                     preferred_mode,
                     configured_mode,
@@ -212,11 +247,7 @@ impl PciDeviceInfo {
             address: self.address,
             pin,
         };
-        let route = self
+        let route = self.segment.irq_translation_map.map_interrupt(&src)?;
-            .segment
-            .irq_translation_map
-            .get(&src)
-            .ok_or(Error::InvalidOperation)?;
 
         log::debug!(
             "PCI {} pin {:?} -> system IRQ #{}",

kernel/driver/bus/pci/src/driver.rs

@@ -0,0 +1,59 @@
+use alloc::{sync::Arc, vec::Vec};
+use device_api::device::Device;
+use libk::error::Error;
+use libk_util::sync::spin_rwlock::IrqSafeRwLock;
+
+use crate::device::PciDeviceInfo;
+
+pub enum PciMatch {
+    Generic(fn(&PciDeviceInfo) -> bool),
+    Vendor(u16, u16),
+    Class(u8, Option<u8>, Option<u8>),
+}
+
+pub struct PciDriverMatch {
+    pub driver: &'static dyn PciDriver,
+    pub check: PciMatch,
+}
+
+pub trait PciDriver: Sync {
+    fn probe(&self, info: &PciDeviceInfo) -> Result<Arc<dyn Device>, Error>;
+    fn driver_name(&self) -> &str;
+}
+
+impl PciMatch {
+    pub fn check_device(&self, info: &PciDeviceInfo) -> bool {
+        match self {
+            Self::Generic(f) => f(info),
+            &Self::Vendor(vendor_, device_) => {
+                info.vendor_id == vendor_ && info.device_id == device_
+            }
+            &Self::Class(class_, Some(subclass_), Some(prog_if_)) => {
+                class_ == info.class && subclass_ == info.subclass && prog_if_ == info.prog_if
+            }
+            &Self::Class(class_, Some(subclass_), _) => {
+                class_ == info.class && subclass_ == info.subclass
+            }
+            &Self::Class(class_, _, _) => class_ == info.class,
+        }
+    }
+}
+
+pub fn register_match(pmatch: PciMatch, driver: &'static dyn PciDriver) {
+    DRIVERS.write().push(PciDriverMatch {
+        check: pmatch,
+        driver,
+    });
+}
+
+pub fn lookup_driver(info: &PciDeviceInfo) -> Option<&'static dyn PciDriver> {
+    DRIVERS.read().iter().find_map(|pmatch| {
+        if pmatch.check.check_device(info) {
+            Some(pmatch.driver)
+        } else {
+            None
+        }
+    })
+}
+
+static DRIVERS: IrqSafeRwLock<Vec<PciDriverMatch>> = IrqSafeRwLock::new(Vec::new());

kernel/driver/bus/pci/src/interrupt.rs

@@ -0,0 +1,63 @@
+use alloc::collections::btree_map::BTreeMap;
+use libk::error::Error;
+
+use crate::device::{PciInterrupt, PciInterruptRoute};
+
+#[derive(Debug)]
+pub enum PciInterruptMap {
+    Fixed(BTreeMap<PciInterrupt, PciInterruptRoute>),
+    #[cfg(any(target_arch = "x86_64", rust_analyzer))]
+    Acpi(alloc::string::String),
+    Legacy,
+}
+
+impl PciInterruptMap {
+    pub fn map_interrupt(&self, interrupt: &PciInterrupt) -> Result<PciInterruptRoute, Error> {
+        match self {
+            Self::Fixed(map) => map.get(interrupt).cloned().ok_or(Error::DoesNotExist),
+            #[cfg(any(target_arch = "x86_64", rust_analyzer))]
+            Self::Acpi(aml_object_name) => {
+                use device_api::interrupt::{IrqLevel, IrqOptions, IrqTrigger};
+
+                use crate::device::PciInterruptPin;
+
+                let aml_pin = match interrupt.pin {
+                    PciInterruptPin::A => ygg_driver_acpi::PciPin::IntA,
+                    PciInterruptPin::B => ygg_driver_acpi::PciPin::IntB,
+                    PciInterruptPin::C => ygg_driver_acpi::PciPin::IntC,
+                    PciInterruptPin::D => ygg_driver_acpi::PciPin::IntD,
+                };
+                let aml_route = ygg_driver_acpi::get_pci_route(
+                    aml_object_name.as_str(),
+                    interrupt.address.device as u16,
+                    interrupt.address.function as u16,
+                    aml_pin,
+                )
+                .or_else(|| {
+                    ygg_driver_acpi::get_pci_route(
+                        aml_object_name.as_str(),
+                        interrupt.address.device as u16,
+                        0xFFFF,
+                        aml_pin,
+                    )
+                })
+                .ok_or(Error::DoesNotExist)?;
+
+                let trigger = match aml_route.trigger {
+                    ygg_driver_acpi::InterruptTrigger::Edge => IrqTrigger::Edge,
+                    ygg_driver_acpi::InterruptTrigger::Level => IrqTrigger::Level,
+                };
+                let level = match aml_route.polarity {
+                    ygg_driver_acpi::InterruptPolarity::ActiveLow => IrqLevel::ActiveLow,
+                    ygg_driver_acpi::InterruptPolarity::ActiveHigh => IrqLevel::ActiveHigh,
+                };
+
+                Ok(PciInterruptRoute {
+                    options: IrqOptions { trigger, level },
+                    number: aml_route.irq,
+                })
+            }
+            Self::Legacy => todo!(),
+        }
+    }
+}

kernel/driver/bus/pci/src/lib.rs

@@ -1,25 +1,34 @@
 //! PCI/PCIe bus interfaces
 #![no_std]
+#![feature(let_chains, decl_macro)]
 #![allow(clippy::missing_transmute_annotations)]
 
 extern crate alloc;
 
 use core::fmt;
 
-#[cfg(target_arch = "x86_64")]
+#[cfg(any(target_arch = "x86_64", rust_analyzer))]
 use acpi::mcfg::McfgEntry;
-use alloc::{collections::BTreeMap, format, sync::Arc, vec::Vec};
+use alloc::{format, sync::Arc, vec::Vec};
+
 use bitflags::bitflags;
-use device::{PciBusDevice, PciDeviceInfo, PciDriver, PciInterrupt, PciInterruptRoute, PciMatch};
+use device::{PciBusDevice, PciDeviceInfo};
 use device_api::device::Device;
+use interrupt::PciInterruptMap;
 use libk::fs::sysfs::{self, object::KObject};
 use libk_mm::address::PhysicalAddress;
-use libk_util::{sync::IrqSafeSpinlock, OneTimeInit};
+use libk_util::{
+    sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock},
+    OneTimeInit,
+};
 use space::legacy;
-use yggdrasil_abi::error::Error;
+use yggdrasil_abi::{error::Error, primitive_enum};
 
 pub mod capability;
 pub mod device;
+pub mod driver;
+pub mod interrupt;
+pub mod macros;
 mod nodes;
 mod space;
 
@@ -31,6 +40,7 @@ pub use space::{
 
 bitflags! {
     /// Command register of the PCI configuration space
+    #[derive(PartialEq, Clone, Copy)]
     pub struct PciCommandRegister: u16 {
         /// If set, I/O access to the device is enabled
         const ENABLE_IO = 1 << 0;
@@ -75,19 +85,13 @@ pub enum PciBaseAddress {
     Io(u16),
 }
 
-/// Unique ID assigned to PCI capability structures
+primitive_enum! {
-#[derive(Clone, Copy, PartialEq, Eq, Debug)]
+    pub enum PciCapabilityId: u8 {
-#[non_exhaustive]
+        PowerManagement = 0x01,
-#[repr(u8)]
+        Msi = 0x05,
-pub enum PciCapabilityId {
+        VendorSpecific = 0x09,
-    /// MSI (32-bit or 64-bit)
+        MsiX = 0x11,
-    Msi = 0x05,
+    }
-    /// Vendor-specific capability
-    VendorSpecific = 0x09,
-    /// MSI-X
-    MsiX = 0x11,
-    /// Unknown capability missing from this list
-    Unknown,
 }
 
 /// Interface used for querying PCI capabilities
@@ -202,7 +206,7 @@ pub struct PciSegmentInfo {
     pub bus_number_end: u8,
     pub ecam_phys_base: Option<PhysicalAddress>,
 
-    pub irq_translation_map: BTreeMap<PciInterrupt, PciInterruptRoute>,
+    pub irq_translation_map: PciInterruptMap,
     pub has_msi: bool,
 }
 
@@ -426,9 +430,9 @@ impl PciBusManager {
     /// Walks the bus device list and calls init/init_irq functions on any devices with associated
     /// drivers
     pub fn setup_bus_devices() -> Result<(), Error> {
-        log::info!("Setting up bus devices");
+        // log::info!("Setting up bus devices");
         Self::walk_bus_devices(|device| {
-            log::info!("Set up {}", device.info.address);
+            // log::info!("Set up {}", device.info.address);
             setup_bus_device(device)?;
             Ok(true)
         })
@@ -462,7 +466,7 @@ impl PciBusManager {
                 bus_number_start: 0,
                 bus_number_end: 255,
                 ecam_phys_base: None,
-                irq_translation_map: BTreeMap::new(),
+                irq_translation_map: PciInterruptMap::Legacy,
                 has_msi: false,
             }),
             allocator: None,
@@ -486,8 +490,8 @@ impl PciBusManager {
                 bus_number_end: entry.bus_number_end,
                 ecam_phys_base: Some(PhysicalAddress::from_u64(entry.base_address)),
 
-                // TODO obtain this from ACPI SSDT
+                // TODO get the segment's PCI root bridge AML name
-                irq_translation_map: BTreeMap::new(),
+                irq_translation_map: PciInterruptMap::Acpi("\\_SB.PCI0._PRT".into()),
                 has_msi: true,
             }),
             // Firmware done this for us
@@ -596,88 +600,18 @@ fn setup_bus_device(device: &mut PciBusDevice) -> Result<(), Error> {
         return Ok(());
     }
 
-    log::debug!(
+    if let Some(driver) = driver::lookup_driver(&device.info) {
-        "{}: {:04x}:{:04x}",
+        log::info!("{} -> {}", device.info.address, driver.driver_name());
-        device.info.address,
+        let instance = driver.probe(&device.info)?;
-        device.info.vendor_id,
-        device.info.device_id
-    );
 
-    let drivers = PCI_DRIVERS.lock();
+        unsafe { instance.clone().init() }?;
-    for driver in drivers.iter() {
-        if driver.check.check_device(&device.info) {
-            // TODO add the device to the bus
-            log::debug!(" -> {:?}", driver.name);
-            let instance = (driver.probe)(&device.info)?;
 
-            unsafe { instance.clone().init() }?;
+        device.device.replace(instance);
-
+        device.driver_name.replace(driver.driver_name());
-            device.device.replace(instance);
-            device.driver_name.replace(driver.name);
-            break;
-        }
     }
 
     Ok(())
 }
 
-impl PciMatch {
-    pub fn check_device(&self, info: &PciDeviceInfo) -> bool {
-        match self {
-            Self::Generic(f) => f(info),
-            &Self::Vendor(vendor_, device_) => {
-                info.vendor_id == vendor_ && info.device_id == device_
-            }
-            &Self::Class(class_, Some(subclass_), Some(prog_if_)) => {
-                class_ == info.class && subclass_ == info.subclass && prog_if_ == info.prog_if
-            }
-            &Self::Class(class_, Some(subclass_), _) => {
-                class_ == info.class && subclass_ == info.subclass
-            }
-            &Self::Class(class_, _, _) => class_ == info.class,
-        }
-    }
-}
-
-pub fn register_class_driver(
-    name: &'static str,
-    class: u8,
-    subclass: Option<u8>,
-    prog_if: Option<u8>,
-    probe: fn(&PciDeviceInfo) -> Result<Arc<dyn Device>, Error>,
-) {
-    PCI_DRIVERS.lock().push(PciDriver {
-        name,
-        check: PciMatch::Class(class, subclass, prog_if),
-        probe,
-    });
-}
-
-pub fn register_vendor_driver(
-    name: &'static str,
-    vendor_id: u16,
-    device_id: u16,
-    probe: fn(&PciDeviceInfo) -> Result<Arc<dyn Device>, Error>,
-) {
-    PCI_DRIVERS.lock().push(PciDriver {
-        name,
-        check: PciMatch::Vendor(vendor_id, device_id),
-        probe,
-    });
-}
-
-pub fn register_generic_driver(
-    name: &'static str,
-    check: fn(&PciDeviceInfo) -> bool,
-    probe: fn(&PciDeviceInfo) -> Result<Arc<dyn Device>, Error>,
-) {
-    PCI_DRIVERS.lock().push(PciDriver {
-        name,
-        check: PciMatch::Generic(check),
-        probe,
-    });
-}
-
-static PCI_DRIVERS: IrqSafeSpinlock<Vec<PciDriver>> = IrqSafeSpinlock::new(Vec::new());
 static PCI_MANAGER: IrqSafeSpinlock<PciBusManager> = IrqSafeSpinlock::new(PciBusManager::new());
 static PCI_SYSFS_NODE: OneTimeInit<Arc<KObject<()>>> = OneTimeInit::new();

kernel/driver/bus/pci/src/macros.rs

@@ -0,0 +1,35 @@
+pub macro pci_driver_match {
+    (class ($class:literal:$subclass:literal:$prog_if:literal)) => {
+        $crate::driver::PciMatch::Class($class, Some($subclass), Some($prog_if))
+    },
+    (class ($class:literal:$subclass:literal)) => {
+        $crate::driver::PciMatch::Class($class, Some($subclass), None)
+    },
+    (class $class:literal) => {
+        $crate::driver::PciMatch::Class($class, None, None)
+    },
+    (device ($vendor:literal:$device:literal)) => {
+        $crate::driver::PciMatch::Vendor($vendor, $device)
+    }
+}
+
+pub macro pci_driver(
+    matches: [$($kind:ident $match:tt),+],
+    driver: $driver:tt
+) {
+    #[link_section = ".init_array"]
+    #[used]
+    static __REGISTER_FN: extern "C" fn() = __register_fn;
+
+    extern "C" fn __register_fn() {
+        struct Driver;
+        impl $crate::driver::PciDriver for Driver $driver
+        static DRIVER: Driver = Driver;
+
+        log::info!("register pci driver: {:?}", $crate::driver::PciDriver::driver_name(&Driver));
+        $(
+            let pmatch = $crate::macros::pci_driver_match!($kind $match);
+            $crate::driver::register_match(pmatch, &DRIVER);
+        )+
+    }
+}

kernel/driver/bus/pci/src/nodes.rs

@@ -107,10 +107,13 @@ pub(crate) fn make_sysfs_object(
             for (capability, offset, _) in state.info.config_space.capability_iter() {
                 write!(output, "{offset:04x}:").ok();
                 match capability {
-                    PciCapabilityId::Msi => write!(output, "msi").ok(),
+                    Some(PciCapabilityId::Msi) => write!(output, "msi").ok(),
-                    PciCapabilityId::MsiX => write!(output, "msix").ok(),
+                    Some(PciCapabilityId::MsiX) => write!(output, "msix").ok(),
-                    PciCapabilityId::VendorSpecific => write!(output, "vendor-specific").ok(),
+                    Some(PciCapabilityId::VendorSpecific) => write!(output, "vendor-specific").ok(),
-                    PciCapabilityId::Unknown => write!(output, "unknown").ok(),
+                    Some(PciCapabilityId::PowerManagement) => {
+                        write!(output, "power-management").ok()
+                    }
+                    None => write!(output, "unknown").ok(),
                 };
                 writeln!(output).ok();
             }

kernel/driver/bus/pci/src/space/mod.rs

@@ -75,12 +75,12 @@ pub struct CapabilityIterator<'s, S: PciConfigurationSpace + ?Sized> {
 }
 
 impl<S: PciConfigurationSpace + ?Sized> Iterator for CapabilityIterator<'_, S> {
-    type Item = (PciCapabilityId, usize, usize);
+    type Item = (Option<PciCapabilityId>, usize, usize);
 
     fn next(&mut self) -> Option<Self::Item> {
         let offset = self.current? & !0x3;
 
-        let id = unsafe { core::mem::transmute(self.space.read_u8(offset)) };
+        let id = PciCapabilityId::try_from(self.space.read_u8(offset)).ok();
         let len = self.space.read_u8(offset + 2);
         let next_pointer = self.space.read_u8(offset + 1);
 
@@ -374,7 +374,7 @@ pub trait PciConfigurationSpace {
     /// Locates a capability within this configuration space
     fn capability<C: PciCapability>(&self) -> Option<C::CapabilityData<'_, Self>> {
         self.capability_iter().find_map(|(id, offset, len)| {
-            if id == C::ID && C::check(self, offset, len) {
+            if id.map_or(false, |id| id == C::ID) && C::check(self, offset, len) {
                 Some(C::data(self, offset, len))
             } else {
                 None

kernel/driver/bus/usb/Cargo.toml

@@ -12,7 +12,10 @@ libk-mm.workspace = true
 libk.workspace = true
 
 ygg_driver_input = { path = "../../input" }
+# For mass storage
+ygg_driver_scsi = { path = "../../block/scsi" }
 
 log.workspace = true
 bytemuck.workspace = true
 futures-util.workspace = true
+async-trait.workspace = true

kernel/driver/bus/usb/src/bus.rs

@@ -24,6 +24,7 @@ impl UsbBusManager {
     }
 
     pub fn register_device(device: Arc<UsbDeviceAccess>) {
+        log::info!("usb: register device {}", device.bus_address());
         BUS_MANAGER
             .devices
             .write()

kernel/driver/bus/usb/src/class_driver/hid_keyboard.rs

@@ -0,0 +1,177 @@
+use core::mem::MaybeUninit;
+
+use alloc::{boxed::Box, sync::Arc};
+use async_trait::async_trait;
+use libk_mm::PageBox;
+use yggdrasil_abi::io::{KeyboardKey, KeyboardKeyEvent};
+
+use crate::{device::UsbDeviceAccess, error::UsbError, info::UsbDeviceClass};
+
+use super::{UsbClassInfo, UsbDriver};
+
+pub struct UsbHidKeyboardDriver;
+
+const MODIFIER_MAP: &[KeyboardKey] = &[
+    KeyboardKey::LControl,
+    KeyboardKey::LShift,
+    KeyboardKey::LAlt,
+    KeyboardKey::Unknown,
+    KeyboardKey::RControl,
+    KeyboardKey::RShift,
+    KeyboardKey::RAlt,
+    KeyboardKey::Unknown,
+];
+
+#[derive(Default)]
+struct KeyboardState {
+    state: [u64; 4],
+    mods: u8,
+}
+
+impl KeyboardState {
+    pub fn new() -> Self {
+        Self::default()
+    }
+
+    pub fn translate_key(k: u8) -> KeyboardKey {
+        match k {
+            4..=29 => KeyboardKey::Char(k - 4 + b'a'),
+            30..=38 => KeyboardKey::Char(k - 30 + b'1'),
+            39 => KeyboardKey::Char(b'0'),
+
+            40 => KeyboardKey::Enter,
+            41 => KeyboardKey::Escape,
+            42 => KeyboardKey::Backspace,
+            43 => KeyboardKey::Tab,
+            44 => KeyboardKey::Char(b' '),
+            45 => KeyboardKey::Char(b'-'),
+            46 => KeyboardKey::Char(b'='),
+            47 => KeyboardKey::Char(b'['),
+            48 => KeyboardKey::Char(b']'),
+            49 => KeyboardKey::Char(b'\\'),
+            51 => KeyboardKey::Char(b';'),
+            52 => KeyboardKey::Char(b'\''),
+            53 => KeyboardKey::Char(b'`'),
+            54 => KeyboardKey::Char(b','),
+            55 => KeyboardKey::Char(b'.'),
+            56 => KeyboardKey::Char(b'/'),
+
+            58..=69 => KeyboardKey::F(k - 58),
+
+            _ => {
+                log::debug!("Unknown key: {}", k);
+                KeyboardKey::Unknown
+            }
+        }
+    }
+
+    pub fn retain_modifiers(
+        &mut self,
+        m: u8,
+        events: &mut [MaybeUninit<KeyboardKeyEvent>],
+    ) -> usize {
+        let mut count = 0;
+        let released = self.mods & !m;
+        for (i, modifier) in MODIFIER_MAP.iter().enumerate().take(8) {
+            if released & (1 << i) != 0 {
+                events[count].write(KeyboardKeyEvent::Released(*modifier));
+                count += 1;
+            }
+        }
+        self.mods &= m;
+        count
+    }
+
+    pub fn press_modifiers(
+        &mut self,
+        m: u8,
+        events: &mut [MaybeUninit<KeyboardKeyEvent>],
+    ) -> usize {
+        let mut count = 0;
+        let pressed = m & !self.mods;
+        for (i, modifier) in MODIFIER_MAP.iter().enumerate().take(8) {
+            if pressed & (1 << i) != 0 {
+                events[count].write(KeyboardKeyEvent::Pressed(*modifier));
+                count += 1;
+            }
+        }
+        self.mods = m;
+        count
+    }
+
+    pub fn retain(&mut self, keys: &[u8], events: &mut [MaybeUninit<KeyboardKeyEvent>]) -> usize {
+        let mut count = 0;
+        for i in 1..256 {
+            if self.state[i / 64] & (1 << (i % 64)) != 0 && !keys.contains(&(i as u8)) {
+                events[count].write(KeyboardKeyEvent::Released(Self::translate_key(i as u8)));
+                self.state[i / 64] &= !(1 << (i % 64));
+                count += 1;
+            }
+        }
+        count
+    }
+
+    pub fn press(&mut self, keys: &[u8], events: &mut [MaybeUninit<KeyboardKeyEvent>]) -> usize {
+        let mut count = 0;
+        for &k in keys {
+            let index = (k as usize) / 64;
+            if self.state[index] & (1 << (k % 64)) == 0 {
+                self.state[index] |= 1 << (k % 64);
+                events[count].write(KeyboardKeyEvent::Pressed(Self::translate_key(k)));
+                count += 1;
+            }
+        }
+        count
+    }
+}
+
+#[async_trait]
+impl UsbDriver for UsbHidKeyboardDriver {
+    async fn run(self: Arc<Self>, device: Arc<UsbDeviceAccess>) -> Result<(), UsbError> {
+        // TODO not sure whether to use boot protocol (easy) or GetReport
+        let config = device.select_configuration(|_| true).await?.unwrap();
+
+        log::info!("Setup HID keyboard");
+        let pipe = device
+            .open_interrupt_in_pipe(1, config.endpoints[0].max_packet_size as u16)
+            .await?;
+
+        let mut buffer = PageBox::new_slice(0, 8).map_err(UsbError::MemoryError)?;
+        let mut state = KeyboardState::new();
+        let mut events = [MaybeUninit::uninit(); 16];
+
+        loop {
+            let mut event_count = 0;
+
+            let len = pipe.read(buffer.as_slice_mut()).await?;
+            if len < 8 {
+                continue;
+            }
+
+            event_count += state.retain_modifiers(buffer[0], &mut events);
+            event_count += state.press_modifiers(buffer[0], &mut events[event_count..]);
+            event_count += state.retain(&buffer[2..], &mut events[event_count..]);
+            event_count += state.press(&buffer[2..], &mut events[event_count..]);
+
+            let events = unsafe { MaybeUninit::slice_assume_init_ref(&events[..event_count]) };
+
+            for &event in events {
+                log::info!("Generic Keyboard: {:?}", event);
+                ygg_driver_input::send_event(event);
+            }
+        }
+    }
+
+    fn name(&self) -> &'static str {
+        "USB HID Keyboard"
+    }
+
+    fn probe(&self, class: &UsbClassInfo, _device: &UsbDeviceAccess) -> bool {
+        log::info!(
+            "class = {:?}, subclass = {:02x}",
+            class.class,
+            class.subclass
+        );
+        class.class == UsbDeviceClass::Hid && (class.subclass == 0x00 || class.subclass == 0x01)
+    }
+}

kernel/driver/bus/usb/src/class_driver/mass_storage.rs

@@ -0,0 +1,235 @@
+use alloc::{boxed::Box, sync::Arc};
+use async_trait::async_trait;
+use bytemuck::{Pod, Zeroable};
+use libk::error::Error;
+use libk_mm::PageBox;
+use ygg_driver_scsi::{transport::ScsiTransport, ScsiDevice};
+
+use crate::{
+    communication::UsbDirection,
+    device::{UsbDeviceAccess, UsbDeviceDetachHandler},
+    error::UsbError,
+    info::{UsbDeviceClass, UsbEndpointType},
+    pipe::{
+        control::UsbClassSpecificRequest,
+        normal::{UsbBulkInPipeAccess, UsbBulkOutPipeAccess},
+    },
+};
+
+use super::{UsbClassInfo, UsbDriver};
+
+pub struct UsbMassStorageDriverBulkOnly;
+
+#[derive(Debug, Clone, Copy, Zeroable, Pod)]
+#[repr(C)]
+struct Cbw {
+    signature: u32,       // 0x00
+    tag: u32,             // 0x04
+    transfer_length: u32, // 0x08
+    flags: u8,            // 0x0C
+    lun: u8,              // 0x0D
+    cb_length: u8,        // 0x0E
+    cb_data: [u8; 16],    // 0x0F
+    // Not sent
+    _0: u8,
+}
+
+#[derive(Debug, Clone, Copy, Zeroable, Pod)]
+#[repr(C)]
+struct Csw {
+    signature: u32,
+    tag: u32,
+    data_residue: u32,
+    status: u8,
+    _0: [u8; 3],
+}
+
+struct Bbb {
+    #[allow(unused)]
+    device: Arc<UsbDeviceAccess>,
+    in_pipe: UsbBulkInPipeAccess,
+    out_pipe: UsbBulkOutPipeAccess,
+    buffer: PageBox<[u8]>,
+    last_tag: u32,
+}
+
+struct DetachHandler(Arc<ScsiDevice>);
+
+impl Bbb {
+    pub fn new(
+        device: Arc<UsbDeviceAccess>,
+        in_pipe: UsbBulkInPipeAccess,
+        out_pipe: UsbBulkOutPipeAccess,
+    ) -> Result<Self, UsbError> {
+        let buffer = PageBox::new_slice(0, 4096).map_err(UsbError::MemoryError)?;
+        Ok(Self {
+            device,
+            in_pipe,
+            out_pipe,
+            buffer,
+            last_tag: 0,
+        })
+    }
+}
+
+impl Bbb {
+    async fn send_cbw(
+        &mut self,
+        lun: u8,
+        host_to_dev: bool,
+        command: &[u8],
+        response_len: usize,
+    ) -> Result<u32, Error> {
+        self.last_tag = self.last_tag.wrapping_add(1);
+
+        self.buffer[..size_of::<Cbw>()].fill(0);
+
+        let cbw = bytemuck::from_bytes_mut::<Cbw>(&mut self.buffer[..size_of::<Cbw>()]);
+        let tag = self.last_tag;
+
+        cbw.signature = 0x43425355;
+        cbw.tag = tag;
+        cbw.transfer_length = response_len as u32;
+        if !host_to_dev {
+            cbw.flags = 1 << 7;
+        }
+        cbw.lun = lun;
+        cbw.cb_length = command.len() as u8;
+        cbw.cb_data[..command.len()].copy_from_slice(command);
+
+        self.out_pipe
+            .write(self.buffer.as_slice().subslice(..31))
+            .await
+            .inspect_err(|error| log::error!("msc: out pipe error: {error:?}"))?;
+
+        Ok(tag)
+    }
+
+    async fn read_csw(&mut self, tag: u32) -> Result<(), Error> {
+        self.in_pipe
+            .read(self.buffer.as_slice_mut().subslice_mut(..13))
+            .await
+            .inspect_err(|error| log::error!("msc: in pipe error: {error:?}"))?;
+
+        let csw = bytemuck::from_bytes::<Csw>(&self.buffer[..size_of::<Csw>()]);
+        if csw.signature != 0x53425355 {
+            log::warn!("msc: invalid csw signature");
+            return Err(Error::InvalidArgument);
+        }
+        if csw.tag != tag {
+            let csw_tag = csw.tag;
+            log::warn!("msc: invalid csw tag (got {}, expected {tag})", csw_tag);
+            return Err(Error::InvalidArgument);
+        }
+        if csw.status != 0x00 {
+            log::warn!("msc: csw error status {:#02x}", csw.status);
+            return Err(Error::InvalidArgument);
+        }
+        Ok(())
+    }
+
+    async fn read_response_data(&mut self, buffer: &mut [u8]) -> Result<usize, Error> {
+        // TODO limit by max_packet_size
+        let len = self
+            .in_pipe
+            .read(self.buffer.as_slice_mut().subslice_mut(..buffer.len()))
+            .await
+            .inspect_err(|error| log::error!("msc: in pipe error: {error:?}"))?;
+        buffer[..len].copy_from_slice(&self.buffer[..len]);
+        Ok(len)
+    }
+}
+
+#[async_trait]
+impl ScsiTransport for Bbb {
+    async fn perform_request_raw(
+        &mut self,
+        lun: u8,
+        request_data: &[u8],
+        response_buffer: &mut [u8],
+    ) -> Result<usize, Error> {
+        if request_data.len() > 16 || response_buffer.len() > self.buffer.len() {
+            return Err(Error::InvalidArgument);
+        }
+
+        let tag = self
+            .send_cbw(lun, false, request_data, response_buffer.len())
+            .await?;
+        let response_len = if response_buffer.is_empty() {
+            0
+        } else {
+            self.read_response_data(response_buffer).await?
+        };
+        self.read_csw(tag).await?;
+        Ok(response_len)
+    }
+}
+
+impl UsbDeviceDetachHandler for DetachHandler {
+    fn handle_device_detach(&self) {
+        log::info!("Mass storage detached");
+        self.0.detach();
+    }
+}
+
+#[derive(Debug, Pod, Zeroable, Clone, Copy)]
+#[repr(C)]
+pub struct BulkOnlyMassStorageReset;
+
+impl UsbClassSpecificRequest for BulkOnlyMassStorageReset {
+    const BM_REQUEST_TYPE: u8 = 0b00100001;
+    const B_REQUEST: u8 = 0b11111111;
+}
+
+#[async_trait]
+impl UsbDriver for UsbMassStorageDriverBulkOnly {
+    async fn run(self: Arc<Self>, device: Arc<UsbDeviceAccess>) -> Result<(), UsbError> {
+        // TODO filter to only accept BBB config
+        let config = device.select_configuration(|_| true).await?.unwrap();
+        // Bulk-in, bulk-out
+        assert_eq!(config.endpoints.len(), 2);
+        // TODO those indices may be different
+        let control_pipe = device.control_pipe();
+        let (in_index, in_info) = config
+            .find_endpoint(|ep| ep.is(UsbEndpointType::Bulk, UsbDirection::In))
+            .ok_or(UsbError::InvalidConfiguration)?;
+        let (out_index, out_info) = config
+            .find_endpoint(|ep| ep.is(UsbEndpointType::Bulk, UsbDirection::Out))
+            .ok_or(UsbError::InvalidConfiguration)?;
+        let in_pipe = device
+            .open_bulk_in_pipe(in_index, in_info.max_packet_size as u16)
+            .await?;
+        let out_pipe = device
+            .open_bulk_out_pipe(out_index, out_info.max_packet_size as u16)
+            .await?;
+
+        // Perform a Bulk-Only Mass Storage Reset
+        // TODO interface id?
+        control_pipe
+            .class_specific_request::<BulkOnlyMassStorageReset>(0, 0)
+            .await?;
+
+        // TODO get max LUN
+
+        let bbb = Bbb::new(device.clone(), in_pipe, out_pipe)?;
+        let scsi = ScsiDevice::setup(bbb)
+            .await
+            .inspect_err(|error| log::error!("msc: scsi error {error:?}"))
+            .map_err(|_| UsbError::DriverError)?;
+        let detach = DetachHandler(scsi.clone());
+        device.set_detach_handler(Arc::new(detach));
+
+        ygg_driver_scsi::attach(scsi).ok();
+
+        Ok(())
+    }
+
+    fn name(&self) -> &'static str {
+        "USB Mass Storage"
+    }
+
+    fn probe(&self, class: &UsbClassInfo, _device: &UsbDeviceAccess) -> bool {
+        // TODO support other protocols
+        class.class == UsbDeviceClass::MassStorage && class.interface_protocol_number == 0x50
+    }
+}

kernel/driver/bus/usb/src/class_driver/mod.rs

@@ -1,5 +1,5 @@
-use alloc::{sync::Arc, vec::Vec};
+use alloc::{boxed::Box, sync::Arc, vec::Vec};
-use futures_util::future::BoxFuture;
+use async_trait::async_trait;
 use libk::task::runtime;
 use libk_util::sync::spin_rwlock::IrqSafeRwLock;
 
@@ -9,30 +9,34 @@ use crate::{
     info::{UsbDeviceClass, UsbDeviceProtocol},
 };
 
+pub mod hid_keyboard;
+pub mod mass_storage;
+
+#[derive(Debug)]
 pub struct UsbClassInfo {
     pub class: UsbDeviceClass,
     pub subclass: u8,
     pub protocol: UsbDeviceProtocol,
+    pub device_protocol_number: u8,
+    pub interface_protocol_number: u8,
 }
 
+#[async_trait]
 pub trait UsbDriver: Send + Sync {
-    fn name(&self) -> &'static str;
+    async fn run(self: Arc<Self>, device: Arc<UsbDeviceAccess>) -> Result<(), UsbError>;
-    fn run(
-        self: Arc<Self>,
-        device: Arc<UsbDeviceAccess>,
-    ) -> BoxFuture<'static, Result<(), UsbError>>;
 
+    fn name(&self) -> &'static str;
     fn probe(&self, class: &UsbClassInfo, device: &UsbDeviceAccess) -> bool;
 }
 
 async fn extract_class_info(device: &UsbDeviceAccess) -> Result<Option<UsbClassInfo>, UsbError> {
-    if device.num_configurations != 1 {
+    if device.info.num_configurations != 1 {
         return Ok(None);
     }
     let device_info = &device.info;
     let config_info = device.query_configuration_info(0).await?;
 
-    if config_info.interfaces.len() == 1 {
+    if config_info.interfaces.len() >= 1 {
         let if_info = &config_info.interfaces[0];
 
         let class = if device_info.device_class == UsbDeviceClass::FromInterface {
@@ -55,6 +59,8 @@ async fn extract_class_info(device: &UsbDeviceAccess) -> Result<Option<UsbClassI
             class,
             subclass,
             protocol,
+            interface_protocol_number: if_info.interface_protocol_number,
+            device_protocol_number: device_info.device_protocol_number,
         }))
     } else {
         Ok(None)
@@ -104,191 +110,8 @@ pub fn register_driver(driver: Arc<dyn UsbDriver + 'static>) {
 
 pub fn register_default_class_drivers() {
     register_driver(Arc::new(hid_keyboard::UsbHidKeyboardDriver));
+    register_driver(Arc::new(mass_storage::UsbMassStorageDriverBulkOnly));
 }
 
 static USB_DEVICE_DRIVERS: IrqSafeRwLock<Vec<Arc<dyn UsbDriver + 'static>>> =
     IrqSafeRwLock::new(Vec::new());
-
-pub mod hid_keyboard {
-    use core::mem::MaybeUninit;
-
-    use alloc::sync::Arc;
-    use futures_util::{future::BoxFuture, FutureExt};
-    use libk_mm::PageBox;
-    use yggdrasil_abi::io::{KeyboardKey, KeyboardKeyEvent};
-
-    use crate::{device::UsbDeviceAccess, error::UsbError, info::UsbDeviceClass};
-
-    use super::{UsbClassInfo, UsbDriver};
-
-    pub struct UsbHidKeyboardDriver;
-
-    const MODIFIER_MAP: &[KeyboardKey] = &[
-        KeyboardKey::LControl,
-        KeyboardKey::LShift,
-        KeyboardKey::LAlt,
-        KeyboardKey::Unknown,
-        KeyboardKey::RControl,
-        KeyboardKey::RShift,
-        KeyboardKey::RAlt,
-        KeyboardKey::Unknown,
-    ];
-
-    #[derive(Default)]
-    struct KeyboardState {
-        state: [u64; 4],
-        mods: u8,
-    }
-
-    impl KeyboardState {
-        pub fn new() -> Self {
-            Self::default()
-        }
-
-        pub fn translate_key(k: u8) -> KeyboardKey {
-            match k {
-                4..=29 => KeyboardKey::Char(k - 4 + b'a'),
-                30..=38 => KeyboardKey::Char(k - 30 + b'1'),
-                39 => KeyboardKey::Char(b'0'),
-
-                40 => KeyboardKey::Enter,
-                41 => KeyboardKey::Escape,
-                42 => KeyboardKey::Backspace,
-                43 => KeyboardKey::Tab,
-                44 => KeyboardKey::Char(b' '),
-                45 => KeyboardKey::Char(b'-'),
-                46 => KeyboardKey::Char(b'='),
-                47 => KeyboardKey::Char(b'['),
-                48 => KeyboardKey::Char(b']'),
-                49 => KeyboardKey::Char(b'\\'),
-                51 => KeyboardKey::Char(b';'),
-                52 => KeyboardKey::Char(b'\''),
-                53 => KeyboardKey::Char(b'`'),
-                54 => KeyboardKey::Char(b','),
-                55 => KeyboardKey::Char(b'.'),
-                56 => KeyboardKey::Char(b'/'),
-
-                58..=69 => KeyboardKey::F(k - 58),
-
-                _ => {
-                    log::debug!("Unknown key: {}", k);
-                    KeyboardKey::Unknown
-                }
-            }
-        }
-
-        pub fn retain_modifiers(
-            &mut self,
-            m: u8,
-            events: &mut [MaybeUninit<KeyboardKeyEvent>],
-        ) -> usize {
-            let mut count = 0;
-            let released = self.mods & !m;
-            for (i, modifier) in MODIFIER_MAP.iter().enumerate().take(8) {
-                if released & (1 << i) != 0 {
-                    events[count].write(KeyboardKeyEvent::Released(*modifier));
-                    count += 1;
-                }
-            }
-            self.mods &= m;
-            count
-        }
-
-        pub fn press_modifiers(
-            &mut self,
-            m: u8,
-            events: &mut [MaybeUninit<KeyboardKeyEvent>],
-        ) -> usize {
-            let mut count = 0;
-            let pressed = m & !self.mods;
-            for (i, modifier) in MODIFIER_MAP.iter().enumerate().take(8) {
-                if pressed & (1 << i) != 0 {
-                    events[count].write(KeyboardKeyEvent::Pressed(*modifier));
-                    count += 1;
-                }
-            }
-            self.mods = m;
-            count
-        }
-
-        pub fn retain(
-            &mut self,
-            keys: &[u8],
-            events: &mut [MaybeUninit<KeyboardKeyEvent>],
-        ) -> usize {
-            let mut count = 0;
-            for i in 1..256 {
-                if self.state[i / 64] & (1 << (i % 64)) != 0 && !keys.contains(&(i as u8)) {
-                    events[count].write(KeyboardKeyEvent::Released(Self::translate_key(i as u8)));
-                    self.state[i / 64] &= !(1 << (i % 64));
-                    count += 1;
-                }
-            }
-            count
-        }
-
-        pub fn press(
-            &mut self,
-            keys: &[u8],
-            events: &mut [MaybeUninit<KeyboardKeyEvent>],
-        ) -> usize {
-            let mut count = 0;
-            for &k in keys {
-                let index = (k as usize) / 64;
-                if self.state[index] & (1 << (k % 64)) == 0 {
-                    self.state[index] |= 1 << (k % 64);
-                    events[count].write(KeyboardKeyEvent::Pressed(Self::translate_key(k)));
-                    count += 1;
-                }
-            }
-            count
-        }
-    }
-
-    impl UsbDriver for UsbHidKeyboardDriver {
-        fn run(
-            self: Arc<Self>,
-            device: Arc<UsbDeviceAccess>,
-        ) -> BoxFuture<'static, Result<(), UsbError>> {
-            async move {
-                // TODO not sure whether to use boot protocol (easy) or GetReport
-                let config = device.select_configuration(|_| true).await?.unwrap();
-                assert_eq!(config.endpoints.len(), 1);
-
-                let pipe = device.open_interrupt_in_pipe(1).await?;
-
-                let mut buffer = PageBox::new_slice(0, 8).map_err(UsbError::MemoryError)?;
-                let mut state = KeyboardState::new();
-                let mut events = [MaybeUninit::uninit(); 16];
-
-                loop {
-                    let mut event_count = 0;
-
-                    let data = pipe.read(&mut buffer).await?;
-
-                    event_count += state.retain_modifiers(data[0], &mut events);
-                    event_count += state.press_modifiers(data[0], &mut events[event_count..]);
-                    event_count += state.retain(&data[2..], &mut events[event_count..]);
-                    event_count += state.press(&data[2..], &mut events[event_count..]);
-
-                    let events =
-                        unsafe { MaybeUninit::slice_assume_init_ref(&events[..event_count]) };
-
-                    for &event in events {
-                        log::trace!("Generic Keyboard: {:?}", event);
-                        ygg_driver_input::send_event(event);
-                    }
-                }
-            }
-            .boxed()
-        }
-
-        fn name(&self) -> &'static str {
-            "USB HID Keyboard"
-        }
-
-        fn probe(&self, class: &UsbClassInfo, _device: &UsbDeviceAccess) -> bool {
-            class.class == UsbDeviceClass::Hid && class.subclass == 0x01
-        }
-    }
-}

kernel/driver/bus/usb/src/communication.rs

@@ -1,124 +1,11 @@
-use core::{
-    future::poll_fn,
-    sync::atomic::{AtomicU32, Ordering},
-    task::{Context, Poll},
-};
-
-use alloc::{sync::Arc, vec::Vec};
-use futures_util::task::AtomicWaker;
-use libk_mm::address::PhysicalAddress;
-
-use crate::error::UsbError;
-
 #[derive(Clone, Copy, PartialEq, Eq, Debug)]
 pub enum UsbDirection {
     Out,
     In,
 }
 
-#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
-#[repr(transparent)]
-pub struct UsbTransferToken(pub u64);
-
-#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
-#[repr(transparent)]
-pub struct UsbTransferResult(pub u32);
-
-pub struct UsbTransferStatus {
-    pub result: AtomicU32,
-    pub notify: AtomicWaker,
-}
-
-pub struct UsbControlTransfer {
-    pub id: UsbTransferToken,
-    pub length: usize,
-    pub direction: UsbDirection,
-    pub elements: Vec<PhysicalAddress>,
-    pub status: Arc<UsbTransferStatus>,
-}
-
-pub struct UsbInterruptTransfer {
-    pub address: PhysicalAddress,
-    pub length: usize,
-    pub direction: UsbDirection,
-    pub status: Arc<UsbTransferStatus>,
-}
-
 impl UsbDirection {
     pub const fn is_device_to_host(self) -> bool {
         matches!(self, UsbDirection::In)
     }
 }
-
-// TODO this is xHCI-specific
-impl UsbTransferResult {
-    pub fn is_aborted(&self) -> bool {
-        self.0 == u32::MAX
-    }
-
-    pub fn is_success(&self) -> bool {
-        (self.0 >> 24) & 0xFF == 1
-    }
-
-    pub fn sub_length(&self) -> usize {
-        (self.0 & 0xFFFFFF) as _
-    }
-}
-
-impl UsbControlTransfer {
-    pub async fn wait(&self) -> Result<usize, UsbError> {
-        let sub_length = self.status.wait().await?;
-        Ok(self.length.saturating_sub(sub_length))
-    }
-}
-
-impl UsbInterruptTransfer {
-    pub async fn wait(&self) -> Result<usize, UsbError> {
-        let sub_length = self.status.wait().await?;
-        Ok(self.length.saturating_sub(sub_length))
-    }
-}
-
-impl UsbTransferStatus {
-    pub fn new() -> Self {
-        Self {
-            result: AtomicU32::new(0),
-            notify: AtomicWaker::new(),
-        }
-    }
-
-    pub(crate) async fn wait(&self) -> Result<usize, UsbError> {
-        poll_fn(|cx| {
-            self.poll(cx).map(|v| {
-                if v.is_success() {
-                    Ok(v.sub_length())
-                } else if v.is_aborted() {
-                    Err(UsbError::DeviceDisconnected)
-                } else {
-                    Err(UsbError::TransferFailed)
-                }
-            })
-        })
-        .await
-    }
-
-    pub fn signal(&self, status: u32) {
-        self.result.store(status, Ordering::Release);
-        self.notify.wake();
-    }
-
-    pub fn abort(&self) {
-        self.result.store(u32::MAX, Ordering::Release);
-        self.notify.wake();
-    }
-
-    pub fn poll(&self, cx: &mut Context<'_>) -> Poll<UsbTransferResult> {
-        self.notify.register(cx.waker());
-        let value = self.result.load(Ordering::Acquire);
-        if value != 0 {
-            Poll::Ready(UsbTransferResult(value))
-        } else {
-            Poll::Pending
-        }
-    }
-}

kernel/driver/bus/usb/src/descriptor.rs

@@ -1,9 +1,10 @@
 use bytemuck::{Pod, Zeroable};
 
 use crate::{
+    communication::UsbDirection,
+    device::UsbSpeed,
     error::UsbError,
-    info::{UsbDeviceClass, UsbDeviceProtocol, UsbEndpointType},
+    info::{UsbDeviceClass, UsbDeviceProtocol, UsbEndpointType, UsbVersion},
-    UsbDirection,
 };
 
 #[derive(Clone, Copy, Debug, Default, Pod, Zeroable)]
@@ -134,12 +135,14 @@ impl UsbDeviceDescriptor {
         UsbDeviceProtocol::try_from(self.device_protocol).unwrap_or(UsbDeviceProtocol::Unknown)
     }
 
-    pub fn max_packet_size(&self) -> Result<usize, UsbError> {
+    pub fn max_packet_size(&self, version: UsbVersion, speed: UsbSpeed) -> Result<usize, UsbError> {
-        match self.max_packet_size_0 {
+        match (version.is_version_3(), speed, self.max_packet_size_0) {
-            8 => Ok(8),
+            (true, UsbSpeed::Super, 9) => Ok(1 << 9),
-            16 => Ok(16),
+            (true, _, _) => todo!("Non-GenX speed USB3+ maxpacketsize0"),
-            32 => Ok(32),
+            (false, _, 8) => Ok(8),
-            64 => Ok(64),
+            (false, _, 16) => Ok(16),
+            (false, _, 32) => Ok(32),
+            (false, _, 64) => Ok(64),
             _ => Err(UsbError::InvalidDescriptorField),
         }
     }

kernel/driver/bus/usb/src/device.rs

@@ -1,16 +1,22 @@
 use core::{fmt, ops::Deref};
 
-use alloc::{boxed::Box, vec::Vec};
+use alloc::{boxed::Box, sync::Arc, vec::Vec};
-use futures_util::future::BoxFuture;
+use async_trait::async_trait;
 use libk_mm::PageBox;
 use libk_util::sync::spin_rwlock::{IrqSafeRwLock, IrqSafeRwLockReadGuard};
 
 use crate::{
     error::UsbError,
-    info::{UsbConfigurationInfo, UsbDeviceInfo, UsbEndpointInfo, UsbInterfaceInfo},
+    info::{
+        UsbConfigurationInfo, UsbDeviceInfo, UsbEndpointInfo, UsbEndpointType, UsbInterfaceInfo,
+        UsbVersion,
+    },
     pipe::{
         control::{ConfigurationDescriptorEntry, UsbControlPipeAccess},
-        interrupt::UsbInterruptInPipeAccess,
+        normal::{
+            UsbBulkInPipeAccess, UsbBulkOutPipeAccess, UsbInterruptInPipeAccess, UsbNormalPipeIn,
+            UsbNormalPipeOut,
+        },
     },
     UsbHostController,
 };
@@ -24,9 +30,8 @@ pub struct UsbBusAddress {
 }
 
 pub struct UsbDeviceAccess {
-    pub device: Box<dyn UsbDevice>,
+    pub device: Arc<dyn UsbDevice>,
     pub info: UsbDeviceInfo,
-    pub num_configurations: u8,
     pub current_configuration: IrqSafeRwLock<Option<UsbConfigurationInfo>>,
 }
 
@@ -38,23 +43,36 @@ pub enum UsbSpeed {
     Super,
 }
 
+pub trait UsbDeviceDetachHandler: Send + Sync {
+    fn handle_device_detach(&self);
+}
+
+#[async_trait]
 #[allow(unused)]
 pub trait UsbDevice: Send + Sync {
     // Endpoint "0"
     fn control_pipe(&self) -> &UsbControlPipeAccess;
 
-    fn open_interrupt_in_pipe(
+    async fn open_normal_in_pipe(
         &self,
         number: u8,
-    ) -> BoxFuture<Result<UsbInterruptInPipeAccess, UsbError>> {
+        max_packet_size: u16,
-        unimplemented!()
+        ty: UsbEndpointType,
-    }
+    ) -> Result<Box<dyn UsbNormalPipeIn>, UsbError>;
+
+    async fn open_normal_out_pipe(
+        &self,
+        number: u8,
+        max_packet_size: u16,
+        ty: UsbEndpointType,
+    ) -> Result<Box<dyn UsbNormalPipeOut>, UsbError>;
 
     fn port_number(&self) -> u8;
     fn bus_address(&self) -> UsbBusAddress;
     fn speed(&self) -> UsbSpeed;
     fn controller_ref(&self) -> &dyn UsbHostController;
 
+    fn set_detach_handler(&self, handler: Arc<dyn UsbDeviceDetachHandler>);
     fn handle_detach(&self);
 
     fn debug(&self) {}
@@ -67,12 +85,23 @@ impl UsbDeviceAccess {
     /// * Device is not yet configured
     /// * Control pipe for the device has been properly set up
     /// * Device has been assigned a bus address
-    pub async fn setup(raw: Box<dyn UsbDevice>) -> Result<Self, UsbError> {
+    pub async fn setup(raw: Arc<dyn UsbDevice>) -> Result<Self, UsbError> {
         let control = raw.control_pipe();
         let mut string_buffer = PageBox::new_uninit().map_err(UsbError::MemoryError)?;
 
         let device_desc = control.query_device_descriptor().await?;
 
+        let bcd_usb = device_desc.bcd_usb;
+        let usb_version = UsbVersion::from_bcd_usb(device_desc.bcd_usb)
+            .ok_or(UsbError::InvalidDescriptorField)
+            .inspect_err(|_| {
+                log::error!(
+                    "{}: unsupported/invalid USB version: {:#x}",
+                    raw.bus_address(),
+                    bcd_usb
+                )
+            })?;
+
         let manufacturer = control
             .query_string(device_desc.manufacturer_str, &mut string_buffer)
             .await?;
@@ -83,6 +112,7 @@ impl UsbDeviceAccess {
         let info = UsbDeviceInfo {
             manufacturer,
             product,
+            usb_version,
 
             id_vendor: device_desc.id_vendor,
             id_product: device_desc.id_product,
@@ -90,18 +120,56 @@ impl UsbDeviceAccess {
             device_class: device_desc.class(),
             device_subclass: device_desc.device_subclass,
             device_protocol: device_desc.protocol(),
+            device_protocol_number: device_desc.device_protocol,
 
-            max_packet_size: device_desc.max_packet_size()?,
+            num_configurations: device_desc.num_configurations,
+
+            max_packet_size: device_desc.max_packet_size(usb_version, raw.speed())?,
         };
 
         Ok(Self {
             device: raw,
             info,
-            num_configurations: device_desc.num_configurations,
             current_configuration: IrqSafeRwLock::new(None),
         })
     }
 
+    pub async fn open_interrupt_in_pipe(
+        &self,
+        number: u8,
+        max_packet_size: u16,
+    ) -> Result<UsbInterruptInPipeAccess, UsbError> {
+        let pipe = self
+            .device
+            .open_normal_in_pipe(number, max_packet_size, UsbEndpointType::Interrupt)
+            .await?;
+        Ok(UsbInterruptInPipeAccess(pipe))
+    }
+
+    pub async fn open_bulk_in_pipe(
+        &self,
+        number: u8,
+        max_packet_size: u16,
+    ) -> Result<UsbBulkInPipeAccess, UsbError> {
+        let pipe = self
+            .device
+            .open_normal_in_pipe(number, max_packet_size, UsbEndpointType::Bulk)
+            .await?;
+        Ok(UsbBulkInPipeAccess(pipe))
+    }
+
+    pub async fn open_bulk_out_pipe(
+        &self,
+        number: u8,
+        max_packet_size: u16,
+    ) -> Result<UsbBulkOutPipeAccess, UsbError> {
+        let pipe = self
+            .device
+            .open_normal_out_pipe(number, max_packet_size, UsbEndpointType::Bulk)
+            .await?;
+        Ok(UsbBulkOutPipeAccess(pipe))
+    }
+
     pub fn read_current_configuration(
         &self,
     ) -> IrqSafeRwLockReadGuard<Option<UsbConfigurationInfo>> {
@@ -115,7 +183,7 @@ impl UsbDeviceAccess {
         let mut current_config = self.current_configuration.write();
         let control_pipe = self.control_pipe();
 
-        for i in 0..self.num_configurations {
+        for i in 0..self.info.num_configurations {
             let info = self.query_configuration_info(i).await?;
 
             if predicate(&info) {
@@ -137,7 +205,7 @@ impl UsbDeviceAccess {
         &self,
         index: u8,
     ) -> Result<UsbConfigurationInfo, UsbError> {
-        if index >= self.num_configurations {
+        if index >= self.info.num_configurations {
             return Err(UsbError::InvalidConfiguration);
         }
         let mut string_buffer = PageBox::new_uninit().map_err(UsbError::MemoryError)?;
@@ -172,6 +240,7 @@ impl UsbDeviceAccess {
                         interface_class: iface.class(),
                         interface_subclass: iface.interface_subclass,
                         interface_protocol: iface.protocol(),
+                        interface_protocol_number: iface.interface_protocol,
                     });
                 }
                 _ => (),
@@ -187,6 +256,10 @@ impl UsbDeviceAccess {
 
         Ok(info)
     }
+
+    pub fn set_detach_handler(&self, handler: Arc<dyn UsbDeviceDetachHandler>) {
+        self.device.set_detach_handler(handler);
+    }
 }
 
 impl Deref for UsbDeviceAccess {

kernel/driver/bus/usb/src/error.rs

@@ -1,5 +1,15 @@
 use yggdrasil_abi::error::Error;
 
+#[derive(Debug, Clone, Copy)]
+pub enum TransferError {
+    InvalidTransfer,
+    ShortPacket(usize),
+    BufferError,
+    UsbTransactionError,
+    Stall,
+    Other(u8),
+}
+
 #[derive(Debug)]
 pub enum UsbError {
     /// Could not allocate memory for some device structure
@@ -8,16 +18,25 @@ pub enum UsbError {
     SystemError(Error),
     // HC-side init stage errors
     OutOfAddresses,
-    HostControllerCommandFailed(u8),
+    HostControllerCommandFailed(u8, u32),
     PortResetFailed,
     PortInitFailed,
     // Setup stage errors
     InvalidConfiguration,
     InvalidDescriptorField,
     // Runtime errors
+    TimedOut,
     DeviceBusy,
     DeviceDisconnected,
-    TransferFailed,
+    TransferFailed(TransferError),
+    // Driver errors
+    DriverError,
+}
+
+impl From<TransferError> for UsbError {
+    fn from(value: TransferError) -> Self {
+        Self::TransferFailed(value)
+    }
 }
 
 impl From<UsbError> for Error {

kernel/driver/bus/usb/src/info.rs

@@ -1,9 +1,11 @@
+use core::fmt;
+
 use alloc::{string::String, vec::Vec};
 use yggdrasil_abi::primitive_enum;
 
-use crate::UsbDirection;
+use crate::communication::UsbDirection;
 
-#[derive(Debug, Clone, Copy)]
+#[derive(Debug, Clone, Copy, PartialEq)]
 pub enum UsbEndpointType {
     Control,
     Isochronous,
@@ -27,10 +29,21 @@ pub enum UsbUsageType {
     Reserved,
 }
 
+#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)]
+pub enum UsbVersion {
+    Usb11,
+    Usb20,
+    Usb21,
+    Usb30,
+    Usb31,
+    Usb32,
+}
+
 primitive_enum! {
     pub enum UsbDeviceClass: u8 {
         FromInterface = 0x00,
         Hid = 0x03,
+        MassStorage = 0x08,
         Unknown = 0xFF,
     }
 }
@@ -50,6 +63,7 @@ pub struct UsbInterfaceInfo {
     pub interface_class: UsbDeviceClass,
     pub interface_subclass: u8,
     pub interface_protocol: UsbDeviceProtocol,
+    pub interface_protocol_number: u8,
 }
 
 #[derive(Debug, Clone)]
@@ -73,12 +87,67 @@ pub struct UsbDeviceInfo {
     pub manufacturer: String,
     pub product: String,
 
+    pub usb_version: UsbVersion,
+
     pub id_vendor: u16,
     pub id_product: u16,
 
     pub device_class: UsbDeviceClass,
     pub device_subclass: u8,
     pub device_protocol: UsbDeviceProtocol,
+    pub device_protocol_number: u8,
 
+    /// Max packet size for endpoint zero
     pub max_packet_size: usize,
+
+    pub num_configurations: u8,
+}
+
+impl UsbVersion {
+    pub fn is_version_3(&self) -> bool {
+        matches!(self, Self::Usb30 | Self::Usb31 | Self::Usb32)
+    }
+
+    pub fn from_bcd_usb(value: u16) -> Option<Self> {
+        match value {
+            0x110 => Some(UsbVersion::Usb11),
+            0x200..=0x20F => Some(UsbVersion::Usb20),
+            0x210..=0x21F => Some(UsbVersion::Usb21),
+            0x300 => Some(UsbVersion::Usb30),
+            0x310 => Some(UsbVersion::Usb31),
+            0x320 => Some(UsbVersion::Usb32),
+            _ => None,
+        }
+    }
+}
+
+impl fmt::Display for UsbVersion {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let string = match self {
+            Self::Usb11 => "USB1.1",
+            Self::Usb20 => "USB2.0",
+            Self::Usb21 => "USB2.1",
+            Self::Usb30 => "USB3.0",
+            Self::Usb31 => "USB3.1",
+            Self::Usb32 => "USB3.2",
+        };
+        f.write_str(string)
+    }
+}
+
+impl UsbEndpointInfo {
+    pub fn is(&self, ty: UsbEndpointType, dir: UsbDirection) -> bool {
+        self.ty == ty && self.direction == dir
+    }
+}
+
+impl UsbConfigurationInfo {
+    pub fn find_endpoint<P: Fn(&UsbEndpointInfo) -> bool>(
+        &self,
+        predicate: P,
+    ) -> Option<(u8, &UsbEndpointInfo)> {
+        let index = self.endpoints.iter().position(predicate)?;
+        let info = &self.endpoints[index];
+        Some((index as u8 + 1, info))
+    }
 }

kernel/driver/bus/usb/src/lib.rs

@@ -15,7 +15,7 @@ pub mod util;
 
 pub mod class_driver;
 
-pub use communication::{UsbControlTransfer, UsbDirection, UsbTransferStatus, UsbTransferToken};
+// pub use communication::{UsbControlTransfer, UsbDirection, UsbTransferStatus, UsbTransferToken};
 
 pub trait UsbEndpoint: Sync {}
 

kernel/driver/bus/usb/src/pipe/control.rs

@@ -5,23 +5,19 @@ use core::{
 };
 
 use alloc::{boxed::Box, string::String};
+use async_trait::async_trait;
 use bytemuck::{Pod, Zeroable};
-use libk_mm::{
+use libk_mm::{PageBox, PageSlice};
-    address::{AsPhysicalAddress, PhysicalAddress},
-    PageBox,
-};
 
 use crate::{
+    communication::UsbDirection,
     descriptor::{
         UsbConfigurationDescriptor, UsbDeviceDescriptor, UsbDeviceQualifier, UsbEndpointDescriptor,
         UsbInterfaceDescriptor, UsbOtherSpeedConfiguration,
     },
     error::UsbError,
-    UsbControlTransfer, UsbDirection,
 };
 
-use super::UsbGenericPipe;
-
 #[derive(Debug)]
 pub struct ControlTransferSetup {
     pub bm_request_type: u8,
@@ -40,6 +36,11 @@ pub trait UsbDeviceRequest: Sized + Pod {
     const B_REQUEST: u8;
 }
 
+pub trait UsbClassSpecificRequest: Sized + Pod {
+    const BM_REQUEST_TYPE: u8;
+    const B_REQUEST: u8;
+}
+
 pub trait UsbDescriptorRequest: UsbDeviceRequest {
     const DESCRIPTOR_TYPE: u8;
 }
@@ -83,28 +84,17 @@ fn decode_usb_string(bytes: &[u8]) -> Result<String, UsbError> {
 
 // Pipe impl
 
-pub trait UsbControlPipe: UsbGenericPipe + Send + Sync {
+#[async_trait]
-    fn start_transfer(
+pub trait UsbControlPipe: Send + Sync {
+    async fn control_transfer(
         &self,
         setup: ControlTransferSetup,
-        data: Option<(PhysicalAddress, usize, UsbDirection)>,
+        data: Option<(&mut PageSlice<MaybeUninit<u8>>, UsbDirection)>,
-    ) -> Result<UsbControlTransfer, UsbError>;
+    ) -> Result<usize, UsbError>;
-
-    fn complete_transfer(&self, transfer: UsbControlTransfer);
 }
 
 pub struct UsbControlPipeAccess(pub Box<dyn UsbControlPipe>);
 
-fn input_buffer<T: Pod>(
-    data: &mut PageBox<MaybeUninit<T>>,
-) -> (PhysicalAddress, usize, UsbDirection) {
-    (
-        unsafe { data.as_physical_address() },
-        size_of::<T>(),
-        UsbDirection::In,
-    )
-}
-
 #[derive(Debug)]
 pub enum ConfigurationDescriptorEntry<'a> {
     Configuration(&'a UsbConfigurationDescriptor),
@@ -183,15 +173,40 @@ impl ConfigurationDescriptorQuery {
 }
 
 impl UsbControlPipeAccess {
-    pub async fn perform_value_control(
+    pub async fn query_device_descriptor2(&self) -> Result<PageBox<UsbDeviceDescriptor>, UsbError> {
-        &self,
+        let mut output = PageBox::new_uninit().map_err(UsbError::MemoryError)?;
-        setup: ControlTransferSetup,
+
-        buffer: Option<(PhysicalAddress, usize, UsbDirection)>,
+        self.control_transfer(
-    ) -> Result<(), UsbError> {
+            ControlTransferSetup {
-        let transfer = self.start_transfer(setup, buffer)?;
+                bm_request_type: 0b10000000,
-        transfer.status.wait().await?;
+                b_request: 0xFF,
-        self.complete_transfer(transfer);
+                // b_request: 0x06,
-        Ok(())
+                w_value: 0x100,
+                w_index: 0,
+                w_length: size_of::<UsbDeviceDescriptor>() as _,
+            },
+            Some((PageBox::as_bytes_mut(&mut output), UsbDirection::In)),
+        )
+        .await?;
+
+        Ok(unsafe { output.assume_init() })
+    }
+    pub async fn query_device_descriptor(&self) -> Result<PageBox<UsbDeviceDescriptor>, UsbError> {
+        let mut output = PageBox::new_uninit().map_err(UsbError::MemoryError)?;
+
+        self.control_transfer(
+            ControlTransferSetup {
+                bm_request_type: 0b10000000,
+                b_request: 0x06,
+                w_value: 0x100,
+                w_index: 0,
+                w_length: size_of::<UsbDeviceDescriptor>() as _,
+            },
+            Some((PageBox::as_bytes_mut(&mut output), UsbDirection::In)),
+        )
+        .await?;
+
+        Ok(unsafe { output.assume_init() })
     }
 
     async fn fill_configuation_descriptor(
@@ -199,7 +214,7 @@ impl UsbControlPipeAccess {
         index: u8,
         buffer: &mut PageBox<[MaybeUninit<u8>]>,
     ) -> Result<(), UsbError> {
-        self.perform_value_control(
+        self.control_transfer(
             ControlTransferSetup {
                 bm_request_type: 0b10000000,
                 b_request: 0x06,
@@ -207,13 +222,33 @@ impl UsbControlPipeAccess {
                 w_index: 0,
                 w_length: buffer.len().try_into().unwrap(),
             },
-            Some((
+            Some((buffer.as_slice_mut(), UsbDirection::In)),
-                unsafe { buffer.as_physical_address() },
-                buffer.len(),
-                UsbDirection::In,
-            )),
         )
-        .await
+        .await?;
+        Ok(())
+    }
+
+    pub async fn query_string(
+        &self,
+        index: u8,
+        buffer: &mut PageBox<MaybeUninit<[u8; 4096]>>,
+    ) -> Result<String, UsbError> {
+        self.control_transfer(
+            ControlTransferSetup {
+                bm_request_type: 0b10000000,
+                b_request: 0x06,
+                w_value: 0x300 | (index as u16),
+                w_index: 0,
+                w_length: 4096,
+            },
+            Some((PageBox::as_bytes_mut(buffer), UsbDirection::In)),
+        )
+        .await?;
+        let data = unsafe { buffer.assume_init_ref() };
+
+        let len = data[0] as usize;
+
+        decode_usb_string(&data[2..len])
     }
 
     pub async fn query_configuration_descriptor(
@@ -254,52 +289,12 @@ impl UsbControlPipeAccess {
         Ok(ConfigurationDescriptorQuery { buffer })
     }
 
-    pub async fn query_device_descriptor(&self) -> Result<PageBox<UsbDeviceDescriptor>, UsbError> {
-        let mut output = PageBox::new_uninit().map_err(UsbError::MemoryError)?;
-        self.perform_value_control(
-            ControlTransferSetup {
-                bm_request_type: 0b10000000,
-                b_request: 0x06,
-                w_value: 0x100,
-                w_index: 0,
-                w_length: size_of::<UsbDeviceDescriptor>() as _,
-            },
-            Some(input_buffer(&mut output)),
-        )
-        .await?;
-
-        Ok(unsafe { output.assume_init() })
-    }
-
-    pub async fn query_string(
-        &self,
-        index: u8,
-        buffer: &mut PageBox<MaybeUninit<[u8; 4096]>>,
-    ) -> Result<String, UsbError> {
-        self.perform_value_control(
-            ControlTransferSetup {
-                bm_request_type: 0b10000000,
-                b_request: 0x06,
-                w_value: 0x300 | (index as u16),
-                w_index: 0,
-                w_length: 4096,
-            },
-            Some(input_buffer(buffer)),
-        )
-        .await?;
-        let data = unsafe { buffer.assume_init_ref() };
-
-        let len = data[0] as usize;
-
-        decode_usb_string(&data[2..len])
-    }
-
     pub async fn perform_action<D: UsbDeviceRequest>(
         &self,
         w_value: u16,
         w_index: u16,
     ) -> Result<(), UsbError> {
-        self.perform_value_control(
+        self.control_transfer(
             ControlTransferSetup {
                 bm_request_type: D::BM_REQUEST_TYPE,
                 b_request: D::B_REQUEST,
@@ -309,7 +304,27 @@ impl UsbControlPipeAccess {
             },
             None,
         )
-        .await
+        .await?;
+        Ok(())
+    }
+
+    pub async fn class_specific_request<D: UsbClassSpecificRequest>(
+        &self,
+        w_value: u16,
+        w_index: u16,
+    ) -> Result<(), UsbError> {
+        self.control_transfer(
+            ControlTransferSetup {
+                bm_request_type: D::BM_REQUEST_TYPE,
+                b_request: D::B_REQUEST,
+                w_value,
+                w_index,
+                w_length: 0,
+            },
+            None,
+        )
+        .await?;
+        Ok(())
     }
 
     pub async fn set_configuration(&self, value: u16) -> Result<(), UsbError> {

kernel/driver/bus/usb/src/pipe/interrupt.rs

@@ -1,32 +0,0 @@
-use core::ops::Deref;
-
-use alloc::boxed::Box;
-use libk_mm::PageBox;
-
-use crate::{communication::UsbInterruptTransfer, error::UsbError};
-
-use super::UsbGenericPipe;
-
-pub trait UsbInterruptInPipe: UsbGenericPipe + Send + Sync {
-    fn start_read(&self, buffer: &mut PageBox<[u8]>) -> Result<UsbInterruptTransfer, UsbError>;
-    fn complete_transfer(&self, transfer: UsbInterruptTransfer);
-}
-
-pub struct UsbInterruptInPipeAccess(pub Box<dyn UsbInterruptInPipe>);
-
-impl UsbInterruptInPipeAccess {
-    pub async fn read<'a>(&self, buffer: &'a mut PageBox<[u8]>) -> Result<&'a [u8], UsbError> {
-        let transfer = self.start_read(buffer)?;
-        let len = transfer.wait().await?;
-        self.complete_transfer(transfer);
-        Ok(&buffer[..len])
-    }
-}
-
-impl Deref for UsbInterruptInPipeAccess {
-    type Target = dyn UsbInterruptInPipe;
-
-    fn deref(&self) -> &Self::Target {
-        &*self.0
-    }
-}

kernel/driver/bus/usb/src/pipe/mod.rs

@@ -1,8 +1,2 @@
 pub mod control;
-pub mod interrupt;
+pub mod normal;
-
-pub trait UsbGenericPipe {}
-
-pub enum UsbPipe {
-    Control(control::UsbControlPipeAccess),
-}

kernel/driver/bus/usb/src/pipe/normal.rs

@@ -0,0 +1,62 @@
+use core::ops::Deref;
+
+use alloc::boxed::Box;
+use async_trait::async_trait;
+use libk_mm::PageSlice;
+
+use crate::error::{TransferError, UsbError};
+
+#[async_trait]
+pub trait UsbNormalPipeIn: Send + Sync {
+    async fn read(&self, buffer: &mut PageSlice<u8>) -> Result<usize, UsbError>;
+    async fn read_exact(&self, buffer: &mut PageSlice<u8>) -> Result<(), UsbError> {
+        match self.read(buffer).await {
+            Ok(len) if len == buffer.len() => Ok(()),
+            Ok(len) => Err(UsbError::TransferFailed(TransferError::ShortPacket(len))),
+            Err(error) => Err(error),
+        }
+    }
+}
+
+#[async_trait]
+pub trait UsbNormalPipeOut: Send + Sync {
+    async fn write(&self, buffer: &PageSlice<u8>) -> Result<usize, UsbError>;
+}
+
+pub struct UsbBulkInPipeAccess(pub Box<dyn UsbNormalPipeIn>);
+pub struct UsbBulkOutPipeAccess(pub Box<dyn UsbNormalPipeOut>);
+
+pub struct UsbInterruptInPipeAccess(pub Box<dyn UsbNormalPipeIn>);
+pub struct UsbInterruptOutPipeAccess(pub Box<dyn UsbNormalPipeOut>);
+
+impl Deref for UsbBulkInPipeAccess {
+    type Target = dyn UsbNormalPipeIn;
+
+    fn deref(&self) -> &Self::Target {
+        self.0.deref()
+    }
+}
+
+impl Deref for UsbBulkOutPipeAccess {
+    type Target = dyn UsbNormalPipeOut;
+
+    fn deref(&self) -> &Self::Target {
+        self.0.deref()
+    }
+}
+
+impl Deref for UsbInterruptInPipeAccess {
+    type Target = dyn UsbNormalPipeIn;
+
+    fn deref(&self) -> &Self::Target {
+        self.0.deref()
+    }
+}
+
+impl Deref for UsbInterruptOutPipeAccess {
+    type Target = dyn UsbNormalPipeOut;
+
+    fn deref(&self) -> &Self::Target {
+        self.0.deref()
+    }
+}

kernel/driver/net/core/src/ethernet.rs

@@ -1,4 +1,4 @@
-use core::mem::size_of;
+use core::{fmt, mem::size_of};
 
 use alloc::sync::Arc;
 use bytemuck::Pod;
@@ -27,6 +27,37 @@ pub struct L2Packet {
     pub data: Arc<PageBox<[u8]>>,
 }
 
+/// Defines an Ethernet link speed
+#[derive(Debug, Clone, Copy)]
+pub enum EthernetSpeed {
+    /// 1Gbps link
+    Speed1000,
+    /// 100Mbps link
+    Speed100,
+    /// 10Mbps link
+    Speed10,
+    /// Link speed not available/unknown
+    Unknown,
+}
+
+/// Defines whether an Ethernet link is capable of transmiting data both ways simultaneously
+#[derive(Debug, Clone, Copy)]
+pub enum Duplex {
+    /// Half-duplex link with multiplexed Tx and Rx
+    Half,
+    /// Full-duplex link capable of simultaneous Tx and Rx
+    Full,
+    /// Duplex mode not available/unknown
+    Unknown,
+}
+
+/// Represents the state of an Ethernet link
+#[derive(Debug, Clone, Copy)]
+pub enum EthernetLinkState {
+    Up(EthernetSpeed, Duplex),
+    Down,
+}
+
 impl L2Packet {
     pub fn ethernet_frame(&self) -> &EthernetFrame {
         bytemuck::from_bytes(
@@ -78,3 +109,39 @@ pub fn handle(packet: L2Packet) {
         }
     }
 }
+
+impl fmt::Display for EthernetSpeed {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let text = match self {
+            Self::Speed10 => "10Mbps",
+            Self::Speed100 => "100Mbps",
+            Self::Speed1000 => "1Gbps",
+            Self::Unknown => "N/A",
+        };
+        f.write_str(text)
+    }
+}
+
+impl fmt::Display for Duplex {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let text = match self {
+            Self::Half => "half-duplex",
+            Self::Full => "full-duplex",
+            Self::Unknown => "N/A duplex mode",
+        };
+        f.write_str(text)
+    }
+}
+
+impl fmt::Display for EthernetLinkState {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::Up(speed, duplex) => {
+                write!(f, "up, speed {speed}, {duplex}")
+            }
+            Self::Down => {
+                write!(f, "down")
+            }
+        }
+    }
+}

kernel/driver/net/rtl81xx/Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+name = "ygg_driver_net_rtl81xx"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+device-api.workspace = true
+yggdrasil-abi.workspace = true
+libk-mm.workspace = true
+libk-util.workspace = true
+libk.workspace = true
+
+ygg_driver_pci.path = "../../bus/pci"
+ygg_driver_net_core.path = "../core"
+
+tock-registers.workspace = true
+log.workspace = true

kernel/driver/net/rtl81xx/src/lib.rs

@@ -0,0 +1,72 @@
+#![no_std]
+
+use alloc::sync::Arc;
+use device_api::device::Device;
+use libk::error::Error;
+use rtl8139::Rtl8139;
+use rtl8168::Rtl8168;
+use ygg_driver_pci::{
+    device::{PciDeviceInfo, PreferredInterruptMode},
+    macros::pci_driver,
+    PciBaseAddress, PciCommandRegister, PciConfigurationSpace,
+};
+
+extern crate alloc;
+
+pub mod rtl8139;
+pub mod rtl8168;
+
+pci_driver! {
+    matches: [device (0x10EC:0x8139)],
+    driver: {
+        fn driver_name(&self) -> &str {
+            "rtl8139"
+        }
+
+        fn probe(&self, info: &PciDeviceInfo) -> Result<Arc<dyn Device>, Error> {
+            info.init_interrupts(PreferredInterruptMode::Msi(false))?;
+
+            // Enable MMIO + interrupts + bus mastering
+            let mut command = info.config_space.command();
+            command |= (PciCommandRegister::BUS_MASTER | PciCommandRegister::ENABLE_MEMORY).bits();
+            command &= !(PciCommandRegister::ENABLE_IO | PciCommandRegister::DISABLE_INTERRUPTS).bits();
+            info.config_space.set_command(command);
+
+            let base = info
+                .config_space
+                .bar(1)
+                .and_then(PciBaseAddress::as_memory)
+                .ok_or(Error::InvalidArgument)?;
+
+            let device = Rtl8139::new(base, info.clone())?;
+            Ok(Arc::new(device))
+        }
+    }
+}
+
+pci_driver! {
+    matches: [device (0x10EC:0x8168)],
+    driver: {
+        fn driver_name(&self) -> &str {
+            "rtl816x"
+        }
+
+        fn probe(&self, info: &PciDeviceInfo) -> Result<Arc<dyn Device>, Error> {
+            let base = info
+                .config_space
+                .bar(2)
+                .and_then(PciBaseAddress::as_memory)
+                .ok_or(Error::InvalidArgument)?;
+
+            // if let Some(power) = info.config_space.capability::<PowerManagementCapability>() {
+            //     power.set_device_power_state(DevicePowerState::D0);
+            // }
+
+            // Enable MMIO + interrupts + bus mastering
+            info.set_command(true, true, false, true);
+
+            let device = Rtl8168::new(base, info.clone())?;
+            Ok(Arc::new(device))
+        }
+    }
+}

kernel/driver/net/rtl81xx/src/rtl8139.rs

@@ -0,0 +1,422 @@
+use core::mem::MaybeUninit;
+
+use alloc::sync::Arc;
+use device_api::{device::Device, interrupt::InterruptHandler};
+use libk::error::Error;
+use libk_mm::{
+    address::{AsPhysicalAddress, PhysicalAddress},
+    device::DeviceMemoryIo,
+    PageBox,
+};
+use libk_util::{queue::BoundedQueue, sync::IrqSafeSpinlock, OneTimeInit};
+use tock_registers::{
+    interfaces::{ReadWriteable, Readable, Writeable},
+    register_bitfields, register_structs,
+    registers::{ReadOnly, ReadWrite},
+};
+use ygg_driver_net_core::{
+    interface::{NetworkDevice, NetworkInterfaceType},
+    Packet,
+};
+use ygg_driver_pci::device::PciDeviceInfo;
+use yggdrasil_abi::net::MacAddress;
+
+register_bitfields! {
+    u8,
+
+    pub CR [
+        /// Reset. Setting this bit forces the RTL8139D(L) to perform a software reset. This bit is
+        /// cleared by the RTL8139D(L) after the reset completes.
+        RST     OFFSET(4)   NUMBITS(1)  [],
+        /// Receiver enable
+        RE      OFFSET(3)   NUMBITS(1)  [],
+        /// Transmitter enable
+        TE      OFFSET(2)   NUMBITS(1)  [],
+        /// Rx buffer empty
+        BUFE    OFFSET(0)   NUMBITS(1)  [],
+    ],
+
+    pub CONFIG1 [
+        /// LED pin control
+        LEDS1   OFFSET(7)   NUMBITS(1)  [],
+        /// LED pin control
+        LEDS0   OFFSET(6)   NUMBITS(1)  [],
+        /// Driver load. Software may use this bit to make sure that the driver has been loaded.
+        DVRLOAD OFFSET(5)   NUMBITS(1)  [],
+        /// LWAKE active mode. The LWACT bit and LWPTN bit in CONFIG4 are used to program the LWAKE
+        /// pin's output signal.
+        LWACT   OFFSET(4)   NUMBITS(1)  [],
+        /// The operational registers are mapped into PCI memory space
+        MEMMAP  OFFSET(3)   NUMBITS(1)  [],
+        /// The operational registers are mapped into PCI I/O space
+        IOMAP   OFFSET(2)   NUMBITS(1)  [],
+        /// Set to enable Vital Product Data
+        VPD     OFFSET(1)   NUMBITS(1)  [],
+        /// Power management enable
+        PMEn    OFFSET(0)   NUMBITS(1)  [],
+    ],
+}
+
+register_bitfields! {
+    u16,
+
+    pub IMR_ISR [
+        /// System error interrupt
+        SERR        OFFSET(15)  NUMBITS(1)  [],
+        /// Time out interrupt
+        TIMEOUT     OFFSET(14)  NUMBITS(1)  [],
+        /// Cable length change interrupt
+        LENCHG      OFFSET(13)  NUMBITS(1)  [],
+        /// Rx FIFO overflow interrupt
+        FOVW        OFFSET(6)   NUMBITS(1)  [],
+        /// Packet underrun/Link change interrupt
+        PUN_LINKCHG OFFSET(5)   NUMBITS(1)  [],
+        /// Rx buffer overflow
+        RXOVW       OFFSET(4)   NUMBITS(1)  [],
+        /// Tx error interrupt
+        TER         OFFSET(3)   NUMBITS(1)  [],
+        /// Tx OK interrupt
+        TOK         OFFSET(2)   NUMBITS(1)  [],
+        /// Rx error interrupt
+        RER         OFFSET(1)   NUMBITS(1)  [],
+        /// Rx OK interrupt
+        ROK         OFFSET(0)   NUMBITS(1)  [],
+    ],
+}
+
+register_bitfields! {
+    u32,
+
+    pub TSDn [
+        /// Carrier sense lost
+        CRS         OFFSET(31)  NUMBITS(1)  [],
+        /// Tx aborted
+        TABT        OFFSET(30)  NUMBITS(1)  [],
+        /// Out of window collision
+        OWC         OFFSET(29)  NUMBITS(1)  [],
+        /// CD heart beat
+        CDH         OFFSET(28)  NUMBITS(1)  [],
+        /// Number of collision count
+        NCC         OFFSET(24)  NUMBITS(4)  [],
+        /// Early Tx threshold
+        ERTXTH      OFFSET(16)  NUMBITS(6)  [],
+        /// Tx OK
+        TOK         OFFSET(15)  NUMBITS(1)  [],
+        /// Tx FIFO underrun
+        TUN         OFFSET(14)  NUMBITS(1)  [],
+        /// Tx descriptor is owned by the DMA. Set to 0 by the driver to initiate a send
+        OWN         OFFSET(13)  NUMBITS(1)  [],
+        /// Tx size
+        SIZE        OFFSET(0)   NUMBITS(13) [],
+    ],
+
+    pub RCR [
+        /// Early Rx threshold bits
+        ERTH        OFFSET(24)  NUMBITS(4)  [],
+        /// Multiple early interrupt select
+        MULERINT    OFFSET(17)  NUMBITS(1)  [],
+        /// TBD, didn't really understand the docs, lmao
+        RER8        OFFSET(16)  NUMBITS(1)  [],
+        /// Rx FIFO threshold
+        RXFTH       OFFSET(13)  NUMBITS(3)  [],
+        /// Rx buffer length (+16 bytes)
+        RBLEN       OFFSET(11)  NUMBITS(2)  [
+            Len8K = 0,
+            Len16K = 1,
+            Len32K = 2,
+            Len64K = 3,
+        ],
+        /// Max DMA burst size per Rx DMA burst
+        MXDMA       OFFSET(8)   NUMBITS(3)  [],
+        /// When 0: the packet will get split if crossing the Rx buffer end
+        /// When 1: the packet will get placed contiguously
+        WRAP        OFFSET(7)   NUMBITS(1)  [],
+        /// Accept error packet
+        AER         OFFSET(5)   NUMBITS(1)  [],
+        /// Accept runt (packets smaller than 64 bytes)
+        AR          OFFSET(4)   NUMBITS(1)  [],
+        /// Accept broadcast packets
+        AB          OFFSET(3)   NUMBITS(1)  [],
+        /// Accept multicast packets
+        AM          OFFSET(2)   NUMBITS(1)  [],
+        /// Accept physical match
+        APM         OFFSET(1)   NUMBITS(1)  [],
+        /// Accept all packets
+        AAP         OFFSET(0)   NUMBITS(1)  [],
+    ],
+}
+
+register_structs! {
+    #[allow(non_snake_case)]
+    pub Regs {
+        (0x0000 => pub IDRn: [ReadWrite<u32>; 2]),
+        (0x0008 => pub MARn: [ReadWrite<u8>; 8]),
+        (0x0010 => pub TSDn: [ReadWrite<u32, TSDn::Register>; 4]),
+        (0x0020 => pub TSADn: [ReadWrite<u32>; 4]),
+        (0x0030 => pub RBSTART: ReadWrite<u32>),
+        (0x0034 => pub ERBCR: ReadOnly<u16>),
+        (0x0036 => pub ERSR: ReadOnly<u8>),
+        (0x0037 => pub CR: ReadWrite<u8, CR::Register>),
+        (0x0038 => pub CAPR: ReadWrite<u16>),
+        (0x003A => pub CBR: ReadOnly<u16>),
+        (0x003C => pub IMR: ReadWrite<u16, IMR_ISR::Register>),
+        (0x003E => pub ISR: ReadWrite<u16, IMR_ISR::Register>),
+        (0x0040 => pub TCR: ReadWrite<u32>),
+        (0x0044 => pub RCR: ReadWrite<u32, RCR::Register>),
+        (0x0048 => pub TCTR: ReadWrite<u32>),
+        (0x004C => pub MPC: ReadWrite<u32>),
+        (0x0050 => pub r9346CR: ReadWrite<u8>),
+        (0x0051 => pub CONFIG0: ReadOnly<u8>),
+        (0x0052 => pub CONFIG1: ReadWrite<u8, CONFIG1::Register>),
+        (0x0053 => _1),
+        (0x0054 => pub TIMERINT: ReadWrite<u32>),
+        (0x0058 => pub MSR: ReadWrite<u8>),
+        (0x0059 => pub CONFIG3: ReadWrite<u8>),
+        (0x005A => pub CONFIG4: ReadWrite<u8>),
+        (0x005B => _2),
+        (0x005C => pub MULINT: ReadWrite<u16>),
+        (0x005E => pub RERID: ReadOnly<u8>),
+        (0x005F => _3),
+        (0x0060 => pub TSAD: ReadOnly<u16>),
+        (0x0062 => pub BMCR: ReadWrite<u16>),
+        (0x0064 => pub BMSR: ReadOnly<u16>),
+        (0x0066 => pub ANAR: ReadWrite<u16>),
+        (0x0068 => pub ANLPAR: ReadOnly<u16>),
+        (0x006A => pub ANER: ReadOnly<u16>),
+        (0x006C => pub DIS: ReadOnly<u16>),
+        (0x006E => pub FCSC: ReadOnly<u16>),
+        (0x0070 => pub NWAYTR: ReadWrite<u16>),
+        (0x0072 => pub REC: ReadOnly<u16>),
+        (0x0074 => pub CSCR: ReadWrite<u16>),
+        (0x0076 => _4),
+        (0x0078 => pub PHY1_PARM: ReadWrite<u32>),
+        (0x007C => pub TW_PARM: ReadWrite<u32>),
+        (0x0080 => pub PHY2_PARM: ReadWrite<u8>),
+        (0x0081 => _5),
+        (0x0084 => pub CRCn: [ReadWrite<u8>; 8]),
+        (0x008C => pub WAKEUPn: [ReadWrite<u32>; 16]),
+        (0x00CC => pub LSBCRCn: [ReadWrite<u8>; 8]),
+        (0x00D4 => _6),
+        (0x00D8 => pub CONFIG5: ReadWrite<u8>),
+        (0x00D9 => _7),
+        (0x0100 => @END),
+    }
+}
+
+struct Rx {
+    buffer: PageBox<[MaybeUninit<u8>]>,
+    rd: usize,
+}
+
+// TODO place a secondary Tx queue here, to send the queued packets when more slots become
+// available
+struct Tx {
+    buffers: [Option<PageBox<[u8]>>; 4],
+    wr: usize,
+    rd: usize,
+    queue: BoundedQueue<PageBox<[u8]>>,
+}
+
+pub struct Rtl8139 {
+    regs: IrqSafeSpinlock<DeviceMemoryIo<'static, Regs>>,
+    mac: MacAddress,
+    pci: PciDeviceInfo,
+
+    nic: OneTimeInit<u32>,
+    rx: OneTimeInit<IrqSafeSpinlock<Rx>>,
+    tx: IrqSafeSpinlock<Tx>,
+}
+
+impl Tx {
+    pub fn tx_now(&mut self, regs: &Regs, packet: PageBox<[u8]>) -> Result<(), Error> {
+        let packet_address = unsafe { packet.as_physical_address() }
+            .try_into_u32()
+            .map_err(|_| Error::InvalidArgument)?;
+        let packet_len = packet.len();
+        if packet_len > 1500 {
+            return Err(Error::InvalidArgument);
+        }
+
+        self.buffers[self.wr] = Some(packet);
+        regs.TSADn[self.wr].set(packet_address);
+        regs.TSDn[self.wr].write(TSDn::SIZE.val(packet_len as u32));
+
+        self.wr = (self.wr + 1) % 4;
+
+        Ok(())
+    }
+
+    pub fn is_full(&self) -> bool {
+        (self.wr + 1) % 4 == self.rd
+    }
+}
+
+impl Rtl8139 {
+    // 8K + overflow space
+    const RX_BUFFER_LEN: usize = 8192;
+    const RX_BUFFER_OVERFLOW: usize = 4096;
+
+    pub fn new(base: PhysicalAddress, info: PciDeviceInfo) -> Result<Self, Error> {
+        let regs = unsafe { DeviceMemoryIo::<Regs>::map(base, Default::default()) }?;
+        let mac0 = regs.IDRn[0].get().to_le_bytes();
+        let mac1 = regs.IDRn[1].get().to_le_bytes();
+        let mac = MacAddress::from([mac0[0], mac0[1], mac0[2], mac0[3], mac1[0], mac1[1]]);
+
+        Ok(Self {
+            mac,
+            regs: IrqSafeSpinlock::new(regs),
+            pci: info,
+
+            nic: OneTimeInit::new(),
+            rx: OneTimeInit::new(),
+            tx: IrqSafeSpinlock::new(Tx {
+                buffers: [const { None }; 4],
+                wr: 0,
+                rd: 0,
+                queue: BoundedQueue::new(64),
+            }),
+        })
+    }
+}
+
+impl InterruptHandler for Rtl8139 {
+    fn handle_irq(self: Arc<Self>, _vector: Option<usize>) -> bool {
+        let regs = self.regs.lock();
+        let status = regs.ISR.extract();
+        // Clear ISR bits
+        regs.ISR.write(IMR_ISR::TOK::SET + IMR_ISR::ROK::SET);
+
+        let mut any = false;
+        if status.matches_all(IMR_ISR::TOK::SET) {
+            let mut tx = self.tx.lock();
+
+            tx.rd = (tx.rd + 1) % 4;
+            if let Some(packet) = tx.queue.pop() {
+                // Should not fail, we just freed a single descriptor
+                tx.tx_now(&regs, packet).unwrap();
+            }
+
+            any = true;
+        }
+        if status.matches_all(IMR_ISR::ROK::SET) {
+            let nic = *self.nic.get();
+            let mut rx = self.rx.get().lock();
+            let cbr = regs.CBR.get();
+
+            loop {
+                let rx_pos = rx.rd % Self::RX_BUFFER_LEN;
+                if rx_pos == cbr as usize {
+                    break;
+                }
+
+                // 4-byte preamble
+                let rx_len_0 = unsafe { rx.buffer[rx_pos + 2].assume_init() };
+                let rx_len_1 = unsafe { rx.buffer[rx_pos + 3].assume_init() };
+                let rx_len = u16::from_le_bytes([rx_len_0, rx_len_1]) as usize;
+
+                if rx_len >= 16 {
+                    if let Ok(mut packet_buf) = PageBox::new_uninit_slice(rx_len) {
+                        packet_buf.copy_from_slice(&rx.buffer[rx_pos + 4..rx_pos + rx_len + 4]);
+                        let packet_buf = unsafe { packet_buf.assume_init_slice() };
+                        let packet = Packet::new(packet_buf, 0, nic);
+                        ygg_driver_net_core::receive_packet(packet).ok();
+                    }
+                }
+
+                // rx_len + 4, aligned to 4 bytes
+                let total_len = (rx_len + 7) & !3;
+                rx.rd = rx.rd.wrapping_add(total_len);
+            }
+
+            regs.CAPR.set((rx.rd as u16).max(cbr).wrapping_sub(0x10));
+            rx.rd %= Self::RX_BUFFER_LEN;
+
+            any = true;
+        }
+
+        any
+    }
+}
+
+impl Device for Rtl8139 {
+    unsafe fn init(self: Arc<Self>) -> Result<(), Error> {
+        log::info!("Initialize rtl8139 driver");
+        log::info!("MAC: {}", self.mac);
+
+        // Setup the initial Rx buffer
+        let rx_buffer = PageBox::new_uninit_slice(Self::RX_BUFFER_LEN + Self::RX_BUFFER_OVERFLOW)?;
+        let rx_buffer_address = unsafe { rx_buffer.as_physical_address() }
+            .try_into_u32()
+            .map_err(|_| Error::InvalidArgument)?;
+
+        self.pci.map_interrupt(Default::default(), self.clone())?;
+
+        let regs = self.regs.lock();
+
+        // Power up the device
+        regs.CONFIG1.set(0);
+
+        // Reset the device
+        let mut timeout = 1000000;
+        regs.CR.write(CR::RST::SET);
+        while timeout > 0 && regs.CR.matches_all(CR::RST::SET) {
+            core::hint::spin_loop();
+            timeout -= 1;
+        }
+        if timeout == 0 {
+            log::error!("rtl8139: reset timed out");
+            return Err(Error::TimedOut);
+        }
+
+        // Configure Rx
+        regs.RBSTART.set(rx_buffer_address);
+        regs.RCR
+            .write(RCR::WRAP::SET + RCR::AB::SET + RCR::APM::SET + RCR::AR::SET + RCR::AM::SET);
+
+        // Enable Rx/Tx interrupts
+        regs.IMR.write(IMR_ISR::ROK::SET + IMR_ISR::TOK::SET);
+
+        // Start Rx/Tx
+        regs.CR.modify(CR::TE::SET + CR::RE::SET);
+
+        self.rx.init(IrqSafeSpinlock::new(Rx {
+            buffer: rx_buffer,
+            rd: 0,
+        }));
+        let nic =
+            ygg_driver_net_core::register_interface(NetworkInterfaceType::Ethernet, self.clone());
+        self.nic.init(nic.id());
+
+        Ok(())
+    }
+
+    fn display_name(&self) -> &str {
+        "Realtek RTL8139 10/100Mbps Ethernet Controller"
+    }
+}
+
+impl NetworkDevice for Rtl8139 {
+    fn transmit(&self, packet: PageBox<[u8]>) -> Result<(), Error> {
+        let mut tx = self.tx.lock();
+
+        // Buffer still in Tx, cannot send
+        if tx.is_full() {
+            if tx.queue.push(packet).is_err() {
+                log::warn!("rtl8139: out of tx buffers + queue full");
+                return Err(Error::WouldBlock);
+            } else {
+                return Ok(());
+            }
+        }
+        let regs = self.regs.lock();
+        tx.tx_now(&regs, packet)
+    }
+
+    fn packet_prefix_size(&self) -> usize {
+        0
+    }
+
+    fn read_hardware_address(&self) -> MacAddress {
+        self.mac
+    }
+}

kernel/driver/net/rtl81xx/src/rtl8168.rs

@@ -0,0 +1,722 @@
+use core::{
+    mem::{self, MaybeUninit},
+    sync::atomic::{self, Ordering},
+};
+
+use alloc::{sync::Arc, vec::Vec};
+use device_api::{device::Device, interrupt::InterruptHandler};
+use libk::error::Error;
+use libk_mm::{
+    address::{AsPhysicalAddress, PhysicalAddress},
+    device::DeviceMemoryIo,
+    PageBox,
+};
+use libk_util::{sync::IrqSafeSpinlock, OneTimeInit};
+use tock_registers::{
+    interfaces::{ReadWriteable, Readable, Writeable},
+    register_bitfields, register_structs,
+    registers::{ReadOnly, ReadWrite, WriteOnly},
+};
+use ygg_driver_net_core::{
+    ethernet::{Duplex, EthernetLinkState, EthernetSpeed},
+    interface::{NetworkDevice, NetworkInterfaceType},
+    Packet,
+};
+use ygg_driver_pci::device::{PciDeviceInfo, PreferredInterruptMode};
+use yggdrasil_abi::{bitflags, net::MacAddress};
+
+register_bitfields! {
+    u8,
+    CR [
+        /// Software reset bit. Set by driver, cleared by device
+        RST     OFFSET(4)   NUMBITS(1)  [],
+        /// Rx enable
+        RE      OFFSET(3)   NUMBITS(1)  [],
+        /// Tx enable
+        TE      OFFSET(2)   NUMBITS(1)  [],
+    ],
+    TPPOLL [
+        HPQ     OFFSET(7)   NUMBITS(1)  [],
+        NPQ     OFFSET(6)   NUMBITS(1)  [],
+        FSWInt  OFFSET(0)   NUMBITS(1)  [],
+    ],
+    PHYSTATUS [
+        TXFLOW  OFFSET(6)   NUMBITS(1)  [],
+        RXFLOW  OFFSET(5)   NUMBITS(1)  [],
+        MF1000  OFFSET(4)   NUMBITS(1)  [],
+        M100    OFFSET(3)   NUMBITS(1)  [],
+        M10     OFFSET(2)   NUMBITS(1)  [],
+        LINKSTS OFFSET(1)   NUMBITS(1)  [],
+        FULLDUP OFFSET(0)   NUMBITS(1)  [],
+    ],
+}
+
+register_bitfields! {
+    u16,
+    IMR_ISR [
+        TIMEOUT OFFSET(14)  NUMBITS(1)  [],
+        FEMP    OFFSET(9)   NUMBITS(1)  [],
+        SWINT   OFFSET(8)   NUMBITS(1)  [],
+        TDU     OFFSET(7)   NUMBITS(1)  [],
+        FOVW    OFFSET(6)   NUMBITS(1)  [],
+        LINKCHG OFFSET(5)   NUMBITS(1)  [],
+        RDU     OFFSET(4)   NUMBITS(1)  [],
+        TER     OFFSET(3)   NUMBITS(1)  [],
+        TOK     OFFSET(2)   NUMBITS(1)  [],
+        RER     OFFSET(1)   NUMBITS(1)  [],
+        ROK     OFFSET(0)   NUMBITS(1)  [],
+    ],
+    CPCR [
+        /// (rtl8169sc) PCI endianness mode
+        ENDIAN      OFFSET(9)   NUMBITS(1)  [
+            Little = 0,
+            Big = 1,
+        ],
+        /// (no docs found) TBD
+        MACSTATDIS  OFFSET(7)   NUMBITS(1)  [],
+        /// Rx VLAN de-tagging enable
+        RXVLAN      OFFSET(6)   NUMBITS(1)  [],
+        /// Rx checksum offload enable
+        RXCHKSUM    OFFSET(5)   NUMBITS(1)  [],
+        /// PCI dual address cycle enable
+        DAC         OFFSET(4)   NUMBITS(1)  [],
+        /// PCI multiple read/write enable
+        MULRW       OFFSET(3)   NUMBITS(1)  [],
+
+        // Not documented, found in OpenBSD's re driver
+        /// C+ Rx mode enable
+        RXENB       OFFSET(1)   NUMBITS(1)  [],
+        /// C+ Tx mode enable
+        TXENB       OFFSET(0)   NUMBITS(1)  [],
+    ],
+}
+
+register_bitfields! {
+    u32,
+    TCR [
+        HWVERID0    OFFSET(28)  NUMBITS(2)  [],
+        HWVERID1    OFFSET(26)  NUMBITS(1)  [],
+        IFG         OFFSET(24)  NUMBITS(2)  [],
+        HWVERID2    OFFSET(23)  NUMBITS(1)  [],
+        TX_NOCRC    OFFSET(16)  NUMBITS(1)  [],
+        MXDMA       OFFSET(8)   NUMBITS(1)  [
+            Burst16 = 0b000,
+            Burst32 = 0b001,
+            Burst64 = 0b010,
+            Burst128 = 0b011,
+            Burst256 = 0b100,
+            Burst512 = 0b101,
+            Unlimited = 0b111,
+        ],
+    ],
+    RCR [
+        RXFTH       OFFSET(13)  NUMBITS(3)  [
+            Threshold64 = 0b010,
+            Threshold128 = 0b011,
+            Threshold256 = 0b100,
+            Threshold512 = 0b101,
+            Threshold1024 = 0b110,
+            NoThreshold = 0b111,
+        ],
+        EARLYOFFV2  OFFSET(11)  NUMBITS(1)  [],
+        MXDMA       OFFSET(8)   NUMBITS(3)  [
+            Burst64 = 0b010,
+            Burst128 = 0b011,
+            Burst256 = 0b100,
+            Burst512 = 0b101,
+            Burst1024 = 0b110,
+            Unlimited = 0b111
+        ],
+        R9356SEL    OFFSET(6)   NUMBITS(1)  [],
+        AER         OFFSET(5)   NUMBITS(1)  [],
+        AR          OFFSET(4)   NUMBITS(1)  [],
+        AB          OFFSET(3)   NUMBITS(1)  [],
+        AM          OFFSET(2)   NUMBITS(1)  [],
+        APM         OFFSET(1)   NUMBITS(1)  [],
+        AAP         OFFSET(0)   NUMBITS(1)  [],
+    ],
+    PHYAR [
+        FLAG        OFFSET(31)  NUMBITS(1)  [
+            Read = 0,
+            Write = 1,
+        ],
+        REGADDR     OFFSET(16)  NUMBITS(5)  [],
+        DATA        OFFSET(0)   NUMBITS(16) [],
+    ],
+}
+
+register_structs! {
+    #[allow(non_snake_case)]
+    Regs {
+        (0x0000 => IDRn: [ReadWrite<u32>; 2]),
+        (0x0008 => MARn: [ReadWrite<u32>; 2]),
+        (0x0010 => DTCCR: ReadWrite<u64>),
+        (0x0018 => _0),
+        (0x0020 => TNPDS: [ReadWrite<u32>; 2]),
+        (0x0028 => THPDS: [ReadWrite<u32>; 2]),
+        (0x0030 => _1),
+        (0x0037 => CR: ReadWrite<u8, CR::Register>),
+        (0x0038 => TPPOLL: WriteOnly<u8, TPPOLL::Register>),
+        (0x0039 => _2),
+        (0x003C => IMR: ReadWrite<u16, IMR_ISR::Register>),
+        (0x003E => ISR: ReadWrite<u16, IMR_ISR::Register>),
+        (0x0040 => TCR: ReadWrite<u32, TCR::Register>),
+        (0x0044 => RCR: ReadWrite<u32, RCR::Register>),
+        (0x0048 => TCTR: ReadWrite<u32>),
+        (0x004C => MPKT: ReadWrite<u32>),
+        (0x0050 => R9346CR: ReadWrite<u8>),
+        (0x0051 => CONFIG0: ReadWrite<u8>),
+        (0x0052 => CONFIG1: ReadWrite<u8>),
+        (0x0053 => CONFIG2: ReadWrite<u8>),
+        (0x0054 => CONFIG3: ReadWrite<u8>),
+        (0x0055 => CONFIG4: ReadWrite<u8>),
+        (0x0056 => CONFIG5: ReadWrite<u8>),
+        (0x0057 => _4),
+        (0x0058 => TIMERINT: ReadWrite<u32>),
+        (0x005C => _5),
+        (0x0060 => PHYAR: ReadWrite<u32, PHYAR::Register>),
+        (0x0064 => _6),
+        (0x006C => PHYSTATUS: ReadOnly<u8, PHYSTATUS::Register>),
+        (0x006D => _7),
+        (0x0084 => WAKEUPn: [ReadWrite<u32>; 16]),
+        (0x00C4 => CRCn: [ReadWrite<u16>; 5]),
+        (0x00CE => _8),
+        (0x00DA => RMS: ReadWrite<u16>),
+        (0x00DC => _9),
+        (0x00E0 => CPCR: ReadWrite<u16, CPCR::Register>),
+        (0x00E2 => _10),
+        (0x00E4 => RDSAR: [ReadWrite<u32>; 2]),
+        (0x00EC => MTPS: ReadWrite<u8>),
+        (0x00ED => _11),
+        (0x00F0 => MISC: ReadWrite<u32>),
+        (0x00F4 => _12),
+        (0x0100 => @END),
+    }
+}
+
+bitflags! {
+    struct ChipFlags: u32 {
+        const MACSTAT: bit 0;
+        const NEED_PHY_RESET: bit 1;
+        const RXDV_GATED: bit 2;
+        const EARLYOFFV2: bit 3;
+    }
+}
+
+#[derive(Clone, Copy, PartialEq, Debug)]
+#[allow(non_camel_case_types)]
+enum Revision {
+    rtl8168gu,
+    rtl8168h,
+    other,
+}
+
+#[repr(C)]
+struct Descriptor {
+    cmd: u32,
+    vlan_cmd: u32,
+    address: PhysicalAddress,
+}
+
+struct RxRing {
+    entries: PageBox<[Descriptor]>,
+    buffers: Vec<PageBox<[MaybeUninit<u8>]>>,
+    rd: usize,
+}
+
+struct TxRing {
+    entries: PageBox<[Descriptor]>,
+    buffers: Vec<Option<PageBox<[u8]>>>,
+    wr: usize,
+    rd: usize,
+}
+
+pub struct Rtl8168 {
+    regs: IrqSafeSpinlock<DeviceMemoryIo<'static, Regs>>,
+    mac: MacAddress,
+    pci: PciDeviceInfo,
+
+    nic: OneTimeInit<u32>,
+    rx: OneTimeInit<IrqSafeSpinlock<RxRing>>,
+    tx: OneTimeInit<IrqSafeSpinlock<TxRing>>,
+}
+
+impl RxRing {
+    pub fn with_capacity(capacity: usize) -> Result<Self, Error> {
+        let buffers = (0..capacity)
+            .map(|_| PageBox::new_uninit_slice(4096))
+            .collect::<Result<Vec<_>, _>>()?;
+        let entries = PageBox::new_slice_with(
+            |i| {
+                Descriptor::new_rx(
+                    unsafe { buffers[i].as_physical_address() },
+                    i == capacity - 1,
+                )
+            },
+            capacity,
+        )?;
+        unsafe { core::arch::asm!("wbinvd") };
+        Ok(Self {
+            entries,
+            buffers,
+            rd: 0,
+        })
+    }
+
+    pub fn consume<F: Fn(PageBox<[u8]>)>(&mut self, handler: F) -> Result<usize, Error> {
+        let mut count = 0;
+        loop {
+            let index = self.rd % self.entries.len();
+            let entry = &self.entries[index];
+
+            if entry.is_host_owned() {
+                let new_buffer = PageBox::new_uninit_slice(4096)?;
+                let new_buffer_address = unsafe { new_buffer.as_physical_address() };
+                let buffer = mem::replace(&mut self.buffers[index], new_buffer);
+                let buffer = unsafe { buffer.assume_init_slice() };
+                handler(buffer);
+
+                self.entries[index].setup_rx(new_buffer_address);
+
+                self.rd = self.rd.wrapping_add(1);
+                count += 1;
+            } else {
+                break;
+            }
+        }
+        Ok(count)
+    }
+
+    pub fn base_address(&self) -> PhysicalAddress {
+        unsafe { self.entries.as_physical_address() }
+    }
+}
+
+impl TxRing {
+    pub fn with_capacity(capacity: usize) -> Result<Self, Error> {
+        let entries =
+            PageBox::new_slice_with(|i| Descriptor::empty_tx(i == capacity - 1), capacity)?;
+        let buffers = (0..capacity).map(|_| None).collect();
+
+        Ok(Self {
+            entries,
+            buffers,
+            wr: 0,
+            rd: 0,
+        })
+    }
+
+    pub fn is_full(&self) -> bool {
+        self.wr.wrapping_add(1) % self.entries.len() == self.rd % self.entries.len()
+    }
+
+    pub fn push(&mut self, packet: PageBox<[u8]>) -> Result<(), Error> {
+        let packet_base = unsafe { packet.as_physical_address() };
+        let packet_size = packet.len();
+
+        // TODO packet size checks
+
+        if self.is_full() {
+            log::warn!("rtl81xx: tx ring full: wr={}, rd={}", self.wr, self.rd);
+            return Err(Error::WouldBlock);
+        }
+
+        let index = self.wr % self.entries.len();
+        self.buffers[index] = Some(packet);
+        self.entries[index].setup_tx(packet_base, packet_size);
+
+        self.wr = self.wr.wrapping_add(1);
+
+        Ok(())
+    }
+
+    pub fn consume(&mut self) -> Result<usize, Error> {
+        let mut count = 0;
+        loop {
+            let index = self.rd % self.entries.len();
+            let entry = &mut self.entries[index];
+
+            if self.rd != self.wr && entry.is_host_owned() {
+                self.rd = self.rd.wrapping_add(1);
+                count += 1;
+            } else {
+                break;
+            }
+        }
+        Ok(count)
+    }
+
+    pub fn base_address(&self) -> PhysicalAddress {
+        unsafe { self.entries.as_physical_address() }
+    }
+}
+
+impl Descriptor {
+    // Descriptor owned by DMA
+    const CMD_OWN: u32 = 1 << 31;
+    // Last descriptor in the ring
+    const CMD_END: u32 = 1 << 30;
+    // Last segment of a packet
+    const CMD_LS: u32 = 1 << 28;
+    // First segment of a packet
+    const CMD_FS: u32 = 1 << 29;
+
+    pub fn new_rx(buffer: PhysicalAddress, last: bool) -> Self {
+        let mut cmd = Self::CMD_OWN | 0x1000;
+        if last {
+            cmd |= Self::CMD_END;
+        }
+        Self {
+            cmd,
+            vlan_cmd: 0,
+            address: buffer,
+        }
+    }
+
+    pub fn empty_tx(last: bool) -> Self {
+        let cmd = if last { Self::CMD_END } else { 0 };
+        Self {
+            cmd,
+            vlan_cmd: 0,
+            address: PhysicalAddress::ZERO,
+        }
+    }
+
+    pub fn setup_rx(&mut self, buffer: PhysicalAddress) {
+        let cmd = self.cmd;
+        self.address = buffer;
+        self.vlan_cmd = 0;
+        unsafe {
+            atomic::fence(Ordering::Release);
+            core::arch::asm!("wbinvd");
+            core::ptr::write_volatile(&mut self.cmd, cmd | Self::CMD_OWN);
+        }
+    }
+
+    pub fn setup_tx(&mut self, buffer: PhysicalAddress, size: usize) {
+        let mut cmd = self.cmd;
+        cmd |= Self::CMD_OWN | Self::CMD_FS | Self::CMD_LS | (size as u32);
+        self.address = buffer;
+        self.vlan_cmd = 0;
+        unsafe {
+            atomic::fence(Ordering::Release);
+            core::arch::asm!("wbinvd");
+            core::ptr::write_volatile(&mut self.cmd, cmd);
+        }
+    }
+
+    pub fn is_host_owned(&self) -> bool {
+        self.cmd & Self::CMD_OWN == 0
+    }
+}
+
+impl Regs {
+    const TCR_REVISION_MASK: u32 = 0x7C800000;
+
+    pub fn gmii_write(&self, reg: u8, value: u16, mut timeout: u64) -> Result<(), Error> {
+        self.PHYAR.write(
+            PHYAR::REGADDR.val(reg as u32) + PHYAR::DATA.val(value as u32) + PHYAR::FLAG::Write,
+        );
+        while timeout > 0 && self.PHYAR.matches_all(PHYAR::FLAG::SET) {
+            core::hint::spin_loop();
+            timeout -= 1;
+        }
+        if timeout == 0 {
+            Err(Error::TimedOut)
+        } else {
+            Ok(())
+        }
+    }
+
+    #[allow(unused)]
+    pub fn gmii_read(&self, reg: u8, mut timeout: u64) -> Result<u16, Error> {
+        self.PHYAR
+            .write(PHYAR::REGADDR.val(reg as u32) + PHYAR::FLAG::Read);
+
+        loop {
+            if timeout == 0 {
+                return Err(Error::TimedOut);
+            }
+
+            let status = self.PHYAR.extract();
+            if status.matches_all(PHYAR::FLAG::SET) {
+                return Ok(status.read(PHYAR::DATA) as u16);
+            }
+
+            core::hint::spin_loop();
+            timeout -= 1;
+        }
+    }
+
+    pub fn get_link_state(&self) -> EthernetLinkState {
+        let phystatus = self.PHYSTATUS.extract();
+        if phystatus.matches_all(PHYSTATUS::LINKSTS::SET) {
+            let duplex = if phystatus.matches_all(PHYSTATUS::FULLDUP::SET) {
+                Duplex::Full
+            } else {
+                Duplex::Half
+            };
+            let (speed, duplex) = if phystatus.matches_all(PHYSTATUS::MF1000::SET) {
+                // 1000MF is always Full-Duplex
+                (EthernetSpeed::Speed1000, Duplex::Full)
+            } else if phystatus.matches_all(PHYSTATUS::M100::SET) {
+                (EthernetSpeed::Speed100, duplex)
+            } else if phystatus.matches_all(PHYSTATUS::M10::SET) {
+                (EthernetSpeed::Speed10, duplex)
+            } else {
+                (EthernetSpeed::Unknown, duplex)
+            };
+
+            EthernetLinkState::Up(speed, duplex)
+        } else {
+            EthernetLinkState::Down
+        }
+    }
+
+    pub fn get_chip_revision(&self) -> Revision {
+        let rev = self.TCR.get() & Self::TCR_REVISION_MASK;
+        match rev {
+            0x50800000 => Revision::rtl8168gu,
+            0x54000000 => Revision::rtl8168h,
+            _ => Revision::other,
+        }
+    }
+
+    pub fn reset_device(&self, mut timeout: u64) -> Result<(), Error> {
+        self.CR.write(CR::RST::SET);
+
+        while timeout > 0 && self.CR.matches_all(CR::RST::SET) {
+            core::hint::spin_loop();
+            timeout -= 1;
+        }
+
+        if timeout == 0 {
+            Err(Error::TimedOut)
+        } else {
+            Ok(())
+        }
+    }
+
+    pub fn reset_phy(&self, timeout: u64) -> Result<(), Error> {
+        // Set the reset bit in bmcr
+        self.gmii_write(0x00, 1 << 15, timeout)?;
+        for _ in 0..timeout {
+            core::hint::spin_loop();
+        }
+        // Disable power off mode + start auto-negotiation
+        self.gmii_write(0x00, 1 << 12, timeout)?;
+        for _ in 0..timeout {
+            core::hint::spin_loop();
+        }
+
+        self.gmii_write(0x1F, 0x00, timeout)?;
+        self.gmii_write(0x0E, 0x00, timeout)?;
+
+        Ok(())
+    }
+
+    // #[inline]
+    // pub fn lock_config(&self) {
+    //     self.R9346CR.set(0x00);
+    // }
+
+    // #[inline]
+    // pub fn unlock_config(&self) {
+    //     self.R9346CR.set(0xC0);
+    // }
+}
+
+impl Rtl8168 {
+    pub fn new(base: PhysicalAddress, info: PciDeviceInfo) -> Result<Self, Error> {
+        let regs = unsafe { DeviceMemoryIo::<Regs>::map(base, Default::default()) }?;
+        let mac0 = regs.IDRn[0].get().to_le_bytes();
+        let mac1 = regs.IDRn[1].get().to_le_bytes();
+        let mac = MacAddress::from([mac0[0], mac0[1], mac0[2], mac0[3], mac1[0], mac1[1]]);
+        Ok(Self {
+            regs: IrqSafeSpinlock::new(regs),
+            mac,
+            pci: info,
+
+            nic: OneTimeInit::new(),
+            rx: OneTimeInit::new(),
+            tx: OneTimeInit::new(),
+        })
+    }
+}
+
+impl InterruptHandler for Rtl8168 {
+    fn handle_irq(self: Arc<Self>, _vector: Option<usize>) -> bool {
+        let regs = self.regs.lock();
+        let status = regs.ISR.extract();
+
+        if status.get() == 0 {
+            return false;
+        }
+        regs.ISR.set(status.get());
+
+        let mut any = false;
+        if status.matches_all(IMR_ISR::LINKCHG::SET) {
+            let state = regs.get_link_state();
+            log::info!("rtl8168: link is {state}");
+            any = true;
+        }
+        if status.matches_all(IMR_ISR::ROK::SET) {
+            let mut rx = self.rx.get().lock();
+            let nic = *self.nic.get();
+            let count = rx
+                .consume(|buffer| {
+                    // TODO add packet len hint to packets
+                    let packet = Packet::new(buffer, 0, nic);
+                    ygg_driver_net_core::receive_packet(packet).ok();
+                })
+                .unwrap_or(0);
+
+            any |= count != 0;
+        }
+        if status.matches_all(IMR_ISR::TOK::SET) {
+            let mut tx = self.tx.get().lock();
+            let count = tx.consume().unwrap_or(0);
+            any |= count != 0;
+        }
+        if status.matches_all(IMR_ISR::RDU::SET) {
+            log::info!("rtl8168: rdu");
+            any |= true;
+        }
+
+        if !any {
+            log::warn!("rtl8168: possibly unhandled IRQ, ISR={:#x}", status.get());
+        }
+
+        any
+    }
+}
+
+impl Device for Rtl8168 {
+    unsafe fn init(self: Arc<Self>) -> Result<(), Error> {
+        log::info!("Initialize rtl8168");
+        log::info!("MAC: {}", self.mac);
+
+        let rx_ring = RxRing::with_capacity(256)?;
+        let tx_ring = TxRing::with_capacity(256)?;
+        let rx_ring_base = rx_ring.base_address().into_u64();
+        let tx_ring_base = tx_ring.base_address().into_u64();
+
+        let regs = self.regs.lock();
+        let hwrev = regs.get_chip_revision();
+
+        log::info!("Revision: {:?}", hwrev);
+        let flags = hwrev.flags();
+
+        // Perform software reset
+        regs.reset_device(10000000)
+            .inspect_err(|_| log::warn!("rtl81xx: reset timed out"))?;
+
+        // Enable C+ mode for Rx/Tx
+        let mut cpcr = CPCR::TXENB::SET + CPCR::MULRW::SET + CPCR::RXCHKSUM::SET;
+        if flags.contains(ChipFlags::MACSTAT) {
+            cpcr += CPCR::MACSTATDIS::SET;
+        } else {
+            cpcr += CPCR::RXENB::SET;
+        }
+        regs.CPCR.write(cpcr);
+
+        // Reset the PHY
+        if flags.contains(ChipFlags::NEED_PHY_RESET) {
+            regs.reset_phy(10000000)
+                .inspect_err(|_| log::warn!("rtl81xx: PHY reset timed out"))?;
+        }
+
+        regs.RDSAR[1].set((rx_ring_base >> 32) as u32);
+        regs.RDSAR[0].set(rx_ring_base as u32);
+        regs.TNPDS[1].set((tx_ring_base >> 32) as u32);
+        regs.TNPDS[0].set(tx_ring_base as u32);
+
+        if flags.contains(ChipFlags::RXDV_GATED) {
+            regs.MISC.set(regs.MISC.get() & !0x80000);
+        }
+
+        log::info!("rx_ring_base = {rx_ring_base:#x}");
+
+        unsafe { core::arch::asm!("wbinvd") };
+
+        // Setup Tx
+        regs.TCR.modify(TCR::MXDMA::Unlimited);
+        // Setup Rx
+        let mut rcr = RCR::MXDMA::Unlimited
+            + RCR::RXFTH::NoThreshold
+            + RCR::AB::SET
+            + RCR::AM::SET
+            + RCR::APM::SET;
+        if flags.contains(ChipFlags::EARLYOFFV2) {
+            rcr += RCR::EARLYOFFV2::SET;
+        }
+        regs.RCR.write(rcr);
+
+        // Enable Rx/Tx
+        regs.CR.write(CR::RE::SET + CR::TE::SET);
+
+        // Setup interrupts
+        regs.IMR.set(0xFFFF);
+        regs.ISR.set(0xFFFF);
+
+        // Clear missed packet counter (?)
+        regs.MPKT.set(0);
+
+        regs.MTPS.set(31);
+        regs.RMS.set(0x1FFF);
+
+        regs.CONFIG1.set(regs.CONFIG1.get() | 0x20);
+
+        self.rx.init(IrqSafeSpinlock::new(rx_ring));
+        self.tx.init(IrqSafeSpinlock::new(tx_ring));
+
+        // Bind IRQ
+        self.pci
+            .init_interrupts(PreferredInterruptMode::Msi(false))?;
+        self.pci.map_interrupt(Default::default(), self.clone())?;
+
+        let interface =
+            ygg_driver_net_core::register_interface(NetworkInterfaceType::Ethernet, self.clone());
+        self.nic.init(interface.id());
+
+        Ok(())
+    }
+
+    fn display_name(&self) -> &str {
+        "Realtek RTL8111/8168 Gigabit Ethernet Controller"
+    }
+}
+
+impl NetworkDevice for Rtl8168 {
+    fn transmit(&self, packet: PageBox<[u8]>) -> Result<(), Error> {
+        let mut tx = self.tx.get().lock();
+        let regs = self.regs.lock();
+
+        tx.push(packet)?;
+        regs.TPPOLL.write(TPPOLL::NPQ::SET);
+
+        Ok(())
+    }
+
+    fn packet_prefix_size(&self) -> usize {
+        0
+    }
+
+    fn read_hardware_address(&self) -> MacAddress {
+        self.mac
+    }
+}
+
+impl Revision {
+    fn flags(&self) -> ChipFlags {
+        match self {
+            Self::rtl8168h | Self::rtl8168gu => {
+                ChipFlags::NEED_PHY_RESET
+                    | ChipFlags::MACSTAT
+                    | ChipFlags::RXDV_GATED
+                    | ChipFlags::EARLYOFFV2
+            }
+            Self::other => ChipFlags::empty(),
+        }
+    }
+}

kernel/driver/usb/xhci/Cargo.toml

@@ -16,8 +16,9 @@ libk = { path = "../../../libk" }
 
 xhci_lib = { git = "https://github.com/rust-osdev/xhci.git", package = "xhci" }
 
-atomic_enum = "0.3.0"
+async-trait.workspace = true
-log = "0.4.22"
+log.workspace = true
-tock-registers = "0.8.1"
+atomic_enum.workspace = true
-bytemuck = { version = "1.16.1", features = ["derive"] }
+tock-registers.workspace = true
-futures-util = { version = "0.3.30", default-features = false, features = ["alloc", "async-await"] }
+bytemuck.workspace = true
+futures-util.workspace = true

kernel/driver/usb/xhci/src/context.rs

@@ -0,0 +1,129 @@
+use core::ops::{Deref, DerefMut};
+
+use libk_mm::{
+    address::{AsPhysicalAddress, PhysicalAddress},
+    PageBox,
+};
+use libk_util::sync::spin_rwlock::IrqSafeRwLock;
+use xhci_lib::context::{self, DeviceHandler, InputHandler};
+use ygg_driver_usb::error::UsbError;
+
+use crate::regs::{ContextSize, PortNumber};
+
+pub enum XhciDeviceContext {
+    Context32(IrqSafeRwLock<PageBox<context::Device32Byte>>),
+    Context64(IrqSafeRwLock<PageBox<context::Device64Byte>>),
+}
+
+pub enum XhciInputContext {
+    Context32(PageBox<context::Input32Byte>),
+    Context64(PageBox<context::Input64Byte>),
+}
+
+impl XhciDeviceContext {
+    pub fn new(size: ContextSize) -> Result<Self, UsbError> {
+        match size {
+            ContextSize::Context32 => PageBox::new(context::Device::new_32byte())
+                .map(IrqSafeRwLock::new)
+                .map(Self::Context32),
+            ContextSize::Context64 => PageBox::new(context::Device::new_64byte())
+                .map(IrqSafeRwLock::new)
+                .map(Self::Context64),
+        }
+        .map_err(UsbError::MemoryError)
+    }
+
+    pub fn map<T, F: FnOnce(&dyn DeviceHandler) -> T>(&self, mapper: F) -> T {
+        match self {
+            Self::Context32(cx) => mapper(&**cx.read()),
+            Self::Context64(cx) => mapper(&**cx.read()),
+        }
+    }
+
+    pub fn physical_address(&self) -> PhysicalAddress {
+        match self {
+            Self::Context32(cx) => unsafe { cx.read().as_physical_address() },
+            Self::Context64(cx) => unsafe { cx.read().as_physical_address() },
+        }
+    }
+}
+
+impl XhciInputContext {
+    pub fn new(size: ContextSize) -> Result<Self, UsbError> {
+        match size {
+            ContextSize::Context32 => {
+                PageBox::new(context::Input::new_32byte()).map(Self::Context32)
+            }
+            ContextSize::Context64 => {
+                PageBox::new(context::Input::new_64byte()).map(Self::Context64)
+            }
+        }
+        .map_err(UsbError::MemoryError)
+    }
+
+    pub fn new_address_device(
+        size: ContextSize,
+        root_hub_port_number: PortNumber,
+        max_packet_size: usize,
+        speed: u8,
+        dequeue_pointer: PhysicalAddress,
+    ) -> Result<Self, UsbError> {
+        let mut cx = Self::new(size)?;
+
+        {
+            let control = cx.control_mut();
+
+            control.set_add_context_flag(0); // Enable slot context
+            control.set_add_context_flag(1); // Enable endpoint 0 context
+        }
+
+        {
+            let slot = cx.device_mut().slot_mut();
+
+            slot.set_context_entries(1);
+            slot.set_interrupter_target(0);
+            slot.set_root_hub_port_number(root_hub_port_number.into());
+            slot.set_speed(speed);
+        }
+
+        {
+            let ep0 = cx.device_mut().endpoint_mut(1);
+
+            ep0.set_endpoint_type(context::EndpointType::Control);
+            ep0.set_error_count(3);
+            // Use the provided max_packet_size, or the default one for the given speed
+            ep0.set_max_packet_size(max_packet_size as u16);
+            ep0.set_tr_dequeue_pointer(dequeue_pointer.into_u64());
+            ep0.set_dequeue_cycle_state();
+        }
+
+        Ok(cx)
+    }
+
+    pub fn physical_address(&self) -> PhysicalAddress {
+        match self {
+            Self::Context32(cx) => unsafe { cx.as_physical_address() },
+            Self::Context64(cx) => unsafe { cx.as_physical_address() },
+        }
+    }
+}
+
+impl Deref for XhciInputContext {
+    type Target = dyn InputHandler;
+
+    fn deref(&self) -> &Self::Target {
+        match self {
+            Self::Context32(cx) => &**cx,
+            Self::Context64(cx) => &**cx,
+        }
+    }
+}
+
+impl DerefMut for XhciInputContext {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        match self {
+            Self::Context32(cx) => &mut **cx,
+            Self::Context64(cx) => &mut **cx,
+        }
+    }
+}

kernel/driver/usb/xhci/src/controller.rs

@@ -1,315 +1,396 @@
-use core::{future::poll_fn, sync::atomic::Ordering, task::Poll, time::Duration};
+use core::sync::atomic::{AtomicU8, Ordering};
 
-use alloc::{boxed::Box, collections::BTreeMap, sync::Arc, vec, vec::Vec};
+use alloc::{boxed::Box, collections::btree_map::BTreeMap, sync::Arc, vec::Vec};
-use atomic_enum::atomic_enum;
+use async_trait::async_trait;
 use device_api::{device::Device, interrupt::InterruptHandler};
-use futures_util::task::AtomicWaker;
+use libk::{error::Error, task::runtime};
-use libk::task::runtime;
 use libk_mm::{
     address::{AsPhysicalAddress, PhysicalAddress},
     PageBox,
 };
-use libk_util::{sync::spin_rwlock::IrqSafeRwLock, OneTimeInit};
+use libk_util::{
+    sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock},
+    OneTimeInit,
+};
+use tock_registers::{
+    interfaces::{ReadWriteable, Readable, Writeable},
+    LocalRegisterCopy,
+};
+use ygg_driver_pci::device::PciDeviceInfo;
 use ygg_driver_usb::{
     bus::UsbBusManager,
-    device::{UsbBusAddress, UsbDevice, UsbDeviceAccess},
+    device::{UsbBusAddress, UsbDeviceAccess, UsbSpeed},
     error::UsbError,
+    info::UsbVersion,
     pipe::control::UsbControlPipeAccess,
-    util::UsbAddressAllocator,
     UsbHostController,
 };
-use yggdrasil_abi::error::Error;
 
 use crate::{
+    context::{XhciDeviceContext, XhciInputContext},
     device::XhciBusDevice,
     pipe::ControlPipe,
-    regs::{Mapper, PortSpeed, Regs},
+    regs::{
-    ring::{
+        self,
-        CommandExecutor, CommandRing, ControlTransferRing, Event, EventRing, EventRingSegmentTable,
+        extended::ExtendedCapability,
-        GenericTransferRing,
+        operational::{PortRegs, CONFIG, PORTSC, USBCMD, USBSTS},
+        PortNumber, Regs,
     },
-    XhciContext,
+    ring::{
+        command::CommandRing,
+        event::{Event, EventRing, EventRingSegmentTable},
+        transfer::TransferRing,
+        CommandExecutor, GenericRing,
+    },
+    util::EventBitmap,
 };
 
-#[atomic_enum]
+#[allow(unused)]
-#[derive(PartialEq, Eq)]
+struct ScratchpadArray {
-pub enum PortState {
+    buffers: Vec<PageBox<[u8]>>,
-    Disconnected, // Default + set by "handle detach"
+    array: PageBox<[PhysicalAddress]>,
-    Init,         // Set by "port task"
-    Running,      // Set by "port task"
 }
 
-struct PortStruct {
+struct RootHubPort {
-    state: AtomicPortState,
+    version: UsbVersion,
-    notify: AtomicWaker,
+    slot_type: u8,
-    address: IrqSafeRwLock<Option<UsbBusAddress>>,
 }
 
 pub struct Xhci {
-    regs: Regs,
+    pub(crate) regs: Regs,
-
-    bus_address: OneTimeInit<u16>,
-    address_allocator: UsbAddressAllocator,
-
-    port_count: usize,
-    // TODO use to allocate proper contexts
     #[allow(unused)]
-    context_size: usize,
+    pci: PciDeviceInfo,
 
     dcbaa: IrqSafeRwLock<PageBox<[PhysicalAddress]>>,
-    endpoints: IrqSafeRwLock<BTreeMap<(u8, u8), Arc<dyn GenericTransferRing>>>,
+    #[allow(unused)]
-    event_ring: EventRing,
+    scratchpads: Option<ScratchpadArray>,
     pub(crate) command_ring: CommandRing,
+    event_ring: EventRing,
+    erst: EventRingSegmentTable,
 
-    port_states: Vec<PortStruct>,
+    root_hub_ports: Vec<Option<RootHubPort>>,
+    pub(crate) endpoints: IrqSafeRwLock<BTreeMap<(u8, u8), Arc<TransferRing>>>,
+    pub(crate) slots: Vec<IrqSafeRwLock<Option<Arc<XhciBusDevice>>>>,
+    pub(crate) port_slot_map: Vec<AtomicU8>,
+    bus_index: OneTimeInit<u16>,
+    port_event_map: EventBitmap,
+}
+
+impl ScratchpadArray {
+    pub fn new(capacity: usize, element_size: usize) -> Result<Self, UsbError> {
+        let buffers = (0..capacity)
+            .map(|_| PageBox::new_slice(0, element_size))
+            .collect::<Result<Vec<_>, _>>()
+            .map_err(UsbError::MemoryError)?;
+        let array =
+            PageBox::new_slice_with(|i| unsafe { buffers[i].as_physical_address() }, capacity)
+                .map_err(UsbError::MemoryError)?;
+        Ok(Self { buffers, array })
+    }
 }
 
 impl Xhci {
-    pub fn new(regs: xhci_lib::Registers<Mapper>) -> Result<Self, UsbError> {
+    pub fn new(pci: PciDeviceInfo, regs: Regs) -> Result<Self, UsbError> {
-        let event_ring = EventRing::new(128)?;
+        let mut dcbaa = PageBox::new_slice(PhysicalAddress::ZERO, regs.slot_count + 1)
-        let command_ring = CommandRing::new(128)?;
-
-        let regs = Regs::from(regs);
-
-        let port_count = regs.port_count();
-        let slot_count = regs.max_slot_count();
-        let context_size = regs.context_size();
-
-        let dcbaa = PageBox::new_slice(PhysicalAddress::ZERO, slot_count + 1)
             .map_err(UsbError::MemoryError)?;
+        let command_ring = CommandRing::new(128)?;
+        let event_ring = EventRing::new(128)?;
+        let erst = EventRingSegmentTable::for_event_rings(&[&event_ring])?;
+
+        // Setup scratch buffers
+        // TODO: Linux seems to just ignore the PAGESIZE, it's (1 << 0) everywhere
+        let scratchpads = if regs.scratch_count != 0 {
+            let array = ScratchpadArray::new(regs.scratch_count, 0x1000)?;
+            dcbaa[0] = unsafe { array.array.as_physical_address() };
+            Some(array)
+        } else {
+            None
+        };
+
+        let mut root_hub_ports: Vec<Option<_>> = (0..regs.port_count).map(|_| None).collect();
+
+        for cap in regs.extended_capabilities.iter() {
+            match cap {
+                ExtendedCapability::ProtocolSupport(support) => {
+                    let Some(version) = support.usb_revision() else {
+                        continue;
+                    };
+
+                    for port in support.port_range() {
+                        log::info!("* Port {port}: {version}");
+                        root_hub_ports[port as usize - 1] = Some(RootHubPort {
+                            version,
+                            slot_type: support.slot_type(),
+                        });
+                    }
+                }
+                ExtendedCapability::LegacySupport(legsup) => {
+                    legsup.write().perform_bios_handoff(10000000)?;
+                }
+            }
+        }
+
+        let port_slot_map = (0..regs.port_count).map(|_| AtomicU8::new(0)).collect();
+        let slots = (0..regs.slot_count)
+            .map(|_| IrqSafeRwLock::new(None))
+            .collect();
 
         Ok(Self {
             regs,
-
+            pci,
-            bus_address: OneTimeInit::new(),
-            address_allocator: UsbAddressAllocator::new(),
-
-            port_count,
-            context_size,
-
-            event_ring,
-            command_ring,
 
             dcbaa: IrqSafeRwLock::new(dcbaa),
-            endpoints: IrqSafeRwLock::new(BTreeMap::new()),
+            scratchpads,
+            command_ring,
+            event_ring,
+            erst,
 
-            port_states: Vec::from_iter((0..port_count).map(|_| PortStruct {
+            root_hub_ports,
-                state: AtomicPortState::new(PortState::Disconnected),
+            bus_index: OneTimeInit::new(),
-                notify: AtomicWaker::new(),
+            endpoints: IrqSafeRwLock::new(BTreeMap::new()),
-                address: IrqSafeRwLock::new(None),
+            slots,
-            })),
+            port_slot_map,
+            port_event_map: EventBitmap::new(),
         })
     }
 
-    pub fn register_device_context(&self, slot_id: u8, context: PhysicalAddress) {
+    fn notify_endpoint(&self, slot_id: u8, endpoint_id: u8, address: PhysicalAddress, status: u32) {
-        self.dcbaa.write()[slot_id as usize] = context;
+        if let Some(endpoint) = self.endpoints.read().get(&(slot_id, endpoint_id)) {
+            endpoint.notify(address, status);
+        } else {
+            log::warn!("Endpoint {slot_id}:{endpoint_id} does not exist");
+        }
     }
 
-    pub fn register_endpoint(
+    fn kill_endpoint(&self, slot_id: u8, endpoint_id: u8) {
-        &self,
+        let endpoint = self.endpoints.write().remove(&(slot_id, endpoint_id));
-        slot_id: u8,
+        if let Some(endpoint) = endpoint {
-        endpoint_id: u8,
-        ring: Arc<dyn GenericTransferRing>,
-    ) {
-        self.endpoints.write().insert((slot_id, endpoint_id), ring);
-    }
-
-    pub fn shutdown_endpoint(&self, slot_id: u8, endpoint_id: u8) {
-        if let Some(endpoint) = self.endpoints.write().remove(&(slot_id, endpoint_id)) {
             endpoint.shutdown();
-        } else {
-            log::warn!(
-                "Endpoint {}:{} does not exist, maybe already shut down?",
-                slot_id,
-                endpoint_id
-            );
         }
     }
 
-    pub fn notify_transfer(
+    async fn allocate_device_slot(
-        &self,
+        self: &Arc<Self>,
-        slot_id: u8,
+        port_id: PortNumber,
-        endpoint_id: u8,
+        slot_type: u8,
-        address: PhysicalAddress,
+        speed: UsbSpeed,
-        status: u32,
+    ) -> Result<(u8, Arc<XhciBusDevice>, Arc<TransferRing>), UsbError> {
-    ) {
+        let device_context = XhciDeviceContext::new(self.regs.context_size)?;
-        if let Some(ep) = self.endpoints.read().get(&(slot_id, endpoint_id)) {
+        let slot_id = self.command_ring.enable_slot(&**self, slot_type).await?;
-            ep.notify(address, status);
+        let (control_pipe, control_ring) = ControlPipe::new(self.clone(), slot_id, 1, 128)?;
-        }
+        let control_pipe = UsbControlPipeAccess(Box::new(control_pipe));
-    }
 
-    async fn assign_device(
+        self.dcbaa.write()[slot_id as usize] = device_context.physical_address();
-        self: Arc<Self>,
+        self.endpoints
-        speed: PortSpeed,
+            .write()
-        slot_id: u8,
+            .insert((slot_id, 1), control_ring.clone());
-        root_hub_port_number: u8,
-    ) -> Result<Box<dyn UsbDevice>, UsbError> {
-        let address = self.address_allocator.allocate().unwrap();
-        let ring = Arc::new(ControlTransferRing::new(slot_id, 1, 128)?);
 
-        let context =
+        let slot = Arc::new(XhciBusDevice {
-            XhciContext::new_32byte_address_device(&ring, speed, address, root_hub_port_number)?;
+            port_id,
-        let mut input = context.input.write();
-
-        self.register_device_context(slot_id, unsafe { context.output.as_physical_address() });
-
-        self.command_ring
-            .address_device(&*self, slot_id, &mut input)
-            .await?;
-
-        self.register_endpoint(slot_id, 1, ring.clone());
-
-        let pipe = ControlPipe::new(self.clone(), slot_id, ring.clone());
-
-        // TODO: If the device is a Full-speed one, determine its max packet size for the control
-        // endpoint
-        if speed == PortSpeed::Full {
-            todo!()
-        }
-
-        drop(input);
-
-        let bus_address = UsbBusAddress {
-            bus: *self.bus_address.get(),
-            device: address,
-        };
-
-        let device = XhciBusDevice {
-            xhci: self.clone(),
             slot_id,
-            port_id: root_hub_port_number,
+            device_context,
-            bus_address,
+            control_pipe,
             speed,
-            context: Arc::new(context),
+            detach_handler: IrqSafeSpinlock::new(None),
-            rings: IrqSafeRwLock::new(vec![ring]),
+            endpoints: IrqSafeRwLock::new(BTreeMap::from_iter([(1, control_ring.clone())])),
-            control_pipe: UsbControlPipeAccess(Box::new(pipe)),
+            xhci: self.clone(),
-        };
+            bus_address: OneTimeInit::new(),
-
-        Ok(Box::new(device))
-    }
-
-    async fn port_task(self: Arc<Self>, index: usize) -> Result<(), UsbError> {
-        let state = &self.port_states[index];
-
-        self.reset_port(index).await?;
-
-        let regs = self.regs.ports.read(index);
-        let speed =
-            PortSpeed::try_from(regs.portsc.port_speed()).map_err(|_| UsbError::PortInitFailed)?;
-
-        let slot_id = self.command_ring.enable_slot(self.as_ref()).await?;
-
-        let device = self
-            .clone()
-            .assign_device(speed, slot_id, (index + 1) as _)
-            .await?;
-        let device = UsbDeviceAccess::setup(device).await?;
-
-        let old = state.address.write().replace(device.bus_address());
-        assert!(old.is_none());
-
-        UsbBusManager::register_device(Arc::new(device));
-
-        state.state.store(PortState::Running, Ordering::Release);
-
-        Ok(())
-    }
-
-    fn handle_device_attached(self: Arc<Self>, port: usize) -> Result<(), UsbError> {
-        log::info!("Port {}: device attached", port);
-
-        if let Err(err) = self.port_states[port].state.compare_exchange(
-            PortState::Disconnected,
-            PortState::Init,
-            Ordering::Acquire,
-            Ordering::Relaxed,
-        ) {
-            log::warn!("Could not start port init task: port state is {:?}", err);
-            return Err(UsbError::DeviceBusy);
-        }
-
-        runtime::spawn(async move { self.port_task(port).await }).map_err(UsbError::SystemError)?;
-        Ok(())
-    }
-
-    fn handle_device_detached(&self, port: usize) -> Result<(), UsbError> {
-        let state = &self.port_states[port];
-
-        match state.state.swap(PortState::Disconnected, Ordering::Release) {
-            PortState::Init => {
-                log::warn!("USB device detach received while in init state");
-                Ok(())
-            }
-            PortState::Running => {
-                log::info!("Port {}: device detached", port);
-                let address = state
-                    .address
-                    .write()
-                    .take()
-                    .expect("Port marked as Running, but has no address");
-
-                UsbBusManager::detach_device(address);
-                state.notify.wake();
-
-                Ok(())
-            }
-            // Already disconnected
-            PortState::Disconnected => Ok(()),
-        }
-    }
-
-    fn handle_port_event(self: Arc<Self>, port: usize) -> Result<(), UsbError> {
-        let state = &self.port_states[port];
-        let port_regs = self.regs.ports.read(port);
-
-        if port_regs.portsc.connect_status_change() {
-            if port_regs.portsc.current_connect_status() {
-                self.handle_device_attached(port)
-            } else {
-                self.handle_device_detached(port)
-            }
-        } else {
-            // Some other event
-            state.notify.wake();
-            Ok(())
-        }
-    }
-
-    async fn reset_port(&self, port: usize) -> Result<(), UsbError> {
-        log::debug!("Reset port {}", port);
-
-        self.regs.ports.update(port, |u| {
-            u.portsc.set_port_reset();
         });
 
-        // Wait for port reset
+        *self.slots[slot_id as usize].write() = Some(slot.clone());
-        // TODO handle disconnect during reset?
+        self.port_slot_map[port_id.index()].store(slot_id, Ordering::Release);
-        let result = runtime::with_timeout(
-            poll_fn(|cx| {
-                let state = &self.port_states[port];
 
-                state.notify.register(cx.waker());
+        Ok((slot_id, slot, control_ring))
-                if !self.regs.ports.read(port).portsc.port_reset() {
+    }
-                    Poll::Ready(())
+
-                } else {
+    // TODO proper timeout
-                    Poll::Pending
+    async fn reset_port(&self, regs: &PortRegs) -> Result<(), UsbError> {
+        // Reset the port
+        regs.modify_portsc_preserving(PORTSC::PR::SET);
+
+        let mut timeout = 10000000;
+        while timeout > 0 {
+            let status = regs.PORTSC.extract();
+
+            if status.matches_all(PORTSC::PR::CLEAR) {
+                break;
+            }
+
+            timeout -= 1;
+        }
+
+        // Clear PRC
+        regs.modify_portsc_preserving(PORTSC::PRC::SET);
+        Ok(())
+    }
+
+    // TODO clean up resources if init fails
+    async fn setup_port(
+        self: &Arc<Self>,
+        regs: &PortRegs,
+        number: PortNumber,
+    ) -> Result<(), UsbError> {
+        let root_hub_port = self.root_hub_ports[number.index()]
+            .as_ref()
+            .ok_or(UsbError::PortInitFailed)?;
+
+        let need_reset = !root_hub_port.version.is_version_3();
+
+        if need_reset {
+            self.reset_port(regs).await?;
+        }
+
+        let status = regs.PORTSC.extract();
+
+        let speed = status.read(PORTSC::PS) as u8;
+        let pls = status.read(PORTSC::PLS);
+
+        let (usb_speed, max_packet_size) = match speed {
+            1 => (UsbSpeed::Full, 8),
+            2 => (UsbSpeed::Low, 8),
+            3 => (UsbSpeed::High, 64),
+            4 => (UsbSpeed::Super, 512),
+            _ => {
+                log::error!("Port {number} invalid speed value {speed}");
+                return Err(UsbError::DeviceDisconnected);
+            }
+        };
+
+        // PLS != 0 after reset
+        if pls != 0 {
+            log::error!("Port {number} pls!=0 after reset");
+            return Err(UsbError::DeviceDisconnected);
+        }
+
+        let (slot_id, slot, control_ring) = self
+            .allocate_device_slot(number, root_hub_port.slot_type, usb_speed)
+            .await?;
+
+        let input_cx = XhciInputContext::new_address_device(
+            self.regs.context_size,
+            number,
+            max_packet_size,
+            speed,
+            control_ring.base(),
+        )?;
+        self.command_ring
+            .address_device(&**self, slot_id, input_cx.physical_address(), false)
+            .await
+            .inspect_err(|error| {
+                log::error!("Port {number} Address Device TRB (BSR=0) failed: {error:?}")
+            })?;
+
+        // TODO update max_packet_size for default control endpoint after fetching the device
+        //      descriptor
+
+        let bus_address = slot
+            .device_context
+            .map(|device| device.slot().usb_device_address());
+        slot.bus_address.init(UsbBusAddress {
+            bus: *self.bus_index.get(),
+            device: bus_address,
+        });
+
+        let device = UsbDeviceAccess::setup(slot).await?;
+        UsbBusManager::register_device(device.into());
+
+        Ok(())
+    }
+
+    async fn handle_disconnect(&self, number: PortNumber) -> Result<(), UsbError> {
+        let slot_id = self.port_slot_map[number.index()].swap(0, Ordering::Acquire);
+        if slot_id == 0 {
+            return Ok(());
+        }
+        log::info!("Port {number} disconnected");
+
+        let slot = self.slots[slot_id as usize].write().take();
+        if let Some(slot) = slot {
+            UsbBusManager::detach_device(*slot.bus_address.get());
+
+            // Clean up xHC resources
+            for (&endpoint_id, endpoint) in slot.endpoints.read().iter() {
+                endpoint.shutdown();
+                self.kill_endpoint(slot_id, endpoint_id);
+            }
+        } else {
+            log::warn!("No device in slot {slot_id}");
+        }
+
+        // TODO stop endpoints on the xHC side?
+        self.command_ring.disable_slot(self, slot_id).await?;
+
+        Ok(())
+    }
+
+    async fn port_handler_task(self: Arc<Self>) -> Result<(), Error> {
+        // Inject notify for ports that are already connected
+        {
+            let ports = self.regs.ports.write();
+            for (index, port) in self.root_hub_ports.iter().enumerate() {
+                if port.is_none() {
+                    continue;
                 }
-            }),
-            Duration::from_secs(1),
-        )
-        .await;
 
-        match result {
+                if ports[index].PORTSC.matches_all(PORTSC::CCS::SET) {
-            Ok(()) => Ok(()),
+                    self.port_event_map.signal(index);
-            Err(_) => Err(UsbError::PortResetFailed),
+                }
+            }
+        }
+
+        loop {
+            let events = self.port_event_map.wait_any(self.regs.port_count).await;
+
+            for index in events {
+                let number = PortNumber::from_index(index);
+                let ports = self.regs.ports.write();
+                let regs = &ports[index];
+
+                let status = regs.PORTSC.extract();
+                let neutral = regs::portsc_to_neutral(status);
+                let mut clear = LocalRegisterCopy::<u32, PORTSC::Register>::new(0);
+
+                // Clear affected change bits
+                let connected = if status.matches_all(PORTSC::CSC::SET) {
+                    clear.modify(PORTSC::CSC::SET);
+                    Some(status.matches_all(PORTSC::CCS::SET))
+                } else {
+                    None
+                };
+                if status.matches_all(PORTSC::PEC::SET) {
+                    clear.modify(PORTSC::PEC::SET);
+                }
+                if status.matches_all(PORTSC::PLC::SET) {
+                    clear.modify(PORTSC::PLC::SET);
+                }
+
+                regs.PORTSC.set(neutral.get() | clear.get());
+
+                if let Some(connected) = connected {
+                    if connected {
+                        log::info!("Port {number} connected");
+                        if let Err(error) = self.setup_port(regs, number).await {
+                            log::error!("Port {number} setup error: {error:?}");
+                        }
+                    }
+
+                    if !connected {
+                        if let Err(error) = self.handle_disconnect(number).await {
+                            log::error!("Port {number} disconnect error: {error:?}");
+                        }
+                    }
+                }
+            }
         }
     }
 
-    fn handle_event(self: Arc<Self>) {
+    fn handle_event(&self) {
         while let Some(event) = self.event_ring.try_dequeue() {
             match event {
-                Event::PortChange(port) => {
+                Event::PortChange(number) => {
-                    self.clone().handle_port_event(port - 1).ok();
+                    if number > 0 {
+                        self.port_event_map.signal(number - 1);
+                    }
                 }
                 Event::CommandCompletion { address, reply } => {
                     self.command_ring.notify(address, reply);
@@ -320,65 +401,98 @@ impl Xhci {
                     endpoint_id,
                     status,
                 } => {
-                    self.notify_transfer(slot_id, endpoint_id, address, status);
+                    self.notify_endpoint(slot_id, endpoint_id, address, status);
                 }
             }
         }
 
         self.regs
-            .set_interrupter_0_dequeue_pointer(self.event_ring.dequeue_pointer());
+            .runtime
+            .write()
+            .set_interrupter_dequeue_pointer(0, self.event_ring.dequeue_pointer());
     }
 }
 
-impl UsbHostController for Xhci {}
+#[async_trait]
-
 impl CommandExecutor for Xhci {
     fn ring_doorbell(&self, index: usize, target: u8) {
-        self.regs.ring_doorbell(index, target);
+        self.regs.doorbells[index].store(target as u32, Ordering::Release);
+    }
+
+    async fn reset_endpoint(
+        &self,
+        slot_id: u8,
+        endpoint_id: u8,
+        dequeue_pointer: PhysicalAddress,
+        dequeue_cycle: bool,
+    ) -> Result<(), UsbError> {
+        log::warn!("xhci: reset stalled endpoint {slot_id}:{endpoint_id}");
+
+        self.command_ring
+            .reset_endpoint(&*self, slot_id, endpoint_id, false)
+            .await?;
+        self.command_ring
+            .set_tr_dequeue_pointer(&*self, slot_id, endpoint_id, dequeue_pointer, dequeue_cycle)
+            .await?;
+
+        Ok(())
     }
 }
 
 impl Device for Xhci {
     unsafe fn init(self: Arc<Self>) -> Result<(), Error> {
-        log::info!("Init USB xHCI");
+        self.regs.hc_reset(10000000)?;
+        log::info!("xHC reset complete");
 
-        self.regs.reset();
+        // Configure the HC
-        self.regs.set_max_slot_count();
+        let dcbaap = unsafe { self.dcbaa.read().as_physical_address() };
+        let cr_base = self.command_ring.base();
 
-        let erst = EventRingSegmentTable::for_event_rings(&[&self.event_ring])?;
+        let op = self.regs.operational.write();
-        let dcbaa = self.dcbaa.read();
+        let rt = self.regs.runtime.write();
 
-        self.regs
+        log::info!("xhci: configure HC");
-            .configure(&dcbaa, &self.command_ring, &self.event_ring, &erst);
+        op.CONFIG
+            .modify(CONFIG::MaxSlotsEn.val(self.regs.slot_count as u32));
+        op.set_dcbaap(dcbaap);
+        op.set_crcr(cr_base, true);
+
+        log::info!("xhci: configure interrupter");
+        rt.configure_interrupter(0, &self.event_ring, &self.erst);
+
+        // Enable interrupts and start the HC
+        log::info!("xhci: start HC");
+        op.USBCMD.modify(USBCMD::INTE::SET + USBCMD::HSEE::SET);
+        op.USBCMD.modify(USBCMD::RS::SET);
+        op.wait_usbsts_bit(USBSTS::CNR::CLEAR, 100000000)?;
 
         let bus = UsbBusManager::register_bus(self.clone());
-        self.bus_address.init(bus);
+        self.bus_index.init(bus);
 
-        for port in 0..self.port_count {
+        runtime::spawn(self.clone().port_handler_task()).ok();
-            let p = self.regs.ports.read(port);
-            if p.portsc.current_connect_status() {
-                self.clone().handle_device_attached(port).ok();
-            }
-        }
 
         Ok(())
     }
 
     fn display_name(&self) -> &str {
-        "USB xHCI"
+        "xHCI"
     }
 }
 
 impl InterruptHandler for Xhci {
     fn handle_irq(self: Arc<Self>, _vector: Option<usize>) -> bool {
-        if let Some(status) = self.regs.handle_interrupt() {
+        let status = self.regs.handle_interrupt(0);
-            if status.event_interrupt() {
-                self.handle_event();
-            }
 
-            true
+        if status.matches_all(USBSTS::HSE::SET) {
-        } else {
+            log::error!("xhci: Host System Error occurred");
-            false
         }
+
+        if status.matches_all(USBSTS::EINT::SET) {
+            self.handle_event();
+        }
+
+        true
     }
 }
+
+impl UsbHostController for Xhci {}

kernel/driver/usb/xhci/src/device.rs

@@ -1,38 +1,177 @@
-use alloc::{boxed::Box, sync::Arc, vec::Vec};
+use alloc::{boxed::Box, collections::btree_map::BTreeMap, sync::Arc};
-use futures_util::{future::BoxFuture, FutureExt};
+use async_trait::async_trait;
-use libk_util::sync::spin_rwlock::IrqSafeRwLock;
+use libk_util::{
-use xhci_lib::context::{self, InputHandler};
+    sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock},
+    OneTimeInit,
+};
+use xhci_lib::context;
 use ygg_driver_usb::{
-    device::{UsbBusAddress, UsbDevice, UsbSpeed},
+    communication::UsbDirection,
+    device::{UsbBusAddress, UsbDevice, UsbDeviceDetachHandler, UsbSpeed},
     error::UsbError,
     info::UsbEndpointType,
-    pipe::{control::UsbControlPipeAccess, interrupt::UsbInterruptInPipeAccess},
+    pipe::{
-    UsbDirection, UsbHostController,
+        control::UsbControlPipeAccess,
+        normal::{UsbNormalPipeIn, UsbNormalPipeOut},
+    },
+    UsbHostController,
 };
 
 use crate::{
-    pipe::InterruptInPipe,
+    context::{XhciDeviceContext, XhciInputContext},
-    regs::PortSpeed,
+    pipe::{NormalInPipe, NormalOutPipe},
-    ring::{transfer::InterruptInTransferRing, GenericTransferRing},
+    regs::PortNumber,
-    Xhci, XhciContext,
+    ring::transfer::TransferRing,
+    Xhci,
 };
 
-// TODO device context information
 pub struct XhciBusDevice {
-    pub(crate) port_id: u8,
+    pub(crate) port_id: PortNumber,
     pub(crate) slot_id: u8,
-    pub(crate) bus_address: UsbBusAddress,
+    pub(crate) speed: UsbSpeed,
-
+    pub(crate) bus_address: OneTimeInit<UsbBusAddress>,
-    pub(crate) speed: PortSpeed,
 
     pub(crate) xhci: Arc<Xhci>,
 
-    pub(crate) context: Arc<XhciContext<8>>,
+    pub(crate) device_context: XhciDeviceContext,
-    pub(crate) rings: IrqSafeRwLock<Vec<Arc<dyn GenericTransferRing>>>,
+    pub(crate) endpoints: IrqSafeRwLock<BTreeMap<u8, Arc<TransferRing>>>,
     pub(crate) control_pipe: UsbControlPipeAccess,
+    pub(crate) detach_handler: IrqSafeSpinlock<Option<Arc<dyn UsbDeviceDetachHandler>>>,
+}
+
+#[async_trait]
+impl UsbDevice for XhciBusDevice {
+    fn port_number(&self) -> u8 {
+        self.port_id.into()
+    }
+
+    fn bus_address(&self) -> UsbBusAddress {
+        *self.bus_address.get()
+    }
+
+    fn control_pipe(&self) -> &UsbControlPipeAccess {
+        &self.control_pipe
+    }
+
+    fn handle_detach(&self) {
+        if let Some(handler) = self.detach_handler.lock().as_ref() {
+            handler.handle_device_detach();
+        }
+    }
+
+    fn set_detach_handler(&self, handler: Arc<dyn UsbDeviceDetachHandler>) {
+        *self.detach_handler.lock() = Some(handler);
+    }
+
+    fn controller_ref(&self) -> &dyn UsbHostController {
+        self.xhci.as_ref()
+    }
+
+    fn debug(&self) {}
+
+    fn speed(&self) -> UsbSpeed {
+        self.speed
+    }
+
+    async fn open_normal_in_pipe(
+        &self,
+        number: u8,
+        max_packet_size: u16,
+        ty: UsbEndpointType,
+    ) -> Result<Box<dyn UsbNormalPipeIn>, UsbError> {
+        let dci = Self::dci(ty, UsbDirection::In, number) as u8;
+        let (pipe, ring) = NormalInPipe::new(self.xhci.clone(), self.slot_id, dci, 128)?;
+        self.add_endpoint(number, ring.clone(), ty, UsbDirection::In, max_packet_size)
+            .await?;
+        Ok(Box::new(pipe))
+    }
+
+    async fn open_normal_out_pipe(
+        &self,
+        number: u8,
+        max_packet_size: u16,
+        ty: UsbEndpointType,
+    ) -> Result<Box<dyn UsbNormalPipeOut>, UsbError> {
+        let dci = Self::dci(ty, UsbDirection::Out, number) as u8;
+        let (pipe, ring) = NormalOutPipe::new(self.xhci.clone(), self.slot_id, dci, 128)?;
+        self.add_endpoint(number, ring.clone(), ty, UsbDirection::Out, max_packet_size)
+            .await?;
+        Ok(Box::new(pipe))
+    }
 }
 
 impl XhciBusDevice {
+    pub async fn add_endpoint(
+        &self,
+        endpoint_number: u8,
+        ring: Arc<TransferRing>,
+        ty: UsbEndpointType,
+        direction: UsbDirection,
+        max_packet_size: u16,
+    ) -> Result<(), UsbError> {
+        let dci = Self::dci(ty, direction, endpoint_number) as u8;
+
+        log::debug!(
+            "Setup endpoint dci #{}: {} {:?} {:?}",
+            dci,
+            endpoint_number,
+            ty,
+            direction
+        );
+
+        let ep_type = match (ty, direction) {
+            (UsbEndpointType::Interrupt, UsbDirection::In) => context::EndpointType::InterruptIn,
+            (UsbEndpointType::Interrupt, UsbDirection::Out) => context::EndpointType::InterruptOut,
+            (UsbEndpointType::Bulk, UsbDirection::In) => context::EndpointType::BulkIn,
+            (UsbEndpointType::Bulk, UsbDirection::Out) => context::EndpointType::BulkOut,
+            _ => todo!(),
+        };
+
+        let mut input_cx = XhciInputContext::new(self.xhci.regs.context_size)?;
+
+        {
+            let control = input_cx.control_mut();
+
+            control.set_add_context_flag(0);
+            control.set_add_context_flag(dci as usize);
+        }
+
+        {
+            let slot = input_cx.device_mut().slot_mut();
+
+            slot.set_context_entries(dci + 1);
+        }
+
+        {
+            let endpoint = input_cx.device_mut().endpoint_mut(dci as usize);
+
+            endpoint.set_endpoint_type(ep_type);
+            endpoint.set_error_count(3);
+            // TODO Pick a better value here
+            endpoint.set_interval(3);
+            endpoint.set_max_packet_size(max_packet_size);
+            endpoint.set_tr_dequeue_pointer(ring.base().into_u64());
+            endpoint.set_dequeue_cycle_state();
+        }
+
+        self.xhci
+            .command_ring
+            .configure_endpoint(
+                self.xhci.as_ref(),
+                self.slot_id,
+                input_cx.physical_address(),
+            )
+            .await?;
+
+        self.endpoints.write().insert(dci, ring.clone());
+        self.xhci
+            .endpoints
+            .write()
+            .insert((self.slot_id, dci), ring);
+
+        Ok(())
+    }
+
     fn dci(ty: UsbEndpointType, dir: UsbDirection, number: u8) -> usize {
         match ty {
             UsbEndpointType::Control => (number as usize) * 2 + 1,
@@ -46,112 +185,4 @@ impl XhciBusDevice {
             }
         }
     }
-
-    async fn setup_endpoint_inner(
-        &self,
-        ring: Arc<dyn GenericTransferRing>,
-        dci: u8,
-        ty: UsbEndpointType,
-        direction: UsbDirection,
-    ) -> Result<(), UsbError> {
-        log::debug!("Setup endpoint dci #{}: {:?} {:?}", dci, ty, direction);
-
-        let mut input = self.context.input.write();
-
-        let ep_type = match (ty, direction) {
-            (UsbEndpointType::Interrupt, UsbDirection::In) => context::EndpointType::InterruptIn,
-            _ => todo!(),
-        };
-
-        {
-            let control = input.control_mut();
-
-            control.set_add_context_flag(0);
-            control.clear_add_context_flag(1);
-            control.set_add_context_flag(dci as _);
-        }
-
-        {
-            let slot = input.device_mut().slot_mut();
-
-            slot.set_context_entries(31);
-        }
-
-        {
-            let ep_cx = input.device_mut().endpoint_mut(dci as _);
-
-            ep_cx.set_tr_dequeue_pointer(ring.dequeue_pointer().into());
-            ep_cx.set_dequeue_cycle_state();
-            ep_cx.set_endpoint_type(ep_type);
-            ep_cx.set_error_count(3);
-
-            // TODO get from endpoint info
-            ep_cx.set_max_packet_size(8);
-        }
-
-        self.xhci
-            .command_ring
-            .configure_endpoint(self.xhci.as_ref(), self.slot_id, &mut input)
-            .await?;
-
-        self.rings.write().push(ring.clone());
-        self.xhci.register_endpoint(self.slot_id, dci, ring);
-
-        Ok(())
-    }
-}
-
-impl UsbDevice for XhciBusDevice {
-    fn control_pipe(&self) -> &UsbControlPipeAccess {
-        &self.control_pipe
-    }
-
-    fn port_number(&self) -> u8 {
-        self.port_id
-    }
-
-    fn bus_address(&self) -> UsbBusAddress {
-        self.bus_address
-    }
-
-    fn controller_ref(&self) -> &dyn UsbHostController {
-        self.xhci.as_ref()
-    }
-
-    fn open_interrupt_in_pipe(
-        &self,
-        number: u8,
-    ) -> BoxFuture<Result<UsbInterruptInPipeAccess, UsbError>> {
-        async move {
-            let dci = Self::dci(UsbEndpointType::Interrupt, UsbDirection::In, number) as u8;
-            let ring = Arc::new(InterruptInTransferRing::new(self.slot_id, dci as _, 128)?);
-
-            self.setup_endpoint_inner(
-                ring.clone(),
-                dci,
-                UsbEndpointType::Interrupt,
-                UsbDirection::In,
-            )
-            .await?;
-
-            let pipe = InterruptInPipe::new(self.xhci.clone(), self.slot_id, number, dci, ring);
-
-            Ok(UsbInterruptInPipeAccess(Box::new(pipe)))
-        }
-        .boxed()
-    }
-
-    fn handle_detach(&self) {
-        log::info!("Device detach handler");
-        for ring in self.rings.write().drain(..) {
-            self.xhci
-                .shutdown_endpoint(ring.slot_id(), ring.endpoint_id());
-        }
-    }
-
-    fn speed(&self) -> UsbSpeed {
-        self.speed.into()
-    }
-
-    fn debug(&self) {}
 }

kernel/driver/usb/xhci/src/lib.rs

@@ -1,110 +1,71 @@
 #![no_std]
 #![allow(clippy::new_without_default)]
-#![feature(iter_array_chunks)]
+#![feature(iter_array_chunks, let_chains)]
 
 extern crate alloc;
 
+use alloc::sync::Arc;
+use controller::Xhci;
+use device_api::{device::Device, interrupt::InterruptAffinity};
+use regs::Regs;
+use ygg_driver_pci::{
+    capability::{DevicePowerState, PowerManagementCapability},
+    device::{PciDeviceInfo, PreferredInterruptMode},
+    macros::pci_driver,
+    PciCommandRegister, PciConfigurationSpace,
+};
+use yggdrasil_abi::error::Error;
+
+mod context;
 mod controller;
 mod device;
 mod pipe;
 mod regs;
 mod ring;
+mod util;
 
-use alloc::sync::Arc;
+// Only x86-64 supports xHCI currently, because only it has MSI-X implemented
-pub use controller::Xhci;
+#[cfg(target_arch = "x86_64")]
-use device_api::{device::Device, interrupt::InterruptAffinity};
+pci_driver! {
-use libk_mm::PageBox;
+    matches: [class (0x0C:0x03:0x30)],
-use libk_util::sync::spin_rwlock::IrqSafeRwLock;
+    driver: {
-use regs::{Mapper, PortSpeed};
+        fn driver_name(&self) -> &str {
-use ring::{ControlTransferRing, GenericTransferRing};
+            "xhci"
-use xhci_lib::context::{self, InputHandler};
-use ygg_driver_pci::{
-    device::{PciDeviceInfo, PreferredInterruptMode},
-    PciCommandRegister, PciConfigurationSpace,
-};
-use ygg_driver_usb::error::UsbError;
-use yggdrasil_abi::error::Error;
-
-pub struct XhciContext<const N: usize> {
-    pub(crate) input: IrqSafeRwLock<PageBox<context::Input<N>>>,
-    pub(crate) output: PageBox<context::Device<N>>,
-}
-
-impl XhciContext<8> {
-    pub fn new_32byte() -> Result<Self, UsbError> {
-        let input = PageBox::new(context::Input::new_32byte()).map_err(UsbError::MemoryError)?;
-        let output = PageBox::new(context::Device::new_32byte()).map_err(UsbError::MemoryError)?;
-
-        Ok(Self {
-            input: IrqSafeRwLock::new(input),
-            output,
-        })
-    }
-
-    pub fn new_32byte_address_device(
-        default_control_ring: &ControlTransferRing,
-        speed: PortSpeed,
-        address: u8,
-        root_hub_port_number: u8,
-    ) -> Result<Self, UsbError> {
-        let mut input =
-            PageBox::new(context::Input::new_32byte()).map_err(UsbError::MemoryError)?;
-        let output = PageBox::new(context::Device::new_32byte()).map_err(UsbError::MemoryError)?;
-
-        // Setup input context
-        {
-            let control = input.control_mut();
-
-            control.set_add_context_flag(0);
-            control.set_add_context_flag(1);
         }
 
-        {
+        fn probe(&self, info: &PciDeviceInfo) -> Result<Arc<dyn Device>, Error> {
-            let slot = input.device_mut().slot_mut();
+            // TODO Chip Hardware Reset
+            let bar0 = info
+                .config_space
+                .bar(0)
+                .expect("xHCI doesn't have BAR0 configured")
+                .as_memory()
+                .expect("xHCI's BAR0 is not memory-type");
 
-            slot.set_context_entries(1);
+            if let Some(power) = info.config_space.capability::<PowerManagementCapability>() {
-            slot.set_interrupter_target(0);
+                log::info!("xHC has power management capability");
-            slot.set_usb_device_address(address);
+                let power_state = power.get_device_power_state();
-            slot.set_root_hub_port_number(root_hub_port_number);
+
-            slot.set_speed(speed.into());
+                if power_state != DevicePowerState::D0 {
+                    power.set_device_power_state(DevicePowerState::D0);
+                }
+                // Enable PME# signal generation
+                power.set_pme_en(true);
+            }
+
+            let mut cmd = PciCommandRegister::from_bits_retain(info.config_space.command());
+            cmd &= !(PciCommandRegister::DISABLE_INTERRUPTS | PciCommandRegister::ENABLE_IO);
+            cmd |= PciCommandRegister::ENABLE_MEMORY | PciCommandRegister::BUS_MASTER;
+            info.config_space.set_command(cmd.bits());
+
+            info.init_interrupts(PreferredInterruptMode::Msi(true))?;
+
+            let regs = Regs::map(bar0)?;
+            let xhci = Arc::new(Xhci::new(info.clone(), regs)?);
+
+            info.map_interrupt(InterruptAffinity::Any, xhci.clone())?;
+
+            Ok(xhci)
         }
-
-        {
-            let ep0 = input.device_mut().endpoint_mut(1);
-
-            ep0.set_endpoint_type(context::EndpointType::Control);
-            ep0.set_tr_dequeue_pointer(default_control_ring.dequeue_pointer().into());
-            ep0.set_dequeue_cycle_state();
-            ep0.set_error_count(3);
-            ep0.set_max_packet_size(speed.default_max_packet_size() as _);
-        }
-
-        Ok(Self {
-            input: IrqSafeRwLock::new(input),
-            output,
-        })
     }
 }
-
-pub fn probe(info: &PciDeviceInfo) -> Result<Arc<dyn Device>, Error> {
-    // TODO Chip Hardware Reset
-    let bar0 = info
-        .config_space
-        .bar(0)
-        .expect("xHCI doesn't have BAR0 configured")
-        .as_memory()
-        .expect("xHCI's BAR0 is not memory-type");
-
-    let mut cmd = PciCommandRegister::from_bits_retain(info.config_space.command());
-    cmd &= !(PciCommandRegister::DISABLE_INTERRUPTS | PciCommandRegister::ENABLE_IO);
-    cmd |= PciCommandRegister::ENABLE_MEMORY | PciCommandRegister::BUS_MASTER;
-    info.config_space.set_command(cmd.bits());
-
-    let regs = unsafe { xhci_lib::Registers::new(bar0.try_into_usize().unwrap(), Mapper::new()) };
-    let xhci = Arc::new(Xhci::new(regs)?);
-
-    info.init_interrupts(PreferredInterruptMode::Msi)?;
-    info.map_interrupt(InterruptAffinity::Any, xhci.clone())?;
-
-    Ok(xhci)
-}

kernel/driver/usb/xhci/src/pipe.rs

@@ -1,85 +1,139 @@
-use alloc::sync::Arc;
+use core::mem::MaybeUninit;
-use libk_mm::{address::PhysicalAddress, PageBox};
+
+use alloc::{boxed::Box, sync::Arc};
+use async_trait::async_trait;
+use libk_mm::{address::AsPhysicalAddress, PageSlice};
 use ygg_driver_usb::{
-    communication::UsbInterruptTransfer,
+    communication::UsbDirection,
-    error::UsbError,
+    error::{TransferError, UsbError},
     pipe::{
         control::{ControlTransferSetup, UsbControlPipe},
-        interrupt::UsbInterruptInPipe,
+        normal::{UsbNormalPipeIn, UsbNormalPipeOut},
-        UsbGenericPipe,
     },
-    UsbControlTransfer, UsbDirection,
 };
 
-use crate::{
+use crate::{controller::Xhci, ring::transfer::TransferRing};
-    ring::{transfer::InterruptInTransferRing, ControlTransferRing},
-    Xhci,
-};
 
 pub struct ControlPipe {
     xhci: Arc<Xhci>,
-    ring: Arc<ControlTransferRing>,
+    pub(crate) ring: Arc<TransferRing>,
 }
 
-#[allow(unused)]
+pub struct NormalInPipe {
-pub struct InterruptInPipe {
     xhci: Arc<Xhci>,
-
+    pub(crate) ring: Arc<TransferRing>,
-    slot_id: u8,
-    endpoint_id: u8,
-    dci: u8,
-
-    ring: Arc<InterruptInTransferRing>,
 }
 
-impl UsbGenericPipe for ControlPipe {}
+pub struct NormalOutPipe {
-
+    xhci: Arc<Xhci>,
-impl UsbControlPipe for ControlPipe {
+    pub(crate) ring: Arc<TransferRing>,
-    fn start_transfer(
-        &self,
-        setup: ControlTransferSetup,
-        data: Option<(PhysicalAddress, usize, UsbDirection)>,
-    ) -> Result<UsbControlTransfer, UsbError> {
-        self.ring.start_transfer(self.xhci.as_ref(), setup, data)
-    }
-
-    fn complete_transfer(&self, transfer: UsbControlTransfer) {
-        self.ring.complete_transfer(transfer)
-    }
 }
 
 impl ControlPipe {
-    pub fn new(xhci: Arc<Xhci>, _slot_id: u8, ring: Arc<ControlTransferRing>) -> Self {
-        Self { xhci, ring }
-    }
-}
-
-impl UsbGenericPipe for InterruptInPipe {}
-
-impl UsbInterruptInPipe for InterruptInPipe {
-    fn start_read(&self, buffer: &mut PageBox<[u8]>) -> Result<UsbInterruptTransfer, UsbError> {
-        self.ring.start_transfer(self.xhci.as_ref(), buffer)
-    }
-
-    fn complete_transfer(&self, transfer: UsbInterruptTransfer) {
-        self.ring.complete_transfer(transfer)
-    }
-}
-
-impl InterruptInPipe {
     pub fn new(
         xhci: Arc<Xhci>,
         slot_id: u8,
         endpoint_id: u8,
-        dci: u8,
+        capacity: usize,
-        ring: Arc<InterruptInTransferRing>,
+    ) -> Result<(Self, Arc<TransferRing>), UsbError> {
-    ) -> Self {
+        let ring = Arc::new(TransferRing::new(slot_id, endpoint_id, capacity)?);
-        Self {
+        Ok((
-            xhci,
+            Self {
-            slot_id,
+                xhci,
-            endpoint_id,
+                ring: ring.clone(),
-            dci,
+            },
             ring,
-        }
+        ))
+    }
+}
+
+impl NormalInPipe {
+    pub fn new(
+        xhci: Arc<Xhci>,
+        slot_id: u8,
+        endpoint_id: u8,
+        capacity: usize,
+    ) -> Result<(Self, Arc<TransferRing>), UsbError> {
+        let ring = Arc::new(TransferRing::new(slot_id, endpoint_id, capacity)?);
+        Ok((
+            Self {
+                xhci,
+                ring: ring.clone(),
+            },
+            ring,
+        ))
+    }
+}
+
+impl NormalOutPipe {
+    pub fn new(
+        xhci: Arc<Xhci>,
+        slot_id: u8,
+        endpoint_id: u8,
+        capacity: usize,
+    ) -> Result<(Self, Arc<TransferRing>), UsbError> {
+        let ring = Arc::new(TransferRing::new(slot_id, endpoint_id, capacity)?);
+        Ok((
+            Self {
+                xhci,
+                ring: ring.clone(),
+            },
+            ring,
+        ))
+    }
+}
+
+#[async_trait]
+impl UsbControlPipe for ControlPipe {
+    async fn control_transfer(
+        &self,
+        setup: ControlTransferSetup,
+        data: Option<(&mut PageSlice<MaybeUninit<u8>>, UsbDirection)>,
+    ) -> Result<usize, UsbError> {
+        let data_len = data.as_ref().map_or(0, |(data, _)| data.len());
+        let result = self
+            .ring
+            .control_transfer(self.xhci.as_ref(), setup, data)
+            .await;
+        allow_short_packet(data_len, result)
+    }
+}
+
+#[async_trait]
+impl UsbNormalPipeIn for NormalInPipe {
+    async fn read(&self, buffer: &mut PageSlice<u8>) -> Result<usize, UsbError> {
+        let data_len = buffer.len();
+        let result = self
+            .ring
+            .normal_transfer(
+                self.xhci.as_ref(),
+                unsafe { buffer.as_physical_address() },
+                buffer.len(),
+            )
+            .await;
+        allow_short_packet(data_len, result)
+    }
+}
+
+#[async_trait]
+impl UsbNormalPipeOut for NormalOutPipe {
+    async fn write(&self, buffer: &PageSlice<u8>) -> Result<usize, UsbError> {
+        self.ring
+            .normal_transfer(
+                self.xhci.as_ref(),
+                unsafe { buffer.as_physical_address() },
+                buffer.len(),
+            )
+            .await
+    }
+}
+
+fn allow_short_packet(data_len: usize, result: Result<usize, UsbError>) -> Result<usize, UsbError> {
+    match result {
+        // Short packets are okay for control transfers
+        Err(UsbError::TransferFailed(TransferError::ShortPacket(residual))) => {
+            Ok(data_len.saturating_sub(residual))
+        }
+        result => result,
     }
 }

kernel/driver/usb/xhci/src/regs.rs

@@ -1,283 +0,0 @@
-use core::{cell::UnsafeCell, num::NonZeroUsize};
-
-use alloc::{sync::Arc, vec::Vec};
-use libk_mm::{
-    address::{AsPhysicalAddress, PhysicalAddress},
-    device::RawDeviceMemoryMapping,
-    PageBox,
-};
-use libk_util::sync::spin_rwlock::IrqSafeRwLock;
-use xhci_lib::{
-    accessor::{array, marker},
-    registers::{
-        operational::UsbStatusRegister, Capability, Doorbell, InterrupterRegisterSet, Operational,
-        PortRegisterSet,
-    },
-};
-use ygg_driver_usb::device::UsbSpeed;
-use yggdrasil_abi::primitive_enum;
-
-use crate::ring::{CommandRing, EventRing, EventRingSegmentTable, GenericRing};
-
-#[derive(Clone)]
-pub struct Mapper {
-    mappings: Vec<Arc<RawDeviceMemoryMapping>>,
-}
-
-pub struct LockedArray<T> {
-    array: UnsafeCell<array::Generic<T, Mapper, marker::ReadWrite>>,
-    locks: Vec<IrqSafeRwLock<()>>,
-}
-
-unsafe impl<T> Sync for LockedArray<T> {}
-unsafe impl<T> Send for LockedArray<T> {}
-
-pub struct Regs {
-    operational: IrqSafeRwLock<Operational<Mapper>>,
-    interrupters: IrqSafeRwLock<InterrupterRegisterSet<Mapper>>,
-    capability: Capability<Mapper>,
-    doorbells: LockedArray<Doorbell>,
-    pub ports: LockedArray<PortRegisterSet>,
-}
-
-impl<T> LockedArray<T> {
-    #[inline]
-    #[allow(clippy::mut_from_ref)]
-    unsafe fn get_mut(&self) -> &mut array::Generic<T, Mapper, marker::ReadWrite> {
-        &mut *self.array.get()
-    }
-
-    pub fn update<U: FnOnce(&mut T)>(&self, index: usize, f: U) {
-        let _guard = self.locks[index].write();
-        unsafe { self.get_mut() }.update_volatile_at(index, f);
-    }
-
-    pub fn read(&self, index: usize) -> T {
-        let _guard = self.locks[index].read();
-        unsafe { self.get_mut() }.read_volatile_at(index)
-    }
-}
-
-impl<T> From<array::Generic<T, Mapper, marker::ReadWrite>> for LockedArray<T> {
-    fn from(value: array::Generic<T, Mapper, marker::ReadWrite>) -> Self {
-        let locks = Vec::from_iter((0..value.len()).map(|_| IrqSafeRwLock::new(())));
-
-        Self {
-            array: UnsafeCell::new(value),
-            locks,
-        }
-    }
-}
-
-impl From<xhci_lib::Registers<Mapper>> for Regs {
-    fn from(value: xhci_lib::Registers<Mapper>) -> Self {
-        Self {
-            operational: IrqSafeRwLock::new(value.operational),
-            capability: value.capability,
-            interrupters: IrqSafeRwLock::new(value.interrupter_register_set),
-            doorbells: LockedArray::from(value.doorbell),
-            ports: LockedArray::from(value.port_register_set),
-        }
-    }
-}
-
-impl Regs {
-    pub fn reset(&self) {
-        let mut o = self.operational.write();
-
-        // TODO Get ownership from firmware
-
-        // Stop the controller
-        o.usbcmd.update_volatile(|u| {
-            u.clear_run_stop();
-        });
-
-        while !o.usbsts.read_volatile().hc_halted() {
-            core::hint::spin_loop();
-        }
-
-        // Reset the controller
-        o.usbcmd.update_volatile(|u| {
-            u.set_host_controller_reset();
-        });
-        while o.usbcmd.read_volatile().host_controller_reset()
-            || o.usbsts.read_volatile().controller_not_ready()
-        {
-            core::hint::spin_loop();
-        }
-    }
-
-    pub fn max_slot_count(&self) -> usize {
-        self.capability
-            .hcsparams1
-            .read_volatile()
-            .number_of_device_slots() as _
-    }
-
-    pub fn set_max_slot_count(&self) -> usize {
-        let device_slot_count = self.max_slot_count();
-        let mut o = self.operational.write();
-        // Set max slots enabled
-        o.config.update_volatile(|u| {
-            u.set_max_device_slots_enabled(device_slot_count as _);
-        });
-
-        device_slot_count as _
-    }
-
-    pub fn context_size(&self) -> usize {
-        match self.capability.hccparams1.read_volatile().context_size() {
-            true => 64,
-            false => 32,
-        }
-    }
-
-    pub fn port_count(&self) -> usize {
-        self.capability.hcsparams1.read_volatile().number_of_ports() as _
-    }
-
-    pub fn configure(
-        &self,
-        dcbaa: &PageBox<[PhysicalAddress]>,
-        cmd_ring: &CommandRing,
-        evt_ring: &EventRing,
-        erst: &EventRingSegmentTable,
-    ) {
-        let mut o = self.operational.write();
-        let mut i = self.interrupters.write();
-
-        o.dcbaap.update_volatile(|u| unsafe {
-            u.set(dcbaa.as_physical_address().into());
-        });
-        o.crcr.update_volatile(|u| {
-            u.set_command_ring_pointer(cmd_ring.base().into());
-            u.set_ring_cycle_state();
-        });
-
-        let mut intr0 = i.interrupter_mut(0);
-        intr0.erstsz.update_volatile(|u| {
-            u.set(erst.capacity().try_into().unwrap());
-        });
-        intr0.erdp.update_volatile(|u| {
-            log::debug!("::: Dequeue Pointer: {:#x}", evt_ring.dequeue_pointer());
-            u.set_event_ring_dequeue_pointer(evt_ring.dequeue_pointer().into());
-        });
-        intr0.erstba.update_volatile(|u| {
-            u.set(erst.physical_address().into());
-        });
-        // intr0.imod.update_volatile(|u| {
-        //     u.set_interrupt_moderation_interval(0)
-        //         .set_interrupt_moderation_counter(0);
-        // });
-        intr0.iman.update_volatile(|u| {
-            u.set_interrupt_enable();
-        });
-
-        o.usbcmd.update_volatile(|u| {
-            u.set_interrupter_enable().set_run_stop();
-        });
-    }
-
-    pub fn handle_interrupt(&self) -> Option<UsbStatusRegister> {
-        let mut o = self.operational.write();
-        let mut i = self.interrupters.write();
-
-        let status = o.usbsts.read_volatile();
-
-        if !status.event_interrupt() {
-            return None;
-        }
-
-        o.usbsts.write_volatile(status);
-
-        if status.host_system_error() {
-            return Some(status);
-        }
-
-        // Acknowledge interrupts
-        let mut intr0 = i.interrupter_mut(0);
-        intr0.iman.update_volatile(|u| {
-            u.clear_interrupt_pending();
-        });
-
-        Some(status)
-    }
-
-    pub fn set_interrupter_0_dequeue_pointer(&self, pointer: PhysicalAddress) {
-        let mut i = self.interrupters.write();
-
-        i.interrupter_mut(0).erdp.update_volatile(|u| {
-            u.set_event_ring_dequeue_pointer(pointer.into());
-            u.clear_event_handler_busy();
-        });
-    }
-
-    pub fn ring_doorbell(&self, index: usize, target: u8) {
-        self.doorbells.update(index, |u| {
-            u.set_doorbell_target(target);
-        });
-    }
-}
-
-impl Mapper {
-    pub fn new() -> Self {
-        Self {
-            mappings: Vec::new(),
-        }
-    }
-}
-
-impl xhci_lib::accessor::Mapper for Mapper {
-    unsafe fn map(&mut self, phys_start: usize, bytes: usize) -> NonZeroUsize {
-        let mapping = RawDeviceMemoryMapping::map(phys_start as u64, bytes, Default::default())
-            .expect("Could not map an USB xHCI region");
-        let address = mapping.address;
-        self.mappings.push(Arc::new(mapping));
-        NonZeroUsize::new_unchecked(address)
-    }
-
-    fn unmap(&mut self, _virt_start: usize, _bytes: usize) {
-        // TODO
-    }
-}
-
-// Register value definitions
-
-primitive_enum! {
-    pub enum PortSpeed: u8 {
-        Full = 1,
-        Low = 2,
-        High = 3,
-        SuperGen1x1 = 4,
-        SuperGen2x1 = 5,
-        SuperGen1x2 = 6,
-        SuperGen2x2 = 7,
-    }
-}
-
-impl PortSpeed {
-    pub fn default_max_packet_size(&self) -> usize {
-        match self {
-            Self::Low => 8,
-            Self::High => 64,
-            Self::SuperGen1x1 | Self::SuperGen1x2 | Self::SuperGen2x1 | Self::SuperGen2x2 => 512,
-
-            // See Section 4.3., point 7. of the initialization list
-            Self::Full => 8,
-        }
-    }
-}
-
-impl From<PortSpeed> for UsbSpeed {
-    fn from(value: PortSpeed) -> Self {
-        match value {
-            PortSpeed::Low => UsbSpeed::Low,
-            PortSpeed::Full => UsbSpeed::Full,
-            PortSpeed::High => UsbSpeed::High,
-            PortSpeed::SuperGen1x1
-            | PortSpeed::SuperGen1x2
-            | PortSpeed::SuperGen2x1
-            | PortSpeed::SuperGen2x2 => UsbSpeed::Super,
-        }
-    }
-}

kernel/driver/usb/xhci/src/regs/capability.rs

@@ -0,0 +1,66 @@
+use tock_registers::{register_bitfields, register_structs, registers::ReadOnly};
+
+register_bitfields! {
+    u32,
+    pub HCSPARAMS1 [
+        MaxSlots            OFFSET(0)   NUMBITS(8)  [],
+        MaxIntrs            OFFSET(8)   NUMBITS(11) [],
+        MaxPorts            OFFSET(24)  NUMBITS(8)  [],
+    ],
+    pub HCSPARAMS2 [
+        IST                 OFFSET(0)   NUMBITS(4)  [],
+        ERSTMax             OFFSET(4)   NUMBITS(4)  [],
+        MaxScratchpadBufsHi OFFSET(21)  NUMBITS(5)  [],
+        ScratchpadRestore   OFFSET(26)  NUMBITS(1)  [],
+        MaxScratchpadBufsLo OFFSET(27)  NUMBITS(5)  [],
+    ],
+    pub HCSPARAMS3 [
+        U1ExitLatency       OFFSET(0)   NUMBITS(8)  [],
+        U2ExitLatency       OFFSET(16)  NUMBITS(16) [],
+    ],
+    pub HCCPARAMS1 [
+        AC64                OFFSET(0)   NUMBITS(1)  [],
+        BNC                 OFFSET(1)   NUMBITS(1)  [],
+        CSZ                 OFFSET(2)   NUMBITS(1)  [],
+        PPC                 OFFSET(3)   NUMBITS(1)  [],
+        PIND                OFFSET(4)   NUMBITS(1)  [],
+        LHRC                OFFSET(5)   NUMBITS(1)  [],
+        LTC                 OFFSET(6)   NUMBITS(1)  [],
+        NSS                 OFFSET(7)   NUMBITS(1)  [],
+        PAE                 OFFSET(8)   NUMBITS(1)  [],
+        SPC                 OFFSET(9)   NUMBITS(1)  [],
+        SEC                 OFFSET(10)  NUMBITS(1)  [],
+        CFC                 OFFSET(11)  NUMBITS(1)  [],
+        MaxPSASize          OFFSET(12)  NUMBITS(4)  [],
+        XECP                OFFSET(16)  NUMBITS(16) [],
+    ],
+    pub HCCPARAMS2 [
+        U3C                 OFFSET(0)   NUMBITS(1)  [],
+        CMC                 OFFSET(1)   NUMBITS(1)  [],
+        FSC                 OFFSET(2)   NUMBITS(1)  [],
+        CTC                 OFFSET(3)   NUMBITS(1)  [],
+        LEC                 OFFSET(4)   NUMBITS(1)  [],
+        CIC                 OFFSET(5)   NUMBITS(1)  [],
+        ETC                 OFFSET(6)   NUMBITS(1)  [],
+        ETC_TSC             OFFSET(7)   NUMBITS(1)  [],
+        GSC                 OFFSET(8)   NUMBITS(1)  [],
+        VTC                 OFFSET(9)   NUMBITS(1)  [],
+    ],
+}
+
+register_structs! {
+    #[allow(non_snake_case)]
+    pub CapabilityRegs {
+        (0x00 => pub CAPLENGTH: ReadOnly<u8>),
+        (0x01 => _0),
+        (0x02 => pub HCIVERSION: ReadOnly<u16>),
+        (0x04 => pub HCSPARAMS1: ReadOnly<u32, HCSPARAMS1::Register>),
+        (0x08 => pub HCSPARAMS2: ReadOnly<u32, HCSPARAMS2::Register>),
+        (0x0C => pub HCSPARAMS3: ReadOnly<u32, HCSPARAMS3::Register>),
+        (0x10 => pub HCCPARAMS1: ReadOnly<u32, HCCPARAMS1::Register>),
+        (0x14 => pub DBOFF: ReadOnly<u32>),
+        (0x18 => pub RTSOFF: ReadOnly<u32>),
+        (0x1C => pub HCCPARAMS2: ReadOnly<u32, HCCPARAMS2::Register>),
+        (0x20 => @END),
+    }
+}

kernel/driver/usb/xhci/src/regs/extended.rs

@@ -0,0 +1,122 @@
+use core::{
+    ops::Range,
+    sync::atomic::{AtomicU32, Ordering},
+};
+
+use alloc::vec::Vec;
+use libk::error::Error;
+use libk_mm::{address::PhysicalAddress, device::DeviceMemoryIo};
+use libk_util::sync::spin_rwlock::IrqSafeRwLock;
+use ygg_driver_usb::{error::UsbError, info::UsbVersion};
+
+pub struct ProtocolSupport {
+    words: [u32; 4],
+}
+
+pub struct LegacySupport(DeviceMemoryIo<'static, [AtomicU32; 4]>);
+
+pub enum ExtendedCapability {
+    LegacySupport(IrqSafeRwLock<LegacySupport>),
+    ProtocolSupport(ProtocolSupport),
+}
+
+impl ExtendedCapability {
+    pub fn parse_list(
+        base: PhysicalAddress,
+        extended_capability_offset: usize,
+    ) -> Result<Vec<Self>, Error> {
+        let mut extended_capabilities = Vec::new();
+
+        if extended_capability_offset != 0 {
+            let extended_capability_base = base.add(extended_capability_offset);
+            let mut current = extended_capability_base;
+
+            loop {
+                let header =
+                    unsafe { DeviceMemoryIo::<[AtomicU32; 4]>::map(current, Default::default()) }?;
+                let word0 = header[0].load(Ordering::Acquire);
+                let id = word0 as u8;
+                let next_pointer = ((word0 >> 8) & 0xFF) as usize;
+
+                match id {
+                    0x01 => extended_capabilities.push(ExtendedCapability::LegacySupport(
+                        IrqSafeRwLock::new(LegacySupport(header)),
+                    )),
+                    0x02 => {
+                        let words = [
+                            word0,
+                            header[1].load(Ordering::Acquire),
+                            header[2].load(Ordering::Acquire),
+                            header[3].load(Ordering::Acquire),
+                        ];
+                        extended_capabilities.push(ExtendedCapability::ProtocolSupport(
+                            ProtocolSupport { words },
+                        ))
+                    }
+                    _ => (),
+                }
+
+                if next_pointer == 0 {
+                    break;
+                } else {
+                    current = current.add(next_pointer << 2);
+                }
+            }
+        }
+
+        Ok(extended_capabilities)
+    }
+}
+
+impl ProtocolSupport {
+    pub fn usb_revision(&self) -> Option<UsbVersion> {
+        UsbVersion::from_bcd_usb((self.words[0] >> 16) as u16)
+    }
+
+    pub fn slot_type(&self) -> u8 {
+        (self.words[3] & 0x1F) as u8
+    }
+
+    pub fn port_range(&self) -> Range<u8> {
+        let port_offset = self.words[2] as u8;
+        let port_count = (self.words[2] >> 8) as u8;
+
+        port_offset.max(1)..port_offset.saturating_add(port_count)
+    }
+}
+
+impl LegacySupport {
+    fn is_bios_owned(&self) -> bool {
+        self.0[0].load(Ordering::Acquire) & (1 << 16) != 0
+    }
+
+    fn set_os_owned(&mut self) {
+        self.0[0].fetch_or(1 << 24, Ordering::Release);
+    }
+
+    pub fn perform_bios_handoff(&mut self, mut timeout_cycles: u64) -> Result<(), UsbError> {
+        if !self.is_bios_owned() {
+            return Ok(());
+        }
+
+        // Clear SMI events
+        self.0[1].fetch_and(
+            !((1 << 0) | (1 << 4) | (1 << 13) | (1 << 14) | (1 << 15)),
+            Ordering::Release,
+        );
+
+        // Set OS owned semaphore
+        self.set_os_owned();
+        while timeout_cycles > 0 && self.is_bios_owned() {
+            core::hint::spin_loop();
+            timeout_cycles -= 1;
+        }
+
+        if timeout_cycles > 0 {
+            Ok(())
+        } else {
+            log::error!("xhci: BIOS handoff timed out");
+            Err(UsbError::DeviceBusy)
+        }
+    }
+}

kernel/driver/usb/xhci/src/regs/mod.rs

@@ -0,0 +1,184 @@
+use core::{fmt, sync::atomic::AtomicU32};
+
+use alloc::vec::Vec;
+use capability::{CapabilityRegs, HCCPARAMS1, HCSPARAMS1, HCSPARAMS2};
+use extended::ExtendedCapability;
+use libk::error::Error;
+use libk_mm::{address::PhysicalAddress, device::DeviceMemoryIo};
+use libk_util::sync::spin_rwlock::IrqSafeRwLock;
+use operational::{OperationalRegs, PortRegs, PORTSC, USBCMD, USBSTS};
+use runtime::{RuntimeRegs, IMAN};
+use tock_registers::{
+    interfaces::{ReadWriteable, Readable, Writeable},
+    LocalRegisterCopy,
+};
+
+pub mod capability;
+pub mod extended;
+pub mod operational;
+pub mod runtime;
+
+pub struct Regs {
+    #[allow(unused)]
+    pub(crate) capability: DeviceMemoryIo<'static, CapabilityRegs>,
+    pub(crate) operational: IrqSafeRwLock<DeviceMemoryIo<'static, OperationalRegs>>,
+    pub(crate) runtime: IrqSafeRwLock<DeviceMemoryIo<'static, RuntimeRegs>>,
+    pub(crate) doorbells: DeviceMemoryIo<'static, [AtomicU32]>,
+    pub(crate) extended_capabilities: Vec<ExtendedCapability>,
+    // TODO per-port locks
+    pub(crate) ports: IrqSafeRwLock<DeviceMemoryIo<'static, [PortRegs]>>,
+    pub(crate) port_count: usize,
+    pub(crate) slot_count: usize,
+    pub(crate) scratch_count: usize,
+    pub(crate) context_size: ContextSize,
+}
+
+#[derive(Clone, Copy, PartialEq, Eq, Debug)]
+pub enum ContextSize {
+    Context32,
+    Context64,
+}
+
+#[derive(Clone, Copy, PartialEq, Eq, Debug)]
+#[repr(transparent)]
+pub struct PortNumber(u8);
+
+impl Regs {
+    const PORT_REGS_OFFSET: usize = 0x400;
+
+    pub fn map(base: PhysicalAddress) -> Result<Self, Error> {
+        let capability =
+            unsafe { DeviceMemoryIo::<CapabilityRegs>::map(base, Default::default()) }?;
+
+        let operational_offset = capability.CAPLENGTH.get() as usize;
+        let doorbell_offset = capability.DBOFF.get() & !0x3;
+        let runtime_offset = capability.RTSOFF.get() & !0x1F;
+        let extended_capability_offset =
+            (capability.HCCPARAMS1.read(HCCPARAMS1::XECP) as usize) << 2;
+
+        let port_count = capability.HCSPARAMS1.read(HCSPARAMS1::MaxPorts) as usize;
+        let slot_count = capability.HCSPARAMS1.read(HCSPARAMS1::MaxSlots) as usize;
+
+        let operational_base = base.add(operational_offset);
+        let doorbell_base = base.add(doorbell_offset as usize);
+        let runtime_base = base.add(runtime_offset as usize);
+
+        let scratch_lo = capability.HCSPARAMS2.read(HCSPARAMS2::MaxScratchpadBufsLo);
+        let scratch_hi = capability.HCSPARAMS2.read(HCSPARAMS2::MaxScratchpadBufsHi);
+        let scratch_count = ((scratch_hi << 5) | scratch_lo) as usize;
+
+        let context_size = if capability.HCCPARAMS1.matches_all(HCCPARAMS1::CSZ::SET) {
+            ContextSize::Context64
+        } else {
+            ContextSize::Context32
+        };
+
+        let operational = unsafe {
+            DeviceMemoryIo::<OperationalRegs>::map(operational_base, Default::default())
+        }?;
+        let runtime =
+            unsafe { DeviceMemoryIo::<RuntimeRegs>::map(runtime_base, Default::default()) }?;
+
+        let ports = unsafe {
+            DeviceMemoryIo::map_slice(
+                operational_base.add(Self::PORT_REGS_OFFSET),
+                port_count,
+                Default::default(),
+            )
+        }?;
+        let doorbells =
+            unsafe { DeviceMemoryIo::map_slice(doorbell_base, 256, Default::default()) }?;
+        let extended_capabilities =
+            ExtendedCapability::parse_list(base, extended_capability_offset)?;
+
+        Ok(Self {
+            capability,
+            operational: IrqSafeRwLock::new(operational),
+            runtime: IrqSafeRwLock::new(runtime),
+            doorbells,
+            ports: IrqSafeRwLock::new(ports),
+            extended_capabilities,
+
+            port_count,
+            slot_count,
+            scratch_count,
+            context_size,
+        })
+    }
+
+    pub fn hc_reset(&self, timeout_cycles: u64) -> Result<(), Error> {
+        let op = self.operational.write();
+
+        // Wait for CNR to get cleared
+        op.wait_usbsts_bit(USBSTS::CNR::CLEAR, timeout_cycles)?;
+
+        // Clear run/stop and wait for the HC to halt
+        op.USBCMD.modify(USBCMD::RS::CLEAR);
+        op.wait_usbsts_bit(USBSTS::HCH::SET, timeout_cycles)?;
+
+        // Reset the HC, wait for CNR to get cleared again
+        op.USBCMD.modify(USBCMD::HCRST::SET);
+        op.wait_usbsts_bit(USBSTS::CNR::CLEAR, timeout_cycles)?;
+
+        // Halt the HC again
+        op.USBCMD.modify(USBCMD::RS::CLEAR);
+        op.wait_usbsts_bit(USBSTS::HCH::SET, timeout_cycles)?;
+
+        Ok(())
+    }
+
+    pub fn handle_interrupt(&self, vector: usize) -> LocalRegisterCopy<u32, USBSTS::Register> {
+        let op = self.operational.write();
+        let rt = self.runtime.write();
+
+        let status = op.USBSTS.extract();
+
+        // Clear the RW1C bits
+        op.USBSTS.set(status.get());
+
+        // Acknowledge interrupts for given vector (vector == interrupter index)
+        let interrupter = &rt.IRn[vector];
+
+        interrupter.IMAN.modify(IMAN::IP::SET);
+
+        status
+    }
+}
+
+pub fn portsc_to_neutral(
+    portsc: LocalRegisterCopy<u32, PORTSC::Register>,
+) -> LocalRegisterCopy<u32, PORTSC::Register> {
+    const RO_MASK: u32 = (1 << 0) | (1 << 3) | (0xF << 10) | (1 << 30);
+    const RWS_MASK: u32 = (0xF << 5)
+        | (1 << 9)
+        | (0x3 << 14)
+        | (1 << 25)
+        | (1 << 26)
+        | (1 << 27)
+        | (1 << 30)
+        | (1 << 31);
+
+    portsc.bitand(RO_MASK | RWS_MASK)
+}
+
+impl Into<u8> for PortNumber {
+    fn into(self) -> u8 {
+        self.0
+    }
+}
+
+impl PortNumber {
+    pub fn from_index(index: usize) -> Self {
+        Self(index as u8 + 1)
+    }
+
+    pub fn index(&self) -> usize {
+        self.0 as usize - 1
+    }
+}
+
+impl fmt::Display for PortNumber {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        fmt::Display::fmt(&self.0, f)
+    }
+}

kernel/driver/usb/xhci/src/regs/operational.rs

@@ -0,0 +1,140 @@
+use libk::error::Error;
+use libk_mm::address::PhysicalAddress;
+use tock_registers::{
+    fields::FieldValue,
+    interfaces::{Readable, Writeable},
+    register_bitfields, register_structs,
+    registers::{ReadOnly, ReadWrite},
+};
+
+register_bitfields! {
+    u32,
+    pub USBCMD [
+        RS                  OFFSET(0)   NUMBITS(1)  [],
+        HCRST               OFFSET(1)   NUMBITS(1)  [],
+        INTE                OFFSET(2)   NUMBITS(1)  [],
+        HSEE                OFFSET(3)   NUMBITS(1)  [],
+        LHCRST              OFFSET(7)   NUMBITS(1)  [],
+        CSS                 OFFSET(8)   NUMBITS(1)  [],
+        CRS                 OFFSET(9)   NUMBITS(1)  [],
+        EWE                 OFFSET(10)  NUMBITS(1)  [],
+        EU3S                OFFSET(11)  NUMBITS(1)  [],
+        CME                 OFFSET(13)  NUMBITS(1)  [],
+        ETE                 OFFSET(14)  NUMBITS(1)  [],
+        TSC_EN              OFFSET(15)  NUMBITS(1)  [],
+        VTIOE               OFFSET(16)  NUMBITS(1)  [],
+    ],
+    pub USBSTS [
+        HCH                 OFFSET(0)   NUMBITS(1)  [],
+        HSE                 OFFSET(2)   NUMBITS(1)  [],
+        EINT                OFFSET(3)   NUMBITS(1)  [],
+        PCD                 OFFSET(4)   NUMBITS(1)  [],
+        SSS                 OFFSET(8)   NUMBITS(1)  [],
+        RSS                 OFFSET(9)   NUMBITS(1)  [],
+        SRE                 OFFSET(10)  NUMBITS(1)  [],
+        CNR                 OFFSET(11)  NUMBITS(1)  [],
+        HCE                 OFFSET(12)  NUMBITS(1)  [],
+    ],
+    pub CONFIG [
+        MaxSlotsEn          OFFSET(0)   NUMBITS(8)  [],
+        U3E                 OFFSET(8)   NUMBITS(1)  [],
+        CIE                 OFFSET(9)   NUMBITS(1)  [],
+    ],
+
+    // Port registers
+    pub PORTSC [
+        CCS                 OFFSET(0)   NUMBITS(1)  [],
+        PED                 OFFSET(1)   NUMBITS(1)  [],
+        OCA                 OFFSET(3)   NUMBITS(1)  [],
+        PR                  OFFSET(4)   NUMBITS(1)  [],
+        PLS                 OFFSET(5)   NUMBITS(4)  [],
+        PP                  OFFSET(9)   NUMBITS(1)  [],
+        PS                  OFFSET(10)  NUMBITS(4)  [],
+        PIC                 OFFSET(14)  NUMBITS(2)  [],
+        LWS                 OFFSET(16)  NUMBITS(1)  [],
+        CSC                 OFFSET(17)  NUMBITS(1)  [],
+        PEC                 OFFSET(18)  NUMBITS(1)  [],
+        WRC                 OFFSET(19)  NUMBITS(1)  [],
+        OCC                 OFFSET(20)  NUMBITS(1)  [],
+        PRC                 OFFSET(21)  NUMBITS(1)  [],
+        PLC                 OFFSET(22)  NUMBITS(1)  [],
+        CEC                 OFFSET(23)  NUMBITS(1)  [],
+    ],
+    pub PORTLI [
+        LEC                 OFFSET(0)   NUMBITS(16) [],
+        RLC                 OFFSET(16)  NUMBITS(4)  [],
+        TLC                 OFFSET(20)  NUMBITS(4)  [],
+    ],
+}
+
+register_structs! {
+    #[allow(non_snake_case)]
+    pub OperationalRegs {
+        (0x000 => pub USBCMD: ReadWrite<u32, USBCMD::Register>),
+        (0x004 => pub USBSTS: ReadWrite<u32, USBSTS::Register>),
+        (0x008 => pub PAGESIZE: ReadOnly<u32>),
+        (0x00C => _0),
+        (0x014 => pub DNCTRL: ReadWrite<u32>),
+        (0x018 => pub CRCR: ReadWrite<u64>),
+        // (0x018 => pub CRCR_LO: ReadWrite<u32>),
+        // (0x01C => pub CRCR_HI: ReadWrite<u32>),
+        (0x020 => _1),
+        (0x030 => pub DCBAAP: ReadWrite<u64>),
+        // (0x030 => pub DCBAAP_LO: ReadWrite<u32>),
+        // (0x034 => pub DCBAAP_HI: ReadWrite<u32>),
+        (0x038 => pub CONFIG: ReadWrite<u32, CONFIG::Register>),
+        (0x03C => _2),
+        (0x400 => @END),
+    }
+}
+
+register_structs! {
+    #[allow(non_snake_case)]
+    pub PortRegs {
+        (0x00 => pub PORTSC: ReadWrite<u32, PORTSC::Register>),
+        (0x04 => pub PORTPMSC: ReadWrite<u32>),
+        (0x08 => pub PORTLI: ReadWrite<u32, PORTLI::Register>),
+        (0x0C => pub PORTHLPMC: ReadWrite<u32>),
+        (0x10 => @END),
+    }
+}
+
+impl OperationalRegs {
+    pub fn wait_usbsts_bit(
+        &self,
+        bit: FieldValue<u32, USBSTS::Register>,
+        mut timeout: u64,
+    ) -> Result<(), Error> {
+        while timeout > 0 && !self.USBSTS.matches_all(bit) {
+            core::hint::spin_loop();
+            timeout -= 1;
+        }
+        if timeout > 0 {
+            Ok(())
+        } else {
+            Err(Error::TimedOut)
+        }
+    }
+
+    pub fn set_dcbaap(&self, value: PhysicalAddress) {
+        let value = value.into_u64();
+        self.DCBAAP.set(value);
+    }
+
+    pub fn set_crcr(&self, value: PhysicalAddress, dcs: bool) {
+        let mut value = value.into_u64();
+        if dcs {
+            value |= 1;
+        }
+        self.CRCR.set(value);
+    }
+}
+
+impl PortRegs {
+    pub fn modify_portsc_preserving(&self, value: FieldValue<u32, PORTSC::Register>) {
+        let neutral = super::portsc_to_neutral(self.PORTSC.extract());
+        self.PORTSC.set(neutral.get() | value.value);
+    }
+}
+
+unsafe impl Sync for PortRegs {}

kernel/driver/usb/xhci/src/regs/runtime.rs

@@ -0,0 +1,69 @@
+use libk_mm::address::PhysicalAddress;
+use tock_registers::{
+    interfaces::{Readable, Writeable},
+    register_bitfields, register_structs,
+    registers::ReadWrite,
+};
+
+use crate::ring::{EventRing, EventRingSegmentTable};
+
+register_bitfields! {
+    u32,
+    pub IMAN [
+        IP      OFFSET(0)   NUMBITS(1)  [],
+        IE      OFFSET(1)   NUMBITS(1)  [],
+    ],
+    pub IMOD [
+        IMODI   OFFSET(0)   NUMBITS(16) [],
+        IMODC   OFFSET(16)  NUMBITS(16) [],
+    ],
+}
+
+register_structs! {
+    #[allow(non_snake_case)]
+    pub InterrupterRegs {
+        (0x00 => pub IMAN: ReadWrite<u32, IMAN::Register>),
+        (0x04 => pub IMOD: ReadWrite<u32, IMOD::Register>),
+        (0x08 => pub ERSTSZ: ReadWrite<u32>),
+        (0x0C => _0),
+        (0x10 => pub ERSTBA: ReadWrite<u64>),
+        (0x18 => pub ERDP: ReadWrite<u64>),
+        (0x20 => @END),
+    }
+}
+
+register_structs! {
+    #[allow(non_snake_case)]
+    pub RuntimeRegs {
+        (0x0000 => pub MFINDEX: ReadWrite<u32>),
+        (0x0004 => _0),
+        // NOTE there're 1024 of them, but I'm not going to use all of those anyway, so use 1023
+        // for a nice round number
+        (0x0020 => pub IRn: [InterrupterRegs; 1023]),
+        (0x8000 => @END),
+    }
+}
+
+impl RuntimeRegs {
+    pub fn configure_interrupter(
+        &self,
+        index: usize,
+        event_ring: &EventRing,
+        erst: &EventRingSegmentTable,
+    ) {
+        let interrupter = &self.IRn[index];
+
+        let erdp = event_ring.dequeue_pointer().into_u64();
+        let erstba = erst.physical_address().into_u64();
+
+        interrupter.ERSTSZ.set(erst.capacity() as u32);
+        interrupter.ERSTBA.set(erstba);
+        interrupter.ERDP.set(erdp);
+        interrupter.IMAN.write(IMAN::IE::SET);
+    }
+
+    pub fn set_interrupter_dequeue_pointer(&self, index: usize, erdp: PhysicalAddress) {
+        let _ = self.IRn[index].ERDP.get();
+        self.IRn[index].ERDP.set(erdp.into_u64() | (1 << 3));
+    }
+}

kernel/driver/usb/xhci/src/ring/command.rs

@@ -15,7 +15,6 @@ use libk_util::{
     sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock},
     waker::QueueWaker,
 };
-use xhci_lib::context;
 use ygg_driver_usb::error::UsbError;
 use yggdrasil_abi::define_bitfields;
 
@@ -110,14 +109,18 @@ impl CommandRing {
         inner.enqueue(trb)
     }
 
-    pub async fn address_device<E: CommandExecutor, const N: usize>(
+    pub async fn address_device<E: CommandExecutor>(
         &self,
         executor: &E,
         slot_id: u8,
-        input: &mut PageBox<context::Input<N>>,
+        cx_physical_address: PhysicalAddress,
+        bsr: bool,
     ) -> Result<(), UsbError> {
-        self.submit_and_wait(executor, AddressDeviceCommandTrb::new(input, slot_id))
+        self.submit_and_wait(
-            .await?;
+            executor,
+            AddressDeviceCommandTrb::new(cx_physical_address, slot_id, bsr),
+        )
+        .await?;
         Ok(())
     }
 
@@ -125,15 +128,68 @@ impl CommandRing {
         &self,
         executor: &E,
         slot_id: u8,
-        input: &mut PageBox<context::Input<8>>,
+        cx_physical_address: PhysicalAddress,
     ) -> Result<(), UsbError> {
-        self.submit_and_wait(executor, ConfigureEndpointCommandTrb::new(input, slot_id))
+        self.submit_and_wait(
-            .await?;
+            executor,
+            ConfigureEndpointCommandTrb::new(cx_physical_address, slot_id),
+        )
+        .await?;
         Ok(())
     }
 
-    pub async fn enable_slot<E: CommandExecutor>(&self, executor: &E) -> Result<u8, UsbError> {
+    pub async fn reset_endpoint<E: CommandExecutor>(
-        self.submit_and_wait(executor, EnableSlotCommandTrb::new())
+        &self,
+        executor: &E,
+        slot_id: u8,
+        endpoint_id: u8,
+        preserve_transaction_state: bool,
+    ) -> Result<(), UsbError> {
+        self.submit_and_wait(
+            executor,
+            ResetEndpointCommandTrb::new(slot_id, endpoint_id, preserve_transaction_state),
+        )
+        .await?;
+        Ok(())
+    }
+
+    pub async fn set_tr_dequeue_pointer<E: CommandExecutor>(
+        &self,
+        executor: &E,
+        slot_id: u8,
+        endpoint_id: u8,
+        dequeue_pointer: PhysicalAddress,
+        dequeue_cycle: bool,
+    ) -> Result<(), UsbError> {
+        self.submit_and_wait(
+            executor,
+            SetTrDequeuePointerCommandTrb::new(
+                slot_id,
+                endpoint_id,
+                dequeue_pointer,
+                dequeue_cycle,
+            ),
+        )
+        .await?;
+        Ok(())
+    }
+
+    pub async fn enable_slot<E: CommandExecutor>(
+        &self,
+        executor: &E,
+        slot_type: u8,
+    ) -> Result<u8, UsbError> {
+        self.submit_and_wait(executor, EnableSlotCommandTrb::new(slot_type))
+            .await
+    }
+
+    #[allow(unused)]
+    pub async fn disable_slot<E: CommandExecutor>(
+        &self,
+        executor: &E,
+        slot_id: u8,
+    ) -> Result<u8, UsbError> {
+        self.submit_and_wait(executor, DisableSlotCommandTrb::new(slot_id))
             .await
     }
 
@@ -153,6 +209,7 @@ impl CommandRing {
                 } else {
                     Poll::Ready(Err(UsbError::HostControllerCommandFailed(
                         status.completion_code,
+                        status.completion_parameter,
                     )))
                 }
             } else {
@@ -181,8 +238,15 @@ define_bitfields! {
     }
 }
 
+define_bitfields! {
+    pub DisableSlotCommandFlags : u32 {
+        (24..32) => slot_id
+    }
+}
+
 define_bitfields! {
     pub AddressDeviceCommandFlags : u32 {
+        9 => bsr + set_bsr,
         (24..32) => slot_id
     }
 }
@@ -193,6 +257,21 @@ define_bitfields! {
     }
 }
 
+define_bitfields! {
+    pub ResetEndpointCommandFlags: u32 {
+        (24..32) => slot_id,
+        (16..21) => endpoint_id,
+        9 => tsp,
+    }
+}
+
+define_bitfields! {
+    pub SetTrDequeuePointerCommandFlags: u32 {
+        (24..32) => slot_id,
+        (16..21) => endpoint_id,
+    }
+}
+
 define_bitfields! {
     pub RawCommandFlags : u32 {
         (10..16) => ty + set_ty,
@@ -207,6 +286,13 @@ pub struct EnableSlotCommandTrb {
     pub flags: EnableSlotCommandFlags,
 }
 
+#[derive(Clone, Copy, Debug, Pod, Zeroable)]
+#[repr(C, align(16))]
+pub struct DisableSlotCommandTrb {
+    _0: [u32; 3],
+    pub flags: DisableSlotCommandFlags,
+}
+
 #[derive(Clone, Copy, Debug, Pod, Zeroable)]
 #[repr(C, align(16))]
 pub struct AddressDeviceCommandTrb {
@@ -223,6 +309,21 @@ pub struct ConfigureEndpointCommandTrb {
     pub flags: ConfigureEndpointCommandFlags,
 }
 
+#[derive(Clone, Copy, Debug, Pod, Zeroable)]
+#[repr(C, align(16))]
+pub struct ResetEndpointCommandTrb {
+    _0: [u32; 3],
+    pub flags: ResetEndpointCommandFlags,
+}
+
+#[derive(Clone, Copy, Debug, Pod, Zeroable)]
+#[repr(C, align(16))]
+pub struct SetTrDequeuePointerCommandTrb {
+    pub value: u64,
+    _0: u32,
+    pub flags: SetTrDequeuePointerCommandFlags,
+}
+
 #[derive(Clone, Copy, Debug, Pod, Zeroable)]
 #[repr(C, align(16))]
 pub struct RawCommandTrb {
@@ -233,6 +334,7 @@ pub struct RawCommandTrb {
 #[derive(Debug, Clone, Copy)]
 pub struct CommandReply {
     pub completion_code: u8,
+    pub completion_parameter: u32,
     pub slot_id: u8,
 }
 
@@ -241,44 +343,83 @@ pub trait CommandTrb: Pod + fmt::Debug {
 }
 
 impl EnableSlotCommandTrb {
-    pub fn new() -> Self {
+    pub fn new(slot_type: u8) -> Self {
         Self {
             _0: [0; 3],
-            flags: EnableSlotCommandFlags::new(0),
+            flags: EnableSlotCommandFlags::new(slot_type as u32),
+        }
+    }
+}
+
+impl DisableSlotCommandTrb {
+    pub fn new(slot_id: u8) -> Self {
+        Self {
+            _0: [0; 3],
+            flags: DisableSlotCommandFlags::new(slot_id as u32),
         }
     }
 }
 
 impl AddressDeviceCommandTrb {
-    pub fn new<const N: usize>(
+    pub fn new(input_context_address: PhysicalAddress, slot_id: u8, bsr: bool) -> Self {
-        input_context: &mut PageBox<context::Input<N>>,
-        slot_id: u8,
-    ) -> Self {
         Self {
-            input_context_address: unsafe { input_context.as_physical_address() },
+            input_context_address,
             _0: 0,
-            flags: AddressDeviceCommandFlags::new(slot_id as _),
+            flags: AddressDeviceCommandFlags::new(bsr, slot_id as _),
         }
     }
 }
 
 impl ConfigureEndpointCommandTrb {
-    pub fn new<const N: usize>(
+    pub fn new(input_context_address: PhysicalAddress, slot_id: u8) -> Self {
-        input_context: &mut PageBox<context::Input<N>>,
-        slot_id: u8,
-    ) -> Self {
         Self {
-            input_context_address: unsafe { input_context.as_physical_address() },
+            input_context_address,
             _0: 0,
             flags: ConfigureEndpointCommandFlags::new(slot_id as _),
         }
     }
 }
 
+impl ResetEndpointCommandTrb {
+    pub fn new(slot_id: u8, endpoint_id: u8, preserve_transaction_state: bool) -> Self {
+        Self {
+            _0: [0; 3],
+            flags: ResetEndpointCommandFlags::new(
+                slot_id as _,
+                endpoint_id as _,
+                preserve_transaction_state,
+            ),
+        }
+    }
+}
+
+impl SetTrDequeuePointerCommandTrb {
+    pub fn new(
+        slot_id: u8,
+        endpoint_id: u8,
+        dequeue_pointer: PhysicalAddress,
+        dequeue_cycle: bool,
+    ) -> Self {
+        let mut value = dequeue_pointer.into_u64();
+        if dequeue_cycle {
+            value |= 1;
+        }
+        Self {
+            value,
+            _0: 0,
+            flags: SetTrDequeuePointerCommandFlags::new(slot_id as _, endpoint_id as _),
+        }
+    }
+}
+
 impl CommandTrb for EnableSlotCommandTrb {
     const TRB_TYPE: u8 = 9;
 }
 
+impl CommandTrb for DisableSlotCommandTrb {
+    const TRB_TYPE: u8 = 10;
+}
+
 impl CommandTrb for AddressDeviceCommandTrb {
     const TRB_TYPE: u8 = 11;
 }
@@ -286,3 +427,11 @@ impl CommandTrb for AddressDeviceCommandTrb {
 impl CommandTrb for ConfigureEndpointCommandTrb {
     const TRB_TYPE: u8 = 12;
 }
+
+impl CommandTrb for ResetEndpointCommandTrb {
+    const TRB_TYPE: u8 = 14;
+}
+
+impl CommandTrb for SetTrDequeuePointerCommandTrb {
+    const TRB_TYPE: u8 = 16;
+}

kernel/driver/usb/xhci/src/ring/event.rs

@@ -218,6 +218,7 @@ impl RawEventTrb {
                     address: command.address,
                     reply: CommandReply {
                         completion_code: command.status.completion_code() as _,
+                        completion_parameter: command.status.completion_parameter(),
                         slot_id: command.flags.slot_id() as _,
                     },
                 })

kernel/driver/usb/xhci/src/ring/mod.rs

@@ -1,17 +1,31 @@
+//use bytemuck::{Pod, Zeroable};
+//use libk_mm::address::PhysicalAddress;
+//use yggdrasil_abi::define_bitfields;
+
+use alloc::boxed::Box;
+use async_trait::async_trait;
 use bytemuck::{Pod, Zeroable};
 use libk_mm::address::PhysicalAddress;
+use ygg_driver_usb::error::UsbError;
 use yggdrasil_abi::define_bitfields;
 
 pub mod command;
 pub mod event;
 pub mod transfer;
 
-pub use command::CommandRing;
+pub use event::{EventRing, EventRingSegmentTable};
-pub use event::{Event, EventRing, EventRingSegmentTable};
-pub use transfer::ControlTransferRing;
 
+#[async_trait]
 pub trait CommandExecutor {
     fn ring_doorbell(&self, index: usize, target: u8);
+
+    async fn reset_endpoint(
+        &self,
+        slot_id: u8,
+        endpoint_id: u8,
+        dequeue_pointer: PhysicalAddress,
+        dequeue_cycle: bool,
+    ) -> Result<(), UsbError>;
 }
 
 pub trait GenericRing {
@@ -19,15 +33,6 @@ pub trait GenericRing {
     fn base(&self) -> PhysicalAddress;
 }
 
-pub trait GenericTransferRing: GenericRing + Send + Sync {
-    fn slot_id(&self) -> u8;
-    fn endpoint_id(&self) -> u8;
-
-    fn dequeue_pointer(&self) -> PhysicalAddress;
-    fn notify(&self, address: PhysicalAddress, status: u32);
-    fn shutdown(&self);
-}
-
 define_bitfields! {
     pub LinkTrbFlags : u32 {
         (10..16) => ty,

kernel/driver/usb/xhci/src/ring/transfer.rs

@@ -1,24 +1,32 @@
 use core::{
-    mem::{size_of, MaybeUninit},
+    future::poll_fn,
-    sync::atomic::{AtomicBool, AtomicU64, Ordering},
+    mem::MaybeUninit,
+    sync::atomic::{AtomicBool, Ordering},
+    task::Poll,
 };
 
-use alloc::{collections::BTreeMap, sync::Arc, vec::Vec};
+use alloc::{
+    sync::{Arc, Weak},
+    vec::Vec,
+};
 use bytemuck::{Pod, Zeroable};
+use futures_util::task::AtomicWaker;
 use libk_mm::{
     address::{AsPhysicalAddress, PhysicalAddress},
-    PageBox,
+    PageBox, PageSlice,
+};
+use libk_util::{
+    queue::BoundedQueue,
+    sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock, IrqSafeSpinlockGuard},
 };
-use libk_util::sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock, IrqSafeSpinlockGuard};
 use ygg_driver_usb::{
-    communication::UsbInterruptTransfer, error::UsbError, pipe::control::ControlTransferSetup,
+    communication::UsbDirection,
-    UsbControlTransfer, UsbDirection, UsbTransferStatus, UsbTransferToken,
+    error::{TransferError, UsbError},
+    pipe::control::ControlTransferSetup,
 };
 use yggdrasil_abi::define_bitfields;
 
-use crate::ring::LinkTrb;
+use super::{CommandExecutor, LinkTrb};
-
-use super::{CommandExecutor, GenericRing, GenericTransferRing};
 
 struct TransferRingInner {
     trbs: PageBox<[MaybeUninit<RawTransferTrb>]>,
@@ -27,111 +35,320 @@ struct TransferRingInner {
     cycle_bit: bool,
 }
 
-// TODO split TransferRing into Normal, Control, etc
+pub struct TransferRing {
-pub struct ControlTransferRing {
     inner: IrqSafeSpinlock<TransferRingInner>,
+    base: PhysicalAddress,
     capacity: usize,
 
-    // TODO this is inefficient and ugly
-    pending_trbs: IrqSafeRwLock<BTreeMap<PhysicalAddress, UsbTransferToken>>,
-    completions: IrqSafeRwLock<BTreeMap<UsbTransferToken, Arc<UsbTransferStatus>>>,
-
     slot_id: u8,
-    ep_id: u8,
+    endpoint_id: u8,
 
-    transfer_id: AtomicU64,
+    transactions: IrqSafeRwLock<Vec<Option<Weak<Transaction>>>>,
     shutdown: AtomicBool,
 }
 
-pub struct InterruptInTransferRing {
+pub struct TransactionBuilder<'a> {
-    inner: IrqSafeSpinlock<TransferRingInner>,
+    inner: IrqSafeSpinlockGuard<'a, TransferRingInner>,
-    capacity: usize,
+    ring: &'a Arc<TransferRing>,
-
+    pending: Vec<PhysicalAddress>,
-    completions: IrqSafeRwLock<BTreeMap<PhysicalAddress, Arc<UsbTransferStatus>>>,
-
-    slot_id: u8,
-    ep_id: u8,
-
-    shutdown: AtomicBool,
 }
 
-struct TransferBuilder<'a> {
+pub struct Transaction {
-    ring: &'a ControlTransferRing,
+    event_queue: BoundedQueue<TransactionEvent>,
-    ring_inner: IrqSafeSpinlockGuard<'a, TransferRingInner>,
+    event_notify: AtomicWaker,
-
+    next_dequeue: usize,
-    token: UsbTransferToken,
+    next_cycle: bool,
-    direction: UsbDirection,
-    addresses: Vec<PhysicalAddress>,
-    status: Arc<UsbTransferStatus>,
 }
 
-impl<'a> TransferBuilder<'a> {
+#[derive(Debug)]
-    pub fn new(ring: &'a ControlTransferRing, direction: UsbDirection) -> Self {
+pub enum TransactionEvent {
-        let ring_inner = ring.inner.lock();
+    Status(usize, u32),
-        let token = UsbTransferToken(ring.transfer_id.fetch_add(1, Ordering::AcqRel));
+    Shutdown,
-        let status = Arc::new(UsbTransferStatus::new());
+}
 
-        ring.completions.write().insert(token, status.clone());
+impl TransferRing {
+    pub fn new(slot_id: u8, endpoint_id: u8, capacity: usize) -> Result<Self, UsbError> {
+        let inner = TransferRingInner::new(capacity)?;
+        let base = unsafe { inner.trbs.as_physical_address() };
+        let transactions = (0..capacity).map(|_| None).collect();
 
-        Self {
+        Ok(Self {
-            ring,
+            inner: IrqSafeSpinlock::new(inner),
-            ring_inner,
+            base,
+            capacity,
 
-            token,
+            slot_id,
-            direction,
+            endpoint_id,
-            status,
+
-            addresses: Vec::new(),
+            transactions: IrqSafeRwLock::new(transactions),
+            shutdown: AtomicBool::new(false),
+        })
+    }
+
+    pub fn transaction_builder(self: &Arc<Self>) -> Result<TransactionBuilder, UsbError> {
+        if self.shutdown.load(Ordering::Acquire) {
+            return Err(UsbError::DeviceDisconnected);
+        }
+
+        Ok(TransactionBuilder {
+            inner: self.inner.lock(),
+            ring: self,
+            pending: Vec::new(),
+        })
+    }
+
+    async fn handle_stall<E: CommandExecutor>(
+        &self,
+        executor: &E,
+        result: &Result<usize, TransferError>,
+        transaction: &Transaction,
+    ) {
+        if let Err(TransferError::Stall) = result {
+            let dequeue = self
+                .base
+                .add(transaction.next_dequeue * size_of::<RawTransferTrb>());
+            if let Err(rerror) = executor
+                .reset_endpoint(
+                    self.slot_id,
+                    self.endpoint_id,
+                    dequeue,
+                    transaction.next_cycle,
+                )
+                .await
+            {
+                log::error!(
+                    "xhci: could not reset endpoint after stall {}:{}: {rerror:?}",
+                    self.slot_id,
+                    self.endpoint_id
+                );
+
+                self.shutdown.store(true, Ordering::Release);
+            }
         }
     }
 
-    pub fn push_trb<C: TransferTrb>(&mut self, trb: C) -> &mut Self {
+    pub async fn normal_transfer<E: CommandExecutor>(
-        let address = self.ring_inner.enqueue(trb);
+        self: &Arc<Self>,
-        self.addresses.push(address);
+        executor: &E,
-        self.ring.pending_trbs.write().insert(address, self.token);
+        buffer: PhysicalAddress,
-        self
+        length: usize,
+    ) -> Result<usize, UsbError> {
+        if length == 0 {
+            return Ok(0);
+        }
+        let mut builder = self.transaction_builder()?;
+        let last_data_trb = builder.enqueue_normal(buffer, length)?;
+        let transaction = builder.submit(executor);
+
+        let status = transaction.wait_normal(last_data_trb).await;
+
+        builder.inner.dequeue_index = builder.inner.enqueue_index;
+        self.handle_stall(executor, &status, &transaction).await;
+
+        let residual = status?;
+        Ok(length.saturating_sub(residual))
     }
 
-    pub fn start<E: CommandExecutor>(self, executor: &E, length: usize) -> UsbControlTransfer {
+    // Helper functions, shorthands for transaction_builder().....finish() + kick()
-        executor.ring_doorbell(self.ring.slot_id as _, self.ring.ep_id);
+    pub async fn control_transfer<E: CommandExecutor>(
+        self: &Arc<Self>,
+        executor: &E,
+        setup: ControlTransferSetup,
+        buffer: Option<(&mut PageSlice<MaybeUninit<u8>>, UsbDirection)>,
+    ) -> Result<usize, UsbError> {
+        let mut builder = self.transaction_builder()?;
 
-        UsbControlTransfer {
+        let data_len = buffer.as_ref().map_or(0, |(buffer, _)| buffer.len());
-            id: self.token,
+        let (setup, data, status) = builder.enqueue_control(setup, buffer)?;
-            length,
 
-            direction: self.direction,
+        let transaction = builder.submit(executor);
-            elements: self.addresses,
+
-            status: self.status,
+        // TODO timeout
+        let status = transaction.wait_control(setup, data, status).await;
+
+        builder.inner.dequeue_index = builder.inner.enqueue_index;
+        self.handle_stall(executor, &status, &transaction).await;
+
+        let residual = status?;
+        Ok(data_len.saturating_sub(residual))
+    }
+
+    pub fn kick<E: CommandExecutor>(&self, executor: &E) {
+        executor.ring_doorbell(self.slot_id as usize, self.endpoint_id);
+    }
+
+    pub fn shutdown(&self) {
+        self.shutdown.store(true, Ordering::Release);
+        // Shutdown transactions
+        let transactions = self.transactions.read();
+        for index in 0..self.capacity {
+            if let Some(tx) = transactions[index].as_ref().and_then(Weak::upgrade) {
+                tx.shutdown();
+            }
         }
     }
+
+    pub fn notify(&self, address: PhysicalAddress, status: u32) {
+        if status == 0 {
+            return;
+        }
+
+        if address < self.base || address - self.base >= size_of::<RawTransferTrb>() * self.capacity
+        {
+            log::warn!("xhci: event outside of trb array: {address:#x}");
+            return;
+        }
+
+        let index = (address - self.base) / size_of::<RawTransferTrb>();
+        if let Some(tx) = self.transactions.write()[index]
+            .take()
+            .and_then(|tx| tx.upgrade())
+        {
+            tx.notify(index, status);
+        } else {
+            log::warn!("xhci: no transaction @ {index} to notify");
+        }
+    }
+
+    pub fn base(&self) -> PhysicalAddress {
+        self.base
+    }
+}
+
+impl TransactionBuilder<'_> {
+    const TRB_SIZE_LIMIT: usize = 65536;
+
+    pub fn enqueue<C: TransferTrb>(&mut self, trb: C, ioc: bool) -> Result<usize, UsbError> {
+        let address = self.inner.enqueue(trb, ioc)?;
+        self.pending.push(address);
+        Ok((address - self.ring.base) / size_of::<RawTransferTrb>())
+    }
+
+    pub fn enqueue_normal(
+        &mut self,
+        buffer: PhysicalAddress,
+        length: usize,
+    ) -> Result<usize, UsbError> {
+        let trb_count = length.div_ceil(Self::TRB_SIZE_LIMIT);
+        if self.inner.free_capacity() <= trb_count || trb_count == 0 {
+            return Err(UsbError::DeviceBusy);
+        }
+
+        let mut last_trb = 0;
+        for i in 0..trb_count {
+            let offset = i * Self::TRB_SIZE_LIMIT;
+            let amount = (length - offset).min(Self::TRB_SIZE_LIMIT);
+
+            last_trb = self
+                .enqueue(
+                    NormalTransferTrb::new(buffer.add(offset), amount),
+                    i == trb_count - 1,
+                )
+                .unwrap();
+        }
+
+        Ok(last_trb)
+    }
+
+    pub fn enqueue_control(
+        &mut self,
+        setup: ControlTransferSetup,
+        buffer: Option<(&mut PageSlice<MaybeUninit<u8>>, UsbDirection)>,
+    ) -> Result<(usize, Option<usize>, usize), UsbError> {
+        // Check ring capacity first
+        // TODO larger DATA stages
+        let trb_count = 2 + if buffer.is_some() { 1 } else { 0 };
+        if self.inner.free_capacity() <= trb_count {
+            return Err(UsbError::DeviceBusy);
+        }
+
+        // unwrap()s are okay here, capacity checked above
+        let setup_stage = self
+            .enqueue(ControlTransferSetupTrb::new(setup), true)
+            .unwrap();
+        let data_stage = if let Some((buffer, direction)) = buffer {
+            Some(
+                self.enqueue(
+                    ControlTransferDataTrb::new(
+                        unsafe { buffer.as_physical_address() },
+                        buffer.len(),
+                        direction,
+                    ),
+                    true,
+                )
+                .unwrap(),
+            )
+        } else {
+            None
+        };
+        let status_stage = self
+            .enqueue(ControlTransferStatusTrb::new(UsbDirection::In), true)
+            .unwrap();
+
+        Ok((setup_stage, data_stage, status_stage))
+    }
+
+    pub fn finish(&mut self) -> Arc<Transaction> {
+        let transaction = Arc::new(Transaction {
+            event_queue: BoundedQueue::new(self.pending.len()),
+            event_notify: AtomicWaker::new(),
+            next_dequeue: self.inner.enqueue_index,
+            next_cycle: self.inner.cycle_bit,
+        });
+
+        let mut transactions = self.ring.transactions.write();
+        for &pending in self.pending.iter() {
+            let index = (pending - self.ring.base) / size_of::<RawTransferTrb>();
+            transactions[index] = Some(Arc::downgrade(&transaction));
+        }
+
+        transaction
+    }
+
+    pub fn submit<E: CommandExecutor>(&mut self, executor: &E) -> Arc<Transaction> {
+        let transaction = self.finish();
+        self.ring.kick(executor);
+        transaction
+    }
 }
 
 impl TransferRingInner {
-    fn enqueue<C: TransferTrb>(&mut self, trb: C) -> PhysicalAddress {
+    fn new(capacity: usize) -> Result<Self, UsbError> {
+        let trbs = PageBox::new_zeroed_slice(capacity).map_err(UsbError::MemoryError)?;
+        Ok(Self {
+            trbs,
+            enqueue_index: 0,
+            dequeue_index: 0,
+            cycle_bit: true,
+        })
+    }
+
+    fn enqueue<C: TransferTrb>(&mut self, trb: C, ioc: bool) -> Result<PhysicalAddress, UsbError> {
         if (self.enqueue_index + 1) % (self.trbs.len() - 1) == self.dequeue_index {
-            todo!("Ring full");
+            log::warn!("xhci: transfer ring full");
+            return Err(UsbError::DeviceBusy);
         }
 
         let mut raw: RawTransferTrb = bytemuck::cast(trb);
 
-        raw.flags.set_ty(C::TRB_TYPE as _);
+        raw.flags.set_ty(C::TRB_TYPE as u32);
         raw.flags.set_cycle(self.cycle_bit);
+        raw.flags.set_ioc(ioc);
 
         self.trbs[self.enqueue_index].write(raw);
 
         let address = unsafe { self.trbs.as_physical_address() }
             .add(self.enqueue_index * size_of::<RawTransferTrb>());
 
-        // Move to the next TRB slot
         self.enqueue_index += 1;
         if self.enqueue_index >= self.trbs.len() - 1 {
             self.enqueue_link();
 
-            // Wrap around
             self.cycle_bit = !self.cycle_bit;
             self.enqueue_index = 0;
         }
 
-        address
+        Ok(address)
     }
 
     fn enqueue_link(&mut self) {
@@ -141,233 +358,109 @@ impl TransferRingInner {
         self.trbs[self.enqueue_index].write(bytemuck::cast(link));
     }
 
-    fn advance(&mut self) {
+    fn free_capacity(&self) -> usize {
-        self.dequeue_index += 1;
+        self.enqueue_index + self.trbs.len() - self.dequeue_index
-
-        if self.dequeue_index >= self.trbs.len() - 1 {
-            self.dequeue_index = 0;
-        }
     }
 }
 
-impl GenericRing for ControlTransferRing {
+impl Transaction {
-    fn base(&self) -> PhysicalAddress {
+    pub fn notify(&self, trb_index: usize, status: u32) {
-        unsafe { self.inner.lock().trbs.as_physical_address() }
+        self.event_queue
+            .push(TransactionEvent::Status(trb_index, status))
+            .ok();
+        self.event_notify.wake();
     }
 
-    fn capacity(&self) -> usize {
+    pub fn shutdown(&self) {
-        self.capacity
+        self.event_queue.push(TransactionEvent::Shutdown).ok();
-    }
+        self.event_notify.wake();
-}
-
-impl GenericTransferRing for ControlTransferRing {
-    fn dequeue_pointer(&self) -> PhysicalAddress {
-        let inner = self.inner.lock();
-        unsafe { inner.trbs.as_physical_address() }
-            .add(inner.dequeue_index * size_of::<RawTransferTrb>())
     }
 
-    fn notify(&self, address: PhysicalAddress, value: u32) {
+    pub async fn wait_normal(&self, last_trb: usize) -> Result<usize, TransferError> {
-        if value == 0 {
+        loop {
-            return;
+            let event = self.next_event().await;
-        }
+            let status = event.to_result();
 
-        let completions = self.completions.read();
+            match event {
-        if let Some(&token) = self.pending_trbs.read().get(&address) {
+                TransactionEvent::Status(trb_index, _) => {
-            let Some(status) = completions.get(&token) else {
+                    if status.is_err() || trb_index == last_trb {
-                log::warn!(
+                        break status;
-                    "Notification received for non-existent transfer: {:?}",
+                    }
-                    token
+                }
-                );
+                TransactionEvent::Shutdown => {
-                return;
+                    log::error!("xhci: abort transaction, endpoint shutdown");
-            };
+                    return Err(TransferError::UsbTransactionError);
-
+                }
-            status.signal(value);
+            }
         }
     }
 
-    fn shutdown(&self) {
+    pub async fn wait_trb(&self, trb: usize) -> Result<usize, TransferError> {
-        self.shutdown.store(true, Ordering::Release);
+        let event = self.next_event().await;
-        let mut completions = self.completions.write();
+        match event {
-        while let Some((_, status)) = completions.pop_first() {
+            TransactionEvent::Status(trb_index, _) => {
-            status.abort();
+                if trb_index != trb {
+                    return Err(TransferError::InvalidTransfer);
+                }
+            }
+            TransactionEvent::Shutdown => {
+                log::error!("xhci: abort transaction, endpoint shutdown");
+                return Err(TransferError::UsbTransactionError);
+            }
         }
+        event.to_result()
     }
 
-    fn slot_id(&self) -> u8 {
+    pub async fn wait_control(
-        self.slot_id
-    }
-
-    fn endpoint_id(&self) -> u8 {
-        self.ep_id
-    }
-}
-
-impl GenericRing for InterruptInTransferRing {
-    fn base(&self) -> PhysicalAddress {
-        unsafe { self.inner.lock().trbs.as_physical_address() }
-    }
-
-    fn capacity(&self) -> usize {
-        self.capacity
-    }
-}
-
-impl GenericTransferRing for InterruptInTransferRing {
-    fn dequeue_pointer(&self) -> PhysicalAddress {
-        let inner = self.inner.lock();
-        unsafe { inner.trbs.as_physical_address() }
-            .add(inner.dequeue_index * size_of::<RawTransferTrb>())
-    }
-
-    fn notify(&self, address: PhysicalAddress, value: u32) {
-        if value == 0 {
-            return;
-        }
-
-        let mut completions = self.completions.write();
-        if let Some(status) = completions.remove(&address) {
-            status.signal(value);
-        }
-    }
-
-    fn shutdown(&self) {
-        self.shutdown.store(true, Ordering::Release);
-        let mut completions = self.completions.write();
-        while let Some((_, status)) = completions.pop_first() {
-            status.abort();
-        }
-    }
-
-    fn slot_id(&self) -> u8 {
-        self.slot_id
-    }
-
-    fn endpoint_id(&self) -> u8 {
-        self.ep_id
-    }
-}
-
-impl InterruptInTransferRing {
-    pub fn new(slot_id: u8, ep_id: u8, capacity: usize) -> Result<Self, UsbError> {
-        let trbs = PageBox::new_zeroed_slice(capacity).map_err(UsbError::MemoryError)?;
-
-        Ok(Self {
-            inner: IrqSafeSpinlock::new(TransferRingInner {
-                trbs,
-                enqueue_index: 0,
-                dequeue_index: 0,
-                cycle_bit: true,
-            }),
-            completions: IrqSafeRwLock::new(BTreeMap::new()),
-            slot_id,
-            ep_id,
-            capacity,
-            shutdown: AtomicBool::new(false),
-        })
-    }
-
-    pub fn start_transfer<E: CommandExecutor>(
         &self,
-        executor: &E,
+        setup_trb: usize,
-        buffer: &mut PageBox<[u8]>,
+        last_data_trb: Option<usize>,
-    ) -> Result<UsbInterruptTransfer, UsbError> {
+        status_trb: usize,
-        // Don't even try to start the transfer
+    ) -> Result<usize, TransferError> {
-        if self.shutdown.load(Ordering::Acquire) {
+        self.wait_trb(setup_trb).await?;
-            return Err(UsbError::DeviceDisconnected);
+        let residual = if let Some(last_data_trb) = last_data_trb {
-        }
+            self.wait_normal(last_data_trb).await?
-
+        } else {
-        let status = Arc::new(UsbTransferStatus::new());
+            0
-        let address = self.inner.lock().enqueue(NormalTransferTrb::new(
+        };
-            unsafe { buffer.as_physical_address() },
+        self.wait_trb(status_trb).await?;
-            buffer.len(),
+        Ok(residual)
-            true,
-        ));
-        self.completions.write().insert(address, status.clone());
-
-        executor.ring_doorbell(self.slot_id as _, self.ep_id);
-
-        Ok(UsbInterruptTransfer {
-            length: buffer.len(),
-            direction: UsbDirection::In,
-            address,
-            status,
-        })
     }
 
-    pub fn complete_transfer(&self, _transfer: UsbInterruptTransfer) {
+    pub async fn next_event(&self) -> TransactionEvent {
-        // Interrupt transfers consist of one TRB each
+        poll_fn(|cx| {
-        // TODO: Can two transfers happen simultaneously? e.g.
+            if let Some(event) = self.event_queue.pop() {
-        //
+                Poll::Ready(event)
-        // [TRBa, TRBb] are queued in the ring, both are executing and
+            } else {
-        // TRBb finishes first
+                self.event_notify.register(cx.waker());
-        self.inner.lock().advance();
+                Poll::Pending
+            }
+        })
+        .await
     }
 }
 
-impl ControlTransferRing {
+impl TransactionEvent {
-    pub fn new(slot_id: u8, ep_id: u8, capacity: usize) -> Result<Self, UsbError> {
+    pub fn to_result(&self) -> Result<usize, TransferError> {
-        let trbs = PageBox::new_zeroed_slice(capacity).map_err(UsbError::MemoryError)?;
+        match self {
-
+            &Self::Status(_, status) => match status >> 24 {
-        Ok(Self {
+                1 => Ok((status as usize) & 0xFFFFFF),
-            inner: IrqSafeSpinlock::new(TransferRingInner {
+                4 => Err(TransferError::UsbTransactionError),
-                trbs,
+                6 => Err(TransferError::Stall),
-                enqueue_index: 0,
+                13 => Err(TransferError::ShortPacket((status as usize) & 0xFFFFFF)),
-                dequeue_index: 0,
+                code => Err(TransferError::Other(code as u8)),
-                cycle_bit: true,
+            },
-            }),
+            Self::Shutdown => Err(TransferError::UsbTransactionError),
-            completions: IrqSafeRwLock::new(BTreeMap::new()),
-            pending_trbs: IrqSafeRwLock::new(BTreeMap::new()),
-            slot_id,
-            ep_id,
-            capacity,
-
-            transfer_id: AtomicU64::new(0),
-
-            shutdown: AtomicBool::new(false),
-        })
-    }
-
-    pub fn start_transfer<E: CommandExecutor>(
-        &self,
-        executor: &E,
-        setup: ControlTransferSetup,
-        buffer: Option<(PhysicalAddress, usize, UsbDirection)>,
-    ) -> Result<UsbControlTransfer, UsbError> {
-        // Don't even try to start the transfer
-        if self.shutdown.load(Ordering::Acquire) {
-            return Err(UsbError::DeviceDisconnected);
         }
-
-        let mut builder = TransferBuilder::new(self, UsbDirection::In);
-
-        builder.push_trb(ControlTransferSetupTrb::new(setup));
-        if let Some((address, length, direction)) = buffer {
-            builder.push_trb(ControlTransferDataTrb::new(address, length, direction));
-        }
-        builder.push_trb(ControlTransferStatusTrb::new(UsbDirection::In, true));
-
-        let transfer = builder.start(executor, 0);
-
-        Ok(transfer)
-    }
-
-    pub fn complete_transfer(&self, transfer: UsbControlTransfer) {
-        let mut pending = self.pending_trbs.write();
-        let mut inner = self.inner.lock();
-        for trb in transfer.elements {
-            pending.remove(&trb);
-            inner.advance();
-        }
-        self.completions.write().remove(&transfer.id);
     }
 }
 
-// TRB implementations
+// TRB definitions
 
 define_bitfields! {
     pub RawTransferFlags : u32 {
         (10..16) => ty + set_ty,
+        5 => ioc + set_ioc,
         0 => cycle + set_cycle
     }
 }
@@ -375,7 +468,6 @@ define_bitfields! {
 define_bitfields! {
     pub NormalTransferFlags: u64 {
         (0..16) => trb_length,
-        37 => interrupt_on_completion,
     }
 }
 
@@ -407,7 +499,6 @@ define_bitfields! {
 define_bitfields! {
     pub ControlTransferStatusFlags : u32 {
         16 => direction,
-        5 => interrupt_on_completion
     }
 }
 
@@ -451,10 +542,10 @@ pub trait TransferTrb: Pod {
 }
 
 impl NormalTransferTrb {
-    pub fn new(buffer: PhysicalAddress, length: usize, interrupt_on_completion: bool) -> Self {
+    pub fn new(buffer: PhysicalAddress, length: usize) -> Self {
         Self {
             buffer,
-            flags: NormalTransferFlags::new(length.try_into().unwrap(), interrupt_on_completion),
+            flags: NormalTransferFlags::new(length.try_into().unwrap()),
         }
     }
 }
@@ -487,13 +578,10 @@ impl ControlTransferDataTrb {
 }
 
 impl ControlTransferStatusTrb {
-    pub const fn new(direction: UsbDirection, interrupt_on_completion: bool) -> Self {
+    pub const fn new(direction: UsbDirection) -> Self {
         Self {
             _0: [0; 3],
-            flags: ControlTransferStatusFlags::new(
+            flags: ControlTransferStatusFlags::new(direction.is_device_to_host()),
-                direction.is_device_to_host(),
-                interrupt_on_completion,
-            ),
         }
     }
 }

kernel/driver/usb/xhci/src/util.rs

@@ -0,0 +1,78 @@
+use core::{
+    future::poll_fn,
+    sync::atomic::{AtomicU64, Ordering},
+    task::Poll,
+};
+
+use futures_util::task::AtomicWaker;
+
+pub struct EventBitmap {
+    bitmap: AtomicU64,
+    waker: AtomicWaker,
+}
+
+struct BitIter {
+    mask: u64,
+    index: usize,
+    limit: usize,
+}
+
+impl Iterator for BitIter {
+    type Item = usize;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        while self.index < self.limit {
+            let bit = self.index;
+            self.index += 1;
+
+            if self.mask & (1 << bit) != 0 {
+                return Some(bit);
+            }
+        }
+        None
+    }
+}
+
+impl EventBitmap {
+    pub fn new() -> Self {
+        Self {
+            bitmap: AtomicU64::new(0),
+            waker: AtomicWaker::new(),
+        }
+    }
+
+    pub fn signal(&self, bit: usize) {
+        self.bitmap.fetch_or(1 << bit, Ordering::Release);
+        self.waker.wake();
+    }
+
+    // pub async fn wait_specific(&self, bit: usize) {
+    //     poll_fn(|cx| {
+    //         let state = self.bitmap.fetch_and(!(1 << bit), Ordering::Acquire);
+    //         if state & (1 << bit) != 0 {
+    //             Poll::Ready(())
+    //         } else {
+    //             self.waker.register(cx.waker());
+    //             Poll::Pending
+    //         }
+    //     })
+    //     .await
+    // }
+
+    pub async fn wait_any(&self, max: usize) -> impl Iterator<Item = usize> {
+        poll_fn(|cx| {
+            let mask = self.bitmap.swap(0, Ordering::Acquire);
+            if mask == 0 {
+                self.waker.register(cx.waker());
+                return Poll::Pending;
+            }
+
+            Poll::Ready(BitIter {
+                mask,
+                index: 0,
+                limit: max,
+            })
+        })
+        .await
+    }
+}

kernel/driver/virtio/gpu/src/lib.rs

@@ -23,7 +23,7 @@ use libk_util::{
     sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock},
     OneTimeInit,
 };
-use ygg_driver_pci::device::PciDeviceInfo;
+use ygg_driver_pci::{device::PciDeviceInfo, macros::pci_driver};
 use ygg_driver_virtio_core::{
     queue::VirtQueue,
     transport::{pci::PciTransport, Transport},
@@ -389,17 +389,26 @@ impl<T: Transport + 'static> DisplayDevice for VirtioGpu<T> {
     }
 }
 
-pub fn probe(info: &PciDeviceInfo) -> Result<Arc<dyn Device>, Error> {
+pci_driver! {
-    let space = &info.config_space;
+    matches: [device (0x1AF4:0x1050)],
+    driver: {
+        fn driver_name(&self) -> &str {
+            "virtio-gpu"
+        }
 
-    let transport = PciTransport::from_config_space(space)
+        fn probe(&self, info: &PciDeviceInfo) -> Result<Arc<dyn Device>, Error> {
-        .inspect_err(|error| {
+            let space = &info.config_space;
-            log::error!("Couldn't set up PCI virtio transport: {error:?}");
-        })
-        .map_err(|_| Error::InvalidArgument)?;
-    let device = VirtioGpu::new(transport, Some(info.clone()))?;
-    let device = Arc::new(device);
-    // let device = Box::leak(Box::new(device));
 
-    Ok(device)
+            let transport = PciTransport::from_config_space(space)
+                .inspect_err(|error| {
+                    log::error!("Couldn't set up PCI virtio transport: {error:?}");
+                })
+                .map_err(|_| Error::InvalidArgument)?;
+            let device = VirtioGpu::new(transport, Some(info.clone()))?;
+            let device = Arc::new(device);
+            // let device = Box::leak(Box::new(device));
+
+            Ok(device)
+        }
+    }
 }

kernel/driver/virtio/net/src/lib.rs

@@ -20,7 +20,10 @@ use ygg_driver_net_core::{
     interface::{NetworkDevice, NetworkInterfaceType},
     Packet,
 };
-use ygg_driver_pci::device::{PciDeviceInfo, PreferredInterruptMode};
+use ygg_driver_pci::{
+    device::{PciDeviceInfo, PreferredInterruptMode},
+    macros::pci_driver,
+};
 use ygg_driver_virtio_core::{
     queue::VirtQueue,
     transport::{pci::PciTransport, Transport},
@@ -176,7 +179,7 @@ impl<T: Transport + 'static> VirtioNet<T> {
         transmit_count: usize,
     ) -> Result<(), Error> {
         let receive_vector = if let Some(pci) = self.pci_device_info.as_ref() {
-            pci.init_interrupts(PreferredInterruptMode::Msi)?;
+            pci.init_interrupts(PreferredInterruptMode::Msi(true))?;
             let info = pci.map_interrupt(InterruptAffinity::Any, self.clone())?;
 
             info.map(|info| info.vector as u16)
@@ -296,13 +299,22 @@ fn cvt_error(error: ygg_driver_virtio_core::error::Error) -> Error {
     }
 }
 
-pub fn probe(info: &PciDeviceInfo) -> Result<Arc<dyn Device>, Error> {
+pci_driver! {
-    let space = &info.config_space;
+    matches: [device (0x1AF4:0x1000)],
+    driver: {
+        fn driver_name(&self) -> &str {
+            "virtio-net"
+        }
 
-    let transport = PciTransport::from_config_space(space).unwrap();
+        fn probe(&self, info: &PciDeviceInfo) -> Result<Arc<dyn Device>, Error> {
-    let device = VirtioNet::new(transport, Some(info.clone()));
+            let space = &info.config_space;
 
-    let device = Arc::new(device);
+            let transport = PciTransport::from_config_space(space).unwrap();
+            let device = VirtioNet::new(transport, Some(info.clone()));
 
-    Ok(device)
+            let device = Arc::new(device);
+
+            Ok(device)
+        }
+    }
 }

kernel/libk/libk-mm/src/device.rs

@@ -158,8 +158,9 @@ impl<T: ?Sized> AsPhysicalAddress for DeviceMemoryIo<'_, T> {
     }
 }
 
-unsafe impl<T: ?Sized> Send for DeviceMemoryIo<'_, T> {}
+unsafe impl<T: ?Sized + Send> Send for DeviceMemoryIo<'_, T> {}
-impl<T: ?Sized> !Sync for DeviceMemoryIo<'_, T> {}
+unsafe impl<T: ?Sized + Sync> Sync for DeviceMemoryIo<'_, T> {}
+//impl<T: ?Sized> !Sync for DeviceMemoryIo<'_, T> {}
 
 impl<'a, T: Sized> DeviceMemoryIoMut<'a, T> {
     /// Maps a physical address as device memory to a slice `[T; len]`

kernel/libk/src/debug.rs

@@ -33,6 +33,7 @@ use crate::{
         object::KObject,
     },
     task::{process::Process, thread::Thread},
+    time::monotonic_time,
 };
 
 const MAX_DEBUG_SINKS: usize = 8;
@@ -160,7 +161,6 @@ impl log::Log for DebugSinkWrapper {
         let level = LogLevel::from(record.level());
         let sink = self.sink();
         let cpu = Cpu::try_local().map(|c| c.id());
-        let file = record.file().unwrap_or("<???>");
         let line = record.line().unwrap_or(0);
         let args = record.args();
 
@@ -171,25 +171,30 @@ impl log::Log for DebugSinkWrapper {
         };
 
         let mut writer = SinkWriter { sink };
+        let now = monotonic_time();
+        let s = now.seconds();
 
         match record.target() {
-            "program" => {
+            ":program" => {
-                write!(writer, "{args}").ok();
+                write!(writer, "{prefix}{s:06}:").ok();
-            }
-            "raw" => {
-                write!(writer, "{prefix}{args}{suffix}").ok();
-            }
-            "io" => {
-                writeln!(writer, "[io] {args}").ok();
-            }
-            _ => {
-                write!(writer, "{prefix}").ok();
                 if let Some(cpu) = cpu {
                     write!(writer, "{cpu}:").ok();
                 } else {
                     write!(writer, "?:").ok();
                 }
-                writeln!(writer, "{file}:{line}: {args}{suffix}").ok();
+                write!(writer, "ptrace:{args}").ok();
+            }
+            ":raw" => {
+                write!(writer, "{prefix}{args}{suffix}").ok();
+            }
+            target => {
+                write!(writer, "{prefix}{s:06}:").ok();
+                if let Some(cpu) = cpu {
+                    write!(writer, "{cpu}:").ok();
+                } else {
+                    write!(writer, "?:").ok();
+                }
+                writeln!(writer, "{target}:{line}: {args}{suffix}").ok();
             }
         }
     }
@@ -317,7 +322,7 @@ impl PanicLoggerSink<'_> {
         self.log(
             &log::Record::builder()
                 .level(log::Level::Error)
-                .target("raw")
+                .target(":raw")
                 .args(args)
                 .build(),
         )
@@ -427,13 +432,12 @@ pub fn disable_early_sinks() {
 }
 
 /// Print a trace message coming from a process
-pub fn program_trace(process: &Process, thread: &Thread, message: &str) {
+pub fn program_trace(process: &Process, _thread: &Thread, message: &str) {
     log::debug!(
-        target: "program",
+        target: ":program",
-        "[trace {}:{}:{:?}] {message}\n",
+        "{} ({}) {message}\n",
+        process.name,
         process.id,
-        thread.id,
-        *thread.name.read()
     );
 }
 

kernel/libk/src/fs/devfs.rs

@@ -9,7 +9,7 @@ use crate::{
     vfs::{
         impls::{fixed_path_symlink, MemoryDirectory},
         path::OwnedFilename,
-        AccessToken, Metadata, Node, NodeFlags, NodeRef,
+        AccessToken, Filename, Metadata, Node, NodeFlags, NodeRef,
     },
 };
 
@@ -41,6 +41,13 @@ pub fn redirect<S: AsRef<str>>(name: S, destination: &str) -> Result<(), Error>
     root.add_child(filename, fixed_path_symlink(destination))
 }
 
+pub fn remove_node<S: AsRef<str>>(name: S) -> Result<(), Error> {
+    let name = name.as_ref();
+    let root = DEVFS_ROOT.get();
+    let filename = Filename::new(name)?;
+    root.remove_file(filename, unsafe { AccessToken::authorized() })
+}
+
 /// Adds a character device with a custom name
 pub fn add_named_char_device<S: AsRef<str>>(
     dev: Arc<dyn CharDevice>,

kernel/libk/src/task/runtime/task.rs

@@ -1,4 +1,4 @@
-use core::fmt;
+use core::{fmt, sync::atomic::AtomicBool};
 
 use alloc::sync::Arc;
 use futures_util::{future::BoxFuture, task::ArcWake, Future, FutureExt};
@@ -12,6 +12,7 @@ pub trait Termination {
 
 pub struct Task {
     pub(super) future: IrqSafeSpinlock<Option<BoxFuture<'static, ()>>>,
+    pub(super) enqueued: AtomicBool,
 }
 
 impl ArcWake for Task {
@@ -28,7 +29,10 @@ impl Task {
             }
             .boxed(),
         ));
-        Arc::new(Self { future })
+        Arc::new(Self {
+            future,
+            enqueued: AtomicBool::new(false),
+        })
     }
 }
 

kernel/libk/src/task/runtime/task_queue.rs

@@ -1,3 +1,5 @@
+use core::sync::atomic::Ordering;
+
 use alloc::sync::Arc;
 use crossbeam_queue::ArrayQueue;
 use libk_util::{sync::IrqGuard, OneTimeInit};
@@ -31,9 +33,13 @@ impl TaskQueue {
     }
 
     pub fn enqueue(&self, task: Arc<Task>) -> Result<(), Error> {
+        // Already enqueued
+        if task.enqueued.swap(true, Ordering::Acquire) {
+            return Ok(());
+        }
         let _irq = IrqGuard::acquire();
         if self.task_queue.push(task).is_err() {
-            todo!();
+            return Err(Error::WouldBlock);
         }
         self.wakeup_one();
         Ok(())
@@ -44,6 +50,7 @@ impl TaskQueue {
         // assert!(PlatformImpl::interrupt_mask());
         loop {
             if let Some(task) = self.task_queue.pop() {
+                task.enqueued.store(false, Ordering::Release);
                 return Ok(task);
             }
 
@@ -59,7 +66,7 @@ impl TaskQueue {
 
 /// Initializes the global async/await task queue
 pub fn init_task_queue() {
-    TASK_QUEUE.init(TaskQueue::new(128));
+    TASK_QUEUE.init(TaskQueue::new(256));
 }
 
 pub(super) fn push_task(task: Arc<Task>) -> Result<(), Error> {

kernel/src/arch/i686/mod.rs

@@ -30,7 +30,7 @@ use libk_mm::{
 };
 use peripherals::textfb::TextFramebuffer;
 
-use crate::arch::x86;
+use crate::{arch::x86, util::call_init_array};
 
 use super::Platform;
 
@@ -162,8 +162,6 @@ impl I686 {
 
         self.init_cpu().expect("Could not initialize CPU");
 
-        x86::register_pci_drivers();
-
         let early = x86::init_platform_early(cmdline)?;
 
         if !config::get().x86.disable_boot_fb {
@@ -173,6 +171,9 @@ impl I686 {
         }
 
         x86::add_legacy_pci();
+
+        call_init_array();
+
         x86::init_platform_devices(early);
 
         Ok(())

kernel/src/arch/i686/peripherals/textfb.rs

@@ -11,7 +11,7 @@ use libk_mm::{
 };
 use libk_util::sync::IrqSafeSpinlock;
 
-use crate::arch::x86::intrinsics::{io_wait, IoPort, IoPortAccess};
+use kernel_arch_x86::intrinsics::{io_wait, IoPort, IoPortAccess};
 
 #[derive(Clone, Copy)]
 #[repr(C)]

kernel/src/arch/x86/mod.rs

@@ -5,6 +5,7 @@
 use abi::error::Error;
 use alloc::{sync::Arc, vec::Vec};
 use device_api::{device::Device, interrupt::Irq};
+use kernel_arch_x86::ISA_IRQ_OFFSET;
 use libk::{
     config, debug,
     fs::{devfs, sysfs},
@@ -22,13 +23,6 @@ use crate::fs::{Initrd, INITRD_DATA};
 
 use super::L3;
 
-#[cfg(any(target_arch = "x86_64", rust_analyzer))]
-pub const ISA_IRQ_OFFSET: u32 = crate::arch::x86_64::ISA_IRQ_OFFSET;
-
-#[cfg(any(target_arch = "x86", rust_analyzer))]
-pub const ISA_IRQ_OFFSET: u32 = 0;
-
-pub mod intrinsics;
 mod pci;
 pub mod peripherals;
 
@@ -56,47 +50,6 @@ pub enum SelectedClockSource {
     Fallback(Arc<I8253>),
 }
 
-// TODO move this to some sort of .init_array-style implicit thing
-pub fn register_pci_drivers() {
-    // XXX: Only works with MSI-X, so no i686
-    #[cfg(any(target_arch = "x86_64", rust_analyzer))]
-    ygg_driver_pci::register_class_driver(
-        "NVMe Host Controller",
-        0x01,
-        Some(0x08),
-        Some(0x02),
-        ygg_driver_nvme::probe,
-    );
-    // XXX: i686 hangs in interrupt handler
-    #[cfg(any(target_arch = "x86_64", rust_analyzer))]
-    ygg_driver_pci::register_class_driver(
-        "AHCI Controller",
-        0x01,
-        Some(0x06),
-        Some(0x01),
-        ygg_driver_ahci::probe,
-    );
-    ygg_driver_pci::register_class_driver(
-        "USB xHCI",
-        0x0C,
-        Some(0x03),
-        Some(0x30),
-        ygg_driver_usb_xhci::probe,
-    );
-    ygg_driver_pci::register_vendor_driver(
-        "Virtio PCI GPU Device",
-        0x1AF4,
-        0x1050,
-        ygg_driver_virtio_gpu::probe,
-    );
-    ygg_driver_pci::register_vendor_driver(
-        "Virtio PCI Network Device",
-        0x1AF4,
-        0x1000,
-        ygg_driver_virtio_net::probe,
-    );
-}
-
 // Initialize the bare minimum required to:
 // * Allocate/manage interrupts
 // * Print debug output

kernel/src/arch/x86/pci.rs

@@ -1,10 +1,7 @@
+use kernel_arch_x86::intrinsics::{IoPort, IoPortAccess};
 use libk_util::sync::IrqSafeSpinlock;
 use ygg_driver_pci::LegacyPciAccess;
 
-use crate::arch::x86::intrinsics::IoPortAccess;
-
-use super::intrinsics::IoPort;
-
 struct LegacyPciInner {
     address: IoPort<u32>,
     data: IoPort<u32>,

kernel/src/arch/x86/peripherals/i8253.rs

@@ -4,13 +4,12 @@ use device_api::{
     device::Device,
     interrupt::{InterruptHandler, Irq},
 };
-use libk::{device::external_interrupt_controller, task::runtime, time};
+use kernel_arch_x86::{
-use libk_util::{sync::IrqSafeSpinlock, OneTimeInit};
-
-use crate::arch::x86::{
     intrinsics::{IoPort, IoPortAccess},
     ISA_IRQ_OFFSET,
 };
+use libk::{device::external_interrupt_controller, task::runtime, time};
+use libk_util::{sync::IrqSafeSpinlock, OneTimeInit};
 
 const FREQUENCY: u32 = 1193180;
 

kernel/src/arch/x86/peripherals/i8259.rs

@@ -9,10 +9,9 @@ use device_api::{
         IrqOptions,
     },
 };
+use kernel_arch_x86::intrinsics::{IoPort, IoPortAccess};
 use libk_util::{sync::IrqSafeSpinlock, OneTimeInit};
 
-use crate::arch::x86::intrinsics::{IoPort, IoPortAccess};
-
 #[cfg(any(target_arch = "x86", rust_analyzer))]
 use crate::arch::i686::exception;
 #[cfg(any(target_arch = "x86", rust_analyzer))]

kernel/src/arch/x86/peripherals/ps2/mod.rs

@@ -8,14 +8,14 @@ use device_api::{
     device::Device,
     interrupt::{InterruptHandler, Irq},
 };
+use kernel_arch_x86::{
+    intrinsics::{IoPort, IoPortAccess},
+    ISA_IRQ_OFFSET,
+};
 use libk::device::external_interrupt_controller;
 use libk_util::sync::IrqSafeSpinlock;
 
-use crate::arch::x86::{
+use codeset::{CODE_SET_1_00, CODE_SET_1_E0};
-    intrinsics::{IoPort, IoPortAccess},
-    peripherals::ps2::codeset::{CODE_SET_1_00, CODE_SET_1_E0},
-    ISA_IRQ_OFFSET,
-};
 
 mod codeset;
 

kernel/src/arch/x86/peripherals/rtc.rs

@@ -5,13 +5,12 @@ use device_api::{
     interrupt::{InterruptHandler, Irq},
 };
 use kernel_arch::{Architecture, ArchitectureImpl};
-use libk::{device::external_interrupt_controller, time};
+use kernel_arch_x86::{
-use libk_util::sync::IrqSafeSpinlock;
-
-use crate::arch::x86::{
     intrinsics::{io_wait, IoPort, IoPortAccess},
     ISA_IRQ_OFFSET,
 };
+use libk::{device::external_interrupt_controller, time};
+use libk_util::sync::IrqSafeSpinlock;
 
 const NMI_DISABLE: u8 = 1 << 7;
 const CMOS_REG_SEC: u8 = 0x00;

kernel/src/arch/x86/peripherals/serial.rs

@@ -5,6 +5,7 @@ use device_api::{
     device::Device,
     interrupt::{InterruptHandler, Irq},
 };
+use kernel_arch_x86::intrinsics::{IoPort, IoPortAccess};
 use libk::{
     debug::DebugSink,
     device::{external_interrupt_controller, manager::DEVICE_REGISTRY},
@@ -12,8 +13,6 @@ use libk::{
 };
 use libk_util::sync::IrqSafeSpinlock;
 
-use crate::arch::x86::intrinsics::{IoPort, IoPortAccess};
-
 // Single port
 struct Regs {
     dr: IoPort<u8>,

kernel/src/arch/x86_64/acpi.rs

@@ -1,390 +0,0 @@
-//! x86-64 implementation of ACPI management interfaces
-use core::{
-    alloc::{AllocError, Allocator, GlobalAlloc, Layout},
-    ptr::NonNull,
-    sync::atomic::Ordering,
-    time::Duration,
-};
-
-use ::acpi::{AcpiHandler, AcpiTables, PhysicalMapping};
-use acpi_system::{
-    AcpiInterruptMethod, AcpiSleepState, AcpiSystem, AcpiSystemError, EventAction, FixedEvent,
-};
-use alloc::{boxed::Box, sync::Arc};
-use device_api::{
-    device::Device,
-    interrupt::{InterruptHandler, IpiDeliveryTarget, IpiMessage, Irq},
-};
-use kernel_arch_x86_64::CPU_COUNT;
-use libk::device::external_interrupt_controller;
-use libk_mm::{
-    address::{PhysicalAddress, Virtualize},
-    heap::GLOBAL_HEAP,
-    pointer::PhysicalRef,
-};
-use libk_util::{sync::IrqSafeSpinlock, OneTimeInit};
-use yggdrasil_abi::error::Error;
-
-use crate::{
-    arch::{
-        x86_64::{apic::ioapic::ISA_IRQ_OFFSET, SHUTDOWN_FENCE},
-        Platform, PLATFORM,
-    },
-    mem::{read_memory, write_memory},
-};
-
-use super::intrinsics;
-
-#[derive(Clone, Copy)]
-#[doc(hidden)]
-pub struct AcpiAllocator;
-#[derive(Clone, Copy)]
-#[doc(hidden)]
-pub struct AcpiHandlerImpl;
-struct SciHandler;
-
-static ACPI_SYSTEM: OneTimeInit<IrqSafeSpinlock<AcpiSystem<AcpiHandlerImpl>>> = OneTimeInit::new();
-
-// impl Device for SciHandler {
-//     fn display_name(&self) -> &'static str {
-//         "ACPI interrupt handler"
-//     }
-// }
-
-impl Device for SciHandler {
-    fn display_name(&self) -> &str {
-        "ACPI SCI Handler"
-    }
-}
-
-impl InterruptHandler for SciHandler {
-    fn handle_irq(self: Arc<Self>, _vector: Option<usize>) -> bool {
-        log::trace!("ACPI SCI received");
-        ACPI_SYSTEM.get().lock().handle_sci();
-        true
-    }
-}
-
-unsafe impl Allocator for AcpiAllocator {
-    fn allocate(&self, layout: Layout) -> Result<NonNull<[u8]>, AllocError> {
-        let ptr = unsafe { GLOBAL_HEAP.alloc(layout) };
-        log::trace!("ACPI alloc: {:?} -> {:p}", layout, ptr);
-
-        if ptr.is_null() {
-            Err(AllocError)
-        } else {
-            unsafe {
-                Ok(NonNull::slice_from_raw_parts(
-                    NonNull::new_unchecked(ptr),
-                    layout.size(),
-                ))
-            }
-        }
-    }
-
-    unsafe fn deallocate(&self, ptr: NonNull<u8>, layout: Layout) {
-        log::trace!("ACPI dealloc: {:?}, {:?}", ptr, layout);
-        GLOBAL_HEAP.dealloc(ptr.as_ptr(), layout);
-    }
-}
-
-impl acpi_system::Handler for AcpiHandlerImpl {
-    type MappedSlice = PhysicalRef<'static, [u8]>;
-
-    unsafe fn map_slice(address: u64, length: u64) -> Self::MappedSlice {
-        PhysicalRef::map_slice(
-            PhysicalAddress::from_u64(address),
-            length.try_into().unwrap(),
-        )
-    }
-
-    fn io_read_u8(port: u16) -> u8 {
-        let value = unsafe { intrinsics::inb(port) };
-        log::trace!("io_read_u8 {:#x} <- {:#x}", port, value);
-        value
-    }
-
-    fn io_read_u16(port: u16) -> u16 {
-        let value = unsafe { intrinsics::inw(port) };
-        log::trace!("io_read_u16 {:#x} <- {:#x}", port, value);
-        value
-    }
-
-    fn io_read_u32(port: u16) -> u32 {
-        let value = unsafe { intrinsics::inl(port) };
-        log::trace!("io_read_u32 {:#x} <- {:#x}", port, value);
-        value
-    }
-
-    fn io_write_u8(port: u16, value: u8) {
-        log::trace!("io_write_u8 {:#x}, {:#x}", port, value);
-        unsafe { intrinsics::outb(port, value) }
-    }
-
-    fn io_write_u16(port: u16, value: u16) {
-        log::trace!("io_write_u16 {:#x}, {:#x}", port, value);
-        unsafe { intrinsics::outw(port, value) }
-    }
-
-    fn io_write_u32(port: u16, value: u32) {
-        log::trace!("io_write_u32 {:#x}, {:#x}", port, value);
-        unsafe { intrinsics::outl(port, value) }
-    }
-
-    fn mem_read_u8(address: u64) -> u8 {
-        let value = unsafe { read_memory(PhysicalAddress::from_u64(address)) };
-        log::trace!("mem_read_u8 {:#x} -> {:#x}", address, value);
-        value
-    }
-
-    fn mem_read_u16(address: u64) -> u16 {
-        let value = unsafe { read_memory(PhysicalAddress::from_u64(address)) };
-        log::trace!("mem_read_u16 {:#x} -> {:#x}", address, value);
-        value
-    }
-
-    fn mem_read_u32(address: u64) -> u32 {
-        let value = unsafe { read_memory(PhysicalAddress::from_u64(address)) };
-        log::trace!("mem_read_u32 {:#x} -> {:#x}", address, value);
-        value
-    }
-
-    fn mem_read_u64(address: u64) -> u64 {
-        let value = unsafe { read_memory(PhysicalAddress::from_u64(address)) };
-        log::trace!("mem_read_u64 {:#x} -> {:#x}", address, value);
-        value
-    }
-
-    fn mem_write_u8(address: u64, value: u8) {
-        log::trace!("mem_write_u8 {:#x}, {:#x}", address, value);
-        unsafe { write_memory(PhysicalAddress::from_u64(address), value) }
-    }
-
-    fn mem_write_u16(address: u64, value: u16) {
-        log::trace!("mem_write_u16 {:#x}, {:#x}", address, value);
-        unsafe { write_memory(PhysicalAddress::from_u64(address), value) }
-    }
-
-    fn mem_write_u32(address: u64, value: u32) {
-        log::trace!("mem_write_u32 {:#x}, {:#x}", address, value);
-        unsafe { write_memory(PhysicalAddress::from_u64(address), value) }
-    }
-
-    fn mem_write_u64(address: u64, value: u64) {
-        log::trace!("mem_write_u64 {:#x}, {:#x}", address, value);
-        unsafe { write_memory(PhysicalAddress::from_u64(address), value) }
-    }
-
-    fn install_interrupt_handler(irq: u32) -> Result<(), AcpiSystemError> {
-        log::info!("Installing ACPI SCI handler at IRQ #{}", irq);
-
-        let intc = external_interrupt_controller().expect("No external intc");
-        let handler = Arc::new(SciHandler);
-        let irq = Irq::External(irq + ISA_IRQ_OFFSET);
-
-        intc.register_irq(irq, Default::default(), handler).unwrap();
-        intc.enable_irq(irq).unwrap();
-
-        Ok(())
-    }
-
-    fn stall(_duration: Duration) {
-        // TODO polling_sleep is not yet implemented properly
-        todo!()
-        // util::polling_sleep(duration).ok();
-    }
-}
-
-impl aml::Handler for AcpiHandlerImpl {
-    fn read_io_u8(&self, port: u16) -> u8 {
-        <Self as acpi_system::Handler>::io_read_u8(port)
-    }
-
-    fn read_io_u16(&self, port: u16) -> u16 {
-        <Self as acpi_system::Handler>::io_read_u16(port)
-    }
-
-    fn read_io_u32(&self, port: u16) -> u32 {
-        <Self as acpi_system::Handler>::io_read_u32(port)
-    }
-
-    fn write_io_u8(&self, port: u16, value: u8) {
-        <Self as acpi_system::Handler>::io_write_u8(port, value)
-    }
-
-    fn write_io_u16(&self, port: u16, value: u16) {
-        <Self as acpi_system::Handler>::io_write_u16(port, value)
-    }
-
-    fn write_io_u32(&self, port: u16, value: u32) {
-        <Self as acpi_system::Handler>::io_write_u32(port, value)
-    }
-
-    fn read_u8(&self, address: usize) -> u8 {
-        <Self as acpi_system::Handler>::mem_read_u8(address as u64)
-    }
-
-    fn read_u16(&self, address: usize) -> u16 {
-        <Self as acpi_system::Handler>::mem_read_u16(address as u64)
-    }
-
-    fn read_u32(&self, address: usize) -> u32 {
-        <Self as acpi_system::Handler>::mem_read_u32(address as u64)
-    }
-
-    fn read_u64(&self, address: usize) -> u64 {
-        <Self as acpi_system::Handler>::mem_read_u64(address as u64)
-    }
-
-    fn write_u8(&self, address: usize, value: u8) {
-        <Self as acpi_system::Handler>::mem_write_u8(address as u64, value)
-    }
-
-    fn write_u16(&self, address: usize, value: u16) {
-        <Self as acpi_system::Handler>::mem_write_u16(address as u64, value)
-    }
-
-    fn write_u32(&self, address: usize, value: u32) {
-        <Self as acpi_system::Handler>::mem_write_u32(address as u64, value)
-    }
-
-    fn write_u64(&self, address: usize, value: u64) {
-        <Self as acpi_system::Handler>::mem_write_u64(address as u64, value)
-    }
-
-    fn read_pci_u8(&self, _segment: u16, _bus: u8, _device: u8, _function: u8, _offset: u16) -> u8 {
-        0xFF
-    }
-
-    fn read_pci_u16(
-        &self,
-        _segment: u16,
-        _bus: u8,
-        _device: u8,
-        _function: u8,
-        _offset: u16,
-    ) -> u16 {
-        0xFFFF
-    }
-
-    fn read_pci_u32(
-        &self,
-        _segment: u16,
-        _bus: u8,
-        _device: u8,
-        _function: u8,
-        _offset: u16,
-    ) -> u32 {
-        0xFFFFFFFF
-    }
-
-    fn write_pci_u8(
-        &self,
-        _segment: u16,
-        _bus: u8,
-        _device: u8,
-        _function: u8,
-        _offset: u16,
-        _value: u8,
-    ) {
-    }
-
-    fn write_pci_u16(
-        &self,
-        _segment: u16,
-        _bus: u8,
-        _device: u8,
-        _function: u8,
-        _offset: u16,
-        _value: u16,
-    ) {
-    }
-
-    fn write_pci_u32(
-        &self,
-        _segment: u16,
-        _bus: u8,
-        _device: u8,
-        _function: u8,
-        _offset: u16,
-        _value: u32,
-    ) {
-    }
-
-    fn read_ec_u8(&self, _address: u64) -> u8 {
-        0x00
-    }
-
-    fn write_ec_u8(&self, _address: u64, _value: u8) {}
-
-    fn sleep(&self, _duration: Duration) {
-        todo!()
-        // util::polling_sleep(duration).unwrap();
-    }
-}
-
-impl AcpiHandler for AcpiHandlerImpl {
-    // No actual address space modification is performed
-    unsafe fn map_physical_region<T>(
-        &self,
-        physical_address: usize,
-        size: usize,
-    ) -> PhysicalMapping<Self, T> {
-        PhysicalMapping::new(
-            physical_address,
-            NonNull::new_unchecked(
-                PhysicalAddress::from_usize(physical_address).virtualize() as *mut T
-            ),
-            size,
-            size,
-            *self,
-        )
-    }
-
-    // Unmap nothing, these addresses are "virtualized" to high address space
-    fn unmap_physical_region<T>(_region: &PhysicalMapping<Self, T>) {}
-}
-
-/// Initializes ACPI management
-#[allow(unused)]
-pub fn init_acpi(tables: &'static AcpiTables<AcpiHandlerImpl>) -> Result<(), Error> {
-    // TODO currently broken for real HW
-    let mut system = AcpiSystem::new(tables, Box::new(AcpiHandlerImpl)).unwrap();
-
-    system.initialize(AcpiInterruptMethod::Apic).unwrap();
-
-    system
-        .enable_fixed_event(
-            &FixedEvent::POWER_BUTTON,
-            Box::new(|_| {
-                log::info!("Power button was pressed");
-
-                // TODO the correct way would be to
-                // 1. Nicely ask all the processes to quit
-                // 2. Wait for some time
-                // 3. Kill the remaining ones
-                // 4. Halt other cores
-                // 5. Sync filesystem
-                // 6. Do something with the devices
-                // 7. Actually enter the S5 state
-
-                unsafe {
-                    PLATFORM
-                        .send_ipi(IpiDeliveryTarget::OtherCpus, IpiMessage::Shutdown)
-                        .unwrap();
-                }
-
-                SHUTDOWN_FENCE.signal();
-                SHUTDOWN_FENCE.wait_all(CPU_COUNT.load(Ordering::Acquire));
-
-                log::info!("CPUs are parked, can shutdown now");
-
-                EventAction::EnterSleepState(AcpiSleepState::S5)
-            }),
-        )
-        .unwrap();
-
-    ACPI_SYSTEM.init(IrqSafeSpinlock::new(system));
-
-    Ok(())
-}

kernel/src/arch/x86_64/apic/ioapic.rs

@@ -9,6 +9,7 @@ use device_api::{
         IrqLevel, IrqOptions, IrqTrigger,
     },
 };
+use kernel_arch_x86::ISA_IRQ_OFFSET;
 use libk_mm::{address::PhysicalAddress, device::DeviceMemoryIo};
 use libk_util::sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock};
 use tock_registers::{
@@ -16,13 +17,12 @@ use tock_registers::{
     register_structs,
     registers::{ReadWrite, WriteOnly},
 };
+use ygg_driver_acpi::AcpiAllocator;
 
-use crate::arch::x86_64::{acpi::AcpiAllocator, apic::local::BSP_APIC_ID};
+use crate::arch::x86_64::apic::local::BSP_APIC_ID;
 
 use super::{APIC_EXTERNAL_OFFSET, POPULATED_EXTERNAL_VECTORS};
 
-pub const ISA_IRQ_OFFSET: u32 = 1024;
-
 // IRQ 0 is timer, IRQ 1 reserved (for now?), +32 offset for exception entries
 const IO_APIC_VECTOR_OFFSET: u32 = 32 + APIC_EXTERNAL_OFFSET;
 
@@ -64,7 +64,7 @@ struct Inner {
 pub struct IoApic {
     inner: IrqSafeSpinlock<Inner>,
     isa_redirections: [Option<IsaRedirection>; 16],
-    table: IrqSafeRwLock<FixedInterruptTable<{ POPULATED_EXTERNAL_VECTORS as usize }>>, // table: IrqSafeSpinlock<FixedInterruptTable<{ POPULATED_EXTERNAL_VECTORS as usize }>>,
+    table: IrqSafeRwLock<FixedInterruptTable<{ POPULATED_EXTERNAL_VECTORS as usize }>>,
 }
 
 impl Regs {
@@ -83,7 +83,7 @@ impl Regs {
 
 impl Inner {
     fn map_gsi(&mut self, gsi: u32, vector: u32, apic_id: u32) -> Result<(), Error> {
-        assert!(gsi < self.max_gsi);
+        assert!(gsi <= self.max_gsi);
         assert!(vector < 0x100);
 
         log::info!("map_irq gsi{}, vec{}, apic{}", gsi, vector, apic_id);
@@ -110,7 +110,7 @@ impl Inner {
     }
 
     fn configure_gsi(&mut self, gsi: u32, level: IrqLevel, trigger: IrqTrigger) {
-        assert!(gsi < self.max_gsi);
+        assert!(gsi <= self.max_gsi);
 
         let mut low = self.regs.read(REG_REDIRECTION_BASE + gsi * 2);
 
@@ -138,7 +138,7 @@ impl Inner {
     }
 
     fn set_gsi_enabled(&mut self, gsi: u32, enabled: bool) {
-        assert!(gsi < self.max_gsi);
+        assert!(gsi <= self.max_gsi);
 
         let low = self.regs.read(REG_REDIRECTION_BASE + gsi * 2);
         if enabled {

kernel/src/arch/x86_64/apic/local.rs

@@ -259,7 +259,7 @@ impl MessageInterruptController for LocalApic {
             );
 
             let value = 32 + APIC_MSI_OFFSET + i as u32;
-            let address = Self::base().into_usize() | ((self.id as usize) << 12);
+            let address = 0xFEE00000 | ((self.id as usize) << 12);
 
             *msi = MsiInfo {
                 address,

kernel/src/arch/x86_64/mod.rs

@@ -3,13 +3,12 @@ use core::{mem::size_of, ops::DerefMut, ptr::null_mut, sync::atomic::Ordering};
 
 use ::acpi::{mcfg::Mcfg, AcpiTables, HpetInfo, InterruptModel};
 use abi::error::Error;
-use acpi::{AcpiAllocator, AcpiHandlerImpl};
 use alloc::{boxed::Box, sync::Arc};
 use apic::{ioapic::IoApic, local::LocalApic};
 use device_api::device::Device;
 use kernel_arch_x86::{
     cpuid::{self, CpuFeatures, EcxFeatures, EdxFeatures, ExtEdxFeatures},
-    gdt,
+    gdt, intrinsics,
 };
 use kernel_arch_x86_64::{
     mem::{
@@ -34,14 +33,14 @@ use libk_mm::{
     phys::{self, reserved::reserve_region, PhysicalMemoryRegion},
     table::{EntryLevel, EntryLevelExt},
 };
-use libk_util::{sync::SpinFence, OneTimeInit};
+use libk_util::OneTimeInit;
 use yboot_proto::{
     v1::{self, AvailableMemoryRegion},
     LoadProtocolV1,
 };
+use ygg_driver_acpi::{AcpiAllocator, AcpiHandlerImpl, EventAction, FixedEvent};
 use ygg_driver_pci::PciBusManager;
 
-mod acpi;
 mod apic;
 mod boot;
 mod exception;
@@ -51,17 +50,12 @@ mod syscall;
 use crate::{
     arch::x86::{self, peripherals::hpet::Hpet},
     device::display::linear_fb::LinearFramebuffer,
+    util::call_init_array,
 };
 
 use self::boot::BootData;
 
-use super::{
+use super::{x86::InitrdSource, Platform};
-    x86::{intrinsics, InitrdSource},
-    Platform,
-};
-
-/// Offset where legacy ISA IRQs are remapped
-pub const ISA_IRQ_OFFSET: u32 = apic::ioapic::ISA_IRQ_OFFSET;
 
 /// x86-64 architecture implementation
 pub struct X86_64 {
@@ -71,8 +65,6 @@ pub struct X86_64 {
     fbconsole: OneTimeInit<Arc<FramebufferConsole>>,
 }
 
-static SHUTDOWN_FENCE: SpinFence = SpinFence::new();
-
 /// Global x86-64 architecture value
 pub static PLATFORM: X86_64 = X86_64 {
     boot_data: OneTimeInit::new(),
@@ -262,8 +254,6 @@ impl X86_64 {
         self.init_local_cpu(cpu_id, available_features, enabled_features);
 
         if cpu_id == 0 {
-            x86::register_pci_drivers();
-
             self.setup_from_boot_data()?;
 
             // TODO yboot doesn't yet pass any command line options
@@ -286,6 +276,8 @@ impl X86_64 {
                 log::info!("HPET disabled by config");
             }
 
+            call_init_array();
+
             x86::init_platform_devices(early);
         }
 
@@ -385,7 +377,18 @@ impl X86_64 {
         let ioapic = IoApic::from_acpi(&apic_info)?;
         register_external_interrupt_controller(ioapic);
 
-        // acpi::init_acpi(acpi).unwrap();
+        if let Err(error) = ygg_driver_acpi::switch_to_acpi(acpi) {
+            log::error!("ACPI initialization error: {error:?}");
+        } else {
+            if let Err(error) =
+                ygg_driver_acpi::add_event_handler(&FixedEvent::POWER_BUTTON, |_| {
+                    log::info!("Power button pressed!");
+                    EventAction::Nothing
+                })
+            {
+                log::error!("Couldn't set ACPI power button handler: {error:?}");
+            }
+        }
 
         if let Ok(mcfg) = acpi.find_table::<Mcfg>() {
             for entry in mcfg.entries() {

kernel/src/arch/x86_64/smp.rs

@@ -18,11 +18,10 @@ use libk_mm::{
     pointer::PhysicalRefMut,
     TableAllocatorImpl,
 };
+use ygg_driver_acpi::AcpiAllocator;
 
 use crate::arch::x86_64::boot::__x86_64_ap_entry;
 
-use super::acpi::AcpiAllocator;
-
 static AP_BOOTSTRAP_BIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/__x86_64_ap_boot.bin"));
 
 const AP_STACK_PAGES: usize = 8;

kernel/src/main.rs

@@ -39,6 +39,7 @@
 use abi::{error::Error, io::FileMode};
 use alloc::string::String;
 use arch::Platform;
+use cfg_if::cfg_if;
 use git_version::git_version;
 use kernel_arch::{Architecture, ArchitectureImpl};
 use libk::{
@@ -61,6 +62,19 @@ extern crate alloc;
 #[cfg(not(rust_analyzer))]
 extern crate compiler_builtins;
 
+cfg_if! {
+    if #[cfg(any(target_arch = "x86_64", rust_analyzer))] {
+        extern crate ygg_driver_net_rtl81xx;
+        extern crate ygg_driver_usb_xhci;
+        extern crate ygg_driver_nvme;
+    }
+
+}
+
+extern crate ygg_driver_ahci;
+extern crate ygg_driver_virtio_gpu;
+extern crate ygg_driver_virtio_net;
+
 #[macro_use]
 pub mod arch;
 

kernel/src/panic.rs

@@ -1,8 +1,8 @@
 //! Kernel panic handler code
 use core::sync::atomic::{AtomicBool, AtomicU32, Ordering};
 
-use device_api::interrupt::{IpiDeliveryTarget, IpiMessage};
+// use device_api::interrupt::{IpiDeliveryTarget, IpiMessage};
-use kernel_arch::{sync::hack_locks, Architecture, ArchitectureImpl};
+use kernel_arch::{Architecture, ArchitectureImpl};
 use libk::{
     arch::{Cpu, LocalCpu},
     debug::{self, PanicLoggerSink},
@@ -65,17 +65,18 @@ pub(crate) fn panic_handler(pi: &core::panic::PanicInfo) -> ! {
         .compare_exchange(false, true, Ordering::Release, Ordering::Acquire)
         .is_ok()
     {
-        // Let other CPUs know we're screwed
+        // TODO IPI leads to triple fault sometimes in x86-64
-        if unsafe {
+        // // Let other CPUs know we're screwed
-            PLATFORM
+        // if unsafe {
-                .send_ipi(IpiDeliveryTarget::OtherCpus, IpiMessage::Panic)
+        //     PLATFORM
-                .unwrap_or(false)
+        //         .send_ipi(IpiDeliveryTarget::OtherCpus, IpiMessage::Panic)
-        } {
+        //         .unwrap_or(false)
-            let ap_count = ArchitectureImpl::cpu_count() - 1;
+        // } {
-            PANIC_HANDLED_FENCE.wait_all(ap_count);
+        //     let ap_count = ArchitectureImpl::cpu_count() - 1;
-        }
+        //     PANIC_HANDLED_FENCE.wait_all(ap_count);
+        // }
 
-        unsafe { hack_locks() };
+        // unsafe { hack_locks() };
 
         dump_panic_info(&cpu, pi);
 

lib/abi/src/process/mod.rs

@@ -134,7 +134,7 @@ impl ProcessId {
 
 impl fmt::Display for ProcessId {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "<Process {}>", self.0)
+        write!(f, "{}", self.0)
     }
 }
 

lib/qemu/src/device.rs

@@ -5,6 +5,7 @@ use crate::IntoArgs;
 #[derive(Debug)]
 pub enum QemuNic {
     VirtioPci { mac: Option<String> },
+    Rtl8139 { mac: Option<String> },
 }
 
 #[derive(Debug, PartialEq, Eq)]
@@ -46,6 +47,15 @@ impl IntoArgs for QemuNic {
                 }
                 command.arg(val);
             }
+            Self::Rtl8139 { mac } => {
+                command.arg("-device");
+                let mut val = "rtl8139,netdev=net0".to_owned();
+                if let Some(mac) = mac {
+                    val.push_str(",mac=");
+                    val.push_str(mac);
+                }
+                command.arg(val);
+            }
         }
     }
 }

xtask/src/qemu.rs

@@ -17,12 +17,21 @@ use crate::{
     util::run_external_command,
 };
 
+#[derive(Debug, Default, serde::Deserialize, serde::Serialize)]
+#[serde(rename_all = "kebab-case")]
+enum QemuNetworkInterface {
+    #[default]
+    VirtioNet,
+    Rtl8139,
+}
+
 #[derive(Debug, serde::Deserialize, serde::Serialize)]
 #[serde(rename_all = "kebab-case", default)]
 struct QemuNetworkConfig {
     enable: bool,
     interface_name: String,
     mac: String,
+    interface: QemuNetworkInterface,
 }
 
 #[derive(Debug, Default, Clone, Copy, serde::Deserialize, serde::Serialize)]
@@ -119,6 +128,7 @@ impl Default for QemuNetworkConfig {
             enable: true,
             interface_name: "qemu-tap0".into(),
             mac: "12:34:56:65:43:21".into(),
+            interface: QemuNetworkInterface::default(),
         }
     }
 }
@@ -307,10 +317,13 @@ fn add_devices_from_config(
     config: &QemuConfig,
 ) -> Result<(), Error> {
     if config.network.enable {
+        let mac = Some(config.network.mac.clone());
+        let nic = match config.network.interface {
+            QemuNetworkInterface::VirtioNet => QemuNic::VirtioPci { mac },
+            QemuNetworkInterface::Rtl8139 => QemuNic::Rtl8139 { mac },
+        };
         devices.push(QemuDevice::NetworkTap {
-            nic: QemuNic::VirtioPci {
+            nic,
-                mac: Some(config.network.mac.clone()),
-            },
             script: Some("xtask/scripts/qemu-ifup".into()),
             ifname: Some(config.network.interface_name.clone()),
         });