pci: better capability handling (todo: split/merge into master

separately)

Cargo.lock

@@ -532,6 +532,12 @@ dependencies = [
  "hashbrown 0.14.5",
 ]
 
+[[package]]
+name = "elf"
+version = "0.7.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4445909572dbd556c457c849c4ca58623d84b27c8fff1e74b0b4227d8b90d17b"
+
 [[package]]
 name = "encode_unicode"
 version = "1.0.0"
@@ -735,6 +741,18 @@ dependencies = [
  "pin-utils",
 ]
 
+[[package]]
+name = "gentables"
+version = "0.1.0"
+dependencies = [
+ "bitflags 2.8.0",
+ "bytemuck",
+ "clap",
+ "elf 0.7.4",
+ "memtables",
+ "thiserror",
+]
+
 [[package]]
 name = "getrandom"
 version = "0.2.15"
@@ -1117,7 +1135,7 @@ dependencies = [
  "device-api",
  "kernel-arch-interface",
  "libk-mm-interface",
- "log",
+ "memtables",
  "static_assertions",
  "tock-registers",
  "yggdrasil-abi",
@@ -1166,6 +1184,7 @@ dependencies = [
  "kernel-arch-interface",
  "libk-mm-interface",
  "log",
+ "memtables",
  "static_assertions",
  "tock-registers",
  "yggdrasil-abi",
@@ -1193,6 +1212,7 @@ dependencies = [
  "kernel-arch-x86",
  "libk-mm-interface",
  "log",
+ "memtables",
  "static_assertions",
  "tock-registers",
  "yggdrasil-abi",
@@ -1250,7 +1270,7 @@ dependencies = [
  "cfg-if",
  "crossbeam-queue",
  "device-api",
- "elf",
+ "elf 0.7.2",
  "futures-util",
  "kernel-arch",
  "libc",
@@ -1445,6 +1465,14 @@ dependencies = [
  "autocfg",
 ]
 
+[[package]]
+name = "memtables"
+version = "0.1.0"
+dependencies = [
+ "bitflags 2.8.0",
+ "bytemuck",
+]
+
 [[package]]
 name = "miniz_oxide"
 version = "0.8.4"
@@ -2680,6 +2708,24 @@ dependencies = [
  "yggdrasil-abi",
 ]
 
+[[package]]
+name = "ygg_driver_intel_hda"
+version = "0.1.0"
+dependencies = [
+ "async-trait",
+ "bytemuck",
+ "device-api",
+ "futures-util",
+ "libk",
+ "libk-mm",
+ "libk-util",
+ "log",
+ "tock-registers",
+ "ygg_driver_pci",
+ "ygg_driver_sound_core",
+ "yggdrasil-abi",
+]
+
 [[package]]
 name = "ygg_driver_net_core"
 version = "0.1.0"
@@ -2808,6 +2854,20 @@ dependencies = [
  "yggdrasil-abi",
 ]
 
+[[package]]
+name = "ygg_driver_sound_core"
+version = "0.1.0"
+dependencies = [
+ "async-trait",
+ "device-api",
+ "futures-util",
+ "libk",
+ "libk-mm",
+ "libk-util",
+ "log",
+ "yggdrasil-abi",
+]
+
 [[package]]
 name = "ygg_driver_usb"
 version = "0.1.0"
@@ -2946,7 +3006,7 @@ dependencies = [
  "device-api",
  "device-api-macros",
  "device-tree",
- "elf",
+ "elf 0.7.2",
  "ext2",
  "futures-util",
  "git-version",
@@ -2962,6 +3022,7 @@ dependencies = [
  "libk-util",
  "log",
  "memfs",
+ "memtables",
  "prettyplease",
  "static_assertions",
  "tock-registers",
@@ -2971,6 +3032,7 @@ dependencies = [
  "ygg_driver_ahci",
  "ygg_driver_fat32",
  "ygg_driver_input",
+ "ygg_driver_intel_hda",
  "ygg_driver_net_core",
  "ygg_driver_net_igbe",
  "ygg_driver_net_loopback",
@@ -2978,6 +3040,7 @@ dependencies = [
  "ygg_driver_net_stmmac",
  "ygg_driver_nvme",
  "ygg_driver_pci",
+ "ygg_driver_sound_core",
  "ygg_driver_usb",
  "ygg_driver_usb_xhci",
  "ygg_driver_virtio_blk",

Cargo.toml

@@ -11,6 +11,7 @@ exclude = [
 ]
 members = [
     "xtask",
+    "kernel/tools/gentables",
     "kernel",
     "lib/abi",
     "lib/libyalloc",
@@ -66,6 +67,7 @@ libk-util.path = "kernel/libk/libk-util"
 libk-mm.path = "kernel/libk/libk-mm"
 libk-mm-interface.path = "kernel/libk/libk-mm/interface"
 libk-device.path = "kernel/libk/libk-device"
+memtables.path = "kernel/lib/memtables"
 vmalloc.path = "kernel/lib/vmalloc"
 device-api-macros.path = "kernel/lib/device-api/macros"
 device-tree.path = "kernel/lib/device-tree"

boot/yboot/src/elf.rs

@@ -1,8 +1,7 @@
 use core::mem::size_of;
 
 use bytemuck::Zeroable;
-use log::{error, info};
-use types::{Rela, SHT_RELA};
+use log::{debug, error, info};
 // TODO use 'elf' crate
 use uefi::{
     prelude::BootServices,
@@ -24,18 +23,14 @@ mod types {
     pub type Half = u16;
     pub type Word = u32;
     pub type XWord = u64;
-    pub type SXWord = i64;
 
     pub const PT_LOAD: Word = 1;
 
     pub const SHT_PROGBITS: Word = 1;
-    pub const SHT_RELA: Word = 4;
 
     pub const SHF_WRITE: XWord = 1 << 0;
     pub const SHF_ALLOC: XWord = 1 << 1;
 
-    pub const R_X86_64_RELATIVE: u32 = 8;
-
     #[derive(Clone, Copy, Zeroable, Pod)]
     #[repr(C)]
     pub struct Ehdr {
@@ -82,20 +77,6 @@ mod types {
         pub memsz: XWord,
         pub align: XWord,
     }
-
-    #[derive(Clone, Copy, Zeroable, Pod)]
-    #[repr(C)]
-    pub struct Rela {
-        pub offset: Addr,
-        pub info: XWord,
-        pub addend: SXWord,
-    }
-
-    impl Rela {
-        pub fn r_type(&self) -> u32 {
-            self.info as u32
-        }
-    }
 }
 
 // Maximum address this loader can map in the target kernel
@@ -111,8 +92,6 @@ pub struct LoadedObject {
     pub image_start: u64,
     pub image_end: u64,
 
-    pub load_address: u64,
-
     pub entry: u64,
 
     pub protocol_struct_paddr: u64,
@@ -126,12 +105,6 @@ struct LocatedProtocol {
     size: usize,
 }
 
-struct RelaSection {
-    offset: u64,
-    entry_count: usize,
-    entry_size: usize,
-}
-
 trait ReadExact {
     fn read_exact(&mut self, buf: &mut [u8]) -> Result<(), Error>;
 }
@@ -146,23 +119,6 @@ impl ReadExact for RegularFile {
     }
 }
 
-impl RelaSection {
-    pub fn from_shdr(shdr: &Shdr) -> Option<Self> {
-        if shdr.type_ != SHT_RELA {
-            return None;
-        }
-
-        let entry_size = shdr.entsize as usize;
-        let entry_count = shdr.size as usize / entry_size;
-
-        Some(Self {
-            offset: shdr.offset,
-            entry_size,
-            entry_count,
-        })
-    }
-}
-
 impl Object {
     pub fn open<D: File>(root: &mut D, path: &CStr16) -> Result<Self, Error> {
         let file = root.open(path, FileMode::Read, FileAttribute::empty())?;
@@ -184,11 +140,11 @@ impl Object {
             return Err(Error::new(Status::LOAD_ERROR, ()));
         }
 
-        // // Check that the entry point is set
-        // if ehdr.entry == 0 {
-        //     error!("Image does not have a valid entry point");
-        //     return Err(Error::new(Status::LOAD_ERROR, ()));
-        // }
+        // Check that the entry point is set
+        if ehdr.entry == 0 {
+            error!("Image does not have a valid entry point");
+            return Err(Error::new(Status::LOAD_ERROR, ()));
+        }
 
         Ok(Self { file, ehdr })
     }
@@ -214,6 +170,11 @@ impl Object {
         self.file
             .read_exact(bytemuck::bytes_of_mut(&mut proto_data))?;
 
+        info!(
+            "Kernel is virtually mapped at {:#x}",
+            proto_data.kernel_virt_offset
+        );
+
         // 2. Find the kernel's range and check that the loaded physical addresses are actually
         //    usable from UEFI
         let mut image_start = u64::MAX;
@@ -248,22 +209,16 @@ impl Object {
         assert_eq!(image_start & 0xFFF, 0);
         assert_eq!(image_end & 0xFFF, 0);
 
-        // Allocate memory to load the kernel into
-        let kernel_load_address = bs
+        info!("Image start: {:#x}, end: {:#x}", image_start, image_end);
+        // Reserve the kernel memory
+        let reserved_addr = bs
             .allocate_pages(
-                AllocateType::MaxAddress(0xFFFFFFFF),
+                AllocateType::Address(image_start),
                 MemoryType::LOADER_DATA,
                 (image_end - image_start) as usize / 0x1000,
             )
-            .expect("Could not allocate memory for the kernel");
-
-        // Print info
-        info!("Image start: {:#x}, end: {:#x}", image_start, image_end);
-        info!(
-            "Kernel virtual offset: {:#x}",
-            proto_data.kernel_virt_offset
-        );
-        info!("Kernel load address: {kernel_load_address:#x}");
+            .expect("Could not allocate memory for kernel image");
+        assert_eq!(reserved_addr, image_start);
 
         // 3. Load the segments
         for i in 0..self.ehdr.phnum {
@@ -273,80 +228,53 @@ impl Object {
                 continue;
             }
 
-            let segment_load_base = phdr.paddr + kernel_load_address;
-
             info!(
-                "[{i}] Load {:#x?}",
-                segment_load_base..segment_load_base + phdr.memsz
+                "Load segment {}: {:#x?}",
+                i,
+                phdr.paddr..phdr.paddr + phdr.memsz
             );
 
             if phdr.filesz > 0 {
-                let dst_slice = unsafe {
-                    core::slice::from_raw_parts_mut(
-                        segment_load_base as *mut u8,
-                        phdr.filesz as usize,
-                    )
+                // The section has load data
+                let dst = unsafe {
+                    core::slice::from_raw_parts_mut(phdr.paddr as *mut u8, phdr.filesz as usize)
                 };
 
+                debug!(
+                    "Load {:#x?} from ELF offset {:#x}",
+                    phdr.paddr..phdr.paddr + phdr.filesz,
+                    phdr.offset
+                );
                 self.file.set_position(phdr.offset)?;
-                self.file.read_exact(dst_slice)?;
+                self.file.read_exact(dst)?;
             }
 
             if phdr.memsz > phdr.filesz {
-                let dst_slice = unsafe {
+                let dst = unsafe {
                     core::slice::from_raw_parts_mut(
-                        (segment_load_base + phdr.filesz) as *mut u8,
+                        (phdr.paddr + phdr.filesz) as *mut u8,
                         (phdr.memsz - phdr.filesz) as usize,
                     )
                 };
 
-                dst_slice.fill(0);
-            }
-        }
+                debug!(
+                    "Zero data {:#x?}",
+                    phdr.paddr + phdr.filesz..phdr.paddr + phdr.memsz
+                );
 
-        // 4. Perform kernel relocation
-        let mut rela_section = None;
-        for i in 0..self.ehdr.shnum as usize {
-            let shdr = self.read_shdr(i)?;
-            if let Some(rela) = RelaSection::from_shdr(&shdr) {
-                rela_section = Some(rela);
-                break;
-            }
-        }
-
-        if let Some(rela_section) = rela_section {
-            info!("Relocating kernel: {image_start:#x} -> {kernel_load_address:#x}");
-            info!("({} relocations)", rela_section.entry_count);
-
-            let b = (kernel_load_address + proto_data.kernel_virt_offset) as i64;
-
-            for i in 0..rela_section.entry_count {
-                let mut rela = Rela::zeroed();
-                self.file
-                    .set_position(rela_section.offset + (i * rela_section.entry_size) as u64)?;
-                self.file.read_exact(bytemuck::bytes_of_mut(&mut rela))?;
-
-                match rela.r_type() {
-                    types::R_X86_64_RELATIVE => {
-                        let qword = (rela.offset + kernel_load_address) as *mut i64;
-                        let value = rela.addend + b;
-                        unsafe { qword.write_volatile(value) };
-                    }
-                    other => todo!("Unsupported relocation type: {other}"),
-                }
+                dst.fill(0);
             }
         }
 
         // Now that the image is in memory, protocol structure can be written in the further steps
-        let protocol_struct_paddr = loc_proto.address as u64 + kernel_load_address; // (loc_proto.address as u64) - proto_data.kernel_virt_offset;
+        let protocol_struct_paddr = (loc_proto.address as u64) - proto_data.kernel_virt_offset;
         let protocol_version = proto_data.header.version;
 
-        let entry = self.ehdr.entry + kernel_load_address;
+        let entry = self.ehdr.entry;
 
         Ok(LoadedObject {
             image_start,
             image_end,
-            load_address: kernel_load_address,
             entry,
             protocol_struct_paddr,
             protocol_version,

boot/yboot/src/main.rs

@@ -16,7 +16,7 @@ use log::{debug, error, info};
 use uefi::{
     prelude::*,
     proto::{
-        console::gop::{self, GraphicsOutput, PixelFormat},
+        console::gop::{GraphicsOutput, PixelFormat},
         device_path::DevicePath,
         loaded_image::LoadedImage,
         media::{file::Directory, fs::SimpleFileSystem},
@@ -32,31 +32,23 @@ use yboot_proto::{
     LoadProtocolV1, LOADER_MAGIC,
 };
 
-use crate::mem::MemoryDescriptorExt;
-
-fn mode_score(mode: &gop::Mode) -> usize {
-    let (w, h) = mode.info().resolution();
-    let mut size_score = w * h;
-    if w > 1920 || h > 1080 {
-        // Don't pick too large sizes
-        size_score = 0;
-    }
-    size_score
-}
+use crate::{mem::MemoryDescriptorExt, protocol_ext::GraphicsOutputExt};
 
 fn setup_framebuffer(bs: &BootServices, fb: &mut FramebufferOption) -> Result<(), Error> {
     let gop_handle = bs.get_handle_for_protocol::<GraphicsOutput>()?;
     let mut gop = bs.open_protocol_exclusive::<GraphicsOutput>(gop_handle)?;
 
-    let mode = gop.modes().max_by_key(mode_score).ok_or_else(|| {
-        error!("No mode found");
+    // Find the requested mode
+    let mode = gop.match_mode(fb.req_width, fb.req_height).ok_or_else(|| {
+        error!(
+            "Requested mode is not supported: {}x{}",
+            fb.req_width, fb.req_height
+        );
         Error::new(Status::INVALID_PARAMETER, ())
     })?;
 
     gop.set_mode(&mode)?;
 
-    let (res_width, res_height) = mode.info().resolution();
-
     let mut result = gop.frame_buffer();
 
     let format = match mode.info().pixel_format() {
@@ -65,8 +57,8 @@ fn setup_framebuffer(bs: &BootServices, fb: &mut FramebufferOption) -> Result<()
         _ => 0,
     };
 
-    fb.res_width = res_width as _;
-    fb.res_height = res_height as _;
+    fb.res_width = fb.req_width;
+    fb.res_height = fb.req_height;
     fb.res_address = result.as_mut_ptr() as _;
     fb.res_stride = mode.info().stride() as u64 * 4;
     fb.res_size = result.size() as _;
@@ -93,7 +85,7 @@ fn locate_rsdp(st: &SystemTable<Boot>) -> Option<u64> {
 fn boot_partition(
     image: Handle,
     bs: &BootServices,
-) -> Result<ScopedProtocol<'_, SimpleFileSystem>, Error> {
+) -> Result<ScopedProtocol<SimpleFileSystem>, Error> {
     let loaded_image = bs.open_protocol_exclusive::<LoadedImage>(image)?;
     let device_handle = loaded_image.device();
 
@@ -113,7 +105,7 @@ fn load_kernel<'a>(
     config: &Config,
     root: &mut Directory,
     st: &SystemTable<Boot>,
-) -> Result<(u64, u64, u64, &'a mut LoadProtocolV1), Error> {
+) -> Result<(u64, u64, &'a mut LoadProtocolV1), Error> {
     let bs = st.boot_services();
 
     let mut kernel_obj = Object::open(root, cstr16!("kernel.elf"))?;
@@ -183,14 +175,13 @@ fn load_kernel<'a>(
 
     let entry = loaded_obj.entry + proto_data.kernel_virt_offset;
 
-    Ok((entry, loaded_obj.load_address, mmap_memory, proto_data))
+    Ok((entry, mmap_memory, proto_data))
 }
 
 unsafe fn map_and_enter_kernel(
     st: SystemTable<Boot>,
     proto_data: &mut LoadProtocolV1,
     mmap_memory: u64,
-    load_base: u64,
     entry: u64,
 ) -> ! {
     let (_, mmap) = st.exit_boot_services();
@@ -217,7 +208,7 @@ unsafe fn map_and_enter_kernel(
     let cr3 = mem::map_image();
     asm!("cli; wbinvd; mov {0}, %cr3", in(reg) cr3, options(att_syntax));
 
-    asm!("jmp *{0}", in(reg) entry, in("eax") LOADER_MAGIC, in("ecx") load_base, options(noreturn, att_syntax));
+    asm!("jmp *{0}", in(reg) entry, in("eax") LOADER_MAGIC, options(noreturn, att_syntax));
 }
 
 #[entry]
@@ -244,16 +235,15 @@ fn efi_main(image_handle: Handle, mut system_table: SystemTable<Boot>) -> Status
         }
     };
 
-    let (entry, load_base, mmap_memory, proto_data) =
-        match load_kernel(&config, &mut root, &system_table) {
-            Ok(e) => e,
-            Err(error) => {
-                error!("Failed to load the kernel/initrd: {error:?}");
-                return Status::LOAD_ERROR;
-            }
-        };
+    let (entry, mmap_memory, proto_data) = match load_kernel(&config, &mut root, &system_table) {
+        Ok(e) => e,
+        Err(error) => {
+            error!("Failed to load the kernel/initrd: {error:?}");
+            return Status::LOAD_ERROR;
+        }
+    };
 
     unsafe {
-        map_and_enter_kernel(system_table, proto_data, mmap_memory, load_base, entry);
+        map_and_enter_kernel(system_table, proto_data, mmap_memory, entry);
     }
 }

boot/yboot/src/mem.rs

@@ -66,7 +66,7 @@ impl MemoryDescriptorExt for MemoryDescriptor {
     }
 }
 
-pub fn memory_map(bs: &BootServices) -> Result<MemoryMap<'_>, Error> {
+pub fn memory_map(bs: &BootServices) -> Result<MemoryMap, Error> {
     bs.memory_map(unsafe { &mut MMAP_BUFFER.data })
 }
 

etc/aarch64-unknown-none.json

@@ -13,7 +13,6 @@
   "panic-strategy": "abort",
   "dynamic-linking": true,
   "relocation-model": "pic",
-  "position-independent-executables": true,
   "eh-frame-header": false,
 
   "linker": "rust-lld",

etc/gdb/aarch64-virt.gdb

@@ -1,4 +0,0 @@
-# Lower-half
-symbol-file -o 0x40080000 target/aarch64-unknown-none/release/yggdrasil-kernel
-symbol-file -o 0xFFFFFF8040080000 target/aarch64-unknown-none/release/yggdrasil-kernel
-target remote :1234

etc/ld/arm/aarch64-unknown-none.ld

@@ -1,67 +0,0 @@
-ENTRY(__aarch64_entry);
-
-/* KERNEL_PHYS_BASE = 0x40080000; */
-KERNEL_VIRT_OFFSET = 0xFFFFFF8000000000;
-
-SECTIONS {
-    /* . = KERNEL_PHYS_BASE; */
-    . = 0x0;
-
-    PROVIDE(__kernel_start = .);
-
-    /* .text.entry : {
-        *(.text.entry)
-    }
-
-    . = ALIGN(16);
-    . = . + KERNEL_VIRT_OFFSET; */
-
-    .text : {
-        KEEP(*(.text.entry));
-        KEEP(*(.text.vectors));
-        *(.text*)
-    }
-
-    . = ALIGN(4K);
-    .rodata : {
-        *(.rodata*)
-        *(.eh_frame*)
-
-        . = ALIGN(16);
-        PROVIDE(__init_array_start = .);
-        KEEP(*(.init_array*))
-        PROVIDE(__init_array_end = .);
-    }
-
-    . = ALIGN(4K);
-    PROVIDE(__rela_start = .);
-    .rela : {
-        *(.rela*)
-    }
-    PROVIDE(__rela_end = .);
-
-    /* . = ALIGN(4K);            */
-    /* .data.tables : {          */
-    /*     KEEP(*(.data.tables)) */
-    /* }                         */
-
-    . = ALIGN(4K);
-    .data : {
-        *(.data*)
-        *(.got*)
-    }
-
-    . = ALIGN(4K);
-    PROVIDE(__bss_start = .);
-    /* PROVIDE(__bss_start_phys = . - KERNEL_VIRT_OFFSET); */
-    .bss : {
-        *(COMMON)
-        *(.bss*)
-    }
-    . = ALIGN(4K);
-    PROVIDE(__bss_end = .);
-    /* PROVIDE(__bss_end_phys = . - KERNEL_VIRT_OFFSET); */
-    /* PROVIDE(__bss_size = __bss_end_phys - __bss_start_phys); */
-
-    PROVIDE(__kernel_end = .);
-};

etc/ld/arm/aarch64-unknown-qemu.ld

@@ -0,0 +1,55 @@
+ENTRY(__aarch64_entry);
+
+KERNEL_PHYS_BASE = 0x40080000;
+KERNEL_VIRT_OFFSET = 0xFFFFFF8000000000;
+
+SECTIONS {
+    . = KERNEL_PHYS_BASE;
+    PROVIDE(__kernel_start = . + KERNEL_VIRT_OFFSET);
+
+    .text.entry : {
+        *(.text.entry)
+    }
+
+    . = ALIGN(16);
+    . = . + KERNEL_VIRT_OFFSET;
+
+    .text : AT(. - KERNEL_VIRT_OFFSET) {
+        KEEP(*(.text.vectors));
+        *(.text*)
+    }
+
+    . = ALIGN(4K);
+    .rodata : AT(. - KERNEL_VIRT_OFFSET) {
+        *(.rodata*)
+        *(.eh_frame*)
+
+        . = ALIGN(16);
+        PROVIDE(__init_array_start = .);
+        KEEP(*(.init_array*))
+        PROVIDE(__init_array_end = .);
+    }
+
+    . = ALIGN(4K);
+    .data.tables : AT (. - KERNEL_VIRT_OFFSET) {
+        KEEP(*(.data.tables))
+    }
+
+    . = ALIGN(4K);
+    .data : AT(. - KERNEL_VIRT_OFFSET) {
+        *(.data*)
+        *(.got*)
+    }
+
+    . = ALIGN(4K);
+    PROVIDE(__bss_start_phys = . - KERNEL_VIRT_OFFSET);
+    .bss : AT(. - KERNEL_VIRT_OFFSET) {
+        *(COMMON)
+        *(.bss*)
+    }
+    . = ALIGN(4K);
+    PROVIDE(__bss_end_phys = . - KERNEL_VIRT_OFFSET);
+    PROVIDE(__bss_size = __bss_end_phys - __bss_start_phys);
+
+    PROVIDE(__kernel_end = .);
+};

etc/ld/arm/aarch64-unknown-raspi4b.ld

@@ -0,0 +1,55 @@
+ENTRY(__aarch64_entry);
+
+KERNEL_PHYS_BASE = 0x80000;
+KERNEL_VIRT_OFFSET = 0xFFFFFF8000000000;
+
+SECTIONS {
+    . = KERNEL_PHYS_BASE;
+    PROVIDE(__kernel_start = . + KERNEL_VIRT_OFFSET);
+
+    .text.entry : {
+        *(.text.entry)
+    }
+
+    . = ALIGN(16);
+    . = . + KERNEL_VIRT_OFFSET;
+
+    .text : AT(. - KERNEL_VIRT_OFFSET) {
+        KEEP(*(.text.vectors));
+        *(.text*)
+    }
+
+    . = ALIGN(4K);
+    .rodata : AT(. - KERNEL_VIRT_OFFSET) {
+        *(.rodata*)
+        *(.eh_frame*)
+
+        . = ALIGN(16);
+        PROVIDE(__init_array_start = .);
+        KEEP(*(.init_array*))
+        PROVIDE(__init_array_end = .);
+    }
+
+    . = ALIGN(4K);
+    .data.tables : AT (. - KERNEL_VIRT_OFFSET) {
+        KEEP(*(.data.tables))
+    }
+
+    . = ALIGN(4K);
+    .data : AT(. - KERNEL_VIRT_OFFSET) {
+        *(.data*)
+        *(.got*)
+    }
+
+    . = ALIGN(4K);
+    PROVIDE(__bss_start_phys = . - KERNEL_VIRT_OFFSET);
+    .bss : AT(. - KERNEL_VIRT_OFFSET) {
+        *(COMMON)
+        *(.bss*)
+    }
+    . = ALIGN(4K);
+    PROVIDE(__bss_end_phys = . - KERNEL_VIRT_OFFSET);
+    PROVIDE(__bss_size = __bss_end_phys - __bss_start_phys);
+
+    PROVIDE(__kernel_end = .);
+};

etc/ld/riscv/riscv64-unknown-jh7110.ld

@@ -1,32 +1,37 @@
 ENTRY(__rv64_entry);
 
+KERNEL_PHYS_BASE = 0x40200000;
 KERNEL_VIRT_OFFSET = 0xFFFFFFF000000000;
 
 SECTIONS {
-    . = 0;
-    PROVIDE(__kernel_start = .);
+    . = KERNEL_PHYS_BASE;
+    PROVIDE(__kernel_start = . + KERNEL_VIRT_OFFSET);
 
-    .text : {
-        KEEP(*(.text.entry));
+    .text.entry : {
+        *(.text.entry)
+    }
+
+    . = ALIGN(16);
+    . = . + KERNEL_VIRT_OFFSET;
+
+    .text : AT(. - KERNEL_VIRT_OFFSET) {
         KEEP(*(.text.vectors));
         *(.text*)
     }
 
     . = ALIGN(4K);
-    .rodata : {
+    .rodata : AT(. - KERNEL_VIRT_OFFSET) {
         *(.rodata*)
         *(.eh_frame*)
     }
 
     . = ALIGN(4K);
-    PROVIDE(__rela_start = .);
-    .rela : {
-        *(.rela*)
+    .data.tables : AT (. - KERNEL_VIRT_OFFSET) {
+        KEEP(*(.data.tables))
     }
-    PROVIDE(__rela_end = .);
 
     . = ALIGN(4K);
-    .data : {
+    .data : AT(. - KERNEL_VIRT_OFFSET) {
         *(.data*)
         . = ALIGN(8);
         /* PROVIDE(__global_pointer = . + 0x800 - KERNEL_VIRT_OFFSET); */
@@ -40,13 +45,14 @@ SECTIONS {
     }
 
     . = ALIGN(4K);
-    PROVIDE(__bss_start = .);
-    .bss : {
+    PROVIDE(__bss_start_phys = . - KERNEL_VIRT_OFFSET);
+    .bss : AT(. - KERNEL_VIRT_OFFSET) {
         *(COMMON)
         *(.bss*)
     }
     . = ALIGN(4K);
-    PROVIDE(__bss_end = .);
+    PROVIDE(__bss_end_phys = . - KERNEL_VIRT_OFFSET);
+    PROVIDE(__bss_size = __bss_end_phys - __bss_start_phys);
 
     PROVIDE(__kernel_end = .);
 };

etc/ld/riscv/riscv64-unknown-qemu.ld

@@ -0,0 +1,58 @@
+ENTRY(__rv64_entry);
+
+KERNEL_PHYS_BASE = 0x80200000;
+KERNEL_VIRT_OFFSET = 0xFFFFFFF000000000;
+
+SECTIONS {
+    . = KERNEL_PHYS_BASE;
+    PROVIDE(__kernel_start = . + KERNEL_VIRT_OFFSET);
+
+    .text.entry : {
+        *(.text.entry)
+    }
+
+    . = ALIGN(16);
+    . = . + KERNEL_VIRT_OFFSET;
+
+    .text : AT(. - KERNEL_VIRT_OFFSET) {
+        KEEP(*(.text.vectors));
+        *(.text*)
+    }
+
+    . = ALIGN(4K);
+    .rodata : AT(. - KERNEL_VIRT_OFFSET) {
+        *(.rodata*)
+        *(.eh_frame*)
+    }
+
+    . = ALIGN(4K);
+    .data.tables : AT (. - KERNEL_VIRT_OFFSET) {
+        KEEP(*(.data.tables))
+    }
+
+    . = ALIGN(4K);
+    .data : AT(. - KERNEL_VIRT_OFFSET) {
+        *(.data*)
+        . = ALIGN(8);
+        /* PROVIDE(__global_pointer = . + 0x800 - KERNEL_VIRT_OFFSET); */
+
+        . = ALIGN(16);
+        PROVIDE(__init_array_start = .);
+        KEEP(*(.init_array*))
+        PROVIDE(__init_array_end = .);
+
+        *(.got*)
+    }
+
+    . = ALIGN(4K);
+    PROVIDE(__bss_start_phys = . - KERNEL_VIRT_OFFSET);
+    .bss : AT(. - KERNEL_VIRT_OFFSET) {
+        *(COMMON)
+        *(.bss*)
+    }
+    . = ALIGN(4K);
+    PROVIDE(__bss_end_phys = . - KERNEL_VIRT_OFFSET);
+    PROVIDE(__bss_size = __bss_end_phys - __bss_start_phys);
+
+    PROVIDE(__kernel_end = .);
+};

etc/ld/x86/x86_64-unknown-none.ld

@@ -1,52 +1,52 @@
 ENTRY(__x86_64_entry);
 
-/* KERNEL_PHYS_BASE = 0x200000; */
+KERNEL_PHYS_BASE = 0x200000;
 KERNEL_VIRT_OFFSET = 0xFFFFFF8000000000;
 
 SECTIONS {
-    /* . = KERNEL_PHYS_BASE; */
-    /* PROVIDE(__kernel_start = . + KERNEL_VIRT_OFFSET); */
+    . = KERNEL_PHYS_BASE;
+    PROVIDE(__kernel_start = . + KERNEL_VIRT_OFFSET);
 
-    . = 0x0;
-    PROVIDE(__kernel_start = .);
-
-    .text : {
+    .text.entry : {
         *(.text.entry)
+    }
+
+    . = ALIGN(16);
+    . = . + KERNEL_VIRT_OFFSET;
+
+    .text : AT(. - KERNEL_VIRT_OFFSET) {
         *(.text*)
     }
 
-    .export.text : {
+    .export.text : AT(. - KERNEL_VIRT_OFFSET) {
         KEEP(*(.export.text*))
     }
 
     . = ALIGN(4K);
-    .rodata : {
+    .rodata : AT(. - KERNEL_VIRT_OFFSET) {
         *(.eh_frame*)
         *(.rodata*)
     }
 
     . = ALIGN(4K);
-    PROVIDE(__rela_start = .);
-    .rela : {
-        *(.rela*)
+    .data.tables : AT (. - KERNEL_VIRT_OFFSET) {
+        KEEP(*(.data.tables))
     }
-    PROVIDE(__rela_end = .);
 
-    . = ALIGN(4K);
-    .data : {
+    .data : AT(. - KERNEL_VIRT_OFFSET) {
         KEEP(*(.data.yboot))
         *(.data*)
         *(.got*)
     }
 
     . = ALIGN(4K);
-    PROVIDE(__bss_start = .);
-    .bss : {
+    PROVIDE(__bss_start_phys = . - KERNEL_VIRT_OFFSET);
+    .bss : AT(. - KERNEL_VIRT_OFFSET) {
         *(COMMON)
         *(.bss*)
     }
     . = ALIGN(4K);
-    PROVIDE(__bss_end = .);
+    PROVIDE(__bss_end_phys = . - KERNEL_VIRT_OFFSET);
 
     PROVIDE(__kernel_end = .);
 };

etc/riscv64-unknown-none.json

@@ -14,7 +14,6 @@
   "panic-strategy": "abort",
   "dynamic-linking": true,
   "relocation-model": "pic",
-  "position-independent-executables": true,
   "code-model": "medium",
   "eh-frame-header": false,
 

etc/x86_64-unknown-none.json

@@ -15,7 +15,6 @@
   "panic-strategy": "abort",
   "dynamic-linking": true,
   "relocation-model": "pic",
-  "position-independent-executables": true,
 
   "has-thread-local": false,
 

kernel/Cargo.toml

@@ -19,6 +19,7 @@ chrono.workspace = true
 device-api = { workspace = true, features = ["derive"] }
 device-api-macros.workspace = true
 
+memtables.workspace = true
 vmalloc.workspace = true
 kernel-arch.workspace = true
 
@@ -35,6 +36,7 @@ ygg_driver_ahci = { path = "driver/block/ahci" }
 ygg_driver_input = { path = "driver/input" }
 ygg_driver_usb_xhci.path = "driver/usb/xhci"
 ygg_driver_net_rtl81xx.path = "driver/net/rtl81xx"
+ygg_driver_sound_core.path = "driver/sound/core"
 
 memfs = { path = "driver/fs/memfs" }
 ext2 = { path = "driver/fs/ext2" }
@@ -69,6 +71,7 @@ kernel-arch-x86.workspace = true
 
 ygg_driver_acpi.path = "driver/acpi"
 ygg_driver_net_igbe.path = "driver/net/igbe"
+ygg_driver_intel_hda.path = "driver/sound/intel-hda"
 
 acpi.workspace = true
 
@@ -98,5 +101,12 @@ ygg_driver_net_stmmac.path = "driver/net/stmmac"
 default = ["fb_console"]
 fb_console = []
 
+# TODO replace this with a better configuration mechanism
+aarch64_board_virt = ["kernel-arch-aarch64/aarch64_board_virt"]
+aarch64_board_raspi4b = ["kernel-arch-aarch64/aarch64_board_raspi4b"]
+
+riscv64_board_virt = ["kernel-arch-riscv64/riscv64_board_virt"]
+riscv64_board_jh7110 = ["kernel-arch-riscv64/riscv64_board_jh7110"]
+
 [lints]
 workspace = true

kernel/arch/aarch64/Cargo.toml

@@ -7,13 +7,13 @@ edition = "2021"
 yggdrasil-abi.workspace = true
 kernel-arch-interface.workspace = true
 libk-mm-interface.workspace = true
+memtables.workspace = true
 device-api = { workspace = true, features = ["derive"] }
 
 bitflags.workspace = true
 static_assertions.workspace = true
 aarch64-cpu.workspace = true
 tock-registers.workspace = true
-log.workspace = true
 
 [build-dependencies]
 cc = "1.0"

kernel/arch/aarch64/src/lib.rs

@@ -1,5 +1,5 @@
 #![no_std]
-#![feature(decl_macro)]
+#![feature(naked_functions, decl_macro)]
 #![allow(clippy::new_without_default)]
 
 extern crate alloc;
@@ -40,9 +40,11 @@ impl CpuData for PerCpuData {}
 static IPI_QUEUES: OneTimeInit<Vec<IpiQueue<ArchitectureImpl>>> = OneTimeInit::new();
 pub static CPU_COUNT: AtomicUsize = AtomicUsize::new(1);
 
-#[unsafe(naked)]
+#[naked]
 extern "C" fn idle_task(_: usize) -> ! {
-    core::arch::naked_asm!("1: nop; b 1b");
+    unsafe {
+        core::arch::naked_asm!("1: nop; b 1b");
+    }
 }
 
 impl ArchitectureImpl {

kernel/arch/aarch64/src/mem/device.rs

@@ -1,95 +0,0 @@
-use core::ops::Range;
-
-use kernel_arch_interface::mem::DeviceMemoryAttributes;
-use libk_mm_interface::{
-    address::PhysicalAddress,
-    table::{DevicePageManager, DevicePageManagerLevel, EntryLevel},
-};
-
-use crate::mem::table::PageEntry;
-
-use super::{
-    table::{PageTable, L2, L3},
-    tlb_flush_range_va, DEVICE_MAPPING_OFFSET, DEVICE_MEMORY_L3_COUNT,
-};
-
-#[repr(transparent)]
-pub struct L2DeviceMemory(pub PageTable<L2>);
-#[repr(transparent)]
-pub struct L3DeviceMemory(pub [PageTable<L3>; DEVICE_MEMORY_L3_COUNT]);
-
-pub(super) static mut DEVICE_MEMORY: DevicePageManager<L3DeviceMemory, L2DeviceMemory> =
-    DevicePageManager::new(
-        L3DeviceMemory([PageTable::zeroed(); DEVICE_MEMORY_L3_COUNT]),
-        L2DeviceMemory(PageTable::zeroed()),
-    );
-
-impl DevicePageManagerLevel for L2DeviceMemory {
-    type Level = L2;
-    const VIRTUAL_BASE: usize = DEVICE_MAPPING_OFFSET;
-    const INDEX_RANGE: Range<usize> = DEVICE_MEMORY_L3_COUNT..512;
-
-    fn map_page(
-        &mut self,
-        index: usize,
-        physical: PhysicalAddress,
-        attrs: &DeviceMemoryAttributes,
-    ) {
-        let _ = index;
-        let _ = physical;
-        let _ = attrs;
-        todo!()
-    }
-
-    fn unmap_page(&mut self, index: usize) {
-        let _ = index;
-        todo!()
-    }
-
-    fn is_mapped(&self, index: usize) -> bool {
-        let _ = index;
-        todo!()
-    }
-
-    fn flush_range(range: Range<usize>) {
-        let start = range.start * L2::SIZE + Self::VIRTUAL_BASE;
-        let size = (range.end - range.start) * L2::SIZE;
-        tlb_flush_range_va(start, size);
-    }
-}
-
-impl DevicePageManagerLevel for L3DeviceMemory {
-    type Level = L3;
-    const VIRTUAL_BASE: usize = DEVICE_MAPPING_OFFSET;
-    const INDEX_RANGE: Range<usize> = 0..512 * DEVICE_MEMORY_L3_COUNT;
-
-    fn map_page(
-        &mut self,
-        index: usize,
-        physical: PhysicalAddress,
-        attrs: &DeviceMemoryAttributes,
-    ) {
-        let _ = attrs;
-        let l2i = index / 512;
-        let l3i = index % 512;
-        self.0[l2i][l3i] = PageEntry::device_page(physical);
-    }
-
-    // TODO
-    fn unmap_page(&mut self, index: usize) {
-        let _ = index;
-        todo!()
-    }
-
-    fn is_mapped(&self, index: usize) -> bool {
-        let l2i = index / 512;
-        let l3i = index % 512;
-        self.0[l2i][l3i].is_present()
-    }
-
-    fn flush_range(range: Range<usize>) {
-        let start = range.start * L3::SIZE + Self::VIRTUAL_BASE;
-        let size = (range.end - range.start) * L3::SIZE;
-        tlb_flush_range_va(start, size);
-    }
-}

kernel/arch/aarch64/src/mem/intrinsics.rs

@@ -1,93 +0,0 @@
-use core::sync::atomic::{self, Ordering};
-
-use aarch64_cpu::{asm::barrier, registers::PAR_EL1};
-use libk_mm_interface::table::{EntryLevel, EntryLevelExt};
-use tock_registers::interfaces::Readable;
-
-use super::table::L3;
-
-#[inline]
-pub fn tlb_flush_asid(asid: u8) {
-    barrier::dsb(barrier::ISHST);
-    let value = (asid as u64) << 48;
-    unsafe {
-        core::arch::asm!("tlbi aside1, {value}", value = in(reg) value);
-    }
-    barrier::dsb(barrier::ISH);
-    barrier::isb(barrier::SY);
-}
-
-#[inline]
-pub fn tlb_flush_all() {
-    barrier::dsb(barrier::ISHST);
-    unsafe {
-        core::arch::asm!("tlbi vmalle1is");
-    }
-    barrier::dsb(barrier::ISH);
-    barrier::isb(barrier::SY);
-}
-
-#[inline]
-pub fn tlb_flush_vaae1(page: usize) {
-    barrier::dsb(barrier::ISHST);
-    let argument = page >> 12;
-    unsafe {
-        core::arch::asm!("tlbi vaae1, {argument}", argument = in(reg) argument);
-    }
-    barrier::dsb(barrier::ISH);
-    barrier::isb(barrier::SY);
-}
-
-pub fn tlb_flush_range_va(base: usize, size: usize) {
-    let end = (base + size).page_align_up::<L3>();
-    let base = base.page_align_down::<L3>();
-    let count = (end - base).page_count::<L3>();
-    for i in 0..count {
-        tlb_flush_vaae1(base + i * L3::SIZE);
-    }
-}
-
-pub fn at_s1e0r(input: usize) -> Option<u64> {
-    barrier::dsb(barrier::ISHST);
-    unsafe {
-        core::arch::asm!("at s1e0r, {address}", address = in(reg) input);
-    }
-    barrier::dsb(barrier::ISH);
-    barrier::isb(barrier::SY);
-    if PAR_EL1.matches_all(PAR_EL1::F::TranslationSuccessfull) {
-        Some(PAR_EL1.read(PAR_EL1::PA))
-    } else {
-        None
-    }
-}
-
-pub fn at_s1e1r(input: usize) -> Option<u64> {
-    barrier::dsb(barrier::ISHST);
-    unsafe {
-        core::arch::asm!("at s1e1r, {address}", address = in(reg) input);
-    }
-    barrier::dsb(barrier::ISH);
-    barrier::isb(barrier::SY);
-    if PAR_EL1.matches_all(PAR_EL1::F::TranslationSuccessfull) {
-        Some(PAR_EL1.read(PAR_EL1::PA))
-    } else {
-        None
-    }
-}
-
-pub fn ic_iallu() {
-    atomic::compiler_fence(Ordering::SeqCst);
-    barrier::dsb(barrier::ISH);
-    barrier::isb(barrier::SY);
-    unsafe {
-        core::arch::asm!("ic iallu");
-    }
-    barrier::isb(barrier::SY);
-}
-
-pub fn dc_cvac(input: usize) {
-    barrier::dsb(barrier::ISHST);
-    unsafe {
-        core::arch::asm!("dc cvac, {address}", address = in(reg) input);
-    }
-}

kernel/arch/aarch64/src/mem/mod.rs

@@ -1,43 +1,108 @@
+use core::{
+    alloc::Layout,
+    ops::{Deref, DerefMut},
+    sync::atomic::{self, AtomicUsize, Ordering},
+};
+
 use aarch64_cpu::{
     asm::barrier,
-    registers::{MAIR_EL1, SCTLR_EL1, TCR_EL1, TTBR0_EL1, TTBR1_EL1},
+    registers::{MAIR_EL1, PAR_EL1, SCTLR_EL1, TTBR0_EL1, TTBR1_EL1},
 };
 use kernel_arch_interface::{
     mem::{DeviceMemoryAttributes, KernelTableManager, RawDeviceMemoryMapping},
-    sync::IrqSafeSpinlock,
-    KERNEL_VIRT_OFFSET,
+    split_spinlock, Architecture, KERNEL_VIRT_OFFSET,
 };
-use libk_mm_interface::{address::PhysicalAddress, table::EntryLevel};
-use table::{PageAttributes, PageEntry, PageTable, L1};
-use tock_registers::interfaces::{ReadWriteable, Writeable};
-
-mod intrinsics;
-
-pub mod device;
-pub mod process;
-pub mod table;
-
-pub use intrinsics::*;
+use libk_mm_interface::{
+    address::PhysicalAddress,
+    table::{page_index, EntryLevel, EntryLevelExt},
+};
+use memtables::aarch64::{FixedTables, KERNEL_L3_COUNT};
+use static_assertions::const_assert_eq;
+use tock_registers::interfaces::{ReadWriteable, Readable, Writeable};
 use yggdrasil_abi::error::Error;
 
 use crate::ArchitectureImpl;
 
+use self::table::{PageAttributes, PageEntry, PageTable, L1, L2, L3};
+
+pub mod process;
+pub mod table;
+
 #[derive(Debug)]
 pub struct KernelTableManagerImpl;
 
+// TODO eliminate this requirement by using precomputed indices
+const MAPPING_OFFSET: usize = KERNEL_VIRT_OFFSET;
+
+#[cfg(any(feature = "aarch64_board_virt", rust_analyzer))]
+const KERNEL_PHYS_BASE: usize = 0x40080000;
+#[cfg(any(feature = "aarch64_board_raspi4b", rust_analyzer))]
+const KERNEL_PHYS_BASE: usize = 0x80000;
+
+// Precomputed mappings
+const KERNEL_L1_INDEX: usize = page_index::<L1>(KERNEL_VIRT_OFFSET + KERNEL_PHYS_BASE);
+const KERNEL_START_L2_INDEX: usize = page_index::<L2>(KERNEL_VIRT_OFFSET + KERNEL_PHYS_BASE);
+const KERNEL_END_L2_INDEX: usize = KERNEL_START_L2_INDEX + KERNEL_L3_COUNT;
+
+// Must not be zero, should be at 4MiB
+const_assert_eq!(KERNEL_START_L2_INDEX, 0);
+// From static mapping
+#[cfg(any(feature = "aarch64_board_raspi4b", rust_analyzer))]
+const_assert_eq!(KERNEL_L1_INDEX, 0);
+#[cfg(any(feature = "aarch64_board_virt", rust_analyzer))]
+const_assert_eq!(KERNEL_L1_INDEX, 1);
+
+// Runtime mappings
+// 2MiB max
+const EARLY_MAPPING_L2I: usize = KERNEL_END_L2_INDEX + 1;
+// 1GiB max
+const DEVICE_MAPPING_L1I: usize = KERNEL_L1_INDEX + 2;
+const DEVICE_MAPPING_L3_COUNT: usize = 4;
+// 16GiB max
+const RAM_MAPPING_START_L1I: usize = KERNEL_L1_INDEX + 3;
+pub const RAM_MAPPING_L1_COUNT: usize = 16;
+
+// 2MiB for early mappings
+const EARLY_MAPPING_OFFSET: usize =
+    MAPPING_OFFSET | (KERNEL_L1_INDEX * L1::SIZE) | (EARLY_MAPPING_L2I * L2::SIZE);
+static mut EARLY_MAPPING_L3: PageTable<L3> = PageTable::zeroed();
+// 1GiB for device MMIO mapping
+const DEVICE_MAPPING_OFFSET: usize = MAPPING_OFFSET | (DEVICE_MAPPING_L1I * L1::SIZE);
+static mut DEVICE_MAPPING_L2: PageTable<L2> = PageTable::zeroed();
+static mut DEVICE_MAPPING_L3S: [PageTable<L3>; DEVICE_MAPPING_L3_COUNT] =
+    [PageTable::zeroed(); DEVICE_MAPPING_L3_COUNT];
+// 16GiB for RAM mapping
+pub const RAM_MAPPING_OFFSET: usize = MAPPING_OFFSET | (RAM_MAPPING_START_L1I * L1::SIZE);
+pub static MEMORY_LIMIT: AtomicUsize = AtomicUsize::new(0);
+
+split_spinlock! {
+    use crate::ArchitectureImpl;
+    use crate::mem::FixedTables;
+    use libk_mm_interface::KernelImageObject;
+
+    #[link_section = ".data.tables"]
+    static KERNEL_TABLES: KernelImageObject<FixedTables> =
+        unsafe { KernelImageObject::new(FixedTables::zeroed()) };
+}
+
 impl KernelTableManager for KernelTableManagerImpl {
-    fn virtualize(phys: u64) -> usize {
-        if phys as usize >= IDENTITY_SIZE {
-            unreachable!("Invalid physical address to virtualize: {phys:#x}");
+    fn virtualize(address: u64) -> usize {
+        let address = address as usize;
+        if address < MEMORY_LIMIT.load(Ordering::Acquire) {
+            address + RAM_MAPPING_OFFSET
+        } else {
+            panic!("Invalid physical address: {:#x}", address);
         }
-        phys as usize + KERNEL_VIRT_OFFSET
     }
 
-    fn physicalize(virt: usize) -> u64 {
-        if virt < KERNEL_VIRT_OFFSET || virt - KERNEL_VIRT_OFFSET >= IDENTITY_SIZE {
-            unreachable!("Invalid virtualized address: {virt:#x}");
+    fn physicalize(address: usize) -> u64 {
+        if address < RAM_MAPPING_OFFSET
+            || address - RAM_MAPPING_OFFSET >= MEMORY_LIMIT.load(Ordering::Acquire)
+        {
+            panic!("Not a virtualized physical address: {:#x}", address);
         }
-        (virt - KERNEL_VIRT_OFFSET) as _
+
+        (address - RAM_MAPPING_OFFSET) as _
     }
 
     unsafe fn map_device_pages(
@@ -45,71 +110,390 @@ impl KernelTableManager for KernelTableManagerImpl {
         count: usize,
         attrs: DeviceMemoryAttributes,
     ) -> Result<RawDeviceMemoryMapping<Self>, Error> {
-        let _guard = DEVICE_MEMORY_LOCK.lock();
-        #[allow(static_mut_refs)]
-        unsafe {
-            device::DEVICE_MEMORY.map_device_pages(PhysicalAddress::from_u64(base), count, attrs)
+        map_device_memory(PhysicalAddress::from_u64(base), count, attrs)
+    }
+
+    unsafe fn unmap_device_pages(mapping: &RawDeviceMemoryMapping<Self>) {
+        unmap_device_memory(mapping)
+    }
+}
+
+/// Memory mapping which may be used for performing early kernel initialization
+pub struct EarlyMapping<'a, T: ?Sized> {
+    value: &'a mut T,
+    page_count: usize,
+}
+
+impl<'a, T: Sized> EarlyMapping<'a, T> {
+    /// # Safety
+    ///
+    /// `physical` address provided must be a valid non-NULL address actually containing `T`.
+    pub unsafe fn map_slice(
+        physical: PhysicalAddress,
+        len: usize,
+    ) -> Result<EarlyMapping<'a, [T]>, Error> {
+        let layout = Layout::array::<T>(len).unwrap();
+        let aligned = physical.page_align_down::<L3>();
+        let offset = physical.page_offset::<L3>();
+        let page_count = (offset + layout.size()).div_ceil(L3::SIZE);
+
+        let virt = map_early_pages(aligned, page_count)?;
+        let value = core::slice::from_raw_parts_mut((virt + offset) as *mut T, len);
+
+        Ok(EarlyMapping { value, page_count })
+    }
+}
+
+impl<T: ?Sized> Deref for EarlyMapping<'_, T> {
+    type Target = T;
+
+    fn deref(&self) -> &Self::Target {
+        self.value
+    }
+}
+
+impl<T: ?Sized> DerefMut for EarlyMapping<'_, T> {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        self.value
+    }
+}
+
+impl<T: ?Sized> Drop for EarlyMapping<'_, T> {
+    fn drop(&mut self) {
+        let address = (self.value as *mut T).addr() & !(L3::SIZE - 1);
+
+        for i in 0..self.page_count {
+            let page = address + i * L3::SIZE;
+
+            unsafe {
+                unmap_early_page(page);
+            }
         }
     }
+}
 
-    unsafe fn unmap_device_pages(_mapping: &RawDeviceMemoryMapping<Self>) {
-        // TODO
+fn kernel_table_flags() -> PageAttributes {
+    PageAttributes::TABLE
+        | PageAttributes::ACCESS
+        | PageAttributes::SH_INNER
+        | PageAttributes::PAGE_ATTR_NORMAL
+        | PageAttributes::PRESENT
+}
+
+fn ram_block_flags() -> PageAttributes {
+    // TODO UXN, PXN
+    PageAttributes::BLOCK
+        | PageAttributes::ACCESS
+        | PageAttributes::SH_OUTER
+        | PageAttributes::PAGE_ATTR_NORMAL
+        | PageAttributes::PRESENT
+}
+
+// Early mappings
+unsafe fn map_early_pages(physical: PhysicalAddress, count: usize) -> Result<usize, Error> {
+    for l3i in 0..512 {
+        let mut taken = false;
+        for i in 0..count {
+            if EARLY_MAPPING_L3[i + l3i].is_present() {
+                taken = true;
+                break;
+            }
+        }
+
+        if taken {
+            continue;
+        }
+
+        for i in 0..count {
+            let page = physical.add(i * L3::SIZE);
+            // TODO NX, NC
+            EARLY_MAPPING_L3[i + l3i] = PageEntry::normal_page(page, PageAttributes::empty());
+            tlb_flush_vaae1(EARLY_MAPPING_OFFSET + (l3i + i) * L3::SIZE);
+        }
+
+        return Ok(EARLY_MAPPING_OFFSET + l3i * L3::SIZE);
+    }
+
+    Err(Error::OutOfMemory)
+}
+
+unsafe fn unmap_early_page(address: usize) {
+    if !(EARLY_MAPPING_OFFSET..EARLY_MAPPING_OFFSET + L2::SIZE).contains(&address) {
+        panic!("Tried to unmap invalid early mapping: {:#x}", address);
+    }
+
+    let l3i = (address - EARLY_MAPPING_OFFSET).page_index::<L3>();
+
+    assert!(EARLY_MAPPING_L3[l3i].is_present());
+    EARLY_MAPPING_L3[l3i] = PageEntry::INVALID;
+}
+
+/// # Safety
+///
+/// Only meant to be used by the architecture initialization functions.
+pub unsafe fn map_ram_l1(index: usize) {
+    if index >= RAM_MAPPING_L1_COUNT {
+        ArchitectureImpl::halt();
+    }
+    let mut tables = KERNEL_TABLES.lock();
+    let table_index = index + RAM_MAPPING_START_L1I;
+
+    if tables.l1.data[table_index] != 0 {
+        ArchitectureImpl::halt();
+    }
+
+    tables.l1.data[table_index] = ((index * L1::SIZE) as u64) | ram_block_flags().bits();
+    tlb_flush_vaae1(RAM_MAPPING_OFFSET + index * L1::SIZE);
+}
+
+// Device mappings
+unsafe fn map_device_memory_l3(
+    base: PhysicalAddress,
+    count: usize,
+    _attrs: DeviceMemoryAttributes,
+) -> Result<usize, Error> {
+    // TODO don't map pages if already mapped
+
+    'l0: for i in 0..DEVICE_MAPPING_L3_COUNT * 512 {
+        for j in 0..count {
+            let l2i = (i + j) / 512;
+            let l3i = (i + j) % 512;
+
+            if DEVICE_MAPPING_L3S[l2i][l3i].is_present() {
+                continue 'l0;
+            }
+        }
+
+        for j in 0..count {
+            let l2i = (i + j) / 512;
+            let l3i = (i + j) % 512;
+
+            // TODO NX, NC
+            DEVICE_MAPPING_L3S[l2i][l3i] = PageEntry::device_page(base.add(j * L3::SIZE));
+            tlb_flush_vaae1(DEVICE_MAPPING_OFFSET + l2i * L2::SIZE + l3i * L3::SIZE);
+        }
+
+        return Ok(DEVICE_MAPPING_OFFSET + i * L3::SIZE);
+    }
+
+    Err(Error::OutOfMemory)
+}
+
+#[allow(unused)]
+unsafe fn map_device_memory_l2(
+    base: PhysicalAddress,
+    count: usize,
+    _attrs: DeviceMemoryAttributes,
+) -> Result<usize, Error> {
+    'l0: for i in DEVICE_MAPPING_L3_COUNT..512 {
+        for j in 0..count {
+            if DEVICE_MAPPING_L2[i + j].is_present() {
+                continue 'l0;
+            }
+        }
+
+        for j in 0..count {
+            DEVICE_MAPPING_L2[i + j] = PageEntry::<L2>::device_block(base.add(j * L2::SIZE));
+            tlb_flush_vaae1(DEVICE_MAPPING_OFFSET + (i + j) * L2::SIZE);
+        }
+
+        return Ok(DEVICE_MAPPING_OFFSET + i * L2::SIZE);
+    }
+
+    Err(Error::OutOfMemory)
+}
+
+pub(crate) unsafe fn map_device_memory(
+    base: PhysicalAddress,
+    size: usize,
+    attrs: DeviceMemoryAttributes,
+) -> Result<RawDeviceMemoryMapping<KernelTableManagerImpl>, Error> {
+    let l3_aligned = base.page_align_down::<L3>();
+    let l3_offset = base.page_offset::<L3>();
+    let page_count = (l3_offset + size).page_count::<L3>();
+
+    if page_count > 256 {
+        // Large mapping, use L2 mapping instead
+        let l2_aligned = base.page_align_down::<L2>();
+        let l2_offset = base.page_offset::<L2>();
+        let page_count = (l2_offset + size).page_count::<L2>();
+
+        let base_address = map_device_memory_l2(l2_aligned, page_count, attrs)?;
+        let address = base_address + l2_offset;
+
+        Ok(RawDeviceMemoryMapping::from_raw_parts(
+            l2_aligned.into_u64(),
+            address,
+            base_address,
+            page_count,
+            L2::SIZE,
+        ))
+    } else {
+        // Just map the pages directly
+        let base_address = map_device_memory_l3(l3_aligned, page_count, attrs)?;
+        let address = base_address + l3_offset;
+
+        Ok(RawDeviceMemoryMapping::from_raw_parts(
+            l3_aligned.into_u64(),
+            address,
+            base_address,
+            page_count,
+            L3::SIZE,
+        ))
     }
 }
 
-// 64GiB of identity-mapped memory
-pub const IDENTITY_SIZE: usize = 64 * L1::SIZE;
-pub const IDENTITY_L1_START: usize = (KERNEL_VIRT_OFFSET >> L1::SHIFT) & 0x1FF;
-// 1GiB of memory-mapped devices
-pub const DEVICE_MEMORY_L1: usize = IDENTITY_L1_START + IDENTITY_SIZE / L1::SIZE;
-pub const DEVICE_MEMORY_L3_COUNT: usize = 32;
-pub const DEVICE_MAPPING_OFFSET: usize = KERNEL_VIRT_OFFSET + DEVICE_MEMORY_L1 * L1::SIZE;
-static DEVICE_MEMORY_LOCK: IrqSafeSpinlock<ArchitectureImpl, ()> = IrqSafeSpinlock::new(());
+pub(crate) unsafe fn unmap_device_memory(map: &RawDeviceMemoryMapping<KernelTableManagerImpl>) {
+    match map.page_size {
+        L3::SIZE => {
+            for i in 0..map.page_count {
+                let page = map.base_address + i * L3::SIZE;
+                let l2i = page.page_index::<L2>();
+                let l3i = page.page_index::<L3>();
+                assert!(DEVICE_MAPPING_L3S[l2i][l3i].is_present());
+                DEVICE_MAPPING_L3S[l2i][l3i] = PageEntry::INVALID;
 
-static mut FIXED_L1: PageTable<L1> = const {
-    if IDENTITY_L1_START != 0 {
-        panic!("Invalid KERNEL_VIRT_OFFSET");
+                tlb_flush_vaae1(page);
+            }
+        }
+        L2::SIZE => todo!(),
+        _ => unimplemented!(),
     }
-    if IDENTITY_L1_START + IDENTITY_SIZE / L1::SIZE >= 500 {
-        panic!("Invalid KERNEL_VIRT_OFFSET + IDENTITY_SIZE");
+}
+
+#[inline]
+pub fn tlb_flush_asid(asid: u8) {
+    barrier::dsb(barrier::ISHST);
+    let value = (asid as u64) << 48;
+    unsafe {
+        core::arch::asm!("tlbi aside1, {value}", value = in(reg) value);
     }
+    barrier::dsb(barrier::ISH);
+    barrier::isb(barrier::SY);
+}
 
-    let mut table = PageTable::<L1>::zeroed();
-    let mut i = 0;
-
-    while i < IDENTITY_SIZE / L1::SIZE {
-        let phys = PhysicalAddress::from_usize(i * L1::SIZE);
-        table.entries[i + IDENTITY_L1_START] =
-            PageEntry::normal_block(phys, PageAttributes::empty());
-        i += 1;
+#[inline]
+pub fn tlb_flush_all() {
+    barrier::dsb(barrier::ISHST);
+    unsafe {
+        core::arch::asm!("tlbi vmalle1is");
     }
+    barrier::dsb(barrier::ISH);
+    barrier::isb(barrier::SY);
+}
 
-    table
-};
+#[inline]
+pub fn tlb_flush_vaae1(page: usize) {
+    barrier::dsb(barrier::ISHST);
+    let argument = page >> 12;
+    unsafe {
+        core::arch::asm!("tlbi vaae1, {argument}", argument = in(reg) argument);
+    }
+    barrier::dsb(barrier::ISH);
+    barrier::isb(barrier::SY);
+}
 
-pub unsafe fn setup_fixed_tables() {
-    let device_l2_physical = auto_lower_address(&raw const device::DEVICE_MEMORY.large);
-    FIXED_L1[DEVICE_MEMORY_L1] = PageEntry::table(
-        PhysicalAddress::from_usize(device_l2_physical),
-        PageAttributes::empty(),
-    );
-    for i in 0..DEVICE_MEMORY_L3_COUNT {
-        let device_l3_physical = auto_lower_address(&raw const device::DEVICE_MEMORY.normal.0[i]);
-        device::DEVICE_MEMORY.large.0[i] = PageEntry::table(
-            PhysicalAddress::from_usize(device_l3_physical),
-            PageAttributes::empty(),
+pub fn at_s1e0r(input: usize) -> Option<u64> {
+    barrier::dsb(barrier::ISHST);
+    unsafe {
+        core::arch::asm!("at s1e0r, {address}", address = in(reg) input);
+    }
+    barrier::dsb(barrier::ISH);
+    barrier::isb(barrier::SY);
+    if PAR_EL1.matches_all(PAR_EL1::F::TranslationSuccessfull) {
+        Some(PAR_EL1.read(PAR_EL1::PA))
+    } else {
+        None
+    }
+}
+
+pub fn at_s1e1r(input: usize) -> Option<u64> {
+    barrier::dsb(barrier::ISHST);
+    unsafe {
+        core::arch::asm!("at s1e1r, {address}", address = in(reg) input);
+    }
+    barrier::dsb(barrier::ISH);
+    barrier::isb(barrier::SY);
+    if PAR_EL1.matches_all(PAR_EL1::F::TranslationSuccessfull) {
+        Some(PAR_EL1.read(PAR_EL1::PA))
+    } else {
+        None
+    }
+}
+
+pub fn ic_iallu() {
+    atomic::compiler_fence(Ordering::SeqCst);
+    barrier::dsb(barrier::ISH);
+    barrier::isb(barrier::SY);
+    unsafe {
+        core::arch::asm!("ic iallu");
+    }
+    barrier::isb(barrier::SY);
+}
+
+pub fn dc_cvac(input: usize) {
+    barrier::dsb(barrier::ISHST);
+    unsafe {
+        core::arch::asm!("dc cvac, {address}", address = in(reg) input);
+    }
+}
+
+fn auto_address<T>(value: *const T) -> usize {
+    let addr = value.addr();
+    if addr < KERNEL_VIRT_OFFSET {
+        // Called from lower half
+        addr
+    } else {
+        // Called from higher-half
+        addr - KERNEL_VIRT_OFFSET
+    }
+}
+
+/// (BSP-early init) loads precomputed kernel mapping tables for the kernel to jump to "higher-half"
+///
+/// # Safety
+///
+/// Unsafe, must only be called by BSP during its early init while still in "lower-half"
+pub unsafe fn load_fixed_tables() {
+    let ttbr0 = auto_address(&raw const KERNEL_TABLES) as u64;
+
+    TTBR0_EL1.set(ttbr0);
+    TTBR1_EL1.set(ttbr0);
+}
+
+/// Sets up additional translation tables for kernel usage
+///
+/// # Safety
+///
+/// Unsafe, must only be called by BSP during its early init, must already be in "higher-half"
+pub unsafe fn init_fixed_tables() {
+    // TODO this could be built in compile-time too?
+    let mut tables = KERNEL_TABLES.grab();
+    let early_mapping_l3_phys = auto_address(&raw const EARLY_MAPPING_L3);
+    let device_mapping_l2_phys = auto_address(&raw const DEVICE_MAPPING_L2);
+
+    for i in 0..DEVICE_MAPPING_L3_COUNT {
+        let device_mapping_l3_phys = PhysicalAddress::from_usize(
+            &DEVICE_MAPPING_L3S[i] as *const _ as usize - KERNEL_VIRT_OFFSET,
         );
+        DEVICE_MAPPING_L2[i] = PageEntry::table(device_mapping_l3_phys, PageAttributes::empty());
     }
+
+    assert_eq!(tables.l2.data[EARLY_MAPPING_L2I], 0);
+    tables.l2.data[EARLY_MAPPING_L2I] =
+        (early_mapping_l3_phys as u64) | kernel_table_flags().bits();
+    tlb_flush_vaae1(EARLY_MAPPING_OFFSET);
+
+    assert_eq!(tables.l1.data[DEVICE_MAPPING_L1I], 0);
+    tables.l1.data[DEVICE_MAPPING_L1I] =
+        (device_mapping_l2_phys as u64) | kernel_table_flags().bits();
+
+    tlb_flush_all();
 }
 
-pub fn load_fixed_tables() {
-    let l1_physical = auto_lower_address(&raw const FIXED_L1);
-    TTBR0_EL1.set_baddr(l1_physical as _);
-    TTBR1_EL1.set_baddr(l1_physical as _);
-}
-
-pub fn configure_mmu() {
+pub fn setup_memory_attributes() {
     // TODO: Figure out why WriteBack_NonTransient_ReadWriteAlloc doesn't work on Pi 4B
+
     MAIR_EL1.write(
         //// Attribute 0 -- normal memory
         MAIR_EL1::Attr0_Normal_Inner::WriteBack_NonTransient +
@@ -120,39 +504,6 @@ pub fn configure_mmu() {
         //// Attribute 2 -- device memory
         MAIR_EL1::Attr1_Device::nonGathering_nonReordering_EarlyWriteAck,
     );
-
-    TCR_EL1.write(
-        TCR_EL1::AS::ASID8Bits +
-             TCR_EL1::A1::TTBR0 +
-             TCR_EL1::HD::CLEAR +
-             // General
-             TCR_EL1::IPS::Bits_48 +
-             // TTBR0
-             TCR_EL1::TG0::KiB_4 + TCR_EL1::T0SZ.val(25) + TCR_EL1::SH0::Inner +
-             // TTBR1
-             TCR_EL1::TG1::KiB_4 + TCR_EL1::T1SZ.val(25) + TCR_EL1::SH1::Inner,
-    );
-    barrier::dsb(barrier::ISHST);
-    barrier::isb(barrier::SY);
-
-    SCTLR_EL1.modify(
-        SCTLR_EL1::E0E::LittleEndian
-            + SCTLR_EL1::EE::LittleEndian
-            + SCTLR_EL1::WXN::Disable
-            + SCTLR_EL1::SA0::Enable
-            + SCTLR_EL1::SA::Enable
-            + SCTLR_EL1::A::Enable
-            + SCTLR_EL1::I::NonCacheable
-            + SCTLR_EL1::C::NonCacheable,
-    );
-
-    barrier::dsb(barrier::ISH);
-    barrier::isb(barrier::SY);
-}
-
-pub fn enable_mmu() {
-    // Enable translation
-    SCTLR_EL1.modify(SCTLR_EL1::M::Enable);
 }
 
 /// Enables data cache.
@@ -198,12 +549,3 @@ pub unsafe fn disable_icache() {
     barrier::dsb(barrier::ISH);
     barrier::isb(barrier::SY);
 }
-
-fn auto_lower_address<T>(ptr: *const T) -> usize {
-    let address = ptr.addr();
-    if address < KERNEL_VIRT_OFFSET {
-        address
-    } else {
-        address - KERNEL_VIRT_OFFSET
-    }
-}

kernel/arch/aarch64/src/mem/table.rs

@@ -56,7 +56,7 @@ bitflags! {
 #[derive(Clone, Copy)]
 #[repr(C, align(0x1000))]
 pub struct PageTable<L: EntryLevel> {
-    pub entries: [PageEntry<L>; 512],
+    entries: [PageEntry<L>; 512],
 }
 
 #[derive(Clone, Copy)]
@@ -262,13 +262,18 @@ impl<L: NonTerminalEntryLevel> PageEntry<L> {
         )
     }
 
-    pub const fn normal_block(phys: PhysicalAddress, attrs: PageAttributes) -> Self {
-        const ATTR: u64 = PageAttributes::BLOCK.bits()
-            | PageAttributes::PRESENT.bits()
-            | PageAttributes::ACCESS.bits()
-            | PageAttributes::SH_OUTER.bits()
-            | PageAttributes::PAGE_ATTR_NORMAL.bits();
-        Self(phys.into_u64() | attrs.bits() | ATTR, PhantomData)
+    pub fn normal_block(phys: PhysicalAddress, attrs: PageAttributes) -> Self {
+        Self(
+            phys.into_u64()
+                | (PageAttributes::BLOCK
+                    | PageAttributes::PRESENT
+                    | PageAttributes::ACCESS
+                    | PageAttributes::SH_OUTER
+                    | PageAttributes::PAGE_ATTR_NORMAL
+                    | attrs)
+                    .bits(),
+            PhantomData,
+        )
     }
 
     pub fn device_block(phys: PhysicalAddress) -> Self {

kernel/arch/interface/src/sync.rs

@@ -42,7 +42,7 @@ impl<A: Architecture, T> Spinlock<A, T> {
         }
     }
 
-    pub fn lock(&self) -> SpinlockGuard<'_, A, T> {
+    pub fn lock(&self) -> SpinlockGuard<A, T> {
         // Loop until the lock can be acquired
         if LOCK_HACK.load(Ordering::Acquire) {
             return SpinlockGuard { lock: self };
@@ -103,7 +103,7 @@ impl<A: Architecture, T> IrqSafeSpinlock<A, T> {
     }
 
     /// Attempts to acquire a lock. IRQs will be disabled until the lock is released.
-    pub fn lock(&self) -> IrqSafeSpinlockGuard<'_, A, T> {
+    pub fn lock(&self) -> IrqSafeSpinlockGuard<A, T> {
         // Disable IRQs to avoid IRQ handler trying to acquire the same lock
         let irq_guard = IrqGuard::acquire();
 

kernel/arch/riscv64/Cargo.toml

@@ -7,6 +7,7 @@ edition = "2024"
 yggdrasil-abi.workspace = true
 kernel-arch-interface.workspace = true
 libk-mm-interface.workspace = true
+memtables.workspace = true
 device-api = { workspace = true, features = ["derive"] }
 
 tock-registers.workspace = true

kernel/arch/riscv64/src/context.rs

@@ -124,7 +124,8 @@ impl<K: KernelTableManager, PA: PhysicalMemoryAllocator<Address = PhysicalAddres
 
         // TODO stack is leaked
         let satp = InMemoryRegister::new(0);
-        let kernel_table_phys = ((&raw const mem::FIXED_L1).addr() - KERNEL_VIRT_OFFSET) as u64;
+        let kernel_table_phys =
+            ((&raw const mem::KERNEL_TABLES).addr() - KERNEL_VIRT_OFFSET) as u64;
         satp.write(SATP::MODE::Sv39 + SATP::ASID.val(0) + SATP::PPN.val(kernel_table_phys >> 12));
 
         Ok(Self {

kernel/arch/riscv64/src/lib.rs

@@ -1,4 +1,4 @@
-#![feature(decl_macro)]
+#![feature(decl_macro, naked_functions)]
 #![no_std]
 
 extern crate alloc;
@@ -60,9 +60,11 @@ impl CpuData for PerCpuData {
     }
 }
 
-#[unsafe(naked)]
+#[naked]
 extern "C" fn idle_task(_: usize) -> ! {
-    core::arch::naked_asm!("1: nop; j 1b");
+    unsafe {
+        core::arch::naked_asm!("1: nop; j 1b");
+    }
 }
 
 impl ArchitectureImpl {

kernel/arch/riscv64/src/mem/mod.rs

@@ -1,38 +1,70 @@
-use kernel_arch_interface::mem::{
-    DeviceMemoryAttributes, KernelTableManager, RawDeviceMemoryMapping,
+use cfg_if::cfg_if;
+use kernel_arch_interface::{
+    mem::{DeviceMemoryAttributes, KernelTableManager, RawDeviceMemoryMapping},
+    split_spinlock,
 };
 use libk_mm_interface::{
     address::PhysicalAddress,
     table::{page_index, EntryLevel, EntryLevelExt},
 };
-use table::{PageEntry, PageTable, L1, L3};
+use memtables::riscv64::PageAttributes;
+use static_assertions::{const_assert, const_assert_eq};
+use table::{PageEntry, PageTable, L1, L2, L3};
 use tock_registers::interfaces::Writeable;
 use yggdrasil_abi::error::Error;
 
+pub use memtables::riscv64::FixedTables;
+
 use crate::registers::SATP;
 
 pub mod process;
 pub mod table;
 
+split_spinlock! {
+    use crate::ArchitectureImpl;
+    use crate::mem::FixedTables;
+    use libk_mm_interface::KernelImageObject;
+
+    #[link_section = ".data.tables"]
+    #[used]
+    static KERNEL_TABLES: KernelImageObject<FixedTables> =
+        unsafe { KernelImageObject::new(FixedTables::zeroed()) };
+}
+
+cfg_if! {
+    if #[cfg(feature = "riscv64_board_virt")] {
+        pub const KERNEL_PHYS_BASE: usize = 0x80200000;
+    } else if #[cfg(feature = "riscv64_board_jh7110")] {
+        pub const KERNEL_PHYS_BASE: usize = 0x40200000;
+    } else if #[cfg(rust_analyzer)] {
+        pub const KERNEL_PHYS_BASE: usize = 0x80200000;
+    }
+}
+
 pub const KERNEL_VIRT_OFFSET: usize = kernel_arch_interface::KERNEL_VIRT_OFFSET;
 pub const SIGN_EXTEND_MASK: usize = 0xFFFFFF80_00000000;
 
-pub const IDENTITY_SIZE: usize = 64 * L1::SIZE;
-pub const IDENTITY_L1_START: usize = page_index::<L1>(KERNEL_VIRT_OFFSET);
+pub const KERNEL_START_L1I: usize = page_index::<L1>(KERNEL_VIRT_OFFSET + KERNEL_PHYS_BASE);
+pub const KERNEL_L2I: usize = page_index::<L2>(KERNEL_VIRT_OFFSET + KERNEL_PHYS_BASE);
+const_assert_eq!(KERNEL_L2I, 1);
 
-pub static mut FIXED_L1: PageTable<L1> = const {
-    let mut table = PageTable::zeroed();
+// Runtime mappings
+// 1GiB of device memory space
+const DEVICE_MAPPING_L1I: usize = KERNEL_START_L1I + 1;
+const DEVICE_MAPPING_L3_COUNT: usize = 4;
+// 32GiB of RAM space
+const RAM_MAPPING_START_L1I: usize = KERNEL_START_L1I + 2;
+const RAM_MAPPING_L1_COUNT: usize = 32;
+const_assert!(RAM_MAPPING_START_L1I + RAM_MAPPING_L1_COUNT <= 512);
+const_assert!(DEVICE_MAPPING_L1I < 512);
 
-    let mut i = 0;
-    while i < IDENTITY_SIZE / L1::SIZE {
-        let entry = PageEntry::identity_block(PhysicalAddress::from_usize(i * L1::SIZE));
-        table.entries[i] = entry;
-        table.entries[i + IDENTITY_L1_START] = entry;
-        i += 1;
-    }
+const DEVICE_MAPPING_OFFSET: usize = (DEVICE_MAPPING_L1I << L1::SHIFT) | SIGN_EXTEND_MASK;
+const RAM_MAPPING_OFFSET: usize = (RAM_MAPPING_START_L1I << L1::SHIFT) | SIGN_EXTEND_MASK;
 
-    table
-};
+// Runtime tables
+static mut DEVICE_MAPPING_L2: PageTable<L2> = PageTable::zeroed();
+static mut DEVICE_MAPPING_L3S: [PageTable<L3>; DEVICE_MAPPING_L3_COUNT] =
+    [const { PageTable::zeroed() }; DEVICE_MAPPING_L3_COUNT];
 
 /// Any VAs above this one are sign-extended
 pub const USER_BOUNDARY: usize = 0x40_00000000;
@@ -43,17 +75,17 @@ pub struct KernelTableManagerImpl;
 impl KernelTableManager for KernelTableManagerImpl {
     fn virtualize(address: u64) -> usize {
         let address = address as usize;
-        if address >= IDENTITY_SIZE {
+        if address >= RAM_MAPPING_OFFSET {
             panic!("Invalid physical address: {address:#x}");
         }
-        address + KERNEL_VIRT_OFFSET
+        address + RAM_MAPPING_OFFSET
     }
 
     fn physicalize(address: usize) -> u64 {
-        if address < KERNEL_VIRT_OFFSET || address - KERNEL_VIRT_OFFSET >= IDENTITY_SIZE {
+        if address < RAM_MAPPING_OFFSET {
             panic!("Invalid \"physicalized\" virtual address {address:#x}");
         }
-        (address - KERNEL_VIRT_OFFSET) as u64
+        (address - RAM_MAPPING_OFFSET) as u64
     }
 
     unsafe fn map_device_pages(
@@ -61,31 +93,146 @@ impl KernelTableManager for KernelTableManagerImpl {
         count: usize,
         attrs: DeviceMemoryAttributes,
     ) -> Result<RawDeviceMemoryMapping<Self>, Error> {
-        let _ = attrs;
-        let base = PhysicalAddress::from_u64(base);
-        let l3_aligned_base = base.page_align_down::<L3>();
-        let l3_aligned_end = base.add(count).page_align_up::<L3>();
-        let l3_offset = base - l3_aligned_base;
-        let l3_page_count = (l3_aligned_end - l3_aligned_base).page_count::<L3>();
-        let l3_aligned_virt = l3_aligned_base.add(KERNEL_VIRT_OFFSET).into_usize();
-
-        Ok(unsafe {
-            RawDeviceMemoryMapping::from_raw_parts(
-                l3_aligned_base.into_u64(),
-                l3_aligned_virt + l3_offset,
-                l3_aligned_virt,
-                l3_page_count,
-                L3::SIZE,
-            )
-        })
+        unsafe { map_device_memory(PhysicalAddress::from_u64(base), count, attrs) }
     }
 
     unsafe fn unmap_device_pages(mapping: &RawDeviceMemoryMapping<Self>) {
-        let _ = mapping;
+        unsafe { unmap_device_memory(mapping) }
     }
 }
 
-pub fn auto_lower_address<T>(x: *const T) -> usize {
+// Device mappings
+unsafe fn map_device_memory_l3(
+    base: PhysicalAddress,
+    count: usize,
+    _attrs: DeviceMemoryAttributes,
+) -> Result<usize, Error> {
+    // TODO don't map pages if already mapped
+
+    'l0: for i in 0..DEVICE_MAPPING_L3_COUNT * 512 {
+        for j in 0..count {
+            let l2i = (i + j) / 512;
+            let l3i = (i + j) % 512;
+
+            unsafe {
+                if DEVICE_MAPPING_L3S[l2i][l3i].is_present() {
+                    continue 'l0;
+                }
+            }
+        }
+
+        for j in 0..count {
+            let l2i = (i + j) / 512;
+            let l3i = (i + j) % 512;
+
+            unsafe {
+                DEVICE_MAPPING_L3S[l2i][l3i] =
+                    PageEntry::page(base.add(j * L3::SIZE), PageAttributes::W);
+            }
+        }
+
+        let start = DEVICE_MAPPING_OFFSET + i * L3::SIZE;
+        tlb_flush_range_va(start, count * L3::SIZE);
+        return Ok(start);
+    }
+
+    Err(Error::OutOfMemory)
+}
+
+#[allow(unused)]
+unsafe fn map_device_memory_l2(
+    base: PhysicalAddress,
+    count: usize,
+    _attrs: DeviceMemoryAttributes,
+) -> Result<usize, Error> {
+    'l0: for i in DEVICE_MAPPING_L3_COUNT..512 {
+        for j in 0..count {
+            unsafe {
+                if DEVICE_MAPPING_L2[i + j].is_present() {
+                    continue 'l0;
+                }
+            }
+        }
+
+        unsafe {
+            for j in 0..count {
+                DEVICE_MAPPING_L2[i + j] =
+                    PageEntry::<L2>::block(base.add(j * L2::SIZE), PageAttributes::W);
+            }
+        }
+
+        let start = DEVICE_MAPPING_OFFSET + i * L2::SIZE;
+        tlb_flush_range_va(start, count * L2::SIZE);
+        return Ok(start);
+    }
+
+    Err(Error::OutOfMemory)
+}
+
+pub(crate) unsafe fn map_device_memory(
+    base: PhysicalAddress,
+    size: usize,
+    attrs: DeviceMemoryAttributes,
+) -> Result<RawDeviceMemoryMapping<KernelTableManagerImpl>, Error> {
+    let l3_aligned = base.page_align_down::<L3>();
+    let l3_offset = base.page_offset::<L3>();
+    let page_count = (l3_offset + size).page_count::<L3>();
+
+    if page_count > 256 {
+        // Large mapping, use L2 mapping instead
+        let l2_aligned = base.page_align_down::<L2>();
+        let l2_offset = base.page_offset::<L2>();
+        let page_count = (l2_offset + size).page_count::<L2>();
+
+        unsafe {
+            let base_address = map_device_memory_l2(l2_aligned, page_count, attrs)?;
+            let address = base_address + l2_offset;
+
+            Ok(RawDeviceMemoryMapping::from_raw_parts(
+                l2_aligned.into_u64(),
+                address,
+                base_address,
+                page_count,
+                L2::SIZE,
+            ))
+        }
+    } else {
+        // Just map the pages directly
+        unsafe {
+            let base_address = map_device_memory_l3(l3_aligned, page_count, attrs)?;
+            let address = base_address + l3_offset;
+
+            Ok(RawDeviceMemoryMapping::from_raw_parts(
+                l3_aligned.into_u64(),
+                address,
+                base_address,
+                page_count,
+                L3::SIZE,
+            ))
+        }
+    }
+}
+
+pub(crate) unsafe fn unmap_device_memory(map: &RawDeviceMemoryMapping<KernelTableManagerImpl>) {
+    match map.page_size {
+        L3::SIZE => {
+            for i in 0..map.page_count {
+                let page = map.base_address + i * L3::SIZE;
+                let l2i = page.page_index::<L2>();
+                let l3i = page.page_index::<L3>();
+                unsafe {
+                    assert!(DEVICE_MAPPING_L3S[l2i][l3i].is_present());
+                    DEVICE_MAPPING_L3S[l2i][l3i] = PageEntry::INVALID;
+                }
+            }
+            tlb_flush_range_va(map.base_address, map.page_count * L3::SIZE);
+        }
+        L2::SIZE => todo!(),
+        _ => unimplemented!(),
+    }
+}
+
+pub fn auto_address<T>(x: *const T) -> usize {
     let x = x.addr();
     if x >= KERNEL_VIRT_OFFSET {
         x - KERNEL_VIRT_OFFSET
@@ -100,7 +247,7 @@ pub fn auto_lower_address<T>(x: *const T) -> usize {
 ///
 /// Only meant to be called once per each HART during their early init.
 pub unsafe fn enable_mmu() {
-    let l1_phys = auto_lower_address(&raw const FIXED_L1) as u64;
+    let l1_phys = auto_address(&raw const KERNEL_TABLES) as u64;
     tlb_flush_full();
     SATP.write(SATP::PPN.val(l1_phys >> 12) + SATP::MODE::Sv39);
 }
@@ -111,10 +258,49 @@ pub unsafe fn enable_mmu() {
 ///
 /// Needs to be called once after secondary HARTs are initialized.
 pub unsafe fn unmap_lower_half() {
-    // for i in 0..(IDENTITY_SIZE / L1::SIZE) {
-    //     unsafe { FIXED_L1[i] = PageEntry::INVALID };
-    // }
-    // tlb_flush_full();
+    let mut tables = KERNEL_TABLES.lock();
+    let kernel_l1i_lower = page_index::<L1>(KERNEL_PHYS_BASE);
+    tables.l1.data[kernel_l1i_lower] = 0;
+    tlb_flush_range_va(0x0, L1::SIZE);
+}
+
+/// Sets up run-time kernel translation tables.
+///
+/// # Safety
+///
+/// The caller must ensure MMU is already enabled.
+pub unsafe fn setup_fixed_tables() {
+    let mut tables = KERNEL_TABLES.lock();
+
+    let device_mapping_l2_phys = auto_address(&raw const DEVICE_MAPPING_L2);
+
+    // Set up static runtime mappings
+    for i in 0..DEVICE_MAPPING_L3_COUNT {
+        unsafe {
+            let device_mapping_l3_phys = PhysicalAddress::from_usize(
+                (&raw const DEVICE_MAPPING_L3S[i]).addr() - KERNEL_VIRT_OFFSET,
+            );
+            DEVICE_MAPPING_L2[i] =
+                PageEntry::table(device_mapping_l3_phys, PageAttributes::empty());
+        }
+    }
+
+    assert_eq!(tables.l1.data[DEVICE_MAPPING_L1I], 0);
+    tables.l1.data[DEVICE_MAPPING_L1I] =
+        ((device_mapping_l2_phys as u64) >> 2) | PageAttributes::V.bits();
+
+    for l1i in 0..RAM_MAPPING_L1_COUNT {
+        let physical = (l1i as u64) << L1::SHIFT;
+        tables.l1.data[l1i + RAM_MAPPING_START_L1I] = (physical >> 2)
+            | (PageAttributes::R
+                | PageAttributes::W
+                | PageAttributes::A
+                | PageAttributes::D
+                | PageAttributes::V)
+                .bits();
+    }
+
+    tlb_flush_full();
 }
 
 pub fn tlb_flush_global_full() {
@@ -166,9 +352,8 @@ pub fn tlb_flush_va_asid(va: usize, asid: usize) {
 }
 
 pub fn clone_kernel_tables(dst: &mut PageTable<L1>) {
-    // let tables = KERNEL_TABLES.lock();
+    let tables = KERNEL_TABLES.lock();
     for l1i in page_index::<L1>(USER_BOUNDARY)..512 {
-        // dst[l1i] = unsafe { PageEntry::from_raw(tables.l1.data[l1i]) };
-        dst[l1i] = unsafe { FIXED_L1[l1i] };
+        dst[l1i] = unsafe { PageEntry::from_raw(tables.l1.data[l1i]) };
     }
 }

kernel/arch/riscv64/src/mem/process.rs

@@ -11,12 +11,13 @@ use libk_mm_interface::{
         EntryLevel, EntryLevelDrop, EntryLevelExt, MapAttributes, NextPageTable, TableAllocator,
     },
 };
+use memtables::riscv64::PageAttributes;
 use yggdrasil_abi::error::Error;
 
 use crate::mem::{clone_kernel_tables, table::PageEntry};
 
 use super::{
-    table::{DroppableRange, PageAttributes, PageTable, L1, L2, L3},
+    table::{DroppableRange, PageTable, L1, L2, L3},
     KernelTableManagerImpl, USER_BOUNDARY,
 };
 

kernel/arch/riscv64/src/mem/table.rs

@@ -3,7 +3,6 @@ use core::{
     ops::{Index, IndexMut, Range},
 };
 
-use bitflags::bitflags;
 use libk_mm_interface::{
     address::{AsPhysicalAddress, PhysicalAddress},
     pointer::{PhysicalRef, PhysicalRefMut},
@@ -17,41 +16,7 @@ use yggdrasil_abi::error::Error;
 
 use super::{KernelTableManagerImpl, USER_BOUNDARY};
 
-// pub use memtables::riscv64::PageAttributes;
-bitflags! {
-    #[derive(Clone, Copy, PartialEq, Eq, Debug)]
-    pub struct PageAttributes: u64 {
-        const N = 1 << 63;
-        /// Software-tracked dirty bit (RSW[0])
-        const SW_DIRTY = 1 << 9;
-        /// Dirty bit
-        const D = 1 << 7;
-        /// Access bit
-        const A = 1 << 6;
-        /// Global mapping bit, implies all lower levels are also global
-        const G = 1 << 5;
-        /// U-mode access permission
-        const U = 1 << 4;
-        /// Execute permission
-        const X = 1 << 3;
-        /// Write permission
-        const W = 1 << 2;
-        /// Read-permission
-        const R = 1 << 1;
-        /// Valid bit
-        const V = 1 << 0;
-    }
-
-    // X W R    Meaning
-    // 0 0 0    Pointer to next level of page table
-    // 0 0 1    Read-only page
-    // 0 1 0    ---
-    // 0 1 1    Read-write page
-    // 1 0 0    Execute only
-    // 1 0 1    Read-execute page
-    // 1 1 0    ---
-    // 1 1 1    Read-write-execute page
-}
+pub use memtables::riscv64::PageAttributes;
 
 /// L3 - entry is 4KiB
 #[derive(Debug, Clone, Copy)]
@@ -75,10 +40,9 @@ impl EntryLevel for L1 {
     const SHIFT: usize = 30;
 }
 
-#[derive(Clone, Copy)]
 #[repr(C, align(0x1000))]
 pub struct PageTable<L: EntryLevel> {
-    pub entries: [PageEntry<L>; 512],
+    entries: [PageEntry<L>; 512],
 }
 
 #[derive(Clone, Copy, Debug, PartialEq)]
@@ -240,17 +204,6 @@ impl<L: NonTerminalEntryLevel + 'static> NextPageTable for PageTable<L> {
 }
 
 impl<L: NonTerminalEntryLevel> PageEntry<L> {
-    pub const fn identity_block(address: PhysicalAddress) -> Self {
-        const ATTR: u64 = PageAttributes::R.bits()
-            | PageAttributes::W.bits()
-            | PageAttributes::X.bits()
-            | PageAttributes::A.bits()
-            | PageAttributes::D.bits()
-            | PageAttributes::V.bits();
-
-        Self((address.into_u64() >> 2) | ATTR, PhantomData)
-    }
-
     pub fn block(address: PhysicalAddress, attrs: PageAttributes) -> Self {
         // TODO validate address alignment
         Self(

kernel/arch/x86_64/Cargo.toml

@@ -7,6 +7,7 @@ edition = "2021"
 yggdrasil-abi.workspace = true
 kernel-arch-interface.workspace = true
 libk-mm-interface.workspace = true
+memtables.workspace = true
 device-api = { workspace = true, features = ["derive"] }
 kernel-arch-x86.workspace = true
 

kernel/arch/x86_64/src/context.rs

@@ -5,14 +5,11 @@ use kernel_arch_interface::{
     task::{ForkFrame, StackBuilder, TaskContext, TaskFrame, UserContextInfo},
 };
 use kernel_arch_x86::registers::{FpuContext, CR3, MSR_IA32_FS_BASE};
-use libk_mm_interface::address::PhysicalAddress;
+use libk_mm_interface::address::{AsPhysicalAddress, PhysicalAddress};
 use tock_registers::interfaces::Writeable;
 use yggdrasil_abi::{arch::SavedFrame, error::Error};
 
-use crate::{
-    mem::{auto_lower_address, FIXED_PML4},
-    ArchitectureImpl,
-};
+use crate::{mem::KERNEL_TABLES, ArchitectureImpl};
 
 /// Frame saved onto the stack when taking an IRQ
 #[derive(Debug)]
@@ -434,7 +431,7 @@ impl<K: KernelTableManager, PA: PhysicalMemoryAllocator<Address = PhysicalAddres
     fn kernel(entry: extern "C" fn(usize) -> !, arg: usize) -> Result<Self, Error> {
         const KERNEL_TASK_PAGES: usize = 32;
 
-        let cr3 = auto_lower_address(&raw const FIXED_PML4);
+        let cr3: usize = unsafe { KERNEL_TABLES.lock().as_physical_address() }.into();
 
         let stack_base_phys = PA::allocate_contiguous_pages(KERNEL_TASK_PAGES)?;
         let stack_base = stack_base_phys.raw_virtualize::<K>();

kernel/arch/x86_64/src/lib.rs

@@ -1,5 +1,6 @@
 #![no_std]
 #![allow(clippy::new_without_default)]
+#![feature(naked_functions)]
 
 extern crate alloc;
 
@@ -68,16 +69,18 @@ impl PerCpuData {
 static IPI_QUEUES: OneTimeInit<Vec<IpiQueue<ArchitectureImpl>>> = OneTimeInit::new();
 pub static CPU_COUNT: AtomicUsize = AtomicUsize::new(1);
 
-#[unsafe(naked)]
+#[naked]
 extern "C" fn idle_task(_: usize) -> ! {
-    core::arch::naked_asm!(
-        r#"
+    unsafe {
+        core::arch::naked_asm!(
+            r#"
             1:
                 nop
                 jmp 1b
             "#,
-        options(att_syntax)
-    );
+            options(att_syntax)
+        );
+    }
 }
 
 impl ArchitectureImpl {

kernel/arch/x86_64/src/mem/device.rs

@@ -1,87 +0,0 @@
-use core::ops::Range;
-
-use kernel_arch_interface::mem::DeviceMemoryAttributes;
-use libk_mm_interface::{
-    address::PhysicalAddress,
-    table::{DevicePageManager, DevicePageManagerLevel},
-};
-
-use crate::mem::table::PageEntry;
-
-use super::{
-    table::{PageAttributes, PageTable, L2, L3},
-    DEVICE_MAPPING_OFFSET, DEVICE_MEMORY_L3_COUNT,
-};
-
-#[repr(transparent)]
-pub struct L2DeviceMemory(pub PageTable<L2>);
-#[repr(transparent)]
-pub struct L3DeviceMemory(pub [PageTable<L3>; DEVICE_MEMORY_L3_COUNT]);
-
-pub(super) static mut DEVICE_MEMORY: DevicePageManager<L3DeviceMemory, L2DeviceMemory> =
-    DevicePageManager::new(
-        L3DeviceMemory([PageTable::zeroed(); DEVICE_MEMORY_L3_COUNT]),
-        L2DeviceMemory(PageTable::zeroed()),
-    );
-
-impl DevicePageManagerLevel for L2DeviceMemory {
-    type Level = L2;
-    const VIRTUAL_BASE: usize = DEVICE_MAPPING_OFFSET;
-    const INDEX_RANGE: Range<usize> = DEVICE_MEMORY_L3_COUNT..512;
-
-    fn map_page(
-        &mut self,
-        index: usize,
-        physical: PhysicalAddress,
-        attrs: &DeviceMemoryAttributes,
-    ) {
-        let _ = attrs;
-        self.0[index] = PageEntry::<L2>::block(physical, PageAttributes::WRITABLE);
-    }
-
-    fn unmap_page(&mut self, index: usize) {
-        self.0[index] = PageEntry::INVALID;
-    }
-
-    fn is_mapped(&self, index: usize) -> bool {
-        self.0[index].is_present()
-    }
-
-    fn flush_range(range: Range<usize>) {
-        let _ = range;
-    }
-}
-
-impl DevicePageManagerLevel for L3DeviceMemory {
-    type Level = L3;
-    const VIRTUAL_BASE: usize = DEVICE_MAPPING_OFFSET;
-    const INDEX_RANGE: Range<usize> = 0..512 * DEVICE_MEMORY_L3_COUNT;
-
-    fn map_page(
-        &mut self,
-        index: usize,
-        physical: PhysicalAddress,
-        attrs: &DeviceMemoryAttributes,
-    ) {
-        let _ = attrs;
-        let l2i = index / 512;
-        let l3i = index % 512;
-        self.0[l2i][l3i] = PageEntry::page(physical, PageAttributes::WRITABLE);
-    }
-
-    fn unmap_page(&mut self, index: usize) {
-        let l2i = index / 512;
-        let l3i = index % 512;
-        self.0[l2i][l3i] = PageEntry::INVALID;
-    }
-
-    fn is_mapped(&self, index: usize) -> bool {
-        let l2i = index / 512;
-        let l3i = index % 512;
-        self.0[l2i][l3i].is_present()
-    }
-
-    fn flush_range(range: Range<usize>) {
-        let _ = range;
-    }
-}

kernel/arch/x86_64/src/mem/mod.rs

@@ -1,39 +1,98 @@
+use core::{
+    alloc::Layout,
+    ops::{Deref, DerefMut},
+    sync::atomic::{AtomicUsize, Ordering},
+};
+
 use kernel_arch_interface::{
     mem::{DeviceMemoryAttributes, KernelTableManager, RawDeviceMemoryMapping},
-    sync::IrqSafeSpinlock,
+    split_spinlock,
 };
 use kernel_arch_x86::registers::CR3;
 use libk_mm_interface::{
     address::PhysicalAddress,
-    table::{EntryLevel, EntryLevelExt},
+    table::{page_index, EntryLevel, EntryLevelExt},
 };
+use static_assertions::{const_assert_eq, const_assert_ne};
 use yggdrasil_abi::error::Error;
 
-use crate::{ArchitectureImpl, KERNEL_VIRT_OFFSET};
+use crate::KERNEL_VIRT_OFFSET;
 
-use self::table::{PageAttributes, PageEntry, PageTable, L0, L1, L2};
+use self::table::{PageAttributes, PageEntry, PageTable, L0, L1, L2, L3};
 
-pub mod device;
 pub mod process;
 pub mod table;
 
 #[derive(Debug)]
 pub struct KernelTableManagerImpl;
 
+const CANONICAL_ADDRESS_MASK: usize = 0xFFFF000000000000;
+const KERNEL_PHYS_BASE: usize = 0x200000;
+
+// Mapped at compile time
+const KERNEL_MAPPING_BASE: usize = KERNEL_VIRT_OFFSET + KERNEL_PHYS_BASE;
+const KERNEL_L0_INDEX: usize = page_index::<L0>(KERNEL_MAPPING_BASE);
+const KERNEL_L1_INDEX: usize = page_index::<L1>(KERNEL_MAPPING_BASE);
+const KERNEL_START_L2_INDEX: usize = page_index::<L2>(KERNEL_MAPPING_BASE);
+
+// Must not be zero, should be at 4MiB
+const_assert_ne!(KERNEL_START_L2_INDEX, 0);
+// From static mapping
+const_assert_eq!(KERNEL_L0_INDEX, 511);
+const_assert_eq!(KERNEL_L1_INDEX, 0);
+
+// Mapped at boot
+const EARLY_MAPPING_L2I: usize = KERNEL_START_L2_INDEX - 1;
+const DEVICE_MAPPING_L1I: usize = KERNEL_L1_INDEX + 2;
+const RAM_MAPPING_L0I: usize = KERNEL_L0_INDEX - 1;
+
+const DEVICE_MAPPING_L3_COUNT: usize = 4;
+
+split_spinlock! {
+    use libk_mm_interface::KernelImageObject;
+    use memtables::x86_64::FixedTables;
+    use crate::ArchitectureImpl;
+
+    #[link_section = ".data.tables"]
+    static KERNEL_TABLES: KernelImageObject<FixedTables> =
+        unsafe { KernelImageObject::new(FixedTables::zeroed()) };
+}
+
+// 2MiB for early mappings
+const EARLY_MAPPING_OFFSET: usize = CANONICAL_ADDRESS_MASK
+    | (KERNEL_L0_INDEX * L0::SIZE)
+    | (KERNEL_L1_INDEX * L1::SIZE)
+    | (EARLY_MAPPING_L2I * L2::SIZE);
+static mut EARLY_MAPPING_L3: PageTable<L3> = PageTable::zeroed();
+// 1GiB for device MMIO mapping
+const DEVICE_MAPPING_OFFSET: usize =
+    CANONICAL_ADDRESS_MASK | (KERNEL_L0_INDEX * L0::SIZE) | (DEVICE_MAPPING_L1I * L1::SIZE);
+static mut DEVICE_MAPPING_L2: PageTable<L2> = PageTable::zeroed();
+static mut DEVICE_MAPPING_L3S: [PageTable<L3>; DEVICE_MAPPING_L3_COUNT] =
+    [PageTable::zeroed(); DEVICE_MAPPING_L3_COUNT];
+// 512GiB for whole RAM mapping
+pub const RAM_MAPPING_OFFSET: usize = CANONICAL_ADDRESS_MASK | (RAM_MAPPING_L0I * L0::SIZE);
+pub static MEMORY_LIMIT: AtomicUsize = AtomicUsize::new(0);
+pub static mut RAM_MAPPING_L1: PageTable<L1> = PageTable::zeroed();
+
 impl KernelTableManager for KernelTableManagerImpl {
     fn virtualize(address: u64) -> usize {
         let address = address as usize;
-        if address >= IDENTITY_SIZE {
-            panic!("Invalid physical address to virtualize: {address:#x}");
+        if address < MEMORY_LIMIT.load(Ordering::Acquire) {
+            address + RAM_MAPPING_OFFSET
+        } else {
+            panic!("Invalid physical address: {:#x}", address);
         }
-        address + KERNEL_VIRT_OFFSET
     }
 
     fn physicalize(address: usize) -> u64 {
-        if address < KERNEL_VIRT_OFFSET || address - KERNEL_VIRT_OFFSET >= IDENTITY_SIZE {
-            panic!("Invalid virtualized address: {address:#x}");
+        if address < RAM_MAPPING_OFFSET
+            || address - RAM_MAPPING_OFFSET >= MEMORY_LIMIT.load(Ordering::Acquire)
+        {
+            panic!("Not a virtualized physical address: {:#x}", address);
         }
-        (address - KERNEL_VIRT_OFFSET) as _
+
+        (address - RAM_MAPPING_OFFSET) as _
     }
 
     unsafe fn map_device_pages(
@@ -41,42 +100,245 @@ impl KernelTableManager for KernelTableManagerImpl {
         count: usize,
         attrs: DeviceMemoryAttributes,
     ) -> Result<RawDeviceMemoryMapping<Self>, Error> {
-        let _guard = DEVICE_MEMORY_LOCK.lock();
-        #[allow(static_mut_refs)]
-        {
-            device::DEVICE_MEMORY.map_device_pages(PhysicalAddress::from_u64(base), count, attrs)
-        }
+        map_device_memory(PhysicalAddress::from_u64(base), count, attrs)
     }
 
     unsafe fn unmap_device_pages(mapping: &RawDeviceMemoryMapping<Self>) {
-        let _guard = DEVICE_MEMORY_LOCK.lock();
-        #[allow(static_mut_refs)]
-        {
-            device::DEVICE_MEMORY.unmap_device_pages(mapping);
+        unmap_device_memory(mapping)
+    }
+}
+
+// Early mappings
+unsafe fn map_early_pages(physical: PhysicalAddress, count: usize) -> Result<usize, Error> {
+    for l3i in 0..512 {
+        let mut taken = false;
+        for i in 0..count {
+            if EARLY_MAPPING_L3[i + l3i].is_present() {
+                taken = true;
+                break;
+            }
+        }
+
+        if taken {
+            continue;
+        }
+
+        for i in 0..count {
+            // TODO NX, NC
+            EARLY_MAPPING_L3[i + l3i] =
+                PageEntry::page(physical.add(i * L3::SIZE), PageAttributes::WRITABLE);
+            flush_tlb_entry(EARLY_MAPPING_OFFSET + (i + l3i) * L3::SIZE);
+        }
+
+        return Ok(EARLY_MAPPING_OFFSET + l3i * L3::SIZE);
+    }
+
+    Err(Error::OutOfMemory)
+}
+
+unsafe fn unmap_early_page(address: usize) {
+    if !(EARLY_MAPPING_OFFSET..EARLY_MAPPING_OFFSET + L2::SIZE).contains(&address) {
+        panic!("Tried to unmap invalid early mapping: {:#x}", address);
+    }
+
+    let l3i = (address - EARLY_MAPPING_OFFSET).page_index::<L3>();
+
+    assert!(EARLY_MAPPING_L3[l3i].is_present());
+    EARLY_MAPPING_L3[l3i] = PageEntry::INVALID;
+}
+
+// Device mappings
+unsafe fn map_device_memory_l3(
+    base: PhysicalAddress,
+    count: usize,
+    _attrs: DeviceMemoryAttributes,
+) -> Result<usize, Error> {
+    // TODO don't map pages if already mapped
+
+    'l0: for i in 0..DEVICE_MAPPING_L3_COUNT * 512 {
+        for j in 0..count {
+            let l2i = (i + j) / 512;
+            let l3i = (i + j) % 512;
+
+            if DEVICE_MAPPING_L3S[l2i][l3i].is_present() {
+                continue 'l0;
+            }
+        }
+
+        for j in 0..count {
+            let l2i = (i + j) / 512;
+            let l3i = (i + j) % 512;
+
+            // TODO NX, NC
+            DEVICE_MAPPING_L3S[l2i][l3i] =
+                PageEntry::page(base.add(j * L3::SIZE), PageAttributes::WRITABLE);
+        }
+
+        return Ok(DEVICE_MAPPING_OFFSET + i * L3::SIZE);
+    }
+
+    Err(Error::OutOfMemory)
+}
+
+unsafe fn map_device_memory_l2(
+    base: PhysicalAddress,
+    count: usize,
+    _attrs: DeviceMemoryAttributes,
+) -> Result<usize, Error> {
+    'l0: for i in DEVICE_MAPPING_L3_COUNT..512 {
+        for j in 0..count {
+            if DEVICE_MAPPING_L2[i + j].is_present() {
+                continue 'l0;
+            }
+        }
+
+        for j in 0..count {
+            DEVICE_MAPPING_L2[i + j] =
+                PageEntry::<L2>::block(base.add(j * L2::SIZE), PageAttributes::WRITABLE);
+        }
+
+        return Ok(DEVICE_MAPPING_OFFSET + i * L2::SIZE);
+    }
+
+    Err(Error::OutOfMemory)
+}
+
+unsafe fn map_device_memory(
+    base: PhysicalAddress,
+    size: usize,
+    attrs: DeviceMemoryAttributes,
+) -> Result<RawDeviceMemoryMapping<KernelTableManagerImpl>, Error> {
+    let l3_aligned = base.page_align_down::<L3>();
+    let l3_offset = base.page_offset::<L3>();
+    let page_count = (l3_offset + size).page_count::<L3>();
+
+    if page_count > 256 {
+        // Large mapping, use L2 mapping instead
+        let l2_aligned = base.page_align_down::<L2>();
+        let l2_offset = base.page_offset::<L2>();
+        let page_count = (l2_offset + size).page_count::<L2>();
+
+        let base_address = map_device_memory_l2(l2_aligned, page_count, attrs)?;
+        let address = base_address + l2_offset;
+
+        Ok(RawDeviceMemoryMapping::from_raw_parts(
+            l2_aligned.into_u64(),
+            address,
+            base_address,
+            page_count,
+            L2::SIZE,
+        ))
+    } else {
+        // Just map the pages directly
+        let base_address = map_device_memory_l3(l3_aligned, page_count, attrs)?;
+        let address = base_address + l3_offset;
+
+        Ok(RawDeviceMemoryMapping::from_raw_parts(
+            l3_aligned.into_u64(),
+            address,
+            base_address,
+            page_count,
+            L3::SIZE,
+        ))
+    }
+}
+
+unsafe fn unmap_device_memory(map: &RawDeviceMemoryMapping<KernelTableManagerImpl>) {
+    match map.page_size {
+        L3::SIZE => {
+            for i in 0..map.page_count {
+                let page = map.base_address + i * L3::SIZE;
+                let l2i = page.page_index::<L2>();
+                let l3i = page.page_index::<L3>();
+                assert!(DEVICE_MAPPING_L3S[l2i][l3i].is_present());
+                DEVICE_MAPPING_L3S[l2i][l3i] = PageEntry::INVALID;
+                flush_tlb_entry(page);
+            }
+        }
+        L2::SIZE => todo!(),
+        _ => unimplemented!(),
+    }
+}
+
+/// Memory mapping which may be used for performing early kernel initialization
+pub struct EarlyMapping<'a, T: ?Sized> {
+    value: &'a mut T,
+    page_count: usize,
+}
+
+impl<'a, T: Sized> EarlyMapping<'a, T> {
+    /// # Safety
+    ///
+    /// `physical` address provided must be a valid non-NULL address actually containing `T`.
+    pub unsafe fn map(physical: PhysicalAddress) -> Result<EarlyMapping<'a, T>, Error> {
+        let layout = Layout::new::<T>();
+        let aligned = physical.page_align_down::<L3>();
+        let offset = physical.page_offset::<L3>();
+        let page_count = (offset + layout.size()).div_ceil(L3::SIZE);
+
+        let virt = map_early_pages(aligned, page_count)?;
+        let value = &mut *((virt + offset) as *mut T);
+
+        Ok(EarlyMapping { value, page_count })
+    }
+
+    /// # Safety
+    ///
+    /// `physical` address provided must be a valid non-NULL address actually containing a `T`
+    /// slice of given `len`.
+    pub unsafe fn map_slice(
+        physical: PhysicalAddress,
+        len: usize,
+    ) -> Result<EarlyMapping<'a, [T]>, Error> {
+        let layout = Layout::array::<T>(len).unwrap();
+        let aligned = physical.page_align_down::<L3>();
+        let offset = physical.page_offset::<L3>();
+        let page_count = (offset + layout.size()).div_ceil(L3::SIZE);
+
+        let virt = map_early_pages(aligned, page_count)?;
+        let value = core::slice::from_raw_parts_mut((virt + offset) as *mut T, len);
+
+        Ok(EarlyMapping { value, page_count })
+    }
+}
+
+impl<T: ?Sized> Deref for EarlyMapping<'_, T> {
+    type Target = T;
+
+    fn deref(&self) -> &Self::Target {
+        self.value
+    }
+}
+
+impl<T: ?Sized> DerefMut for EarlyMapping<'_, T> {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        self.value
+    }
+}
+
+impl<T: ?Sized> Drop for EarlyMapping<'_, T> {
+    fn drop(&mut self) {
+        let address = (self.value as *mut T).addr() & !(L3::SIZE - 1);
+
+        for i in 0..self.page_count {
+            let page = address + i * L3::SIZE;
+
+            unsafe {
+                unmap_early_page(page);
+            }
         }
     }
 }
 
 pub fn clone_kernel_tables(dst: &mut PageTable<L0>) {
+    let tables = KERNEL_TABLES.lock();
     unsafe {
-        let kernel_l0i = KERNEL_VIRT_OFFSET.page_index::<L0>();
-        dst[kernel_l0i] = FIXED_PML4[kernel_l0i];
+        dst[KERNEL_L0_INDEX] = PageEntry::from_raw(tables.l0.data[KERNEL_L0_INDEX]);
+        dst[RAM_MAPPING_L0I] = PageEntry::from_raw(tables.l0.data[RAM_MAPPING_L0I]);
     }
 }
 
-const FIXED_PD_COUNT: usize = 32;
-const IDENTITY_SIZE: usize = FIXED_PD_COUNT * L1::SIZE;
-
-const DEVICE_MAPPING_L1I: usize = FIXED_PD_COUNT;
-const DEVICE_MEMORY_L3_COUNT: usize = 16;
-const DEVICE_MAPPING_OFFSET: usize = KERNEL_VIRT_OFFSET + IDENTITY_SIZE;
-
-static DEVICE_MEMORY_LOCK: IrqSafeSpinlock<ArchitectureImpl, ()> = IrqSafeSpinlock::new(());
-static mut FIXED_PDS: [PageTable<L2>; FIXED_PD_COUNT] = [PageTable::zeroed(); FIXED_PD_COUNT];
-static mut FIXED_PDPT: PageTable<L1> = PageTable::zeroed();
-pub static mut FIXED_PML4: PageTable<L0> = PageTable::zeroed();
-
-pub fn auto_lower_address<T>(pointer: *const T) -> usize {
+pub fn auto_address<T>(pointer: *const T) -> usize {
     let address = pointer.addr();
     if address < KERNEL_VIRT_OFFSET {
         address
@@ -85,80 +347,53 @@ pub fn auto_lower_address<T>(pointer: *const T) -> usize {
     }
 }
 
-pub unsafe fn init_fixed_tables(have_1gib_pages: bool) {
-    if have_1gib_pages {
-        for i in 0..IDENTITY_SIZE / L1::SIZE {
-            FIXED_PDPT[i] = PageEntry::<L1>::block(
-                PhysicalAddress::from_usize(i * L1::SIZE),
-                PageAttributes::WRITABLE,
-            );
-        }
-    } else {
-        for i in 0..IDENTITY_SIZE / L1::SIZE {
-            for j in 0..512 {
-                FIXED_PDS[i][j] = PageEntry::<L2>::block(
-                    PhysicalAddress::from_usize(i * L1::SIZE + j * L2::SIZE),
-                    PageAttributes::WRITABLE,
-                );
-            }
-            let pd_physical =
-                PhysicalAddress::from_usize(auto_lower_address(&raw const FIXED_PDS[i]));
-            FIXED_PDPT[i] = PageEntry::table(pd_physical, PageAttributes::WRITABLE);
-        }
+/// Sets up the following memory map:
+/// ...: KERNEL_TABLES.l0:
+///     * 0xFFFFFF0000000000 .. 0xFFFFFFFF8000000000        : RAM_MAPPING_L1
+///     * 0xFFFFFF8000000000 .. ...                         : KERNEL_TABLES.kernel_l1:
+///         * 0xFFFFFF8000000000 .. 0xFFFFFF8040000000      : KERNEL_TABLES.kernel_l2
+///             * 0xFFFFFF8000000000 .. 0xFFFFFF8000200000  : ---
+///             * 0xFFFFFF8000200000 .. 0xFFFFFF8000400000  : EARLY_MAPPING_L3
+///             * 0xFFFFFF8000400000 .. ...                 : KERNEL_TABLES.kernel_l3s
+///         * 0xFFFFFF8040000000 .. 0xFFFFFF8080000000      : ---
+///         * 0xFFFFFF8080000000 .. 0xFFFFFF8100000000      : DEVICE_MAPPING_L2
+///             * 0xFFFFFF8080000000 .. 0xFFFFFF8080800000  : DEVICE_MAPPING_L3S
+///             * 0xFFFFFF8080800000 .. 0xFFFFFF8100000000  : ...
+///
+/// # Safety
+///
+/// Unsafe, must only be called by BSP during its early init, must already be in "higher-half"
+pub unsafe fn init_fixed_tables() {
+    let mut tables = KERNEL_TABLES.lock();
+
+    // TODO this could be built in compile-time too?
+    let early_mapping_l3_phys = auto_address(&raw const EARLY_MAPPING_L3);
+    let device_mapping_l2_phys = auto_address(&raw const DEVICE_MAPPING_L2);
+    let ram_mapping_l1_phys = auto_address(&raw const RAM_MAPPING_L1);
+
+    for i in 0..DEVICE_MAPPING_L3_COUNT {
+        let device_mapping_l3_phys =
+            PhysicalAddress::from_usize(auto_address(&raw const DEVICE_MAPPING_L3S[i]));
+        DEVICE_MAPPING_L2[i] = PageEntry::table(device_mapping_l3_phys, PageAttributes::WRITABLE);
     }
 
-    // Device memory
-    let device_pd_physical =
-        PhysicalAddress::from_usize(auto_lower_address(&raw const device::DEVICE_MEMORY.large));
+    assert_eq!(tables.kernel_l2.data[EARLY_MAPPING_L2I], 0);
+    tables.kernel_l2.data[EARLY_MAPPING_L2I] = (early_mapping_l3_phys as u64)
+        | (PageAttributes::WRITABLE | PageAttributes::PRESENT).bits();
 
-    for i in 0..DEVICE_MEMORY_L3_COUNT {
-        let device_pt_physical = PhysicalAddress::from_usize(auto_lower_address(
-            &raw const device::DEVICE_MEMORY.normal.0[i],
-        ));
-        device::DEVICE_MEMORY.large.0[i] =
-            PageEntry::table(device_pt_physical, PageAttributes::WRITABLE);
-    }
+    assert_eq!(tables.kernel_l1.data[DEVICE_MAPPING_L1I], 0);
+    tables.kernel_l1.data[DEVICE_MAPPING_L1I] = (device_mapping_l2_phys as u64)
+        | (PageAttributes::WRITABLE | PageAttributes::PRESENT).bits();
 
-    FIXED_PDPT[DEVICE_MAPPING_L1I] =
-        PageEntry::<L1>::table(device_pd_physical, PageAttributes::WRITABLE);
+    assert_eq!(tables.l0.data[RAM_MAPPING_L0I], 0);
+    tables.l0.data[RAM_MAPPING_L0I] =
+        (ram_mapping_l1_phys as u64) | (PageAttributes::WRITABLE | PageAttributes::PRESENT).bits();
 
-    let pdpt_physical = PhysicalAddress::from_usize(auto_lower_address(&raw const FIXED_PDPT));
-
-    FIXED_PML4[KERNEL_VIRT_OFFSET.page_index::<L0>()] =
-        PageEntry::table(pdpt_physical, PageAttributes::WRITABLE);
-
-    let pml4_physical = auto_lower_address(&raw const FIXED_PML4);
-    CR3.set_address(pml4_physical);
+    // TODO ENABLE EFER.NXE
+    let cr3 = auto_address(&raw const tables.l0);
+    CR3.set_address(cr3);
 }
-//     let mut tables = KERNEL_TABLES.lock();
-//
-//     // TODO this could be built in compile-time too?
-//     let early_mapping_l3_phys = auto_address(&raw const EARLY_MAPPING_L3);
-//     let device_mapping_l2_phys = auto_address(&raw const DEVICE_MAPPING_L2);
-//     let ram_mapping_l1_phys = auto_address(&raw const RAM_MAPPING_L1);
-//
-//     for i in 0..DEVICE_MAPPING_L3_COUNT {
-//         let device_mapping_l3_phys =
-//             PhysicalAddress::from_usize(auto_address(&raw const DEVICE_MAPPING_L3S[i]));
-//         DEVICE_MAPPING_L2[i] = PageEntry::table(device_mapping_l3_phys, PageAttributes::WRITABLE);
-//     }
-//
-//     assert_eq!(tables.kernel_l2.data[EARLY_MAPPING_L2I], 0);
-//     tables.kernel_l2.data[EARLY_MAPPING_L2I] = (early_mapping_l3_phys as u64)
-//         | (PageAttributes::WRITABLE | PageAttributes::PRESENT).bits();
-//
-//     assert_eq!(tables.kernel_l1.data[DEVICE_MAPPING_L1I], 0);
-//     tables.kernel_l1.data[DEVICE_MAPPING_L1I] = (device_mapping_l2_phys as u64)
-//         | (PageAttributes::WRITABLE | PageAttributes::PRESENT).bits();
-//
-//     assert_eq!(tables.l0.data[RAM_MAPPING_L0I], 0);
-//     tables.l0.data[RAM_MAPPING_L0I] =
-//         (ram_mapping_l1_phys as u64) | (PageAttributes::WRITABLE | PageAttributes::PRESENT).bits();
-//
-//     // TODO ENABLE EFER.NXE
-//     let cr3 = auto_address(&raw const tables.l0);
-// }
-//
+
 /// # Safety
 ///
 /// `address` must be page-aligned.

kernel/driver/block/ahci/src/port.rs

@@ -235,7 +235,7 @@ impl AhciPort {
         .await
     }
 
-    async fn submit<C: AtaCommand>(&self, command: &C) -> Result<SubmittedCommand<'_>, AhciError> {
+    async fn submit<C: AtaCommand>(&self, command: &C) -> Result<SubmittedCommand, AhciError> {
         if command.prd_count() > 2 {
             log::warn!("TODO: AHCI doesn't like 3+ PRD transfers");
             return Err(AhciError::RegionTooLarge);

kernel/driver/bus/pci/src/capability/express.rs

@@ -0,0 +1,177 @@
+use core::time::Duration;
+
+use libk::{error::Error, task::runtime::psleep};
+use tock_registers::{fields::FieldValue, register_bitfields, LocalRegisterCopy};
+
+use crate::{PciCapability, PciCapabilityId, PciConfigurationSpace};
+
+// bitflags! {
+//     pub struct PcieLinkControl: u16 {
+//         const ASPM_DISABLE = 0 << 0;
+//         // Active state power management control
+//         const ASPM_MASK = 0x3 << 0;
+//         // Enable clock power management
+//         const ECPM = 1 << 8;
+//     }
+// }
+
+register_bitfields! {
+    u32,
+    pub DeviceCapabilities [
+        MAX_PAYLOAD_SIZE            OFFSET(0)   NUMBITS(3)  [],
+        PHANTOM_FUNCTIONS           OFFSET(3)   NUMBITS(2)  [],
+        L0S_ACCEPTABLE_LATENCY      OFFSET(6)   NUMBITS(3)  [],
+        L1_ACCEPTABLE_LATENCY       OFFSET(9)   NUMBITS(3)  [],
+        ROLE_ERROR_REPORTING        OFFSET(15)  NUMBITS(1)  [],
+        CAPTURED_SLOT_POWER_LIMIT   OFFSET(18)  NUMBITS(8)  [],
+        CAPTURED_SLOT_POWER_SCALE   OFFSET(26)  NUMBITS(3)  [],
+        FUNCTION_LEVEL_RESET        OFFSET(28)  NUMBITS(1)  [],
+    ],
+    pub LinkCapabilities [
+        MAX_LINK_SPEED              OFFSET(0)   NUMBITS(4)  [],
+        MAX_LINK_WIDTH              OFFSET(4)   NUMBITS(6)  [],
+        ASPM                        OFFSET(10)  NUMBITS(2)  [],
+        L0S_EXIT_LATENCY            OFFSET(12)  NUMBITS(3)  [],
+        L1_EXIT_LATENCY             OFFSET(15)  NUMBITS(3)  [],
+        CLOCK_PM                    OFFSET(18)  NUMBITS(1)  [],
+        SURPRISE_DOWN_ERROR         OFFSET(19)  NUMBITS(1)  [],
+        DATA_LINK_ACTIVE_REPORTING  OFFSET(20)  NUMBITS(1)  [],
+        LINK_BANDWIDTH_NOTIFICATION OFFSET(21)  NUMBITS(1)  [],
+        ASPM_COMPLIANCE             OFFSET(22)  NUMBITS(1)  [],
+        PORT_NUMBER                 OFFSET(24)  NUMBITS(8)  [],
+    ],
+}
+
+register_bitfields! {
+    u16,
+    pub DeviceControl [
+        CORRECTABLE_ERROR_REPORTING OFFSET(0)   NUMBITS(1)  [],
+        NONFATAL_ERROR_REPORTING    OFFSET(1)   NUMBITS(1)  [],
+        FATAL_ERROR_REPORTING       OFFSET(2)   NUMBITS(1)  [],
+        UNSUPPORTED_REQ_REPORTING   OFFSET(3)   NUMBITS(1)  [],
+        RELAXED_ORDERING            OFFSET(4)   NUMBITS(1)  [],
+        MAX_PAYLOAD_SIZE            OFFSET(5)   NUMBITS(3)  [],
+        EXTENDED_TAG_FIELD          OFFSET(8)   NUMBITS(1)  [],
+        PHANTOM_FUNCTIONS           OFFSET(9)   NUMBITS(1)  [],
+        AUX_POWER_PM_ENABLE         OFFSET(10)  NUMBITS(1)  [],
+        NO_SNOOP                    OFFSET(11)  NUMBITS(1)  [],
+        MAX_READ_REQ_SIZE           OFFSET(12)  NUMBITS(3)  [],
+        FUNCTION_LEVEL_RESET        OFFSET(15)  NUMBITS(1)  [],
+    ],
+    pub LinkControl [
+        ASPM                        OFFSET(0)   NUMBITS(2)  [
+            Disabled = 0,
+            L0sEntryEnabled = 1,
+            L1EntryEnabled = 2,
+            L0sL1EntryEnabled = 3,
+        ],
+        READ_COMPLETION_BOUNDARY    OFFSET(3)   NUMBITS(1)  [],
+        LINK_DISABLE                OFFSET(4)   NUMBITS(1)  [],
+        RETRAIN_LINK                OFFSET(5)   NUMBITS(1)  [],
+        COMMON_CLOCK_CONFIG         OFFSET(6)   NUMBITS(1)  [],
+        EXTENDED_SYNCH              OFFSET(7)   NUMBITS(1)  [],
+        CLOCK_PM                    OFFSET(8)   NUMBITS(1)  [],
+        HARDWARE_ABW_DISABLE        OFFSET(9)   NUMBITS(1)  [],
+        LINK_BW_MANAGEMENT_IRQ      OFFSET(10)  NUMBITS(1)  [],
+        LINK_ABW_IRQ                OFFSET(11)  NUMBITS(1)  [],
+    ],
+}
+
+/// PCIe capability
+pub struct PciExpressCapability;
+
+/// PCI Express capability data structure
+pub struct PciExpressData<'s, S: PciConfigurationSpace + ?Sized + 's> {
+    space: &'s S,
+    offset: usize,
+}
+
+impl PciCapability for PciExpressCapability {
+    const ID: PciCapabilityId = PciCapabilityId::PciExpress;
+    type CapabilityData<'a, S: PciConfigurationSpace + ?Sized + 'a> = PciExpressData<'a, S>;
+
+    fn data<'s, S: PciConfigurationSpace + ?Sized + 's>(
+        space: &'s S,
+        offset: usize,
+        _len: usize,
+    ) -> Self::CapabilityData<'s, S> {
+        PciExpressData { space, offset }
+    }
+}
+
+macro_rules! reg_read {
+    ($self:expr, $offset:literal, u32) => {
+        $self.space.read_u32($self.offset + $offset)
+    };
+    ($self:expr, $offset:literal, u16) => {
+        $self.space.read_u16($self.offset + $offset)
+    };
+}
+macro_rules! reg_write {
+    ($self:expr, $offset:literal, u32, $value:expr) => {
+        $self.space.write_u32($self.offset + $offset, $value)
+    };
+    ($self:expr, $offset:literal, u16, $value:expr) => {
+        $self.space.write_u16($self.offset + $offset, $value)
+    };
+}
+macro_rules! make_register {
+    (
+        $reg:ident : $ty:ident @ $offset:literal {
+            $get:ident
+            $(, $set:ident, $modify:ident)?
+            $(,)?
+        }
+    ) => {
+        pub fn $get(&self) -> LocalRegisterCopy<$ty, $reg::Register> {
+            LocalRegisterCopy::new(reg_read!(self, $offset, $ty))
+        }
+
+        $(
+            pub fn $set(&mut self, value: $ty) {
+                reg_write!(self, $offset, $ty, value)
+            }
+
+            pub fn $modify(&mut self, field: FieldValue<$ty, $reg::Register>) {
+                let mut value = self.$get();
+                value.modify(field);
+                self.$set(value.get());
+            }
+        )?
+    };
+}
+
+impl<'s, S: PciConfigurationSpace + ?Sized + 's> PciExpressData<'s, S> {
+    make_register!(DeviceCapabilities : u32 @ 0x04 { device_capabilities });
+    make_register!(DeviceControl : u16 @ 0x08 {
+        device_control,
+        set_device_control,
+        modify_device_control,
+    });
+    make_register!(LinkCapabilities : u32 @ 0x0C { link_capabilities });
+    make_register!(LinkControl : u16 @ 0x10 {
+        link_control,
+        set_link_control,
+        modify_link_control,
+    });
+
+    pub fn function_level_reset(&mut self) -> Result<(), Error> {
+        if self
+            .device_capabilities()
+            .matches_all(DeviceCapabilities::FUNCTION_LEVEL_RESET::SET)
+        {
+            self.modify_device_control(DeviceControl::FUNCTION_LEVEL_RESET::SET);
+            psleep(Duration::from_millis(10));
+            Ok(())
+        } else {
+            Err(Error::NotImplemented)
+        }
+    }
+
+    pub fn hot_link_reset(&mut self) {
+        self.modify_link_control(LinkControl::LINK_DISABLE::SET);
+        psleep(Duration::from_millis(10));
+        self.modify_link_control(LinkControl::LINK_DISABLE::CLEAR);
+        psleep(Duration::from_millis(100));
+    }
+}

kernel/driver/bus/pci/src/capability/mod.rs

@@ -0,0 +1,14 @@
+//! PCI capability structures and queries
+
+pub mod express;
+pub mod msi;
+pub mod power;
+pub mod virtio;
+
+pub use express::PciExpressCapability;
+pub use msi::{MsiCapability, MsiXCapability};
+pub use power::PowerManagementCapability;
+pub use virtio::{
+    VirtioCapability, VirtioCommonConfigCapability, VirtioDeviceConfigCapability,
+    VirtioInterruptStatusCapability, VirtioNotifyConfigCapability,
+};

kernel/driver/bus/pci/src/capability/msi.rs

@@ -1,91 +1,21 @@
-//! PCI capability structures and queries
-
 use alloc::{sync::Arc, vec, vec::Vec};
-use bitflags::bitflags;
+
 use device_api::interrupt::{
     InterruptAffinity, InterruptHandler, MessageInterruptController, MsiInfo,
 };
+use libk::error::Error;
 use libk_mm::{address::PhysicalAddress, device::DeviceMemoryIoMut};
 use tock_registers::{
     interfaces::{Readable, Writeable},
     registers::{ReadWrite, WriteOnly},
 };
-use yggdrasil_abi::error::Error;
-
-use crate::PciBaseAddress;
-
-use super::{PciCapability, PciCapabilityId, PciConfigurationSpace};
-
-bitflags! {
-    pub struct PcieLinkControl: u16 {
-        const ASPM_DISABLE = 0 << 0;
-        // Active state power management control
-        const ASPM_MASK = 0x3 << 0;
-        // Enable clock power management
-        const ECPM = 1 << 8;
-    }
-}
 
 #[cfg(any(target_arch = "x86", target_arch = "x86_64", rust_analyzer))]
 use core::mem::offset_of;
 #[cfg(any(target_arch = "x86", target_arch = "x86_64", rust_analyzer))]
 use kernel_arch_x86::intrinsics;
 
-pub trait VirtioCapabilityData<'s, S: PciConfigurationSpace + ?Sized + 's>: Sized {
-    fn from_space_offset(space: &'s S, offset: usize) -> Self;
-
-    fn space(&self) -> &'s S;
-    fn offset(&self) -> usize;
-
-    fn bar_index(&self) -> Option<usize> {
-        let value = self.space().read_u8(self.offset() + 4);
-        (value <= 0x5).then_some(value as _)
-    }
-
-    fn bar_offset(&self) -> usize {
-        let value = self.space().read_u32(self.offset() + 8);
-        value as _
-    }
-
-    fn length(&self) -> usize {
-        let value = self.space().read_u32(self.offset() + 12);
-        value as _
-    }
-}
-
-pub trait VirtioCapability {
-    const CFG_TYPE: u8;
-    const MIN_LEN: usize = 0;
-    type Output<'a, S: PciConfigurationSpace + ?Sized + 'a>: VirtioCapabilityData<'a, S>;
-}
-
-/// Power management capability entry
-pub struct PowerManagementCapability;
-/// MSI-X capability query
-pub struct MsiXCapability;
-/// MSI capability query
-pub struct MsiCapability;
-/// PCIe capability
-pub struct PciExpressCapability;
-
-// VirtIO-over-PCI capabilities
-/// VirtIO PCI configuration access
-pub struct VirtioDeviceConfigCapability;
-/// VirtIO common configuration
-pub struct VirtioCommonConfigCapability;
-/// VirtIO notify configuration
-pub struct VirtioNotifyConfigCapability;
-/// VirtIO interrupt status
-pub struct VirtioInterruptStatusCapability;
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum DevicePowerState {
-    D0,
-    D1,
-    D2,
-    D3Cold,
-    D3Hot,
-}
+use crate::{PciBaseAddress, PciCapability, PciCapabilityId, PciConfigurationSpace};
 
 /// Represents an entry in MSI-X vector table
 #[repr(C)]
@@ -109,11 +39,10 @@ pub struct MsiXVectorTable<'a> {
     len: usize,
 }
 
-/// PCI Power Management capability data structure
-pub struct PowerManagementData<'s, S: PciConfigurationSpace + ?Sized + 's> {
-    space: &'s S,
-    offset: usize,
-}
+/// MSI-X capability query
+pub struct MsiXCapability;
+/// MSI capability query
+pub struct MsiCapability;
 
 /// MSI-X capability data structure
 pub struct MsiXData<'s, S: PciConfigurationSpace + ?Sized + 's> {
@@ -127,63 +56,6 @@ pub struct MsiData<'s, S: PciConfigurationSpace + ?Sized + 's> {
     offset: usize,
 }
 
-/// PCI Express capability data structure
-pub struct PcieData<'s, S: PciConfigurationSpace + ?Sized + 's> {
-    space: &'s S,
-    offset: usize,
-}
-
-pub struct VirtioDeviceConfigData<'s, S: PciConfigurationSpace + ?Sized + 's> {
-    space: &'s S,
-    offset: usize,
-}
-
-pub struct VirtioCommonConfigData<'s, S: PciConfigurationSpace + ?Sized + 's> {
-    space: &'s S,
-    offset: usize,
-}
-
-pub struct VirtioNotifyConfigData<'s, S: PciConfigurationSpace + ?Sized + 's> {
-    space: &'s S,
-    offset: usize,
-}
-
-pub struct VirtioInterruptStatusData<'s, S: PciConfigurationSpace + ?Sized + 's> {
-    space: &'s S,
-    offset: usize,
-}
-
-impl<T: VirtioCapability> PciCapability for T {
-    const ID: PciCapabilityId = PciCapabilityId::VendorSpecific;
-    type CapabilityData<'a, S: PciConfigurationSpace + ?Sized + 'a> = T::Output<'a, S>;
-
-    fn check<S: PciConfigurationSpace + ?Sized>(space: &S, offset: usize, len: usize) -> bool {
-        let cfg_type = space.read_u8(offset + 3);
-        cfg_type == T::CFG_TYPE && len >= T::MIN_LEN
-    }
-
-    fn data<'s, S: PciConfigurationSpace + ?Sized + 's>(
-        space: &'s S,
-        offset: usize,
-        _len: usize,
-    ) -> Self::CapabilityData<'s, S> {
-        T::Output::from_space_offset(space, offset)
-    }
-}
-
-impl PciCapability for PowerManagementCapability {
-    const ID: PciCapabilityId = PciCapabilityId::PowerManagement;
-    type CapabilityData<'a, S: PciConfigurationSpace + ?Sized + 'a> = PowerManagementData<'a, S>;
-
-    fn data<'s, S: PciConfigurationSpace + ?Sized + 's>(
-        space: &'s S,
-        offset: usize,
-        _len: usize,
-    ) -> Self::CapabilityData<'s, S> {
-        PowerManagementData { space, offset }
-    }
-}
-
 impl PciCapability for MsiXCapability {
     const ID: PciCapabilityId = PciCapabilityId::MsiX;
     type CapabilityData<'a, S: PciConfigurationSpace + ?Sized + 'a> = MsiXData<'a, S>;
@@ -210,167 +82,6 @@ impl PciCapability for MsiCapability {
     }
 }
 
-impl PciCapability for PciExpressCapability {
-    const ID: PciCapabilityId = PciCapabilityId::PciExpress;
-    type CapabilityData<'a, S: PciConfigurationSpace + ?Sized + 'a> = PcieData<'a, S>;
-
-    fn data<'s, S: PciConfigurationSpace + ?Sized + 's>(
-        space: &'s S,
-        offset: usize,
-        _len: usize,
-    ) -> Self::CapabilityData<'s, S> {
-        PcieData { space, offset }
-    }
-}
-
-impl VirtioCapability for VirtioDeviceConfigCapability {
-    const CFG_TYPE: u8 = 0x04;
-    type Output<'a, S: PciConfigurationSpace + ?Sized + 'a> = VirtioDeviceConfigData<'a, S>;
-}
-
-impl<'s, S: PciConfigurationSpace + ?Sized + 's> VirtioCapabilityData<'s, S>
-    for VirtioDeviceConfigData<'s, S>
-{
-    fn from_space_offset(space: &'s S, offset: usize) -> Self {
-        Self { space, offset }
-    }
-
-    fn space(&self) -> &'s S {
-        self.space
-    }
-
-    fn offset(&self) -> usize {
-        self.offset
-    }
-}
-
-impl VirtioCapability for VirtioCommonConfigCapability {
-    const CFG_TYPE: u8 = 0x01;
-    type Output<'a, S: PciConfigurationSpace + ?Sized + 'a> = VirtioCommonConfigData<'a, S>;
-}
-
-impl<'s, S: PciConfigurationSpace + ?Sized + 's> VirtioCapabilityData<'s, S>
-    for VirtioCommonConfigData<'s, S>
-{
-    fn from_space_offset(space: &'s S, offset: usize) -> Self {
-        Self { space, offset }
-    }
-
-    fn space(&self) -> &'s S {
-        self.space
-    }
-
-    fn offset(&self) -> usize {
-        self.offset
-    }
-}
-
-impl VirtioCapability for VirtioNotifyConfigCapability {
-    const CFG_TYPE: u8 = 0x02;
-    const MIN_LEN: usize = 0x14;
-    type Output<'a, S: PciConfigurationSpace + ?Sized + 'a> = VirtioNotifyConfigData<'a, S>;
-}
-
-impl<'s, S: PciConfigurationSpace + ?Sized + 's> VirtioNotifyConfigData<'s, S> {
-    pub fn offset_multiplier(&self) -> usize {
-        self.space.read_u32(self.offset + 16) as usize
-    }
-}
-
-impl<'s, S: PciConfigurationSpace + ?Sized + 's> VirtioCapabilityData<'s, S>
-    for VirtioNotifyConfigData<'s, S>
-{
-    fn from_space_offset(space: &'s S, offset: usize) -> Self {
-        Self { space, offset }
-    }
-
-    fn space(&self) -> &'s S {
-        self.space
-    }
-
-    fn offset(&self) -> usize {
-        self.offset
-    }
-}
-
-impl VirtioCapability for VirtioInterruptStatusCapability {
-    const CFG_TYPE: u8 = 0x03;
-    const MIN_LEN: usize = 1;
-    type Output<'a, S: PciConfigurationSpace + ?Sized + 'a> = VirtioInterruptStatusData<'a, S>;
-}
-
-impl<'s, S: PciConfigurationSpace + ?Sized + 's> VirtioInterruptStatusData<'s, S> {
-    pub fn read_status(&self) -> (bool, bool) {
-        todo!()
-    }
-}
-
-impl<'s, S: PciConfigurationSpace + ?Sized + 's> VirtioCapabilityData<'s, S>
-    for VirtioInterruptStatusData<'s, S>
-{
-    fn from_space_offset(space: &'s S, offset: usize) -> Self {
-        Self { space, offset }
-    }
-
-    fn space(&self) -> &'s S {
-        self.space
-    }
-
-    fn offset(&self) -> usize {
-        self.offset
-    }
-}
-
-impl<'s, S: PciConfigurationSpace + ?Sized + 's> PowerManagementData<'s, S> {
-    pub fn set_device_power_state(&self, state: DevicePowerState) {
-        let pmcsr = self.space.read_u16(self.offset + 4) & !0x3;
-        let current = self.get_device_power_state();
-
-        if state == current {
-            return;
-        }
-
-        log::info!("Set device power state: {state:?}");
-
-        match state {
-            DevicePowerState::D0 => {
-                // power = 0b00 | PME_EN
-                self.space.write_u16(self.offset + 4, pmcsr);
-            }
-            _ => {
-                log::warn!("TODO: {state:?} power state");
-            }
-        }
-    }
-
-    pub fn set_pme_en(&self, state: bool) {
-        let pmcsr = self.space.read_u16(self.offset + 4);
-        let new = if state {
-            pmcsr | (1 << 8)
-        } else {
-            pmcsr & !(1 << 8)
-        };
-        if pmcsr == new {
-            return;
-        }
-
-        log::info!("Set PMCSR.PME_En = {state}");
-
-        self.space.write_u16(self.offset + 4, new);
-    }
-
-    pub fn get_device_power_state(&self) -> DevicePowerState {
-        let pmcsr = self.space.read_u16(self.offset + 4);
-        match pmcsr & 0x3 {
-            0b00 => DevicePowerState::D0,
-            0b01 => DevicePowerState::D1,
-            0b10 => DevicePowerState::D2,
-            0b11 => DevicePowerState::D3Hot,
-            _ => unreachable!(),
-        }
-    }
-}
-
 impl<'s, S: PciConfigurationSpace + ?Sized + 's> MsiXData<'s, S> {
     // TODO use pending bits as well
     /// Maps and returns the vector table associated with the device's MSI-X capability
@@ -601,13 +312,3 @@ impl<'s, S: PciConfigurationSpace + ?Sized + 's> MsiData<'s, S> {
         Ok(info)
     }
 }
-
-impl<'s, S: PciConfigurationSpace + ?Sized + 's> PcieData<'s, S> {
-    pub fn link_control(&self) -> PcieLinkControl {
-        PcieLinkControl::from_bits_retain(self.space.read_u16(self.offset + 0x10))
-    }
-
-    pub fn set_link_control(&mut self, value: PcieLinkControl) {
-        self.space.write_u16(self.offset + 0x10, value.bits());
-    }
-}

kernel/driver/bus/pci/src/capability/power.rs

@@ -0,0 +1,82 @@
+use crate::{PciCapability, PciCapabilityId, PciConfigurationSpace};
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum DevicePowerState {
+    D0,
+    D1,
+    D2,
+    D3Cold,
+    D3Hot,
+}
+
+/// Power management capability entry
+pub struct PowerManagementCapability;
+
+/// PCI Power Management capability data structure
+pub struct PowerManagementData<'s, S: PciConfigurationSpace + ?Sized + 's> {
+    space: &'s S,
+    offset: usize,
+}
+
+impl PciCapability for PowerManagementCapability {
+    const ID: PciCapabilityId = PciCapabilityId::PowerManagement;
+    type CapabilityData<'a, S: PciConfigurationSpace + ?Sized + 'a> = PowerManagementData<'a, S>;
+
+    fn data<'s, S: PciConfigurationSpace + ?Sized + 's>(
+        space: &'s S,
+        offset: usize,
+        _len: usize,
+    ) -> Self::CapabilityData<'s, S> {
+        PowerManagementData { space, offset }
+    }
+}
+
+impl<'s, S: PciConfigurationSpace + ?Sized + 's> PowerManagementData<'s, S> {
+    pub fn set_device_power_state(&self, state: DevicePowerState) {
+        let pmcsr = self.space.read_u16(self.offset + 4) & !0x3;
+        let current = self.get_device_power_state();
+
+        if state == current {
+            return;
+        }
+
+        log::info!("Set device power state: {state:?}");
+
+        match state {
+            DevicePowerState::D0 => {
+                // power = 0b00 | PME_EN
+                self.space.write_u16(self.offset + 4, pmcsr);
+            }
+            _ => {
+                log::warn!("TODO: {state:?} power state");
+            }
+        }
+    }
+
+    pub fn set_pme_en(&self, state: bool) {
+        let pmcsr = self.space.read_u16(self.offset + 4);
+        let new = if state {
+            pmcsr | (1 << 8)
+        } else {
+            pmcsr & !(1 << 8)
+        };
+        if pmcsr == new {
+            return;
+        }
+
+        log::info!("Set PMCSR.PME_En = {state}");
+
+        self.space.write_u16(self.offset + 4, new);
+    }
+
+    pub fn get_device_power_state(&self) -> DevicePowerState {
+        let pmcsr = self.space.read_u16(self.offset + 4);
+        match pmcsr & 0x3 {
+            0b00 => DevicePowerState::D0,
+            0b01 => DevicePowerState::D1,
+            0b10 => DevicePowerState::D2,
+            0b11 => DevicePowerState::D3Hot,
+            _ => unreachable!(),
+        }
+    }
+}

kernel/driver/bus/pci/src/capability/virtio.rs

@@ -0,0 +1,175 @@
+use crate::{PciCapability, PciCapabilityId, PciConfigurationSpace};
+
+pub trait VirtioCapabilityData<'s, S: PciConfigurationSpace + ?Sized + 's>: Sized {
+    fn from_space_offset(space: &'s S, offset: usize) -> Self;
+
+    fn space(&self) -> &'s S;
+    fn offset(&self) -> usize;
+
+    fn bar_index(&self) -> Option<usize> {
+        let value = self.space().read_u8(self.offset() + 4);
+        (value <= 0x5).then_some(value as _)
+    }
+
+    fn bar_offset(&self) -> usize {
+        let value = self.space().read_u32(self.offset() + 8);
+        value as _
+    }
+
+    fn length(&self) -> usize {
+        let value = self.space().read_u32(self.offset() + 12);
+        value as _
+    }
+}
+
+pub trait VirtioCapability {
+    const CFG_TYPE: u8;
+    const MIN_LEN: usize = 0;
+    type Output<'a, S: PciConfigurationSpace + ?Sized + 'a>: VirtioCapabilityData<'a, S>;
+}
+
+// VirtIO-over-PCI capabilities
+/// VirtIO PCI configuration access
+pub struct VirtioDeviceConfigCapability;
+/// VirtIO common configuration
+pub struct VirtioCommonConfigCapability;
+/// VirtIO notify configuration
+pub struct VirtioNotifyConfigCapability;
+/// VirtIO interrupt status
+pub struct VirtioInterruptStatusCapability;
+
+pub struct VirtioDeviceConfigData<'s, S: PciConfigurationSpace + ?Sized + 's> {
+    space: &'s S,
+    offset: usize,
+}
+
+pub struct VirtioCommonConfigData<'s, S: PciConfigurationSpace + ?Sized + 's> {
+    space: &'s S,
+    offset: usize,
+}
+
+pub struct VirtioNotifyConfigData<'s, S: PciConfigurationSpace + ?Sized + 's> {
+    space: &'s S,
+    offset: usize,
+}
+
+pub struct VirtioInterruptStatusData<'s, S: PciConfigurationSpace + ?Sized + 's> {
+    space: &'s S,
+    offset: usize,
+}
+
+impl<T: VirtioCapability> PciCapability for T {
+    const ID: PciCapabilityId = PciCapabilityId::VendorSpecific;
+    type CapabilityData<'a, S: PciConfigurationSpace + ?Sized + 'a> = T::Output<'a, S>;
+
+    fn check<S: PciConfigurationSpace + ?Sized>(space: &S, offset: usize, len: usize) -> bool {
+        let cfg_type = space.read_u8(offset + 3);
+        cfg_type == T::CFG_TYPE && len >= T::MIN_LEN
+    }
+
+    fn data<'s, S: PciConfigurationSpace + ?Sized + 's>(
+        space: &'s S,
+        offset: usize,
+        _len: usize,
+    ) -> Self::CapabilityData<'s, S> {
+        T::Output::from_space_offset(space, offset)
+    }
+}
+
+impl VirtioCapability for VirtioDeviceConfigCapability {
+    const CFG_TYPE: u8 = 0x04;
+    type Output<'a, S: PciConfigurationSpace + ?Sized + 'a> = VirtioDeviceConfigData<'a, S>;
+}
+
+impl<'s, S: PciConfigurationSpace + ?Sized + 's> VirtioCapabilityData<'s, S>
+    for VirtioDeviceConfigData<'s, S>
+{
+    fn from_space_offset(space: &'s S, offset: usize) -> Self {
+        Self { space, offset }
+    }
+
+    fn space(&self) -> &'s S {
+        self.space
+    }
+
+    fn offset(&self) -> usize {
+        self.offset
+    }
+}
+
+impl VirtioCapability for VirtioCommonConfigCapability {
+    const CFG_TYPE: u8 = 0x01;
+    type Output<'a, S: PciConfigurationSpace + ?Sized + 'a> = VirtioCommonConfigData<'a, S>;
+}
+
+impl<'s, S: PciConfigurationSpace + ?Sized + 's> VirtioCapabilityData<'s, S>
+    for VirtioCommonConfigData<'s, S>
+{
+    fn from_space_offset(space: &'s S, offset: usize) -> Self {
+        Self { space, offset }
+    }
+
+    fn space(&self) -> &'s S {
+        self.space
+    }
+
+    fn offset(&self) -> usize {
+        self.offset
+    }
+}
+
+impl VirtioCapability for VirtioNotifyConfigCapability {
+    const CFG_TYPE: u8 = 0x02;
+    const MIN_LEN: usize = 0x14;
+    type Output<'a, S: PciConfigurationSpace + ?Sized + 'a> = VirtioNotifyConfigData<'a, S>;
+}
+
+impl<'s, S: PciConfigurationSpace + ?Sized + 's> VirtioNotifyConfigData<'s, S> {
+    pub fn offset_multiplier(&self) -> usize {
+        self.space.read_u32(self.offset + 16) as usize
+    }
+}
+
+impl<'s, S: PciConfigurationSpace + ?Sized + 's> VirtioCapabilityData<'s, S>
+    for VirtioNotifyConfigData<'s, S>
+{
+    fn from_space_offset(space: &'s S, offset: usize) -> Self {
+        Self { space, offset }
+    }
+
+    fn space(&self) -> &'s S {
+        self.space
+    }
+
+    fn offset(&self) -> usize {
+        self.offset
+    }
+}
+
+impl VirtioCapability for VirtioInterruptStatusCapability {
+    const CFG_TYPE: u8 = 0x03;
+    const MIN_LEN: usize = 1;
+    type Output<'a, S: PciConfigurationSpace + ?Sized + 'a> = VirtioInterruptStatusData<'a, S>;
+}
+
+impl<'s, S: PciConfigurationSpace + ?Sized + 's> VirtioInterruptStatusData<'s, S> {
+    pub fn read_status(&self) -> (bool, bool) {
+        todo!()
+    }
+}
+
+impl<'s, S: PciConfigurationSpace + ?Sized + 's> VirtioCapabilityData<'s, S>
+    for VirtioInterruptStatusData<'s, S>
+{
+    fn from_space_offset(space: &'s S, offset: usize) -> Self {
+        Self { space, offset }
+    }
+
+    fn space(&self) -> &'s S {
+        self.space
+    }
+
+    fn offset(&self) -> usize {
+        self.offset
+    }
+}

kernel/driver/bus/pci/src/device.rs

@@ -13,7 +13,7 @@ use libk_util::{sync::spin_rwlock::IrqSafeRwLock, OneTimeInit};
 use yggdrasil_abi::error::Error;
 
 use crate::{
-    capability::{MsiCapability, MsiXCapability, MsiXVectorTable},
+    capability::{msi::MsiXVectorTable, MsiCapability, MsiXCapability, PciExpressCapability},
     driver::PciDriver,
     PciAddress, PciCommandRegister, PciConfigSpace, PciConfigurationSpace, PciSegmentInfo,
 };
@@ -142,6 +142,23 @@ impl PciDeviceInfo {
         }
     }
 
+    pub fn function_level_reset(&self) -> Result<(), Error> {
+        if let Some(mut pcie) = self.config_space.capability::<PciExpressCapability>() {
+            pcie.function_level_reset()
+        } else {
+            Err(Error::NotImplemented)
+        }
+    }
+
+    pub fn hot_link_reset(&self) -> Result<(), Error> {
+        if let Some(mut pcie) = self.config_space.capability::<PciExpressCapability>() {
+            pcie.hot_link_reset();
+            Ok(())
+        } else {
+            Err(Error::NotImplemented)
+        }
+    }
+
     pub fn init_interrupts(&self, preferred_mode: PreferredInterruptMode) -> Result<(), Error> {
         self.interrupt_config
             .try_init_with(|| {

kernel/driver/bus/pci/src/space/mod.rs

@@ -349,7 +349,7 @@ pub trait PciConfigurationSpace {
     }
 
     /// Returns an iterator over the PCI capabilities
-    fn capability_iter(&self) -> CapabilityIterator<'_, Self> {
+    fn capability_iter(&self) -> CapabilityIterator<Self> {
         let status = PciStatusRegister::from_bits_retain(self.status());
 
         let current = if status.contains(PciStatusRegister::CAPABILITIES_LIST) {

kernel/driver/bus/usb/src/address.rs

@@ -1,58 +0,0 @@
-use core::fmt;
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
-pub struct UsbRoute {
-    bus: u16,
-    ports: [u8; 8],
-    len: u8,
-}
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
-pub struct UsbBusAddress {
-    pub bus: u16,
-    pub device: u8,
-}
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
-pub struct UsbInterfaceAddress {
-    pub device: UsbBusAddress,
-    pub interface: u8,
-}
-
-impl UsbBusAddress {
-    pub fn with_interface(self, interface: u8) -> UsbInterfaceAddress {
-        UsbInterfaceAddress {
-            device: self,
-            interface,
-        }
-    }
-}
-
-impl fmt::Display for UsbBusAddress {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "<Bus {} Device {}>", self.bus, self.device)
-    }
-}
-
-impl fmt::Display for UsbInterfaceAddress {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(
-            f,
-            "<Bus {} Device {} Interface {}>",
-            self.device.bus, self.device.device, self.interface
-        )
-    }
-}
-
-impl fmt::Display for UsbRoute {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "{}-", self.bus)?;
-        for (i, &port) in self.ports[..self.len as usize].iter().enumerate() {
-            if i != 0 {
-                write!(f, ".")?;
-            }
-            write!(f, "{port}")?;
-        }
-        Ok(())
-    }
-}

kernel/driver/bus/usb/src/bus.rs

@@ -1,56 +1,34 @@
 use core::sync::atomic::{AtomicU16, Ordering};
 
-use alloc::{collections::BTreeMap, format, sync::Arc};
-use libk_util::{queue::UnboundedMpmcQueue, sync::spin_rwlock::IrqSafeRwLock, OneTimeInit};
+use alloc::{collections::BTreeMap, sync::Arc};
+use libk_util::{queue::UnboundedMpmcQueue, sync::spin_rwlock::IrqSafeRwLock};
 
 use crate::{
-    address::UsbBusAddress,
     class_driver,
-    device::{UsbDevice, UsbDeviceAccess},
-    sysfs::{self, UsbBusKObject},
+    device::{UsbBusAddress, UsbDeviceAccess},
     UsbHostController,
 };
 
-pub struct UsbBusWrapper {
-    pub(crate) hc: Arc<dyn UsbHostController>,
-    pub(crate) index: u16,
-    kobject: OneTimeInit<UsbBusKObject>,
-}
-
 pub struct UsbBusManager {
-    busses: IrqSafeRwLock<BTreeMap<u16, Arc<UsbBusWrapper>>>,
+    busses: IrqSafeRwLock<BTreeMap<u16, Arc<dyn UsbHostController>>>,
     devices: IrqSafeRwLock<BTreeMap<UsbBusAddress, Arc<UsbDeviceAccess>>>,
 
     last_bus_address: AtomicU16,
 }
 
-impl UsbBusWrapper {
-    pub fn kobject(&self) -> &UsbBusKObject {
-        self.kobject.get()
-    }
-}
-
 impl UsbBusManager {
-    pub fn register_bus(hc: Arc<dyn UsbHostController>) -> (u16, Arc<UsbBusWrapper>) {
+    pub fn register_bus(hc: Arc<dyn UsbHostController>) -> u16 {
         let i = BUS_MANAGER.last_bus_address.fetch_add(1, Ordering::AcqRel);
-        let wrapper = Arc::new(UsbBusWrapper {
-            hc,
-            index: i,
-            kobject: OneTimeInit::new(),
-        });
-        BUS_MANAGER.busses.write().insert(i, wrapper.clone());
-        wrapper.kobject.init(sysfs::register_bus_kobject(&wrapper));
-        (i, wrapper)
+        BUS_MANAGER.busses.write().insert(i, hc);
+        i
     }
 
     pub fn register_device(device: Arc<UsbDeviceAccess>) {
         log::info!("usb: register device {}", device.bus_address());
-
         BUS_MANAGER
             .devices
             .write()
             .insert(device.bus_address(), device.clone());
-        device.kobject.init(sysfs::register_device_kobject(&device));
 
         QUEUE.push_back(device);
     }
@@ -73,11 +51,7 @@ pub async fn bus_handler() {
             new_device.bus_address()
         );
 
-        let address = new_device.bus_address();
-        if let Err(error) = class_driver::setup_device(new_device).await {
-            log::warn!("USB device {address} setup error: {error:?}",);
-        }
-        // class_driver::spawn_driver(new_device).await.ok();
+        class_driver::spawn_driver(new_device).await.ok();
     }
 }
 

kernel/driver/bus/usb/src/class_driver/hid_keyboard.rs

@@ -4,12 +4,9 @@ use alloc::{boxed::Box, sync::Arc};
 use async_trait::async_trait;
 use yggdrasil_abi::io::{KeyboardKey, KeyboardKeyEvent};
 
-use crate::{
-    class_driver::{UsbInterfaceClass, UsbInterfaceDriver},
-    device::UsbDeviceAccess,
-    error::UsbError,
-    info::UsbInterfaceInfo,
-};
+use crate::{device::UsbDeviceAccess, error::UsbError, info::UsbDeviceClass};
+
+use super::{UsbClassInfo, UsbDriver};
 
 pub struct UsbHidKeyboardDriver;
 
@@ -128,14 +125,10 @@ impl KeyboardState {
 }
 
 #[async_trait]
-impl UsbInterfaceDriver for UsbHidKeyboardDriver {
-    async fn run(
-        self: Arc<Self>,
-        device: Arc<UsbDeviceAccess>,
-        interface: UsbInterfaceInfo,
-    ) -> Result<(), UsbError> {
+impl UsbDriver for UsbHidKeyboardDriver {
+    async fn run(self: Arc<Self>, device: Arc<UsbDeviceAccess>) -> Result<(), UsbError> {
         // TODO not sure whether to use boot protocol (easy) or GetReport
-        let config = device.current_configuration().unwrap();
+        let config = device.select_configuration(|_| true).await?.unwrap();
 
         log::info!("Setup HID keyboard");
         let pipe = device
@@ -163,7 +156,7 @@ impl UsbInterfaceDriver for UsbHidKeyboardDriver {
 
             for &event in events {
                 log::trace!("Generic Keyboard: {:?}", event);
-                ygg_driver_input::send_keyboard_event(event);
+                ygg_driver_input::send_event(event);
             }
         }
     }
@@ -172,18 +165,12 @@ impl UsbInterfaceDriver for UsbHidKeyboardDriver {
         "USB HID Keyboard"
     }
 
-    fn probe(
-        &self,
-        device: &UsbDeviceAccess,
-        interface: &UsbInterfaceInfo,
-        class: UsbInterfaceClass,
-    ) -> bool {
-        let _ = (device, interface);
-        class
-            == UsbInterfaceClass {
-                class: 3,
-                subclass: 1,
-                protocol: 1,
-            }
+    fn probe(&self, class: &UsbClassInfo, _device: &UsbDeviceAccess) -> bool {
+        log::info!(
+            "class = {:?}, subclass = {:02x}",
+            class.class,
+            class.subclass
+        );
+        class.class == UsbDeviceClass::Hid && (class.subclass == 0x00 || class.subclass == 0x01)
     }
 }

kernel/driver/bus/usb/src/class_driver/hid_mouse.rs

@@ -1,64 +0,0 @@
-use alloc::{boxed::Box, sync::Arc};
-use async_trait::async_trait;
-use yggdrasil_abi::io::{ButtonMask, MouseEvent};
-
-use crate::{
-    class_driver::{UsbInterfaceClass, UsbInterfaceDriver},
-    device::UsbDeviceAccess,
-    error::UsbError,
-    info::UsbInterfaceInfo,
-};
-
-pub struct UsbHidMouseDriver;
-
-#[async_trait]
-impl UsbInterfaceDriver for UsbHidMouseDriver {
-    async fn run(
-        self: Arc<Self>,
-        device: Arc<UsbDeviceAccess>,
-        interface: UsbInterfaceInfo,
-    ) -> Result<(), UsbError> {
-        let config = device.current_configuration().unwrap();
-
-        log::info!("Setup HID mouse");
-        let pipe = device
-            .open_interrupt_in_pipe(1, config.endpoints[0].max_packet_size as u16)
-            .await?;
-
-        let mut buffer = [0; 16];
-        let mut button_state = 0;
-
-        loop {
-            let len = pipe.read(&mut buffer).await?;
-            if len < 4 {
-                continue;
-            }
-
-            let event = MouseEvent {
-                buttons: ButtonMask(buffer[0]),
-                dx: (buffer[1] as i8) as i32,
-                dy: (buffer[2] as i8) as i32,
-            };
-
-            ygg_driver_input::send_mouse_event(event);
-        }
-    }
-
-    fn name(&self) -> &'static str {
-        "USB HID Mouse"
-    }
-
-    fn probe(
-        &self,
-        device: &UsbDeviceAccess,
-        interface: &UsbInterfaceInfo,
-        class: UsbInterfaceClass,
-    ) -> bool {
-        class
-            == UsbInterfaceClass {
-                class: 3,
-                subclass: 1,
-                protocol: 2,
-            }
-    }
-}

kernel/driver/bus/usb/src/class_driver/mass_storage.rs

@@ -1,273 +1,273 @@
-// use core::mem::MaybeUninit;
-//
-// use alloc::{boxed::Box, sync::Arc};
-// use async_trait::async_trait;
-// use bytemuck::{Pod, Zeroable};
-// use libk::{
-//     dma::{DmaBuffer, DmaSliceMut},
-//     error::Error,
-// };
-// use ygg_driver_scsi::{transport::ScsiTransport, ScsiEnclosure};
-//
-// use crate::{
-//     communication::UsbDirection,
-//     device::{UsbDeviceAccess, UsbDeviceDetachHandler},
-//     error::UsbError,
-//     info::{UsbDeviceClass, UsbEndpointType},
-//     pipe::{
-//         control::{ControlTransferSetup, UsbClassSpecificRequest},
-//         normal::{UsbBulkInPipeAccess, UsbBulkOutPipeAccess},
-//     },
-// };
-//
-// use super::{UsbClassInfo, UsbDriver};
-//
-// pub struct UsbMassStorageDriverBulkOnly;
-//
-// #[derive(Debug, Clone, Copy, Zeroable, Pod)]
-// #[repr(C)]
-// struct Cbw {
-//     signature: u32,       // 0x00
-//     tag: u32,             // 0x04
-//     transfer_length: u32, // 0x08
-//     flags: u8,            // 0x0C
-//     lun: u8,              // 0x0D
-//     cb_length: u8,        // 0x0E
-//     cb_data: [u8; 16],    // 0x0F
-//     // Not sent
-//     _0: u8,
-// }
-//
-// #[derive(Debug, Clone, Copy, Zeroable, Pod)]
-// #[repr(C)]
-// struct Csw {
-//     signature: u32,
-//     tag: u32,
-//     data_residue: u32,
-//     status: u8,
-//     _0: [u8; 3],
-// }
-//
-// struct Bbb {
-//     #[allow(unused)]
-//     device: Arc<UsbDeviceAccess>,
-//     in_pipe: UsbBulkInPipeAccess,
-//     out_pipe: UsbBulkOutPipeAccess,
-//     last_tag: u32,
-// }
-//
-// struct DetachHandler(Arc<ScsiEnclosure>);
-//
-// impl Bbb {
-//     pub fn new(
-//         device: Arc<UsbDeviceAccess>,
-//         in_pipe: UsbBulkInPipeAccess,
-//         out_pipe: UsbBulkOutPipeAccess,
-//     ) -> Result<Self, UsbError> {
-//         Ok(Self {
-//             device,
-//             in_pipe,
-//             out_pipe,
-//             last_tag: 0,
-//         })
-//     }
-// }
-//
-// impl Bbb {
-//     async fn send_cbw(
-//         &mut self,
-//         lun: u8,
-//         host_to_dev: bool,
-//         command: &[u8],
-//         response_len: usize,
-//     ) -> Result<u32, Error> {
-//         self.last_tag = self.last_tag.wrapping_add(1);
-//
-//         let flags = if !host_to_dev { 1 << 7 } else { 0 };
-//         let tag = self.last_tag;
-//         let mut cbw_bytes = [0; 32];
-//         let cbw = bytemuck::from_bytes_mut::<Cbw>(&mut cbw_bytes);
-//
-//         cbw.signature = 0x43425355;
-//         cbw.transfer_length = response_len as u32;
-//         cbw.flags = flags;
-//         cbw.tag = tag;
-//         cbw.lun = lun;
-//         cbw.cb_length = command.len() as u8;
-//         cbw.cb_data[..command.len()].copy_from_slice(command);
-//
-//         self.out_pipe
-//             .write(&cbw_bytes[..31])
-//             .await
-//             .inspect_err(|error| log::error!("msc: CBW send error: {error:?}"))?;
-//
-//         Ok(tag)
-//     }
-//
-//     async fn read_csw(&mut self, tag: u32) -> Result<(), Error> {
-//         let mut csw_bytes = [0; 16];
-//         self.in_pipe
-//             .read_exact(&mut csw_bytes[..13])
-//             .await
-//             .inspect_err(|error| log::error!("msc: CSW receive error: {error:?}"))?;
-//         let csw = bytemuck::from_bytes::<Csw>(&csw_bytes);
-//
-//         if csw.signature != 0x53425355 {
-//             log::warn!("msc: invalid csw signature");
-//             return Err(Error::InvalidArgument);
-//         }
-//         if csw.tag != tag {
-//             let csw_tag = csw.tag;
-//             log::warn!("msc: invalid csw tag (got {}, expected {tag})", csw_tag);
-//             return Err(Error::InvalidArgument);
-//         }
-//         if csw.status != 0x00 {
-//             return Err(Error::InvalidArgument);
-//         }
-//         Ok(())
-//     }
-//
-//     async fn read_response_data(
-//         &mut self,
-//         buffer: DmaSliceMut<'_, MaybeUninit<u8>>,
-//     ) -> Result<usize, Error> {
-//         if buffer.len() == 0 {
-//             return Ok(0);
-//         }
-//         let len = self
-//             .in_pipe
-//             .read_dma(buffer)
-//             .await
-//             .inspect_err(|error| log::error!("msc: DMA read error: {error:?}"))?;
-//         Ok(len)
-//     }
-// }
-//
-// #[async_trait]
-// impl ScsiTransport for Bbb {
-//     fn allocate_buffer(&self, size: usize) -> Result<DmaBuffer<[MaybeUninit<u8>]>, Error> {
-//         Ok(self.in_pipe.allocate_dma_buffer(size)?)
-//     }
-//
-//     async fn perform_request_raw(
-//         &mut self,
-//         lun: u8,
-//         request_data: &[u8],
-//         response_buffer: DmaSliceMut<'_, MaybeUninit<u8>>,
-//     ) -> Result<usize, Error> {
-//         if request_data.len() > 16 || response_buffer.len() > self.max_bytes_per_request() {
-//             return Err(Error::InvalidArgument);
-//         }
-//
-//         let tag = self
-//             .send_cbw(lun, false, request_data, response_buffer.len())
-//             .await?;
-//         let response_len = self.read_response_data(response_buffer).await?;
-//         self.read_csw(tag).await?;
-//         Ok(response_len)
-//     }
-//
-//     fn max_bytes_per_request(&self) -> usize {
-//         32768
-//     }
-// }
-//
-// impl UsbDeviceDetachHandler for DetachHandler {
-//     fn handle_device_detach(&self) {
-//         log::info!("Mass storage detached");
-//         self.0.detach();
-//     }
-// }
-//
-// #[derive(Debug, Pod, Zeroable, Clone, Copy)]
-// #[repr(C)]
-// pub struct BulkOnlyMassStorageReset;
-//
-// #[derive(Debug, Pod, Zeroable, Clone, Copy)]
-// #[repr(C)]
-// pub struct GetMaxLun;
-//
-// impl UsbClassSpecificRequest for BulkOnlyMassStorageReset {
-//     const BM_REQUEST_TYPE: u8 = 0b00100001;
-//     const B_REQUEST: u8 = 0b11111111;
-// }
-//
-// impl UsbClassSpecificRequest for GetMaxLun {
-//     const BM_REQUEST_TYPE: u8 = 0b10100001;
-//     const B_REQUEST: u8 = 0b11111110;
-// }
-//
-// #[async_trait]
-// impl UsbDriver for UsbMassStorageDriverBulkOnly {
-//     async fn run(self: Arc<Self>, device: Arc<UsbDeviceAccess>) -> Result<(), UsbError> {
-//         // TODO filter to only accept BBB config
-//         let config = device.select_configuration(|_| true).await?.unwrap();
-//         // Bulk-in, bulk-out
-//         assert_eq!(config.endpoints.len(), 2);
-//         let control_pipe = device.control_pipe();
-//         let (in_index, in_info) = config
-//             .find_endpoint(|ep| ep.is(UsbEndpointType::Bulk, UsbDirection::In))
-//             .ok_or(UsbError::InvalidConfiguration)?;
-//         let (out_index, out_info) = config
-//             .find_endpoint(|ep| ep.is(UsbEndpointType::Bulk, UsbDirection::Out))
-//             .ok_or(UsbError::InvalidConfiguration)?;
-//         let in_pipe = device
-//             .open_bulk_in_pipe(in_index, in_info.max_packet_size as u16)
-//             .await?;
-//         let out_pipe = device
-//             .open_bulk_out_pipe(out_index, out_info.max_packet_size as u16)
-//             .await?;
-//
-//         // Perform a Bulk-Only Mass Storage Reset
-//         // TODO interface id?
-//         control_pipe
-//             .control_transfer(ControlTransferSetup {
-//                 bm_request_type: BulkOnlyMassStorageReset::BM_REQUEST_TYPE,
-//                 b_request: BulkOnlyMassStorageReset::B_REQUEST,
-//                 w_value: 0,
-//                 w_index: 0,
-//                 w_length: 0,
-//             })
-//             .await?;
-//
-//         // Get max LUN
-//         // TODO on devices which do not support multiple LUNs, this command may STALL
-//         let mut buffer = [MaybeUninit::uninit()];
-//         let len = control_pipe
-//             .control_transfer_in(
-//                 ControlTransferSetup {
-//                     bm_request_type: GetMaxLun::BM_REQUEST_TYPE,
-//                     b_request: GetMaxLun::B_REQUEST,
-//                     w_value: 0,
-//                     w_index: 0,
-//                     w_length: 1,
-//                 },
-//                 &mut buffer,
-//             )
-//             .await?;
-//         let max_lun = if len < 1 {
-//             0
-//         } else {
-//             unsafe { buffer[0].assume_init() }
-//         };
-//
-//         let bbb = Bbb::new(device.clone(), in_pipe, out_pipe)?;
-//         let scsi = ScsiEnclosure::setup(Box::new(bbb), max_lun as usize + 1)
-//             .await
-//             .inspect_err(|error| log::error!("msc: scsi error {error:?}"))
-//             .map_err(|_| UsbError::DriverError)?;
-//         let detach = DetachHandler(scsi.clone());
-//         device.set_detach_handler(Arc::new(detach));
-//
-//         Ok(())
-//     }
-//
-//     fn name(&self) -> &'static str {
-//         "USB Mass Storage"
-//     }
-//
-//     fn probe(&self, class: &UsbClassInfo, _device: &UsbDeviceAccess) -> bool {
-//         // TODO support other protocols
-//         class.class == UsbDeviceClass::MassStorage && class.interface_protocol_number == 0x50
-//     }
-// }
+use core::mem::MaybeUninit;
+
+use alloc::{boxed::Box, sync::Arc};
+use async_trait::async_trait;
+use bytemuck::{Pod, Zeroable};
+use libk::{
+    dma::{DmaBuffer, DmaSliceMut},
+    error::Error,
+};
+use ygg_driver_scsi::{transport::ScsiTransport, ScsiEnclosure};
+
+use crate::{
+    communication::UsbDirection,
+    device::{UsbDeviceAccess, UsbDeviceDetachHandler},
+    error::UsbError,
+    info::{UsbDeviceClass, UsbEndpointType},
+    pipe::{
+        control::{ControlTransferSetup, UsbClassSpecificRequest},
+        normal::{UsbBulkInPipeAccess, UsbBulkOutPipeAccess},
+    },
+};
+
+use super::{UsbClassInfo, UsbDriver};
+
+pub struct UsbMassStorageDriverBulkOnly;
+
+#[derive(Debug, Clone, Copy, Zeroable, Pod)]
+#[repr(C)]
+struct Cbw {
+    signature: u32,       // 0x00
+    tag: u32,             // 0x04
+    transfer_length: u32, // 0x08
+    flags: u8,            // 0x0C
+    lun: u8,              // 0x0D
+    cb_length: u8,        // 0x0E
+    cb_data: [u8; 16],    // 0x0F
+    // Not sent
+    _0: u8,
+}
+
+#[derive(Debug, Clone, Copy, Zeroable, Pod)]
+#[repr(C)]
+struct Csw {
+    signature: u32,
+    tag: u32,
+    data_residue: u32,
+    status: u8,
+    _0: [u8; 3],
+}
+
+struct Bbb {
+    #[allow(unused)]
+    device: Arc<UsbDeviceAccess>,
+    in_pipe: UsbBulkInPipeAccess,
+    out_pipe: UsbBulkOutPipeAccess,
+    last_tag: u32,
+}
+
+struct DetachHandler(Arc<ScsiEnclosure>);
+
+impl Bbb {
+    pub fn new(
+        device: Arc<UsbDeviceAccess>,
+        in_pipe: UsbBulkInPipeAccess,
+        out_pipe: UsbBulkOutPipeAccess,
+    ) -> Result<Self, UsbError> {
+        Ok(Self {
+            device,
+            in_pipe,
+            out_pipe,
+            last_tag: 0,
+        })
+    }
+}
+
+impl Bbb {
+    async fn send_cbw(
+        &mut self,
+        lun: u8,
+        host_to_dev: bool,
+        command: &[u8],
+        response_len: usize,
+    ) -> Result<u32, Error> {
+        self.last_tag = self.last_tag.wrapping_add(1);
+
+        let flags = if !host_to_dev { 1 << 7 } else { 0 };
+        let tag = self.last_tag;
+        let mut cbw_bytes = [0; 32];
+        let cbw = bytemuck::from_bytes_mut::<Cbw>(&mut cbw_bytes);
+
+        cbw.signature = 0x43425355;
+        cbw.transfer_length = response_len as u32;
+        cbw.flags = flags;
+        cbw.tag = tag;
+        cbw.lun = lun;
+        cbw.cb_length = command.len() as u8;
+        cbw.cb_data[..command.len()].copy_from_slice(command);
+
+        self.out_pipe
+            .write(&cbw_bytes[..31])
+            .await
+            .inspect_err(|error| log::error!("msc: CBW send error: {error:?}"))?;
+
+        Ok(tag)
+    }
+
+    async fn read_csw(&mut self, tag: u32) -> Result<(), Error> {
+        let mut csw_bytes = [0; 16];
+        self.in_pipe
+            .read_exact(&mut csw_bytes[..13])
+            .await
+            .inspect_err(|error| log::error!("msc: CSW receive error: {error:?}"))?;
+        let csw = bytemuck::from_bytes::<Csw>(&csw_bytes);
+
+        if csw.signature != 0x53425355 {
+            log::warn!("msc: invalid csw signature");
+            return Err(Error::InvalidArgument);
+        }
+        if csw.tag != tag {
+            let csw_tag = csw.tag;
+            log::warn!("msc: invalid csw tag (got {}, expected {tag})", csw_tag);
+            return Err(Error::InvalidArgument);
+        }
+        if csw.status != 0x00 {
+            return Err(Error::InvalidArgument);
+        }
+        Ok(())
+    }
+
+    async fn read_response_data(
+        &mut self,
+        buffer: DmaSliceMut<'_, MaybeUninit<u8>>,
+    ) -> Result<usize, Error> {
+        if buffer.len() == 0 {
+            return Ok(0);
+        }
+        let len = self
+            .in_pipe
+            .read_dma(buffer)
+            .await
+            .inspect_err(|error| log::error!("msc: DMA read error: {error:?}"))?;
+        Ok(len)
+    }
+}
+
+#[async_trait]
+impl ScsiTransport for Bbb {
+    fn allocate_buffer(&self, size: usize) -> Result<DmaBuffer<[MaybeUninit<u8>]>, Error> {
+        Ok(self.in_pipe.allocate_dma_buffer(size)?)
+    }
+
+    async fn perform_request_raw(
+        &mut self,
+        lun: u8,
+        request_data: &[u8],
+        response_buffer: DmaSliceMut<'_, MaybeUninit<u8>>,
+    ) -> Result<usize, Error> {
+        if request_data.len() > 16 || response_buffer.len() > self.max_bytes_per_request() {
+            return Err(Error::InvalidArgument);
+        }
+
+        let tag = self
+            .send_cbw(lun, false, request_data, response_buffer.len())
+            .await?;
+        let response_len = self.read_response_data(response_buffer).await?;
+        self.read_csw(tag).await?;
+        Ok(response_len)
+    }
+
+    fn max_bytes_per_request(&self) -> usize {
+        32768
+    }
+}
+
+impl UsbDeviceDetachHandler for DetachHandler {
+    fn handle_device_detach(&self) {
+        log::info!("Mass storage detached");
+        self.0.detach();
+    }
+}
+
+#[derive(Debug, Pod, Zeroable, Clone, Copy)]
+#[repr(C)]
+pub struct BulkOnlyMassStorageReset;
+
+#[derive(Debug, Pod, Zeroable, Clone, Copy)]
+#[repr(C)]
+pub struct GetMaxLun;
+
+impl UsbClassSpecificRequest for BulkOnlyMassStorageReset {
+    const BM_REQUEST_TYPE: u8 = 0b00100001;
+    const B_REQUEST: u8 = 0b11111111;
+}
+
+impl UsbClassSpecificRequest for GetMaxLun {
+    const BM_REQUEST_TYPE: u8 = 0b10100001;
+    const B_REQUEST: u8 = 0b11111110;
+}
+
+#[async_trait]
+impl UsbDriver for UsbMassStorageDriverBulkOnly {
+    async fn run(self: Arc<Self>, device: Arc<UsbDeviceAccess>) -> Result<(), UsbError> {
+        // TODO filter to only accept BBB config
+        let config = device.select_configuration(|_| true).await?.unwrap();
+        // Bulk-in, bulk-out
+        assert_eq!(config.endpoints.len(), 2);
+        let control_pipe = device.control_pipe();
+        let (in_index, in_info) = config
+            .find_endpoint(|ep| ep.is(UsbEndpointType::Bulk, UsbDirection::In))
+            .ok_or(UsbError::InvalidConfiguration)?;
+        let (out_index, out_info) = config
+            .find_endpoint(|ep| ep.is(UsbEndpointType::Bulk, UsbDirection::Out))
+            .ok_or(UsbError::InvalidConfiguration)?;
+        let in_pipe = device
+            .open_bulk_in_pipe(in_index, in_info.max_packet_size as u16)
+            .await?;
+        let out_pipe = device
+            .open_bulk_out_pipe(out_index, out_info.max_packet_size as u16)
+            .await?;
+
+        // Perform a Bulk-Only Mass Storage Reset
+        // TODO interface id?
+        control_pipe
+            .control_transfer(ControlTransferSetup {
+                bm_request_type: BulkOnlyMassStorageReset::BM_REQUEST_TYPE,
+                b_request: BulkOnlyMassStorageReset::B_REQUEST,
+                w_value: 0,
+                w_index: 0,
+                w_length: 0,
+            })
+            .await?;
+
+        // Get max LUN
+        // TODO on devices which do not support multiple LUNs, this command may STALL
+        let mut buffer = [MaybeUninit::uninit()];
+        let len = control_pipe
+            .control_transfer_in(
+                ControlTransferSetup {
+                    bm_request_type: GetMaxLun::BM_REQUEST_TYPE,
+                    b_request: GetMaxLun::B_REQUEST,
+                    w_value: 0,
+                    w_index: 0,
+                    w_length: 1,
+                },
+                &mut buffer,
+            )
+            .await?;
+        let max_lun = if len < 1 {
+            0
+        } else {
+            unsafe { buffer[0].assume_init() }
+        };
+
+        let bbb = Bbb::new(device.clone(), in_pipe, out_pipe)?;
+        let scsi = ScsiEnclosure::setup(Box::new(bbb), max_lun as usize + 1)
+            .await
+            .inspect_err(|error| log::error!("msc: scsi error {error:?}"))
+            .map_err(|_| UsbError::DriverError)?;
+        let detach = DetachHandler(scsi.clone());
+        device.set_detach_handler(Arc::new(detach));
+
+        Ok(())
+    }
+
+    fn name(&self) -> &'static str {
+        "USB Mass Storage"
+    }
+
+    fn probe(&self, class: &UsbClassInfo, _device: &UsbDeviceAccess) -> bool {
+        // TODO support other protocols
+        class.class == UsbDeviceClass::MassStorage && class.interface_protocol_number == 0x50
+    }
+}

kernel/driver/bus/usb/src/class_driver/mod.rs

@@ -1,208 +1,117 @@
-use core::mem::MaybeUninit;
-
 use alloc::{boxed::Box, sync::Arc, vec::Vec};
 use async_trait::async_trait;
 use libk::task::runtime;
 use libk_util::sync::spin_rwlock::IrqSafeRwLock;
 
 use crate::{
-    address::UsbInterfaceAddress,
     device::UsbDeviceAccess,
     error::UsbError,
-    info::{UsbInterfaceInfo, CLASS_FROM_INTERFACE},
-    pipe::control::{ControlTransferSetup, UsbClassSpecificRequest},
+    info::{UsbDeviceClass, UsbDeviceProtocol},
 };
 
-// use alloc::{boxed::Box, sync::Arc, vec::Vec};
-// use async_trait::async_trait;
-// use libk::task::runtime;
-// use libk_util::sync::spin_rwlock::IrqSafeRwLock;
-//
-// use crate::{
-//     device::UsbDeviceAccess,
-//     error::UsbError,
-//     info::{UsbDeviceClass, UsbDeviceProtocol},
-// };
-//
 pub mod hid_keyboard;
-pub mod hid_mouse;
-// pub mod mass_storage;
-//
-// #[derive(Debug)]
-// pub struct UsbClassInfo {
-//     pub class: UsbDeviceClass,
-//     pub subclass: u8,
-//     pub protocol: UsbDeviceProtocol,
-//     pub device_protocol_number: u8,
-//     pub interface_protocol_number: u8,
-// }
-//
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub struct UsbInterfaceClass {
-    pub class: u8,
+pub mod mass_storage;
+
+#[derive(Debug)]
+pub struct UsbClassInfo {
+    pub class: UsbDeviceClass,
     pub subclass: u8,
-    pub protocol: u8,
+    pub protocol: UsbDeviceProtocol,
+    pub device_protocol_number: u8,
+    pub interface_protocol_number: u8,
 }
 
 #[async_trait]
-pub trait UsbInterfaceDriver: Send + Sync {
-    async fn run(
-        self: Arc<Self>,
-        device: Arc<UsbDeviceAccess>,
-        interface: UsbInterfaceInfo,
-    ) -> Result<(), UsbError>;
+pub trait UsbDriver: Send + Sync {
+    async fn run(self: Arc<Self>, device: Arc<UsbDeviceAccess>) -> Result<(), UsbError>;
 
     fn name(&self) -> &'static str;
-    fn probe(
-        &self,
-        device: &UsbDeviceAccess,
-        interface: &UsbInterfaceInfo,
-        class: UsbInterfaceClass,
-    ) -> bool;
+    fn probe(&self, class: &UsbClassInfo, device: &UsbDeviceAccess) -> bool;
 }
 
-// async fn extract_class_info(device: &UsbDeviceAccess) -> Result<Option<UsbClassInfo>, UsbError> {
-//     if device.info.num_configurations != 1 {
-//         return Ok(None);
-//     }
-//     let device_info = &device.info;
-//     let config_info = device.query_configuration_info(0).await?;
-//
-//     if config_info.interfaces.len() >= 1 {
-//         let if_info = &config_info.interfaces[0];
-//
-//         let class = if device_info.device_class == UsbDeviceClass::FromInterface {
-//             if_info.interface_class
-//         } else {
-//             device_info.device_class
-//         };
-//         let subclass = if device_info.device_subclass == 0 {
-//             if_info.interface_subclass
-//         } else {
-//             device_info.device_subclass
-//         };
-//         let protocol = if device_info.device_protocol == UsbDeviceProtocol::FromInterface {
-//             if_info.interface_protocol
-//         } else {
-//             device_info.device_protocol
-//         };
-//
-//         Ok(Some(UsbClassInfo {
-//             class,
-//             subclass,
-//             protocol,
-//             interface_protocol_number: if_info.interface_protocol_number,
-//             device_protocol_number: device_info.device_protocol_number,
-//         }))
-//     } else {
-//         Ok(None)
-//     }
-// }
-//
-// async fn pick_driver(
-//     device: &UsbDeviceAccess,
-// ) -> Result<Option<Arc<dyn UsbDriver + 'static>>, UsbError> {
-//     let Some(class) = extract_class_info(device).await? else {
-//         return Ok(None);
-//     };
-//
-//     for driver in USB_DEVICE_DRIVERS.read().iter() {
-//         if driver.probe(&class, device) {
-//             return Ok(Some(driver.clone()));
-//         }
-//     }
-//     Ok(None)
-// }
-//
-// pub async fn spawn_driver(device: Arc<UsbDeviceAccess>) -> Result<(), UsbError> {
-//     // if let Some(driver) = pick_driver(&device).await? {
-//     //     runtime::spawn(async move {
-//     //         let name = driver.name();
-//     //         match driver.run(device).await {
-//     //             e @ Err(UsbError::DeviceDisconnected) => {
-//     //                 log::warn!(
-//     //                     "Driver {:?} did not exit cleanly: device disconnected",
-//     //                     name,
-//     //                 );
-//
-//     //                 e
-//     //             }
-//     //             e => e,
-//     //         }
-//     //     })
-//     //     .map_err(UsbError::SystemError)?;
-//     // }
-//     Ok(())
-// }
+async fn extract_class_info(device: &UsbDeviceAccess) -> Result<Option<UsbClassInfo>, UsbError> {
+    if device.info.num_configurations != 1 {
+        return Ok(None);
+    }
+    let device_info = &device.info;
+    let config_info = device.query_configuration_info(0).await?;
 
-async fn setup_interface(
-    device: &Arc<UsbDeviceAccess>,
-    address: UsbInterfaceAddress,
-    interface: &UsbInterfaceInfo,
-) -> Result<(), UsbError> {
-    let class = UsbInterfaceClass {
-        class: interface.interface_class,
-        subclass: interface.interface_subclass,
-        protocol: interface.interface_protocol,
+    if config_info.interfaces.len() >= 1 {
+        let if_info = &config_info.interfaces[0];
+
+        let class = if device_info.device_class == UsbDeviceClass::FromInterface {
+            if_info.interface_class
+        } else {
+            device_info.device_class
+        };
+        let subclass = if device_info.device_subclass == 0 {
+            if_info.interface_subclass
+        } else {
+            device_info.device_subclass
+        };
+        let protocol = if device_info.device_protocol == UsbDeviceProtocol::FromInterface {
+            if_info.interface_protocol
+        } else {
+            device_info.device_protocol
+        };
+
+        Ok(Some(UsbClassInfo {
+            class,
+            subclass,
+            protocol,
+            interface_protocol_number: if_info.interface_protocol_number,
+            device_protocol_number: device_info.device_protocol_number,
+        }))
+    } else {
+        Ok(None)
+    }
+}
+
+async fn pick_driver(
+    device: &UsbDeviceAccess,
+) -> Result<Option<Arc<dyn UsbDriver + 'static>>, UsbError> {
+    let Some(class) = extract_class_info(device).await? else {
+        return Ok(None);
     };
-    let drivers = USB_INTERFACE_DRIVERS.read();
-    for driver in drivers.iter() {
-        if driver.probe(device, interface, class) {
-            let driver = driver.clone();
-            let device = device.clone();
-            let interface = interface.clone();
-            runtime::spawn(async move {
-                let name = driver.name();
-                match driver.run(device, interface).await {
-                    e @ Err(UsbError::DeviceDisconnected) => {
-                        log::warn!("{address} did not exit cleanly: device disconnected ({name})");
-                        e
-                    }
-                    e => e,
+
+    for driver in USB_DEVICE_DRIVERS.read().iter() {
+        if driver.probe(&class, device) {
+            return Ok(Some(driver.clone()));
+        }
+    }
+    Ok(None)
+}
+
+pub async fn spawn_driver(device: Arc<UsbDeviceAccess>) -> Result<(), UsbError> {
+    if let Some(driver) = pick_driver(&device).await? {
+        runtime::spawn(async move {
+            let name = driver.name();
+            match driver.run(device).await {
+                e @ Err(UsbError::DeviceDisconnected) => {
+                    log::warn!(
+                        "Driver {:?} did not exit cleanly: device disconnected",
+                        name,
+                    );
+
+                    e
                 }
-            })
-            .map_err(UsbError::SystemError)?;
-            break;
-        }
+                e => e,
+            }
+        })
+        .map_err(UsbError::SystemError)?;
     }
     Ok(())
 }
 
-pub async fn setup_device(device: Arc<UsbDeviceAccess>) -> Result<(), UsbError> {
-    // If device has only one configuration available, use it
-    // TODO support devices with multiple configurations
-    let address = device.bus_address();
-    log::info!("Setup USB device @ {address}");
-
-    let Some(config_info) = device.use_default_configuration().await? else {
-        log::warn!("{address} has multiple configurations, not supported yet",);
-        return Ok(());
-    };
-
-    // Setup drivers for interfaces
-    log::info!("{address}: {config_info:#?}");
-
-    // TODO device-level drivers
-    for interface in config_info.interfaces.iter() {
-        let address = address.with_interface(interface.number);
-        if let Err(error) = setup_interface(&device, address, interface).await {
-            log::error!("{}: {:?}", address, error);
-        }
-    }
-
-    Ok(())
-}
-
-pub fn register_driver(driver: Arc<dyn UsbInterfaceDriver + 'static>) {
+pub fn register_driver(driver: Arc<dyn UsbDriver + 'static>) {
     // TODO check for duplicates
-    USB_INTERFACE_DRIVERS.write().push(driver);
+    USB_DEVICE_DRIVERS.write().push(driver);
 }
 
 pub fn register_default_class_drivers() {
     register_driver(Arc::new(hid_keyboard::UsbHidKeyboardDriver));
-    register_driver(Arc::new(hid_mouse::UsbHidMouseDriver));
-    // register_driver(Arc::new(mass_storage::UsbMassStorageDriverBulkOnly));
+    register_driver(Arc::new(mass_storage::UsbMassStorageDriverBulkOnly));
 }
 
-static USB_INTERFACE_DRIVERS: IrqSafeRwLock<Vec<Arc<dyn UsbInterfaceDriver + 'static>>> =
+static USB_DEVICE_DRIVERS: IrqSafeRwLock<Vec<Arc<dyn UsbDriver + 'static>>> =
     IrqSafeRwLock::new(Vec::new());

kernel/driver/bus/usb/src/descriptor.rs

@@ -2,9 +2,9 @@ use bytemuck::{Pod, Zeroable};
 
 use crate::{
     communication::UsbDirection,
-    device::{self, UsbSpeed},
+    device::UsbSpeed,
     error::UsbError,
-    info::UsbEndpointType,
+    info::{UsbDeviceClass, UsbDeviceProtocol, UsbEndpointType, UsbVersion},
 };
 
 #[derive(Clone, Copy, Debug, Default, Pod, Zeroable)]
@@ -91,15 +91,15 @@ pub struct UsbOtherSpeedConfiguration {
     pub max_power: u8,
 }
 
-// impl UsbInterfaceDescriptor {
-//     pub fn class(&self) -> UsbDeviceClass {
-//         UsbDeviceClass::try_from(self.interface_class).unwrap_or(UsbDeviceClass::Unknown)
-//     }
-//
-//     pub fn protocol(&self) -> UsbDeviceProtocol {
-//         UsbDeviceProtocol::try_from(self.interface_protocol).unwrap_or(UsbDeviceProtocol::Unknown)
-//     }
-// }
+impl UsbInterfaceDescriptor {
+    pub fn class(&self) -> UsbDeviceClass {
+        UsbDeviceClass::try_from(self.interface_class).unwrap_or(UsbDeviceClass::Unknown)
+    }
+
+    pub fn protocol(&self) -> UsbDeviceProtocol {
+        UsbDeviceProtocol::try_from(self.interface_protocol).unwrap_or(UsbDeviceProtocol::Unknown)
+    }
+}
 
 impl UsbEndpointDescriptor {
     pub fn direction(&self) -> UsbDirection {
@@ -127,16 +127,16 @@ impl UsbEndpointDescriptor {
 }
 
 impl UsbDeviceDescriptor {
-    // pub fn class(&self) -> UsbDeviceClass {
-    //     UsbDeviceClass::try_from(self.device_class).unwrap_or(UsbDeviceClass::Unknown)
-    // }
+    pub fn class(&self) -> UsbDeviceClass {
+        UsbDeviceClass::try_from(self.device_class).unwrap_or(UsbDeviceClass::Unknown)
+    }
 
-    // pub fn protocol(&self) -> UsbDeviceProtocol {
-    //     UsbDeviceProtocol::try_from(self.device_protocol).unwrap_or(UsbDeviceProtocol::Unknown)
-    // }
+    pub fn protocol(&self) -> UsbDeviceProtocol {
+        UsbDeviceProtocol::try_from(self.device_protocol).unwrap_or(UsbDeviceProtocol::Unknown)
+    }
 
-    pub fn max_packet_size(&self, version: u16, speed: UsbSpeed) -> Result<usize, UsbError> {
-        match (is_version_3(version), speed, self.max_packet_size_0) {
+    pub fn max_packet_size(&self, version: UsbVersion, speed: UsbSpeed) -> Result<usize, UsbError> {
+        match (version.is_version_3(), speed, self.max_packet_size_0) {
             (true, UsbSpeed::Super, 9) => Ok(1 << 9),
             (true, _, _) => todo!("Non-GenX speed USB3+ maxpacketsize0"),
             (false, _, 8) => Ok(8),
@@ -147,7 +147,3 @@ impl UsbDeviceDescriptor {
         }
     }
 }
-
-pub fn is_version_3(version: u16) -> bool {
-    version & 0xFF00 == 0x300
-}

kernel/driver/bus/usb/src/device.rs

@@ -1,28 +1,14 @@
-use core::{
-    fmt,
-    ops::{Deref, Sub},
-};
+use core::{fmt, ops::Deref};
 
-use alloc::{
-    boxed::Box,
-    format,
-    string::{String, ToString},
-    sync::Arc,
-    vec::Vec,
-};
+use alloc::{boxed::Box, sync::Arc, vec::Vec};
 use async_trait::async_trait;
-use libk::error::Error;
-use libk_util::{
-    sync::spin_rwlock::{IrqSafeRwLock, IrqSafeRwLockReadGuard},
-    OneTimeInit,
-};
+use libk_util::sync::spin_rwlock::{IrqSafeRwLock, IrqSafeRwLockReadGuard};
 
 use crate::{
-    address::UsbBusAddress,
-    bus::UsbBusWrapper,
     error::UsbError,
     info::{
         UsbConfigurationInfo, UsbDeviceInfo, UsbEndpointInfo, UsbEndpointType, UsbInterfaceInfo,
+        UsbVersion,
     },
     pipe::{
         control::{ConfigurationDescriptorEntry, UsbControlPipeAccess},
@@ -31,26 +17,21 @@ use crate::{
             UsbNormalPipeOut,
         },
     },
-    sysfs::UsbDeviceKObject,
     UsbHostController,
 };
 
 // High-level structures for info provided through descriptors
 
-// #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)]
-// pub struct UsbBusAddress {
-//     pub bus: u16,
-//     pub device: u8,
-// }
+#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)]
+pub struct UsbBusAddress {
+    pub bus: u16,
+    pub device: u8,
+}
 
 pub struct UsbDeviceAccess {
     pub device: Arc<dyn UsbDevice>,
-    pub bus: Arc<UsbBusWrapper>,
     pub info: UsbDeviceInfo,
-    pub configurations: Vec<UsbConfigurationInfo>,
-    current_configuration: IrqSafeRwLock<Option<usize>>,
-
-    pub(crate) kobject: OneTimeInit<UsbDeviceKObject>,
+    pub current_configuration: IrqSafeRwLock<Option<UsbConfigurationInfo>>,
 }
 
 #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)]
@@ -88,7 +69,7 @@ pub trait UsbDevice: Send + Sync {
     fn port_number(&self) -> u8;
     fn bus_address(&self) -> UsbBusAddress;
     fn speed(&self) -> UsbSpeed;
-    fn host_controller(&self) -> Arc<dyn UsbHostController>;
+    fn controller_ref(&self) -> &dyn UsbHostController;
 
     fn set_detach_handler(&self, handler: Arc<dyn UsbDeviceDetachHandler>);
     fn handle_detach(&self);
@@ -103,43 +84,47 @@ impl UsbDeviceAccess {
     /// * Device is not yet configured
     /// * Control pipe for the device has been properly set up
     /// * Device has been assigned a bus address
-    pub async fn setup(bus: Arc<UsbBusWrapper>, raw: Arc<dyn UsbDevice>) -> Result<Self, UsbError> {
+    pub async fn setup(raw: Arc<dyn UsbDevice>) -> Result<Self, UsbError> {
         let control = raw.control_pipe();
 
         let device_desc = control.query_device_descriptor().await?;
 
+        let bcd_usb = device_desc.bcd_usb;
+        let usb_version = UsbVersion::from_bcd_usb(device_desc.bcd_usb)
+            .ok_or(UsbError::InvalidDescriptorField)
+            .inspect_err(|_| {
+                log::error!(
+                    "{}: unsupported/invalid USB version: {:#x}",
+                    raw.bus_address(),
+                    bcd_usb
+                )
+            })?;
+
         let manufacturer = control.query_string(device_desc.manufacturer_str).await?;
         let product = control.query_string(device_desc.product_str).await?;
 
-        // Query device
         let info = UsbDeviceInfo {
             manufacturer,
             product,
-            usb_version: device_desc.bcd_usb,
+            usb_version,
 
             id_vendor: device_desc.id_vendor,
             id_product: device_desc.id_product,
 
-            device_class: device_desc.device_class,
+            device_class: device_desc.class(),
             device_subclass: device_desc.device_subclass,
-            device_protocol: device_desc.device_protocol,
+            device_protocol: device_desc.protocol(),
+            device_protocol_number: device_desc.device_protocol,
 
             num_configurations: device_desc.num_configurations,
 
-            max_packet_size: device_desc.max_packet_size(device_desc.bcd_usb, raw.speed())?,
+            max_packet_size: device_desc.max_packet_size(usb_version, raw.speed())?,
         };
 
-        let configurations =
-            Self::query_configurations(control, device_desc.num_configurations).await?;
-
         Ok(Self {
             device: raw,
-            bus,
             info,
             current_configuration: IrqSafeRwLock::new(None),
-            configurations,
-
-            kobject: OneTimeInit::new(),
         })
     }
 
@@ -179,54 +164,45 @@ impl UsbDeviceAccess {
         Ok(UsbBulkOutPipeAccess(pipe))
     }
 
-    pub fn current_configuration(&self) -> Option<&UsbConfigurationInfo> {
-        let index = (*self.current_configuration.read())?;
-        Some(&self.configurations[index])
-    }
-
-    pub async fn use_default_configuration(
+    pub fn read_current_configuration(
         &self,
+    ) -> IrqSafeRwLockReadGuard<Option<UsbConfigurationInfo>> {
+        self.current_configuration.read()
+    }
+
+    pub async fn select_configuration<F: Fn(&UsbConfigurationInfo) -> bool>(
+        &self,
+        predicate: F,
     ) -> Result<Option<UsbConfigurationInfo>, UsbError> {
-        if self.configurations.len() != 1 {
-            return Ok(None);
-        }
-
-        self.set_configuration(0).await.map(Some)
-    }
-
-    pub async fn set_configuration(&self, index: usize) -> Result<UsbConfigurationInfo, UsbError> {
-        if index >= self.configurations.len() {
-            return Err(UsbError::InvalidConfiguration);
-        }
-
-        let mut current = self.current_configuration.write();
+        let mut current_config = self.current_configuration.write();
         let control_pipe = self.control_pipe();
-        let info = self.configurations[index].clone();
 
-        control_pipe
-            .set_configuration(info.config_value as _)
-            .await?;
-        *current = Some(index);
+        for i in 0..self.info.num_configurations {
+            let info = self.query_configuration_info(i).await?;
 
-        Ok(info)
-    }
+            if predicate(&info) {
+                log::debug!("Selected configuration: {:#?}", info);
+                let config = current_config.insert(info);
 
-    async fn query_configurations(
-        control_pipe: &UsbControlPipeAccess,
-        num_configurations: u8,
-    ) -> Result<Vec<UsbConfigurationInfo>, UsbError> {
-        let mut configurations = Vec::new();
-        for i in 0..num_configurations {
-            let configuration = Self::query_configuration(control_pipe, i).await?;
-            configurations.push(configuration);
+                control_pipe
+                    .set_configuration(config.config_value as _)
+                    .await?;
+
+                return Ok(Some(config.clone()));
+            }
         }
-        Ok(configurations)
+
+        Ok(None)
     }
 
-    async fn query_configuration(
-        control_pipe: &UsbControlPipeAccess,
+    pub async fn query_configuration_info(
+        &self,
         index: u8,
     ) -> Result<UsbConfigurationInfo, UsbError> {
+        if index >= self.info.num_configurations {
+            return Err(UsbError::InvalidConfiguration);
+        }
+        let control_pipe = self.control_pipe();
         let query = control_pipe.query_configuration_descriptor(index).await?;
 
         let configuration_name = control_pipe
@@ -252,9 +228,10 @@ impl UsbDeviceAccess {
                         name,
                         number: iface.interface_number,
 
-                        interface_class: iface.interface_class,
+                        interface_class: iface.class(),
                         interface_subclass: iface.interface_subclass,
-                        interface_protocol: iface.interface_protocol,
+                        interface_protocol: iface.protocol(),
+                        interface_protocol_number: iface.interface_protocol,
                     });
                 }
                 _ => (),
@@ -271,13 +248,6 @@ impl UsbDeviceAccess {
         Ok(info)
     }
 
-    // pub async fn query_configuration_info(
-    //     &self,
-    //     index: u8,
-    // ) -> Result<UsbConfigurationInfo, UsbError> {
-    //     let control_pipe = self.control_pipe();
-    // }
-
     pub fn set_detach_handler(&self, handler: Arc<dyn UsbDeviceDetachHandler>) {
         self.device.set_detach_handler(handler);
     }
@@ -290,3 +260,9 @@ impl Deref for UsbDeviceAccess {
         &*self.device
     }
 }
+
+impl fmt::Display for UsbBusAddress {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}:{}", self.bus, self.device)
+    }
+}

kernel/driver/bus/usb/src/info.rs

@@ -29,51 +29,41 @@ pub enum UsbUsageType {
     Reserved,
 }
 
-// #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)]
-// pub enum UsbVersion {
-//     Usb11,
-//     Usb20,
-//     Usb21,
-//     Usb30,
-//     Usb31,
-//     Usb32,
-// }
+#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)]
+pub enum UsbVersion {
+    Usb11,
+    Usb20,
+    Usb21,
+    Usb30,
+    Usb31,
+    Usb32,
+}
 
-pub const CLASS_FROM_INTERFACE: u8 = 0x00;
-pub const CLASS_HID: u8 = 0x03;
-pub const CLASS_MASS_STORAGE: u8 = 0x08;
+primitive_enum! {
+    pub enum UsbDeviceClass: u8 {
+        FromInterface = 0x00,
+        Hid = 0x03,
+        MassStorage = 0x08,
+        Unknown = 0xFF,
+    }
+}
 
-// primitive_enum! {
-//     pub enum UsbDeviceClass: u8 {
-//         FromInterface = 0x00,
-//         Hid = 0x03,
-//         MassStorage = 0x08,
-//         Unknown = 0xFF,
-//     }
-// }
-//
-// primitive_enum! {
-//     pub enum UsbDeviceProtocol: u8 {
-//         FromInterface = 0x00,
-//         Unknown = 0xFF,
-//     }
-// }
+primitive_enum! {
+    pub enum UsbDeviceProtocol: u8 {
+        FromInterface = 0x00,
+        Unknown = 0xFF,
+    }
+}
 
 #[derive(Debug, Clone)]
 pub struct UsbInterfaceInfo {
     pub name: String,
     pub number: u8,
 
-    pub interface_class: u8,
+    pub interface_class: UsbDeviceClass,
     pub interface_subclass: u8,
-    pub interface_protocol: u8,
-    // pub name: String,
-    // pub number: u8,
-
-    // pub interface_class: UsbDeviceClass,
-    // pub interface_subclass: u8,
-    // pub interface_protocol: UsbDeviceProtocol,
-    // pub interface_protocol_number: u8,
+    pub interface_protocol: UsbDeviceProtocol,
+    pub interface_protocol_number: u8,
 }
 
 #[derive(Debug, Clone)]
@@ -97,14 +87,15 @@ pub struct UsbDeviceInfo {
     pub manufacturer: String,
     pub product: String,
 
-    pub usb_version: u16,
+    pub usb_version: UsbVersion,
 
     pub id_vendor: u16,
     pub id_product: u16,
 
-    pub device_class: u8,
+    pub device_class: UsbDeviceClass,
     pub device_subclass: u8,
-    pub device_protocol: u8,
+    pub device_protocol: UsbDeviceProtocol,
+    pub device_protocol_number: u8,
 
     /// Max packet size for endpoint zero
     pub max_packet_size: usize,
@@ -112,37 +103,37 @@ pub struct UsbDeviceInfo {
     pub num_configurations: u8,
 }
 
-// impl UsbVersion {
-//     pub fn is_version_3(&self) -> bool {
-//         matches!(self, Self::Usb30 | Self::Usb31 | Self::Usb32)
-//     }
-//
-//     pub fn from_bcd_usb(value: u16) -> Option<Self> {
-//         match value {
-//             0x110 => Some(UsbVersion::Usb11),
-//             0x200..=0x20F => Some(UsbVersion::Usb20),
-//             0x210..=0x21F => Some(UsbVersion::Usb21),
-//             0x300 => Some(UsbVersion::Usb30),
-//             0x310 => Some(UsbVersion::Usb31),
-//             0x320 => Some(UsbVersion::Usb32),
-//             _ => None,
-//         }
-//     }
-// }
-//
-// impl fmt::Display for UsbVersion {
-//     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-//         let string = match self {
-//             Self::Usb11 => "USB1.1",
-//             Self::Usb20 => "USB2.0",
-//             Self::Usb21 => "USB2.1",
-//             Self::Usb30 => "USB3.0",
-//             Self::Usb31 => "USB3.1",
-//             Self::Usb32 => "USB3.2",
-//         };
-//         f.write_str(string)
-//     }
-// }
+impl UsbVersion {
+    pub fn is_version_3(&self) -> bool {
+        matches!(self, Self::Usb30 | Self::Usb31 | Self::Usb32)
+    }
+
+    pub fn from_bcd_usb(value: u16) -> Option<Self> {
+        match value {
+            0x110 => Some(UsbVersion::Usb11),
+            0x200..=0x20F => Some(UsbVersion::Usb20),
+            0x210..=0x21F => Some(UsbVersion::Usb21),
+            0x300 => Some(UsbVersion::Usb30),
+            0x310 => Some(UsbVersion::Usb31),
+            0x320 => Some(UsbVersion::Usb32),
+            _ => None,
+        }
+    }
+}
+
+impl fmt::Display for UsbVersion {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let string = match self {
+            Self::Usb11 => "USB1.1",
+            Self::Usb20 => "USB2.0",
+            Self::Usb21 => "USB2.1",
+            Self::Usb30 => "USB3.0",
+            Self::Usb31 => "USB3.1",
+            Self::Usb32 => "USB3.2",
+        };
+        f.write_str(string)
+    }
+}
 
 impl UsbEndpointInfo {
     pub fn is(&self, ty: UsbEndpointType, dir: UsbDirection) -> bool {

kernel/driver/bus/usb/src/interface.rs

@@ -1 +0,0 @@
-pub struct UsbInterface {}

kernel/driver/bus/usb/src/lib.rs

@@ -8,20 +8,15 @@
     maybe_uninit_fill
 )]
 
-use crate::sysfs::UsbBusKObject;
-
 extern crate alloc;
 
-pub mod address;
 pub mod bus;
 pub mod communication;
 pub mod descriptor;
 pub mod device;
 pub mod error;
 pub mod info;
-pub mod interface;
 pub mod pipe;
-pub mod sysfs;
 pub mod util;
 
 pub mod class_driver;
@@ -30,8 +25,4 @@ pub mod class_driver;
 
 pub trait UsbEndpoint: Sync {}
 
-pub trait UsbHostController: Sync + Send {
-    fn register_sysfs_properties(&self, kobject: &UsbBusKObject) {
-        let _ = kobject;
-    }
-}
+pub trait UsbHostController: Sync + Send {}

kernel/driver/bus/usb/src/pipe/control.rs

@@ -37,7 +37,6 @@ pub trait UsbDeviceRequest: Sized + Pod {
 pub trait UsbClassSpecificRequest: Sized + Pod {
     const BM_REQUEST_TYPE: u8;
     const B_REQUEST: u8;
-    const W_VALUE: u16 = 0;
 }
 
 pub trait UsbDescriptorRequest: UsbDeviceRequest {

kernel/driver/bus/usb/src/sysfs.rs

@@ -1,129 +0,0 @@
-use alloc::{format, sync::Arc};
-use libk::{
-    error::Error,
-    fs::sysfs::{
-        self,
-        attribute::{IntegerAttribute, IntegerAttributeFormat, IntegerAttributeOps},
-        object::KObject,
-    },
-};
-use libk_util::OneTimeInit;
-
-use crate::{bus::UsbBusWrapper, device::UsbDeviceAccess};
-
-pub type UsbBusKObject = Arc<KObject<Arc<UsbBusWrapper>>>;
-pub type UsbDeviceKObject = Arc<KObject<Arc<UsbDeviceAccess>>>;
-
-pub(crate) fn register_bus_kobject(bus: &Arc<UsbBusWrapper>) -> UsbBusKObject {
-    let root = sysfs_usb_root();
-    let bus_kobject = KObject::new(bus.clone());
-    bus.hc.register_sysfs_properties(&bus_kobject);
-    root.add_object(format!("{}", bus.index), bus_kobject.clone())
-        .ok();
-    bus_kobject
-}
-
-pub(crate) fn register_device_kobject(device: &Arc<UsbDeviceAccess>) -> UsbDeviceKObject {
-    struct Class;
-    struct Subclass;
-    struct Protocol;
-    struct Version;
-    struct IdVendor;
-    struct IdProduct;
-
-    impl IntegerAttributeOps<u8> for Class {
-        type Data = Arc<UsbDeviceAccess>;
-        const NAME: &'static str = "class";
-        const FORMAT: IntegerAttributeFormat = IntegerAttributeFormat::Hex;
-
-        fn read(state: &Self::Data) -> Result<u8, Error> {
-            Ok(state.info.device_class)
-        }
-    }
-    impl IntegerAttributeOps<u8> for Subclass {
-        type Data = Arc<UsbDeviceAccess>;
-        const NAME: &'static str = "subclass";
-        const FORMAT: IntegerAttributeFormat = IntegerAttributeFormat::Hex;
-
-        fn read(state: &Self::Data) -> Result<u8, Error> {
-            Ok(state.info.device_subclass)
-        }
-    }
-    impl IntegerAttributeOps<u8> for Protocol {
-        type Data = Arc<UsbDeviceAccess>;
-        const NAME: &'static str = "protocol";
-        const FORMAT: IntegerAttributeFormat = IntegerAttributeFormat::Hex;
-
-        fn read(state: &Self::Data) -> Result<u8, Error> {
-            Ok(state.info.device_protocol)
-        }
-    }
-    impl IntegerAttributeOps<u16> for Version {
-        type Data = Arc<UsbDeviceAccess>;
-        const NAME: &'static str = "version";
-        const FORMAT: IntegerAttributeFormat = IntegerAttributeFormat::Hex;
-
-        fn read(state: &Self::Data) -> Result<u16, Error> {
-            Ok(state.info.usb_version)
-        }
-    }
-    impl IntegerAttributeOps<u16> for IdVendor {
-        type Data = Arc<UsbDeviceAccess>;
-        const NAME: &'static str = "vendor";
-        const FORMAT: IntegerAttributeFormat = IntegerAttributeFormat::Hex;
-
-        fn read(state: &Self::Data) -> Result<u16, Error> {
-            Ok(state.info.id_vendor)
-        }
-    }
-    impl IntegerAttributeOps<u16> for IdProduct {
-        type Data = Arc<UsbDeviceAccess>;
-        const NAME: &'static str = "product";
-        const FORMAT: IntegerAttributeFormat = IntegerAttributeFormat::Hex;
-
-        fn read(state: &Self::Data) -> Result<u16, Error> {
-            Ok(state.info.id_product)
-        }
-    }
-
-    let bus_kobject = device.bus.kobject();
-    let device_kobject = KObject::new(device.clone());
-
-    device_kobject
-        .add_attribute(IntegerAttribute::from(Class))
-        .ok();
-    device_kobject
-        .add_attribute(IntegerAttribute::from(Subclass))
-        .ok();
-    device_kobject
-        .add_attribute(IntegerAttribute::from(Protocol))
-        .ok();
-    device_kobject
-        .add_attribute(IntegerAttribute::from(Version))
-        .ok();
-    device_kobject
-        .add_attribute(IntegerAttribute::from(IdVendor))
-        .ok();
-    device_kobject
-        .add_attribute(IntegerAttribute::from(IdProduct))
-        .ok();
-
-    let address = device.bus_address();
-
-    bus_kobject
-        .add_object(format!("{}", address.device), device_kobject.clone())
-        .ok();
-
-    device_kobject
-}
-
-fn sysfs_usb_root() -> &'static Arc<KObject<()>> {
-    static USB_ROOT: OneTimeInit<Arc<KObject<()>>> = OneTimeInit::new();
-
-    USB_ROOT.or_init_with(|| {
-        let bus_object = sysfs::bus().expect("bus object");
-        let usb_object = KObject::new(());
-        bus_object.add_object("usb", usb_object.clone()).ok();
-        usb_object
-    })
-}

kernel/driver/fs/ext2/src/dir/walk.rs

@@ -127,7 +127,7 @@ impl<'a> DirentIter<'a> {
         self.offset
     }
 
-    pub fn next_record(&mut self) -> Option<Record<'_, &[u8]>> {
+    pub fn next_record(&mut self) -> Option<Record<&[u8]>> {
         if self.offset + size_of::<Dirent>() > self.block.len() {
             return None;
         }
@@ -183,7 +183,7 @@ impl<'a> DirentIterMut<'a> {
         Self { fs, block, offset }
     }
 
-    pub fn next_record(&mut self) -> Option<Record<'_, &mut [u8]>> {
+    pub fn next_record(&mut self) -> Option<Record<&mut [u8]>> {
         if self.offset + size_of::<Dirent>() > self.block.len() {
             return None;
         }

kernel/driver/fs/ext2/src/lib.rs

@@ -1,6 +1,6 @@
 #![feature(if_let_guard, async_drop, impl_trait_in_assoc_type)]
 #![cfg_attr(not(test), no_std)]
-#![allow(clippy::new_ret_no_self, incomplete_features)]
+#![allow(clippy::new_ret_no_self)]
 
 extern crate alloc;
 

kernel/driver/input/src/lib.rs

@@ -9,20 +9,14 @@ use async_trait::async_trait;
 use device_api::device::Device;
 use libk::{device::char::CharDevice, vfs::FileReadiness};
 use libk_util::{ring::LossyRingQueue, OneTimeInit};
-use yggdrasil_abi::{
-    abi_serde::wire,
-    error::Error,
-    io::{KeyboardKeyEvent, MouseEvent},
-};
+use yggdrasil_abi::{error::Error, io::KeyboardKeyEvent};
 
 #[derive(Clone, Copy)]
 pub struct KeyboardDevice;
-#[derive(Clone, Copy)]
-pub struct MouseDevice;
 
 impl FileReadiness for KeyboardDevice {
     fn poll_read(&self, cx: &mut Context<'_>) -> Poll<Result<(), Error>> {
-        KEYBOARD_INPUT_QUEUE.poll_readable(cx).map(Ok)
+        INPUT_QUEUE.poll_readable(cx).map(Ok)
     }
 }
 
@@ -39,7 +33,7 @@ impl CharDevice for KeyboardDevice {
             return Ok(0);
         }
 
-        let ev = KEYBOARD_INPUT_QUEUE.read().await;
+        let ev = INPUT_QUEUE.read().await;
 
         buf[..4].copy_from_slice(&ev.as_bytes());
 
@@ -51,7 +45,7 @@ impl CharDevice for KeyboardDevice {
             return Ok(0);
         }
 
-        let ev = KEYBOARD_INPUT_QUEUE.try_read().ok_or(Error::WouldBlock)?;
+        let ev = INPUT_QUEUE.try_read().ok_or(Error::WouldBlock)?;
 
         buf[..4].copy_from_slice(&ev.as_bytes());
 
@@ -74,68 +68,15 @@ impl CharDevice for KeyboardDevice {
     }
 }
 
-impl FileReadiness for MouseDevice {
-    fn poll_read(&self, cx: &mut Context<'_>) -> Poll<Result<(), Error>> {
-        MOUSE_INPUT_QUEUE.poll_readable(cx).map(Ok)
-    }
-}
-
-impl Device for MouseDevice {
-    fn display_name(&self) -> &str {
-        "Mouse input pseudo-device"
-    }
-}
-
-#[async_trait]
-impl CharDevice for MouseDevice {
-    async fn read(&self, buf: &mut [u8]) -> Result<usize, Error> {
-        let ev = MOUSE_INPUT_QUEUE.read().await;
-        let len = wire::to_slice(&ev, buf)?;
-        Ok(len)
-    }
-
-    fn read_nonblocking(&self, buf: &mut [u8]) -> Result<usize, Error> {
-        let ev = MOUSE_INPUT_QUEUE.try_read().ok_or(Error::WouldBlock)?;
-        let len = wire::to_slice(&ev, buf)?;
-        Ok(len)
-    }
-
-    fn is_writeable(&self) -> bool {
-        false
-    }
-
-    fn device_request(&self, option: u32, buffer: &mut [u8], len: usize) -> Result<usize, Error> {
-        let _ = option;
-        let _ = buffer;
-        let _ = len;
-        Err(Error::InvalidOperation)
-    }
-
-    fn is_terminal(&self) -> bool {
-        false
-    }
-}
-
-static KEYBOARD_INPUT_QUEUE: LossyRingQueue<KeyboardKeyEvent> = LossyRingQueue::with_capacity(32);
+static INPUT_QUEUE: LossyRingQueue<KeyboardKeyEvent> = LossyRingQueue::with_capacity(32);
 static KEYBOARD_DEVICE: OneTimeInit<Arc<KeyboardDevice>> = OneTimeInit::new();
 
-static MOUSE_INPUT_QUEUE: LossyRingQueue<MouseEvent> = LossyRingQueue::with_capacity(32);
-static MOUSE_DEVICE: OneTimeInit<Arc<MouseDevice>> = OneTimeInit::new();
-
-pub fn setup_keyboard() -> Arc<KeyboardDevice> {
+pub fn setup() -> Arc<KeyboardDevice> {
     KEYBOARD_DEVICE
         .or_init_with(|| Arc::new(KeyboardDevice))
         .clone()
 }
 
-pub fn setup_mouse() -> Arc<MouseDevice> {
-    MOUSE_DEVICE.or_init_with(|| Arc::new(MouseDevice)).clone()
-}
-
-pub fn send_keyboard_event(ev: KeyboardKeyEvent) {
-    KEYBOARD_INPUT_QUEUE.write(ev);
-}
-
-pub fn send_mouse_event(ev: MouseEvent) {
-    MOUSE_INPUT_QUEUE.write(ev);
+pub fn send_event(ev: KeyboardKeyEvent) {
+    INPUT_QUEUE.write(ev);
 }

kernel/driver/net/core/src/ephy.rs

@@ -125,31 +125,24 @@ impl<'a, M: MdioBus> PhyAccess<'a, M> {
         })
     }
 
-    pub fn setup_link(&self, have_pause: bool, force_gbesr: Option<GBESR>) -> Result<(), Error> {
+    pub fn setup_link(&self, have_pause: bool, force_gbesr: GBESR) -> Result<(), Error> {
         let bmsr = BMSR::from(self.read_reg(REG_BMSR)?);
-        let gbesr = if let Some(force_gbesr) = force_gbesr {
-            let mut gbesr = if bmsr.contains(BMSR::EXT_STATUS_1000BASET) {
-                GBESR::from(self.read_reg(REG_GBESR)?)
-            } else {
-                GBESR::empty()
-            };
-            gbesr |= force_gbesr;
-            Some(gbesr)
+        let mut gbesr = if bmsr.contains(BMSR::EXT_STATUS_1000BASET) {
+            GBESR::from(self.read_reg(REG_GBESR)?)
         } else {
-            None
+            GBESR::empty()
         };
+        gbesr |= force_gbesr;
         let mut anar = ANAR::from_capabilities(bmsr);
         if have_pause {
             anar |= ANAR::HAVE_PAUSE | ANAR::ASM_DIR;
         }
         let mut gbcr = GBCR::empty();
-        if let Some(gbesr) = gbesr {
-            if gbesr.contains(GBESR::HAVE_1000BASET_HALF) {
-                gbcr |= GBCR::HAVE_1000BASET_HALF;
-            }
-            if gbesr.contains(GBESR::HAVE_1000BASET_FULL) {
-                gbcr |= GBCR::HAVE_1000BASET_FULL;
-            }
+        if gbesr.contains(GBESR::HAVE_1000BASET_HALF) {
+            gbcr |= GBCR::HAVE_1000BASET_HALF;
+        }
+        if gbesr.contains(GBESR::HAVE_1000BASET_FULL) {
+            gbcr |= GBCR::HAVE_1000BASET_FULL;
         }
 
         self.write_reg(REG_ANAR, anar.bits())?;

kernel/driver/net/core/src/lib.rs

@@ -1,4 +1,4 @@
-#![feature(map_try_insert, let_chains)]
+#![feature(map_try_insert, let_chains, result_flattening)]
 #![allow(clippy::type_complexity, clippy::new_without_default)]
 #![no_std]
 

kernel/driver/net/core/src/socket/local/mod.rs

@@ -15,7 +15,7 @@ enum OwnedAddress {
     Anonymous(u64),
 }
 
-pub fn load_address(bytes: &[u8]) -> Result<LocalSocketAddress<'_>, Error> {
+pub fn load_address(bytes: &[u8]) -> Result<LocalSocketAddress, Error> {
     Ok(wire::from_slice(bytes)?)
 }
 
@@ -76,7 +76,7 @@ pub fn write_ancillary(
 }
 
 impl OwnedAddress {
-    fn to_borrowed(&self) -> LocalSocketAddress<'_> {
+    fn to_borrowed(&self) -> LocalSocketAddress {
         match self {
             Self::Path(path) => LocalSocketAddress::Path(path.as_ref()),
             Self::Anonymous(anon) => LocalSocketAddress::Anonymous(*anon),

kernel/driver/net/core/src/socket/tcp/listener.rs

@@ -70,7 +70,7 @@ impl TcpListener {
     pub(super) fn poll_accept(
         &self,
         cx: &mut Context<'_>,
-    ) -> Poll<IrqSafeSpinlockGuard<'_, Vec<Arc<TcpStream>>>> {
+    ) -> Poll<IrqSafeSpinlockGuard<Vec<Arc<TcpStream>>>> {
         let lock = self.pending_accept.lock();
         self.accept_notify.register(cx.waker());
         if !lock.is_empty() {

kernel/driver/net/core/src/socket/tcp/stream.rs

@@ -214,7 +214,7 @@ impl TcpStream {
     pub(super) fn poll_receive(
         &self,
         cx: &mut Context<'_>,
-    ) -> Poll<Result<IrqSafeRwLockWriteGuard<'_, TcpConnection>, Error>> {
+    ) -> Poll<Result<IrqSafeRwLockWriteGuard<TcpConnection>, Error>> {
         let lock = self.connection.write();
         match lock.poll_receive(cx) {
             Poll::Ready(Ok(())) => Poll::Ready(Ok(lock)),

kernel/driver/net/igbe/src/lib.rs

@@ -26,15 +26,12 @@ use ygg_driver_pci::{
 };
 use yggdrasil_abi::net::{link::LinkState, MacAddress};
 
-use crate::regs::Revision;
-
 extern crate alloc;
 
 mod regs;
 mod ring;
 
 struct Igbe {
-    chip: Revision,
     regs: IrqSafeSpinlock<Regs>,
     dma: Arc<dyn DmaAllocator>,
     pci: PciDeviceInfo,
@@ -46,9 +43,8 @@ struct Igbe {
 }
 
 impl Igbe {
-    pub fn new(dma: Arc<dyn DmaAllocator>, regs: Regs, chip: Revision, pci: PciDeviceInfo) -> Self {
+    pub fn new(dma: Arc<dyn DmaAllocator>, regs: Regs, pci: PciDeviceInfo) -> Self {
         Self {
-            chip,
             dma,
             pci,
             mac: OneTimeInit::new(),
@@ -78,7 +74,7 @@ impl Device for Igbe {
         regs.reset(Duration::from_millis(200))?;
         // Intel 8257x manuals say an additional interrupt disable is needed after a global reset
         regs.disable_interrupts();
-        regs.set_link_up(self.chip)?;
+        regs.set_link_up()?;
 
         // Initialize Rx
         regs.initialize_receiver(&rx_ring);
@@ -179,10 +175,6 @@ impl NetworkDevice for Igbe {
 
 pci_driver! {
     matches: [
-        device (0x8086:0x100E), // 82540EM (E1000)
-        device (0x8086:0x100C), // 82544GC (E1000)
-        device (0x8086:0x100F), // 82545EM (E1000)
-        device (0x8086:0x10D3), // 82574L (E1000E) [[BROKEN]]
         device (0x8086:0x10C9), // 82576 GbE
         device (0x8086:0x1502), // 82579LM GbE (Lewisville)
     ],
@@ -205,22 +197,11 @@ pci_driver! {
                 }
             };
 
-            let chip = match info.device_id {
-                0x100E | 0x100C | 0x100F => Revision::I8254x,
-                0x10D3 => Revision::I82574L,
-                0x10C9 => Revision::I82576,
-                0x1502 => Revision::I82579LM,
-                id => {
-                    log::error!("Invalid igbe chip variant: {id:#04x}");
-                    return Err(Error::InvalidOperation)
-                },
-            };
-
             info.init_interrupts(PreferredInterruptMode::Msi(true))?;
             info.set_command(true, use_mmio, !use_mmio, true);
 
             let regs = unsafe { Regs::map(base) }?;
-            let device = Igbe::new(dma.clone(), regs, chip, info.clone());
+            let device = Igbe::new(dma.clone(), regs, info.clone());
             Ok(Arc::new(device))
         }
 

kernel/driver/net/igbe/src/regs.rs

@@ -42,14 +42,6 @@ pub trait Reg {
     const OFFSET: u16;
 }
 
-#[derive(Debug, Clone, Copy, PartialEq)]
-pub enum Revision {
-    I8254x,
-    I82574L,
-    I82576,
-    I82579LM,
-}
-
 register_bitfields! {
     u32,
     pub CTRL [
@@ -337,7 +329,6 @@ impl MdioBus for Regs {
 
         let mdic = self.inner.extract();
         if mdic.matches_all(MDIC::E::SET) {
-            log::warn!("MDIO read error: phyaddr={phyaddr:#x}, regaddr={regaddr:#x}");
             return Err(Error::InvalidOperation);
         }
 
@@ -359,9 +350,6 @@ impl MdioBus for Regs {
         )?;
 
         if self.inner.matches_all(MDIC::E::SET) {
-            log::warn!(
-                "MDIO write error: phyaddr={phyaddr:#x}, regaddr={regaddr:#x}, value={value:#x}"
-            );
             return Err(Error::InvalidOperation);
         }
 
@@ -415,7 +403,7 @@ impl Regs {
         })
     }
 
-    pub fn set_link_up(&mut self, chip: Revision) -> Result<(), Error> {
+    pub fn set_link_up(&mut self) -> Result<(), Error> {
         self.inner
             .modify(CTRL::SLU::SET + CTRL::RFCE::SET + CTRL::TFCE::SET);
 
@@ -424,14 +412,8 @@ impl Regs {
         let (id0, id1) = phy.id()?;
         log::info!("PHY {:04x}:{:04x}", id0, id1);
 
-        phy.reset(Duration::from_millis(200))
-            .inspect_err(|e| log::error!("PHY reset error {e:?}"))?;
-        let force_gbesr = match chip {
-            Revision::I82576 | Revision::I82579LM => Some(GBESR::empty()),
-            _ => None,
-        };
-        phy.setup_link(true, force_gbesr)
-            .inspect_err(|e| log::error!("PHY setup error: {e:?}"))?;
+        phy.reset(Duration::from_millis(200))?;
+        phy.setup_link(true, GBESR::empty())?;
 
         Ok(())
     }

kernel/driver/net/rtl81xx/src/lib.rs

@@ -7,7 +7,8 @@ use rtl8139::Rtl8139;
 use rtl8168::Rtl8168;
 use ygg_driver_pci::{
     capability::{
-        DevicePowerState, PciExpressCapability, PcieLinkControl, PowerManagementCapability,
+        express::LinkControl, power::DevicePowerState, PciExpressCapability,
+        PowerManagementCapability,
     },
     device::{PciDeviceInfo, PreferredInterruptMode},
     macros::pci_driver,
@@ -30,9 +31,7 @@ pci_driver! {
             info.init_interrupts(PreferredInterruptMode::Msi(false))?;
 
             if let Some(mut pcie) = info.config_space.capability::<PciExpressCapability>() {
-                let mut lcr = pcie.link_control();
-                lcr.remove(PcieLinkControl::ASPM_MASK | PcieLinkControl::ECPM);
-                pcie.set_link_control(lcr);
+                pcie.modify_link_control(LinkControl::ASPM::CLEAR + LinkControl::CLOCK_PM::CLEAR);
             }
 
             // Enable MMIO + interrupts + bus mastering
@@ -76,9 +75,7 @@ pci_driver! {
             }
 
             if let Some(mut pcie) = info.config_space.capability::<PciExpressCapability>() {
-                let mut lcr = pcie.link_control();
-                lcr.remove(PcieLinkControl::ASPM_MASK | PcieLinkControl::ECPM);
-                pcie.set_link_control(lcr);
+                pcie.modify_link_control(LinkControl::ASPM::CLEAR + LinkControl::CLOCK_PM::CLEAR);
             }
 
             let device = Rtl8168::new(dma.clone(), base, info.clone())?;

kernel/driver/net/rtl81xx/src/rtl8168.rs

@@ -498,7 +498,7 @@ impl Regs {
         phy.write_reg(0x0E, 0x00)?;
 
         phy.reset(timeout)?;
-        phy.setup_link(true, Some(GBESR::empty()))?;
+        phy.setup_link(true, GBESR::empty())?;
 
         psleep(Duration::from_millis(100));
 

kernel/driver/net/stmmac/src/lib.rs

@@ -337,7 +337,7 @@ impl Device for Stmmac {
         let (id0, id1) = phy.id()?;
         log::info!("stmmac: PHY {id0:04x}:{id1:04x}");
         phy.reset(Duration::from_millis(100))?;
-        phy.setup_link(true, Some(GBESR::empty()))?;
+        phy.setup_link(true, GBESR::empty())?;
 
         self.inner.init(Inner {
             regs: IrqSafeSpinlock::new(regs),

kernel/driver/sound/core/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "ygg_driver_sound_core"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+device-api.workspace = true
+yggdrasil-abi.workspace = true
+libk-mm.workspace = true
+libk-util.workspace = true
+libk.workspace = true
+
+log.workspace = true
+async-trait.workspace = true
+futures-util.workspace = true

kernel/driver/sound/core/src/lib.rs

@@ -0,0 +1,122 @@
+#![no_std]
+
+use core::{
+    sync::atomic::{AtomicU32, Ordering},
+    task::{Context, Poll},
+};
+
+use alloc::{boxed::Box, collections::btree_map::BTreeMap, format, sync::Arc, vec::Vec};
+use async_trait::async_trait;
+use device_api::device::Device;
+use libk::{block, device::char::CharDevice, error::Error, fs::devfs, vfs::FileReadiness};
+use libk_util::sync::spin_rwlock::IrqSafeRwLock;
+use yggdrasil_abi::io::FileMode;
+
+extern crate alloc;
+
+#[derive(Debug, Clone)]
+pub enum SampleFormat {
+    S8,
+    S16Le,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[repr(usize)]
+pub enum SampleRate {
+    Rate8000 = 8000,
+    Rate11025 = 11025,
+    Rate16000 = 16000,
+    Rate22050 = 22050,
+    Rate32000 = 32000,
+    Rate44100 = 44100,
+    Rate48000 = 48000,
+    Rate88200 = 88200,
+    Rate96000 = 96000,
+    Rate176400 = 176400,
+    Rate192000 = 192000,
+    Rate384000 = 384000,
+}
+
+#[derive(Debug, Clone)]
+pub struct SinkFormat {
+    pub sample_rate: SampleRate,
+    pub sample_format: SampleFormat,
+    pub channels: usize,
+}
+
+#[derive(Debug, Clone)]
+pub struct SinkSupportedFormats {
+    pub rates: Vec<usize>,
+    pub formats: Vec<SampleFormat>,
+    pub channels: usize,
+}
+
+#[async_trait]
+pub trait AudioSink: Sync + Send {
+    async fn set_format(&self, format: SinkFormat) -> Result<(), Error>;
+    async fn current_format(&self) -> Result<SinkFormat, Error>;
+    async fn supported_formats(&self) -> Result<SinkSupportedFormats, Error>;
+
+    async fn write(&self, data: &[u8]) -> Result<usize, Error>;
+
+    async fn start(&self) -> Result<(), Error>;
+    fn stop(&self) -> Result<(), Error>;
+
+    fn display_name(&self) -> &str {
+        "Audio Sink"
+    }
+}
+
+pub struct AudioSinkWrapper(Arc<dyn AudioSink>);
+
+impl Device for AudioSinkWrapper {
+    fn display_name(&self) -> &str {
+        self.0.display_name()
+    }
+}
+
+impl FileReadiness for AudioSinkWrapper {
+    fn poll_read(&self, cx: &mut Context<'_>) -> Poll<Result<(), Error>> {
+        todo!()
+    }
+}
+
+#[async_trait]
+impl CharDevice for AudioSinkWrapper {
+    fn close(&self) -> Result<(), Error> {
+        self.0.stop()
+    }
+
+    async fn write(&self, buffer: &[u8]) -> Result<usize, Error> {
+        self.0.write(buffer).await
+    }
+
+    fn device_request(&self, option: u32, buffer: &mut [u8], len: usize) -> Result<usize, Error> {
+        let _ = option;
+        let _ = buffer;
+        let _ = len;
+        log::warn!("device_request unimplemented: {option:#x}");
+        Err(Error::InvalidOperation)
+    }
+}
+
+static SINKS: IrqSafeRwLock<BTreeMap<u32, Arc<AudioSinkWrapper>>> =
+    IrqSafeRwLock::new(BTreeMap::new());
+static LAST_SINK_ID: AtomicU32 = AtomicU32::new(0);
+
+pub fn register_audio_sink(sink: Arc<dyn AudioSink>) {
+    let id = LAST_SINK_ID.fetch_add(1, Ordering::Relaxed);
+    let name = format!("snd{id}");
+    let sink = Arc::new(AudioSinkWrapper(sink));
+    SINKS.write().insert(id, sink.clone());
+    devfs::add_named_char_device(sink, &name, FileMode::new(0o222)).ok();
+}
+
+impl SampleFormat {
+    pub fn sample_size(&self) -> usize {
+        match self {
+            Self::S8 => 1,
+            Self::S16Le => 2,
+        }
+    }
+}

kernel/driver/sound/intel-hda/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "ygg_driver_intel_hda"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+device-api.workspace = true
+yggdrasil-abi.workspace = true
+libk-mm.workspace = true
+libk-util.workspace = true
+libk.workspace = true
+
+ygg_driver_pci.path = "../../bus/pci"
+ygg_driver_sound_core.path = "../core"
+
+log.workspace = true
+tock-registers.workspace = true
+futures-util.workspace = true
+async-trait.workspace = true
+bytemuck.workspace = true

kernel/driver/sound/intel-hda/src/codec/mod.rs

@@ -0,0 +1,527 @@
+use core::fmt;
+
+use alloc::{sync::Arc, vec::Vec};
+use libk::error::Error;
+use tock_registers::{fields::FieldValue, register_bitfields, LocalRegisterCopy};
+use yggdrasil_abi::bitflags;
+
+use crate::{
+    ring::{Command, PinControl, PinDefaultConfig, PinDevice, Verb},
+    HdAudio,
+};
+
+pub mod parameter;
+pub use parameter::*;
+
+#[derive(Debug, Clone, Copy)]
+pub struct Node {
+    pub codec: u8,
+    pub nid: u8,
+}
+
+pub struct WidgetConnectionChain<'a> {
+    afg: &'a AudioNode,
+    current: Option<u8>,
+}
+
+#[derive(Debug)]
+pub struct AudioOutputWidget {
+    channel_count: usize,
+}
+#[derive(Debug)]
+pub struct AudioInputWidget {
+    channel_count: usize,
+}
+#[derive(Debug)]
+pub struct PinComplexWidget {
+    default_config: PinDefaultConfig,
+}
+#[derive(Debug)]
+pub struct MixerWidget {}
+#[derive(Debug)]
+pub enum AudioWidget {
+    AudioOutput(AudioOutputWidget),
+    AudioInput(AudioInputWidget),
+    PinComplex(PinComplexWidget),
+    Mixer(MixerWidget),
+    Other,
+}
+
+#[derive(Debug)]
+pub enum AudioFormat {
+    Pcm { rates: Vec<usize>, bits: Vec<u8> },
+    Other,
+}
+
+#[derive(Debug)]
+pub struct AudioWidgetNode {
+    parent_nid: u8,
+    node: Node,
+
+    output_amp: Option<LocalRegisterCopy<u32, AmpCapabilities::Register>>,
+    input_amp: Option<LocalRegisterCopy<u32, AmpCapabilities::Register>>,
+    connection: Option<u8>,
+    formats: Vec<AudioFormat>,
+
+    widget: AudioWidget,
+}
+
+pub struct AudioNode {
+    node: Node,
+    widget_connection: Vec<(u8, u8)>,
+    widgets: Vec<AudioWidgetNode>,
+}
+
+pub struct Codec {
+    hda: Arc<HdAudio>,
+
+    root: Node,
+    nodes: Vec<AudioNode>,
+}
+
+impl AudioFormat {
+    async fn query_from_node(hda: &HdAudio, node: Node) -> Result<Vec<Self>, Error> {
+        let supported_formats = node
+            .get_parameter::<SupportedStreamFormats::Register>(hda)
+            .await?;
+
+        let mut formats = Vec::new();
+
+        if supported_formats.matches_all(SupportedStreamFormats::PCM::SET) {
+            formats.push(Self::query_pcm(hda, node).await?);
+        }
+        if supported_formats.matches_all(SupportedStreamFormats::F32::SET) {
+            formats.push(Self::Other);
+        }
+        if supported_formats.matches_all(SupportedStreamFormats::AC3::SET) {
+            formats.push(Self::Other);
+        }
+
+        Ok(formats)
+    }
+
+    async fn query_pcm(hda: &HdAudio, node: Node) -> Result<Self, Error> {
+        // Conversion tables
+        const RATES: &[(usize, FieldValue<u32, SupportedPcmFormats::Register>)] = &[
+            (8000, SupportedPcmFormats::RATE_8::SET),
+            (11024, SupportedPcmFormats::RATE_11_025::SET),
+            (16000, SupportedPcmFormats::RATE_16::SET),
+            (22050, SupportedPcmFormats::RATE_22_05::SET),
+            (32000, SupportedPcmFormats::RATE_32::SET),
+            (44100, SupportedPcmFormats::RATE_44_1::SET),
+            (48000, SupportedPcmFormats::RATE_48::SET),
+            (88200, SupportedPcmFormats::RATE_88_2::SET),
+            (96000, SupportedPcmFormats::RATE_96::SET),
+            (176400, SupportedPcmFormats::RATE_176_4::SET),
+            (192000, SupportedPcmFormats::RATE_192::SET),
+            (384000, SupportedPcmFormats::RATE_384::SET),
+        ];
+        const BITS: &[(u8, FieldValue<u32, SupportedPcmFormats::Register>)] = &[
+            (8, SupportedPcmFormats::BITS_8::SET),
+            (16, SupportedPcmFormats::BITS_16::SET),
+            (20, SupportedPcmFormats::BITS_20::SET),
+            (24, SupportedPcmFormats::BITS_24::SET),
+            (32, SupportedPcmFormats::BITS_32::SET),
+        ];
+
+        let supported_pcm_formats = node
+            .get_parameter::<SupportedPcmFormats::Register>(hda)
+            .await?;
+
+        let mut supported_rates = Vec::new();
+        let mut supported_bits = Vec::new();
+
+        for &(rate, bit) in RATES {
+            if supported_pcm_formats.matches_all(bit) {
+                supported_rates.push(rate);
+            }
+        }
+        for &(bits, bit) in BITS {
+            if supported_pcm_formats.matches_all(bit) {
+                supported_bits.push(bits);
+            }
+        }
+
+        Ok(Self::Pcm {
+            rates: supported_rates,
+            bits: supported_bits,
+        })
+    }
+}
+
+impl Node {
+    pub async fn perform_command(
+        &self,
+        hda: &HdAudio,
+        verb: Verb,
+        parameter: u32,
+    ) -> Result<u32, Error> {
+        hda.perform_command(Command::new(self.codec, self.nid, verb, parameter))
+            .await
+    }
+
+    pub async fn get_parameter<P: NodeParameter>(&self, hda: &HdAudio) -> Result<P::Value, Error> {
+        self.perform_command(hda, Verb::GetParameter, P::NUMBER as u32)
+            .await
+            .map(P::from_response)
+    }
+
+    pub async fn get_connection_list_entry(
+        &self,
+        hda: &HdAudio,
+        index: usize,
+    ) -> Result<u32, Error> {
+        self.perform_command(hda, Verb::GetConnectionListEntry, index as u32)
+            .await
+    }
+
+    pub async fn set_stream(&self, hda: &HdAudio, stream: u8, channel: u8) -> Result<(), Error> {
+        self.perform_command(
+            hda,
+            Verb::SetConverterStreamChannel,
+            ((stream as u32) << 4) | (channel as u32),
+        )
+        .await?;
+        Ok(())
+    }
+
+    pub async fn get_pin_default_config(&self, hda: &HdAudio) -> Result<PinDefaultConfig, Error> {
+        self.perform_command(hda, Verb::GetPinWidgetDefaultConfig, 0)
+            .await
+            .map(PinDefaultConfig::from)
+    }
+
+    pub async fn set_power_state(&self, hda: &HdAudio, power: bool) -> Result<(), Error> {
+        let ps_set = if power { 0x0 } else { 0x4 };
+        self.perform_command(hda, Verb::SetPowerState, ps_set)
+            .await?;
+        Ok(())
+    }
+
+    pub async fn set_unsolicited_response(
+        &self,
+        hda: &HdAudio,
+        tag: Option<u8>,
+    ) -> Result<(), Error> {
+        let value = match tag {
+            Some(tag) => (tag as u32) | (1 << 7),
+            None => 0,
+        };
+        self.perform_command(hda, Verb::SetUnsolicitedResponse, value)
+            .await?;
+        Ok(())
+    }
+
+    pub async fn set_pin_control(&self, hda: &HdAudio, pin_control: u32) -> Result<(), Error> {
+        self.perform_command(hda, Verb::SetPinWidgetControl, pin_control & 0xFF)
+            .await?;
+        Ok(())
+    }
+
+    pub async fn get_pin_control(
+        &self,
+        hda: &HdAudio,
+    ) -> Result<LocalRegisterCopy<u32, PinControl::Register>, Error> {
+        self.perform_command(hda, Verb::GetPinWidgetControl, 0)
+            .await
+            .map(LocalRegisterCopy::new)
+    }
+
+    pub async fn modify_pin_control(
+        &self,
+        hda: &HdAudio,
+        change: FieldValue<u32, PinControl::Register>,
+    ) -> Result<(), Error> {
+        let mut value = self.get_pin_control(hda).await?;
+        value.modify(change);
+        self.set_pin_control(hda, value.get()).await
+    }
+}
+
+impl AudioWidgetNode {
+    pub async fn probe(hda: &HdAudio, parent_nid: u8, node: Node) -> Result<Self, Error> {
+        let capabilities = node
+            .get_parameter::<AudioWidgetCapabilities::Register>(hda)
+            .await?;
+        let channel_count = ((capabilities.read(AudioWidgetCapabilities::EXT_CHANNEL_COUNT) << 1)
+            | capabilities.read(AudioWidgetCapabilities::CHANNEL_COUNT))
+            + 1;
+        let in_amp = capabilities.matches_all(AudioWidgetCapabilities::IN_AMP::SET);
+        let out_amp = capabilities.matches_all(AudioWidgetCapabilities::OUT_AMP::SET);
+
+        let in_amp =
+            if in_amp && capabilities.matches_all(AudioWidgetCapabilities::AMP_OVERRIDE::SET) {
+                node.get_parameter::<InputAmpCapabilities>(hda).await?
+            } else {
+                None
+            };
+        let out_amp =
+            if out_amp && capabilities.matches_all(AudioWidgetCapabilities::AMP_OVERRIDE::SET) {
+                node.get_parameter::<OutputAmpCapabilities>(hda).await?
+            } else {
+                None
+            };
+
+        let connection = if capabilities.matches_all(AudioWidgetCapabilities::CONN_LIST::SET) {
+            let (length, long) = node.get_parameter::<ConnectionListLength>(hda).await?;
+            // if length > 1 || long {
+            //     todo!()
+            // }
+            let connection = node.get_connection_list_entry(hda, 0).await?;
+            Some(connection as u8)
+        } else {
+            None
+        };
+
+        let formats = if capabilities.matches_all(AudioWidgetCapabilities::FORMAT_OVERRIDE::SET) {
+            AudioFormat::query_from_node(hda, node).await?
+        } else {
+            Vec::new()
+        };
+
+        let widget = match capabilities.read_as_enum(AudioWidgetCapabilities::TYPE) {
+            Some(AudioWidgetCapabilities::TYPE::Value::AudioOutput) => {
+                // Disable audio outputs immediately
+                node.set_stream(hda, 0x00, 0x00).await?;
+
+                AudioWidget::AudioOutput(AudioOutputWidget {
+                    channel_count: channel_count as usize,
+                })
+            }
+            Some(AudioWidgetCapabilities::TYPE::Value::AudioInput) => {
+                AudioWidget::AudioInput(AudioInputWidget {
+                    channel_count: channel_count as usize,
+                })
+            }
+            Some(AudioWidgetCapabilities::TYPE::Value::PinComplex) => {
+                let default_config = node.get_pin_default_config(hda).await?;
+                AudioWidget::PinComplex(PinComplexWidget { default_config })
+            }
+            Some(AudioWidgetCapabilities::TYPE::Value::AudioMixer) => {
+                AudioWidget::Mixer(MixerWidget {})
+            }
+            v => {
+                log::warn!("hda: unknown audio widget: {v:?}");
+                AudioWidget::Other
+            }
+        };
+
+        Ok(Self {
+            parent_nid,
+            node,
+            widget,
+
+            output_amp: out_amp,
+            input_amp: in_amp,
+            formats,
+            connection,
+        })
+    }
+}
+
+impl AudioNode {
+    pub async fn probe(hda: &HdAudio, node: Node) -> Result<Self, Error> {
+        // Query widgets
+        let subnodes = node.get_parameter::<SubNodeInfo>(hda).await?;
+
+        let mut widgets = Vec::new();
+        for wid in subnodes.iter() {
+            let widget_node = Node {
+                codec: node.codec,
+                nid: wid,
+            };
+            match AudioWidgetNode::probe(hda, node.nid, widget_node).await {
+                Ok(widget) => {
+                    widgets.push(widget);
+                }
+                Err(error) => {
+                    log::warn!(
+                        "hda: could not enumerate {}.{}.{}: {error:?}",
+                        node.codec,
+                        node.nid,
+                        wid
+                    );
+                }
+            }
+        }
+
+        let mut widget_connection = Vec::new();
+        for widget in &widgets {
+            if let Some(connection) = widget.connection {
+                widget_connection.push((widget.node.nid, connection));
+            }
+        }
+
+        Ok(Self {
+            node,
+            widgets,
+            widget_connection,
+        })
+    }
+
+    pub fn widget(&self, wid: u8) -> Option<&AudioWidgetNode> {
+        self.widgets.iter().find(|w| w.node.nid == wid)
+    }
+
+    pub fn widget_supported_formats<'a>(&self, widget: &'a AudioWidgetNode) -> &'a [AudioFormat] {
+        if !widget.formats.is_empty() {
+            &widget.formats
+        } else {
+            // TODO node supported formats
+            &[]
+        }
+    }
+
+    // Returns (pin, kind) pair
+    pub fn default_output_pin(&self) -> Option<(Node, PinDevice)> {
+        let mut best: Option<(Node, PinDevice)> = None;
+        for widget in &self.widgets {
+            if let AudioWidget::PinComplex(pin) = &widget.widget {
+                let Some(connection) = widget.connection else {
+                    continue;
+                };
+                let dev = pin.default_config.default_device;
+
+                let better = match best {
+                    None => true,
+                    Some((_, other)) => dev.output_score() > other.output_score(),
+                };
+
+                if better {
+                    best = Some((widget.node, dev));
+                }
+            }
+        }
+
+        best
+    }
+
+    pub fn connection_chain(&self, source: u8) -> WidgetConnectionChain<'_> {
+        WidgetConnectionChain {
+            afg: self,
+            current: Some(source),
+        }
+    }
+}
+
+impl fmt::Debug for AudioNode {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("AudioNode")
+            .field("id", &self.node.nid)
+            .field("widgets", &self.widgets)
+            .finish()
+    }
+}
+
+impl Codec {
+    pub async fn probe(hda: Arc<HdAudio>, codec: u8) -> Result<Self, Error> {
+        let root_node = Node { codec, nid: 0 };
+        let id = root_node.get_parameter::<DeviceId>(&*hda).await?;
+        if (id.vendor_id == 0 && id.device_id == 0)
+            || (id.vendor_id == 0xFFFF && id.device_id == 0xFFFF)
+        {
+            return Err(Error::DoesNotExist);
+        }
+
+        let root_subnodes = root_node.get_parameter::<SubNodeInfo>(&*hda).await?;
+
+        let mut nodes = Vec::new();
+
+        for nid in root_subnodes.iter() {
+            let node = Node { codec, nid };
+            let node_type = node.get_parameter::<NodeType>(&*hda).await?;
+
+            // Ignore non-audio nodes
+            if node_type != 0x01 {
+                continue;
+            }
+
+            match AudioNode::probe(&*hda, node).await {
+                Ok(node) => nodes.push(node),
+                Err(error) => {
+                    log::warn!("hda: could not enumerate {}.{}: {error:?}", codec, nid);
+                }
+            }
+        }
+
+        Ok(Self {
+            hda,
+            root: root_node,
+            nodes,
+        })
+    }
+
+    pub fn node(&self, nid: u8) -> Option<&AudioNode> {
+        self.nodes.iter().find(|n| n.node.nid == nid)
+    }
+
+    pub async fn setup_default_output(&self) -> Result<Node, Error> {
+        let (pin_id, dev, node_id) = self
+            .nodes
+            .iter()
+            .find_map(|node| {
+                node.default_output_pin()
+                    .map(|(pin, dev)| (pin, dev, node.node))
+            })
+            .ok_or(Error::DoesNotExist)?;
+
+        log::info!(
+            "hda#{}: default output {pin_id:?}, {dev:?}",
+            self.root.codec
+        );
+        let audio_node = self.node(node_id.nid).ok_or(Error::DoesNotExist)?;
+        let mut audio_widget = None;
+
+        for widget in audio_node.connection_chain(pin_id.nid) {
+            let wid = widget.node;
+
+            match &widget.widget {
+                AudioWidget::PinComplex(_) => {
+                    log::info!("  Configure pin @ {wid:?}");
+                    wid.set_power_state(&*self.hda, true).await?;
+                    // TODO only set for jacks
+                    wid.set_unsolicited_response(&*self.hda, Some(wid.nid))
+                        .await?;
+                    wid.modify_pin_control(
+                        &*self.hda,
+                        PinControl::OutEnable::SET + PinControl::HPhnEnable::SET,
+                    )
+                    .await?;
+                }
+                AudioWidget::AudioOutput(out) => {
+                    log::info!("  Configure audio output @ {wid:?}");
+                    audio_widget = Some(widget.node);
+                    wid.set_power_state(&*self.hda, true).await?;
+                    wid.set_unsolicited_response(&*self.hda, None).await?;
+                }
+                AudioWidget::Mixer(_) => {
+                    log::info!("  Configure mixer @ {wid:?}");
+                    wid.set_power_state(&*self.hda, true).await?;
+                }
+                _ => (),
+            }
+        }
+
+        Ok(audio_widget.unwrap())
+    }
+}
+
+impl fmt::Debug for Codec {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("Codec")
+            .field("id", &self.root.codec)
+            .field("nodes", &self.nodes)
+            .finish()
+    }
+}
+
+impl<'a> Iterator for WidgetConnectionChain<'a> {
+    type Item = &'a AudioWidgetNode;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        let current = self.current?;
+        let widget = self.afg.widget(current)?;
+        self.current = widget.connection;
+        Some(widget)
+    }
+}

kernel/driver/sound/intel-hda/src/codec/parameter.rs

@@ -0,0 +1,198 @@
+use libk::error::Error;
+use tock_registers::{register_bitfields, LocalRegisterCopy};
+
+use crate::ring::NodeParameterNumber;
+
+pub trait NodeParameter {
+    const NUMBER: NodeParameterNumber;
+    type Value;
+
+    fn data(&self) -> u32 {
+        0
+    }
+    fn from_response(response: u32) -> Self::Value;
+}
+
+#[derive(Debug)]
+pub struct DeviceId {
+    pub vendor_id: u16,
+    pub device_id: u16,
+}
+#[derive(Debug)]
+pub struct SubNodeInfo {
+    pub start_number: u8,
+    pub total_number: u8,
+}
+pub struct NodeType;
+register_bitfields! {
+    u32,
+    pub AudioWidgetCapabilities [
+        CHANNEL_COUNT       OFFSET(0)   NUMBITS(1)  [],
+        IN_AMP              OFFSET(1)   NUMBITS(1)  [],
+        OUT_AMP             OFFSET(2)   NUMBITS(1)  [],
+        AMP_OVERRIDE        OFFSET(3)   NUMBITS(1)  [],
+        FORMAT_OVERRIDE     OFFSET(4)   NUMBITS(1)  [],
+        STRIPE              OFFSET(5)   NUMBITS(1)  [],
+        PROC_WIDGET         OFFSET(6)   NUMBITS(1)  [],
+        UNSOL_CAPABLE       OFFSET(7)   NUMBITS(1)  [],
+        CONN_LIST           OFFSET(8)   NUMBITS(1)  [],
+        DIGITAL             OFFSET(9)   NUMBITS(1)  [],
+        POWER_CONTROL       OFFSET(10)  NUMBITS(1)  [],
+        LR_SWAP             OFFSET(11)  NUMBITS(1)  [],
+        CP_CAPS             OFFSET(12)  NUMBITS(1)  [],
+        EXT_CHANNEL_COUNT   OFFSET(13)  NUMBITS(3)  [],
+        DELAY               OFFSET(16)  NUMBITS(4)  [],
+        TYPE                OFFSET(20)  NUMBITS(4)  [
+            AudioOutput     = 0x0,
+            AudioInput      = 0x1,
+            AudioMixer      = 0x2,
+            AudioSelector   = 0x3,
+            PinComplex      = 0x4,
+            Power           = 0x5,
+            VolumeKnob      = 0x6,
+            BeepGenerator   = 0x7,
+            VendorDefined   = 0xF,
+        ],
+    ],
+}
+register_bitfields! {
+    u32,
+    pub AmpCapabilities [
+        OFFSET          OFFSET(0)   NUMBITS(7)  [],
+        NUM_STEPS       OFFSET(8)   NUMBITS(7)  [],
+        STEP_SIZE       OFFSET(16)  NUMBITS(7)  [],
+        MUTE_CAPABLE    OFFSET(31)  NUMBITS(1)  [],
+    ],
+}
+pub struct InputAmpCapabilities;
+pub struct OutputAmpCapabilities;
+pub struct ConnectionListLength;
+register_bitfields! {
+    u32,
+    pub SupportedStreamFormats [
+        PCM OFFSET(0)   NUMBITS(1)  [],
+        F32 OFFSET(1)   NUMBITS(1)  [],
+        AC3 OFFSET(2)   NUMBITS(1)  [],
+    ],
+}
+register_bitfields! {
+    u32,
+    pub SupportedPcmFormats [
+        RATE_8    OFFSET(0)   NUMBITS(1)  [],
+        RATE_11_025 OFFSET(1)   NUMBITS(1)  [],
+        RATE_16   OFFSET(2)   NUMBITS(1)  [],
+        RATE_22_05  OFFSET(3)   NUMBITS(1)  [],
+        RATE_32     OFFSET(4)   NUMBITS(1)  [],
+        RATE_44_1   OFFSET(5)   NUMBITS(1)  [],
+        RATE_48     OFFSET(6)   NUMBITS(1)  [],
+        RATE_88_2   OFFSET(7)   NUMBITS(1)  [],
+        RATE_96     OFFSET(8)   NUMBITS(1)  [],
+        RATE_176_4  OFFSET(9)   NUMBITS(1)  [],
+        RATE_192    OFFSET(10)  NUMBITS(1)  [],
+        RATE_384    OFFSET(11)  NUMBITS(1)  [],
+
+        BITS_8      OFFSET(16)  NUMBITS(1)  [],
+        BITS_16     OFFSET(17)  NUMBITS(1)  [],
+        BITS_20     OFFSET(18)  NUMBITS(1)  [],
+        BITS_24     OFFSET(19)  NUMBITS(1)  [],
+        BITS_32     OFFSET(20)  NUMBITS(1)  [],
+    ],
+}
+
+impl NodeParameter for DeviceId {
+    const NUMBER: NodeParameterNumber = NodeParameterNumber::DeviceId;
+    type Value = Self;
+
+    fn from_response(response: u32) -> Self::Value {
+        Self {
+            vendor_id: (response >> 16) as u16,
+            device_id: response as u16,
+        }
+    }
+}
+impl NodeParameter for SubNodeInfo {
+    const NUMBER: NodeParameterNumber = NodeParameterNumber::NodeCount;
+    type Value = Self;
+
+    fn from_response(response: u32) -> Self::Value {
+        Self {
+            start_number: (response >> 16) as u8,
+            total_number: response as u8,
+        }
+    }
+}
+impl NodeParameter for NodeType {
+    const NUMBER: NodeParameterNumber = NodeParameterNumber::FunctionGroupType;
+    type Value = u8;
+
+    fn from_response(response: u32) -> Self::Value {
+        response as u8
+    }
+}
+impl NodeParameter for AudioWidgetCapabilities::Register {
+    const NUMBER: NodeParameterNumber = NodeParameterNumber::AudioWidgetCapabilities;
+    type Value = LocalRegisterCopy<u32, Self>;
+
+    fn from_response(response: u32) -> Self::Value {
+        LocalRegisterCopy::new(response)
+    }
+}
+impl NodeParameter for InputAmpCapabilities {
+    const NUMBER: NodeParameterNumber = NodeParameterNumber::InputAmplifierCapabilities;
+    type Value = Option<LocalRegisterCopy<u32, AmpCapabilities::Register>>;
+
+    fn from_response(response: u32) -> Self::Value {
+        if response != 0 {
+            Some(LocalRegisterCopy::new(response))
+        } else {
+            None
+        }
+    }
+}
+impl NodeParameter for OutputAmpCapabilities {
+    const NUMBER: NodeParameterNumber = NodeParameterNumber::OutputAmplifierCapabilities;
+    type Value = Option<LocalRegisterCopy<u32, AmpCapabilities::Register>>;
+
+    fn from_response(response: u32) -> Self::Value {
+        if response != 0 {
+            Some(LocalRegisterCopy::new(response))
+        } else {
+            None
+        }
+    }
+}
+impl NodeParameter for ConnectionListLength {
+    const NUMBER: NodeParameterNumber = NodeParameterNumber::ConnectionListLength;
+    type Value = (usize, bool);
+
+    fn from_response(response: u32) -> Self::Value {
+        let long_form = response & (1 << 7) != 0;
+        let length = (response & 0x3F) as usize;
+        (length, long_form)
+    }
+}
+impl NodeParameter for SupportedStreamFormats::Register {
+    const NUMBER: NodeParameterNumber = NodeParameterNumber::SupportedFormats;
+    type Value = LocalRegisterCopy<u32, Self>;
+
+    fn from_response(response: u32) -> Self::Value {
+        LocalRegisterCopy::new(response)
+    }
+}
+impl NodeParameter for SupportedPcmFormats::Register {
+    const NUMBER: NodeParameterNumber = NodeParameterNumber::SupportedPcmRates;
+    type Value = LocalRegisterCopy<u32, Self>;
+
+    fn from_response(response: u32) -> Self::Value {
+        LocalRegisterCopy::new(response)
+    }
+}
+
+impl SubNodeInfo {
+    pub fn iter(&self) -> impl Iterator<Item = u8> {
+        // Exclude root
+        let start_subnode = self.start_number.max(1);
+        let end_subnode = self.start_number.saturating_add(self.total_number);
+        start_subnode..end_subnode
+    }
+}

kernel/driver/sound/intel-hda/src/lib.rs

@@ -0,0 +1,344 @@
+#![no_std]
+#![feature(let_chains)]
+
+use core::{
+    sync::atomic::{AtomicU8, Ordering},
+    time::Duration,
+};
+
+use alloc::{boxed::Box, sync::Arc, vec::Vec};
+use async_trait::async_trait;
+use codec::{AudioFormat, Codec, Node};
+use device_api::{
+    device::{Device, DeviceInitContext},
+    dma::DmaAllocator,
+    interrupt::{InterruptAffinity, InterruptHandler, IrqVector},
+};
+use futures_util::task::AtomicWaker;
+use libk::{
+    dma::DmaBuffer,
+    error::Error,
+    task::runtime::{self, psleep, with_timeout},
+    time::monotonic_time,
+};
+use libk_mm::device::DeviceMemoryIo;
+use libk_util::{
+    event::{BitmapEvent, BoolEvent, CounterEvent},
+    sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock},
+    OneTimeInit,
+};
+use regs::{Regs, SDxCTL0, CORBRP, GCAP, RIRBWP};
+use ring::{Command, CommandRing, NodeParameterNumber, Verb};
+use sink::HdAudioSink;
+use stream::{BufferDescriptorList, OutputStream};
+use tock_registers::interfaces::{ReadWriteable, Readable, Writeable};
+use ygg_driver_pci::{
+    capability::{power::DevicePowerState, PowerManagementCapability},
+    device::{PciDeviceInfo, PreferredInterruptMode},
+    macros::pci_driver,
+    PciBaseAddress, PciConfigurationSpace,
+};
+
+extern crate alloc;
+
+mod codec;
+mod regs;
+mod ring;
+mod sink;
+mod stream;
+
+struct HdAudio {
+    regs: IrqSafeSpinlock<DeviceMemoryIo<'static, Regs>>,
+    dma: Arc<dyn DmaAllocator>,
+    pci: PciDeviceInfo,
+
+    input_stream_count: usize,
+    output_stream_count: usize,
+    bidi_stream_count: usize,
+
+    sinks: Vec<IrqSafeSpinlock<Option<Arc<HdAudioSink>>>>,
+    last_stream_tag: AtomicU8,
+
+    softirq_event: BitmapEvent<AtomicWaker>,
+
+    cring: OneTimeInit<CommandRing>,
+}
+
+impl HdAudio {
+    fn new(
+        pci: PciDeviceInfo,
+        dma: Arc<dyn DmaAllocator>,
+        regs: DeviceMemoryIo<'static, Regs>,
+        input_stream_count: usize,
+        output_stream_count: usize,
+        bidi_stream_count: usize,
+    ) -> Self {
+        Self {
+            pci,
+            dma,
+            regs: IrqSafeSpinlock::new(regs),
+
+            input_stream_count,
+            output_stream_count,
+            bidi_stream_count,
+
+            sinks: (0..output_stream_count)
+                .map(|_| IrqSafeSpinlock::new(None))
+                .collect(),
+            last_stream_tag: AtomicU8::new(1),
+
+            softirq_event: BitmapEvent::new(AtomicWaker::new()),
+
+            cring: OneTimeInit::new(),
+        }
+    }
+}
+
+impl HdAudio {
+    async fn late_init(self: &Arc<Self>) -> Result<(), Error> {
+        // Interrogate present codecs
+        let codecs = self.regs.lock().take_attached_codecs();
+
+        for index in (0..16).filter(|&i| codecs & (1 << i) != 0) {
+            match Codec::probe(self.clone(), index).await {
+                Ok(codec) => {
+                    let codec = Arc::new(codec);
+                    log::info!("{codec:#?}");
+
+                    match codec.setup_default_output().await {
+                        Ok(id) => match self.add_sink(codec, id).await {
+                            Ok(_) => {
+                                log::info!("hda#{}: sink added", id.codec);
+                            }
+                            Err(error) => {
+                                log::info!("hda#{}: could not add sink: {error:?}", id.codec);
+                            }
+                        },
+                        Err(error) => {
+                            log::error!("Default output setup error: {error:?}");
+                        }
+                    }
+                }
+                Err(error) => {
+                    log::error!("hda: codec #{index} setup error: {error:?}");
+                }
+            }
+        }
+
+        loop {
+            runtime::sleep(Duration::from_secs(1)).await;
+        }
+    }
+
+    async fn add_sink(
+        self: &Arc<Self>,
+        codec: Arc<Codec>,
+        audio_widget: Node,
+    ) -> Result<Arc<HdAudioSink>, Error> {
+        for (index, slot) in self.sinks.iter().enumerate() {
+            let mut slot = slot.lock();
+            if slot.is_none() {
+                log::info!("hda: add sink #{index}");
+                let stream_tag = self.last_stream_tag.fetch_add(1, Ordering::Relaxed);
+                let sink = HdAudioSink::create(
+                    self.clone(),
+                    index + self.input_stream_count,
+                    stream_tag,
+                    codec.clone(),
+                    audio_widget,
+                )
+                .await?;
+
+                *slot = Some(sink.clone());
+                return Ok(sink);
+            }
+        }
+        log::warn!("hda: too many sinks");
+        Err(Error::InvalidOperation)
+    }
+
+    async fn softirq(&self) -> Result<(), Error> {
+        let cring = self.cring.get();
+
+        loop {
+            match with_timeout(self.softirq_event.wait(), Duration::from_millis(100)).await {
+                // IRQs happened
+                Ok(events) => {
+                    log::info!("softirq()");
+                    for sink_index in
+                        self.input_stream_count..self.input_stream_count + self.output_stream_count
+                    {
+                        // Handle sink events
+                        if events & (1 << sink_index) != 0
+                            && let Some(sink) = self.sink(sink_index - self.input_stream_count)
+                        {
+                            let position = self.regs.lock().STREAMS[sink_index].SDxLPIB.get();
+                            sink.handle_softirq(position).await;
+                        }
+                    }
+                    //
+                }
+                // Just poll
+                Err(_) => {}
+            }
+
+            let regs = self.regs.lock();
+            let rirb_head = regs.RIRBWP.read(RIRBWP::RIRBWP) as u8;
+            let corb_tail = regs.CORBRP.read(CORBRP::CORBRP) as u8;
+
+            cring.process_completions(corb_tail, rirb_head);
+        }
+    }
+
+    async fn perform_command(&self, command: Command) -> Result<u32, Error> {
+        let cring = self.cring.get();
+        let codec = command.codec();
+        let head = cring.submit_command(command).await?;
+
+        {
+            let regs = self.regs.lock();
+            regs.CORBWP.set(head as u16);
+        }
+
+        with_timeout(cring.wait_codec(codec), Duration::from_secs(3)).await
+    }
+
+    fn sink(&self, index: usize) -> Option<Arc<HdAudioSink>> {
+        self.sinks[index].lock().clone()
+    }
+}
+
+impl InterruptHandler for HdAudio {
+    fn handle_irq(self: Arc<Self>, _vector: IrqVector) -> bool {
+        let regs = self.regs.lock();
+        let rirbsts = regs.RIRBSTS.extract();
+        let status = regs.INTSTS.get() & 0x3FFFFFFF;
+
+        if rirbsts.get() != 0 {
+            regs.RIRBSTS.set(rirbsts.get());
+            self.softirq_event.signal(1 << 30);
+        }
+
+        if status != 0 {
+            for stream in 0..30 {
+                if status & (1 << stream) != 0 {
+                    let stream_status = regs.STREAMS[stream].SDxSTS.extract();
+                    regs.STREAMS[stream].SDxSTS.set(stream_status.get());
+                }
+            }
+            self.softirq_event.signal(status as u64);
+        }
+
+        true
+    }
+}
+
+impl Device for HdAudio {
+    unsafe fn init(self: Arc<Self>, _cx: DeviceInitContext) -> Result<(), Error> {
+        // FLR if capable
+        self.pci.function_level_reset().ok();
+        self.pci
+            .map_interrupt(InterruptAffinity::Any, self.clone())?;
+
+        let regs = self.regs.lock();
+
+        let vmaj = regs.VMAJ.get();
+        let vmin = regs.VMIN.get();
+
+        log::info!(
+            "hda: version={}.{}, iss={}, oss={}, bss={}",
+            vmaj,
+            vmin,
+            self.input_stream_count,
+            self.output_stream_count,
+            self.bidi_stream_count
+        );
+
+        regs.reset(
+            Duration::from_millis(100),
+            self.input_stream_count + self.output_stream_count + self.bidi_stream_count,
+        )?;
+
+        regs.disable_interrupts();
+        regs.disable_control();
+        let corb_size = regs.set_max_corb_size()?;
+        let rirb_size = regs.set_max_rirb_size()?;
+        log::info!("hda: corb size = {corb_size}, rirb size = {rirb_size}");
+
+        let (cring, corb_base, rirb_base) =
+            CommandRing::with_capacity(&*self.dma, corb_size, rirb_size)?;
+        regs.initialize_corb_rirb(corb_base, rirb_base)?;
+        regs.start_corb_rirb();
+        regs.enable_interrupts();
+
+        self.cring.init(cring);
+
+        drop(regs);
+
+        // Spawn softirq worker
+        let this = self.clone();
+        runtime::spawn(async move { this.softirq().await })?;
+
+        // Do the main initialization in background
+        let this = self.clone();
+        runtime::spawn(async move { this.late_init().await })?;
+
+        Ok(())
+    }
+
+    fn display_name(&self) -> &str {
+        "High Definition Audio"
+    }
+}
+
+pci_driver! {
+    matches: [
+        device (0x8086:0x2668), // Intel ICH6 82801FB/FBM/FR/FW/FRW HDA
+        device (0x8086:0x293E), // Intel ICH9 82801I HDA
+
+        device (0x1022:0x1487), // AMD Starship/Matisse HDA
+    ],
+    driver: {
+        fn probe(
+            &self,
+            info: &PciDeviceInfo,
+            dma: &Arc<dyn DmaAllocator>,
+        ) -> Result<Arc<dyn Device>, Error> {
+            info.set_command(true, true, false, true);
+
+            let base = info
+                .config_space
+                .bar(0)
+                .and_then(PciBaseAddress::as_memory)
+                .ok_or(Error::InvalidArgument)?;
+
+            info.init_interrupts(PreferredInterruptMode::Msi(true))?;
+
+            let regs = unsafe { DeviceMemoryIo::<Regs>::map(base, Default::default()) }?;
+
+            let gcap = regs.GCAP.extract();
+            if gcap.matches_all(GCAP::OK64::CLEAR) {
+                log::error!("Non 64-bit HD Audio not supported yet");
+                return Err(Error::NotImplemented);
+            }
+            let input_stream_count = gcap.read(GCAP::ISS) as usize;
+            let output_stream_count = gcap.read(GCAP::OSS).min(30) as usize;
+            let bidi_stream_count = gcap.read(GCAP::BSS) as usize;
+
+            let hda = Arc::new(HdAudio::new(
+                info.clone(),
+                dma.clone(),
+                regs,
+                input_stream_count,
+                output_stream_count,
+                bidi_stream_count,
+            ));
+
+            Ok(hda)
+        }
+
+        fn driver_name(&self) -> &str {
+            "intel-hda"
+        }
+    }
+}

kernel/driver/sound/intel-hda/src/regs.rs

@@ -0,0 +1,453 @@
+use core::time::Duration;
+
+use libk::{
+    dma::BusAddress,
+    error::Error,
+    task::runtime::{psleep, pwait},
+};
+use tock_registers::{
+    fields::FieldValue,
+    interfaces::{ReadWriteable, Readable, Writeable},
+    register_bitfields, register_structs,
+    registers::{ReadOnly, ReadWrite},
+};
+use ygg_driver_sound_core::{SampleFormat, SampleRate, SinkFormat};
+
+use crate::stream::OutputStream;
+
+register_bitfields! {
+    u8,
+    pub CORBCTL [
+        /// Enable CORB DMA engine
+        CORBRUN OFFSET(1)   NUMBITS(1)  [],
+        /// CORB memory error interrupt enable
+        CMEIE   OFFSET(0)   NUMBITS(1)  [],
+    ],
+    pub CORBSTS [
+        /// CORB memory error
+        CMEI    OFFSET(0)   NUMBITS(1)  [],
+    ],
+    pub CORBSIZE [
+        CORBSZCAP   OFFSET(4)   NUMBITS(4)  [],
+        CORBSIZE    OFFSET(0)   NUMBITS(2)  [
+            Size2   = 0b00,
+            Size16  = 0b01,
+            Size256 = 0b10,
+        ],
+    ],
+    pub RIRBCTL [
+        /// RIRB overrun interrupt control
+        RURBOIC     OFFSET(2)   NUMBITS(1)  [],
+        /// Enable RIRB DMA engine
+        RIRBDMAEN   OFFSET(1)   NUMBITS(1)  [],
+        /// Response interrupt control
+        RINTCTL     OFFSET(0)   NUMBITS(1)  [],
+    ],
+    pub RIRBSTS [
+        /// RIRB overrun interrupt
+        RIRBOIS     OFFSET(2)   NUMBITS(1)  [],
+        /// Response interrupt
+        RINTFL      OFFSET(0)   NUMBITS(1)  [],
+    ],
+    pub RIRBSIZE [
+        /// RIRB size capability
+        RIRBSZCAP   OFFSET(4)   NUMBITS(4)  [],
+        RIRBSIZE    OFFSET(0)   NUMBITS(2)  [
+            Size2   = 0b00,
+            Size16  = 0b01,
+            Size256 = 0b10,
+        ],
+    ],
+
+    pub SDxCTL0 [
+        SRST    OFFSET(0)   NUMBITS(1)  [],
+        RUN     OFFSET(1)   NUMBITS(1)  [],
+        IOCE    OFFSET(2)   NUMBITS(1)  [],
+        FEIE    OFFSET(3)   NUMBITS(1)  [],
+        DEIE    OFFSET(4)   NUMBITS(1)  [],
+    ],
+    pub SDxCTL2 [
+        STRIPE  OFFSET(0)   NUMBITS(2)  [],
+        TP      OFFSET(2)   NUMBITS(1)  [],
+        DIR     OFFSET(3)   NUMBITS(1)  [],
+        STRM    OFFSET(4)   NUMBITS(4)  [],
+    ],
+    pub SDxSTS [
+        BCIS    OFFSET(2)   NUMBITS(1)  [],
+        FIFOE   OFFSET(3)   NUMBITS(1)  [],
+        DESE    OFFSET(4)   NUMBITS(1)  [],
+        FIFORDY OFFSET(5)   NUMBITS(1)  [],
+    ],
+}
+
+register_bitfields! {
+    u16,
+    pub GCAP [
+        /// Number of output streams supported
+        OSS     OFFSET(12)  NUMBITS(4)  [],
+        /// Number of input streams supported
+        ISS     OFFSET(8)   NUMBITS(4)  [],
+        /// Number of bidirectional streams supported
+        BSS     OFFSET(3)   NUMBITS(5)  [],
+        /// Number of serial data out signals
+        NSDO    OFFSET(1)   NUMBITS(2)  [],
+        /// 64-bit addresses supported
+        OK64    OFFSET(0)   NUMBITS(1)  [],
+    ],
+    pub GSTS [
+        /// Flush status bit
+        FSTS    OFFSET(1)   NUMBITS(1)  [],
+    ],
+    pub CORBRP [
+        /// CORB read pointer reset
+        CORBRPRST   OFFSET(15)  NUMBITS(1)  [],
+        /// CORB read pointer
+        CORBRP      OFFSET(0)   NUMBITS(8)  [],
+    ],
+    pub RIRBWP [
+        /// RIRB write pointer reset
+        RIRBWPRST   OFFSET(15)  NUMBITS(1)  [],
+        /// RIRB write pointer
+        RIRBWP      OFFSET(0)   NUMBITS(8)  [],
+    ],
+
+    pub SDxFMT [
+        CHAN        OFFSET(0)   NUMBITS(4)  [],
+        BITS        OFFSET(4)   NUMBITS(3)  [
+            Bits8   = 0b000,
+            Bits16  = 0b001,
+            Bits20  = 0b010,
+            Bits24  = 0b011,
+            Bits32  = 0b100,
+        ],
+        DIV         OFFSET(8)   NUMBITS(3)  [],
+        MULT        OFFSET(11)  NUMBITS(3)  [],
+        BASE        OFFSET(14)  NUMBITS(1)  [],
+    ],
+}
+
+register_bitfields! {
+    u32,
+    pub GCTL [
+        /// Accept unsolicited response enable
+        UNSOL   OFFSET(8)   NUMBITS(1)  [],
+        /// Flush control
+        FCNTRL  OFFSET(1)   NUMBITS(1)  [],
+        /// Controller reset
+        CRST    OFFSET(0)   NUMBITS(1)  [],
+    ],
+    pub INTCTL [
+        /// Global interrupt enable
+        GIE     OFFSET(31)  NUMBITS(1)  [],
+        /// Controller interrupt enable
+        CIE     OFFSET(30)  NUMBITS(1)  [],
+        // Stream interrupt enable bits are based on stream counts
+    ],
+    pub INTSTS [
+        /// Global interrupt status
+        GIS     OFFSET(31)  NUMBITS(1)  [],
+        /// Controller interrupt status
+        CIS     OFFSET(30)  NUMBITS(1)  [],
+        // Stream interrupt status bits are based on stream counts
+    ],
+}
+
+register_structs! {
+    #[allow(non_snake_case)]
+    pub Regs {
+        (0x0000 => pub GCAP: ReadOnly<u16, GCAP::Register>),
+        (0x0002 => pub VMIN: ReadOnly<u8>),
+        (0x0003 => pub VMAJ: ReadOnly<u8>),
+        (0x0004 => pub OUTPAY: ReadOnly<u16>),
+        (0x0006 => pub INPAY: ReadOnly<u16>),
+        (0x0008 => pub GCTL: ReadWrite<u32, GCTL::Register>),
+        (0x000C => pub WAKEEN: ReadWrite<u16>),
+        (0x000E => pub WAKESTS: ReadWrite<u16>),
+        (0x0010 => pub GSTS: ReadWrite<u16, GSTS::Register>),
+        (0x0012 => _0),
+        (0x0018 => pub OUTSTRMPAY: ReadOnly<u16>),
+        (0x001A => pub INSTRMPAY: ReadOnly<u16>),
+        (0x001C => _1),
+        (0x0020 => pub INTCTL: ReadWrite<u32, INTCTL::Register>),
+        (0x0024 => pub INTSTS: ReadOnly<u32, INTSTS::Register>),
+        (0x0028 => _2),
+        (0x0030 => pub WALCLK: ReadOnly<u32>),
+        (0x0034 => _3),
+        (0x0038 => pub SSYNC: ReadWrite<u32>),
+        (0x003C => _4),
+        (0x0040 => pub CORBLBASE: ReadWrite<u32>),
+        (0x0044 => pub CORBUBASE: ReadWrite<u32>),
+        (0x0048 => pub CORBWP: ReadWrite<u16>),
+        (0x004A => pub CORBRP: ReadWrite<u16, CORBRP::Register>),
+        (0x004C => pub CORBCTL: ReadWrite<u8, CORBCTL::Register>),
+        (0x004D => pub CORBSTS: ReadWrite<u8, CORBSTS::Register>),
+        (0x004E => pub CORBSIZE: ReadWrite<u8, CORBSIZE::Register>),
+        (0x004F => _5),
+        (0x0050 => pub RIRBLBASE: ReadWrite<u32>),
+        (0x0054 => pub RIRBUBASE: ReadWrite<u32>),
+        (0x0058 => pub RIRBWP: ReadWrite<u16, RIRBWP::Register>),
+        (0x005A => pub RINTCNT: ReadWrite<u16>),
+        (0x005C => pub RIRBCTL: ReadWrite<u8, RIRBCTL::Register>),
+        (0x005D => pub RIRBSTS: ReadWrite<u8, RIRBSTS::Register>),
+        (0x005E => pub RIRBSIZE: ReadWrite<u8, RIRBSIZE::Register>),
+        (0x005F => _6),
+        (0x0060 => pub ICOI: ReadWrite<u32>),
+        (0x0064 => pub ICII: ReadWrite<u32>),
+        (0x0068 => pub ICIS: ReadOnly<u16>),
+        (0x006A => _7),
+        (0x0070 => pub DPLBASE: ReadWrite<u32>),
+        (0x0074 => pub DPUBASE: ReadWrite<u32>),
+        (0x0078 => _8),
+        (0x0080 => pub STREAMS: [StreamRegs; 128]),
+        (0x1080 => _9),
+        (0x2030 => pub WALCLKA: ReadOnly<u32>),
+        (0x2034 => _10),
+        (0x2080 => pub SDxLPIBA: [StreamLinkRegs; 128]),
+        (0x3080 => _11),
+        (0x4000 => @END),
+    }
+}
+
+register_structs! {
+    #[allow(non_snake_case)]
+    pub StreamRegs {
+        (0x00 => pub SDxCTL0: ReadWrite<u8, SDxCTL0::Register>),
+        (0x01 => pub SDxCTL1: ReadWrite<u8>),
+        (0x02 => pub SDxCTL2: ReadWrite<u8, SDxCTL2::Register>),
+        (0x03 => pub SDxSTS: ReadWrite<u8, SDxSTS::Register>),
+        (0x04 => pub SDxLPIB: ReadWrite<u32>),
+        (0x08 => pub SDxCBL: ReadWrite<u32>),
+        (0x0C => pub SDxLVI: ReadWrite<u16>),
+        (0x0E => _0),
+        (0x10 => pub SDxFIFOD: ReadWrite<u16>),
+        (0x12 => pub SDxFMT: ReadWrite<u16, SDxFMT::Register>),
+        (0x14 => _1),
+        (0x18 => pub SDxBDPL: ReadWrite<u32>),
+        (0x1C => pub SDxBDPU: ReadWrite<u32>),
+        (0x20 => @END),
+    }
+}
+
+register_structs! {
+    #[allow(non_snake_case)]
+    pub StreamLinkRegs {
+        (0x00 => _0),
+        (0x04 => pub SDxLPIBA: ReadOnly<u32>),
+        (0x08 => _1),
+        (0x20 => @END),
+    }
+}
+
+impl Regs {
+    pub fn reset(&self, timeout: Duration, stream_count: usize) -> Result<(), Error> {
+        // Go through all the streams, disable their DMA
+        for i in 0..stream_count {
+            let stream = &self.STREAMS[i];
+            stream
+                .SDxCTL0
+                .write(SDxCTL0::RUN::CLEAR + SDxCTL0::SRST::SET);
+        }
+
+        self.CORBCTL.write(CORBCTL::CORBRUN::CLEAR);
+        self.RIRBCTL.write(RIRBCTL::RIRBDMAEN::CLEAR);
+
+        // Flush the FIFO
+        self.GCTL.modify(GCTL::FCNTRL::SET);
+        psleep(Duration::from_millis(10));
+        pwait(timeout, Duration::from_millis(10), || {
+            self.GSTS.matches_all(GSTS::FSTS::SET)
+        })?;
+        self.GCTL.modify(GCTL::FCNTRL::CLEAR);
+        self.GSTS.write(GSTS::FSTS::SET);
+
+        // TODO does FIFO need to be flushed here as well?
+        // Begin controller reset
+        self.GCTL.write(GCTL::CRST::CLEAR);
+        psleep(Duration::from_millis(10));
+        // End controller reset
+        self.GCTL.write(GCTL::CRST::SET);
+        psleep(Duration::from_millis(10));
+        pwait(timeout, Duration::from_millis(10), || {
+            self.GCTL.matches_all(GCTL::CRST::SET)
+        })?;
+
+        Ok(())
+    }
+
+    pub fn disable_interrupts(&self) {
+        self.INTCTL.set(0);
+    }
+
+    pub fn enable_interrupts(&self) {
+        // Also accept unsolicited responses from codecs
+        self.GCTL.modify(GCTL::UNSOL::SET);
+        self.INTCTL.write(INTCTL::GIE::SET + INTCTL::CIE::SET);
+    }
+
+    pub fn enable_stream_interrupts(&self, stream: usize) {
+        self.INTCTL.set(self.INTCTL.get() | (1 << stream));
+    }
+
+    pub fn disable_control(&self) {
+        self.DPLBASE.set(0);
+        self.DPUBASE.set(0);
+
+        self.CORBCTL.write(CORBCTL::CORBRUN::CLEAR);
+        self.RIRBCTL.write(RIRBCTL::RIRBDMAEN::CLEAR);
+    }
+
+    pub fn set_max_corb_size(&self) -> Result<usize, Error> {
+        const SIZES: &[(usize, u8, FieldValue<u8, CORBSIZE::Register>)] = &[
+            (256, 0b0100, CORBSIZE::CORBSIZE::Size256),
+            (16, 0b0010, CORBSIZE::CORBSIZE::Size16),
+            (2, 0b0001, CORBSIZE::CORBSIZE::Size2),
+        ];
+
+        let corbsize = self.CORBSIZE.extract();
+        let cap = corbsize.read(CORBSIZE::CORBSZCAP);
+        for &(size, bit, cfg) in SIZES {
+            if cap & bit != 0 {
+                self.CORBSIZE.modify_no_read(corbsize, cfg);
+                return Ok(size);
+            }
+        }
+
+        log::error!("hda: no supported CORB size");
+        Err(Error::InvalidOperation)
+    }
+
+    pub fn set_max_rirb_size(&self) -> Result<usize, Error> {
+        const SIZES: &[(usize, u8, FieldValue<u8, RIRBSIZE::Register>)] = &[
+            (256, 0b0100, RIRBSIZE::RIRBSIZE::Size256),
+            (16, 0b0010, RIRBSIZE::RIRBSIZE::Size16),
+            (2, 0b0001, RIRBSIZE::RIRBSIZE::Size2),
+        ];
+
+        let rirbsize = self.RIRBSIZE.extract();
+        let cap = rirbsize.read(RIRBSIZE::RIRBSZCAP);
+        for &(size, bit, cfg) in SIZES {
+            if cap & bit != 0 {
+                self.RIRBSIZE.modify_no_read(rirbsize, cfg);
+                return Ok(size);
+            }
+        }
+
+        log::error!("hda: no supported RIRB size");
+        Err(Error::InvalidOperation)
+    }
+
+    pub fn initialize_corb_rirb(
+        &self,
+        corb_base: BusAddress,
+        rirb_base: BusAddress,
+    ) -> Result<(), Error> {
+        let corb_base = corb_base.into_u64();
+        let rirb_base = rirb_base.into_u64();
+
+        self.CORBUBASE.set((corb_base >> 32) as u32);
+        self.CORBLBASE.set(corb_base as u32);
+
+        self.RIRBUBASE.set((rirb_base >> 32) as u32);
+        self.RIRBLBASE.set(rirb_base as u32);
+
+        // Reset write/read pointers
+        self.CORBWP.set(0);
+        self.CORBRP.write(CORBRP::CORBRPRST::SET);
+        self.RIRBSTS
+            .write(RIRBSTS::RIRBOIS::SET + RIRBSTS::RINTFL::SET);
+        self.RINTCNT.set(255);
+
+        // Ensure CORBRP is reset properly
+        pwait(
+            Duration::from_millis(100),
+            Duration::from_millis(10),
+            || self.CORBRP.matches_all(CORBRP::CORBRPRST::SET),
+        )?;
+        self.CORBRP.write(CORBRP::CORBRPRST::CLEAR);
+        pwait(
+            Duration::from_millis(100),
+            Duration::from_millis(10),
+            || self.CORBRP.matches_all(CORBRP::CORBRPRST::CLEAR),
+        )?;
+
+        Ok(())
+    }
+
+    pub fn start_corb_rirb(&self) {
+        self.CORBCTL
+            .modify(CORBCTL::CORBRUN::SET + CORBCTL::CMEIE::SET);
+        self.RIRBCTL
+            .modify(RIRBCTL::RIRBDMAEN::SET + RIRBCTL::RINTCTL::SET);
+    }
+
+    pub fn take_attached_codecs(&self) -> u16 {
+        let state = self.WAKESTS.get();
+        self.WAKESTS.set(0xFFFF);
+        state
+    }
+
+    pub fn configure_stream(
+        &self,
+        index: usize,
+        config: &SinkFormat,
+        bdl_base: BusAddress,
+        buffer_size: usize,
+        buffer_count: usize,
+        number: u8,
+    ) -> Result<(), Error> {
+        log::info!("Configure stream: #{index}");
+        let bdl_base = bdl_base.into_u64();
+
+        // TODO don't use pwait in async context
+        let regs = &self.STREAMS[index];
+
+        // Stop and reset the stream
+        regs.SDxCTL0.write(SDxCTL0::SRST::SET);
+        pwait(
+            Duration::from_millis(100),
+            Duration::from_millis(10),
+            || regs.SDxCTL0.matches_all(SDxCTL0::SRST::SET),
+        )?;
+        regs.SDxCTL0.write(SDxCTL0::SRST::CLEAR);
+        pwait(
+            Duration::from_millis(100),
+            Duration::from_millis(10),
+            || regs.SDxCTL0.matches_all(SDxCTL0::SRST::CLEAR),
+        )?;
+
+        regs.SDxCTL2.write(SDxCTL2::STRM.val(number));
+        regs.SDxSTS.write(SDxSTS::BCIS::SET);
+
+        regs.SDxLVI.set(buffer_count as u16 - 1);
+        regs.SDxCBL.set((buffer_count * buffer_size) as _);
+        regs.SDxBDPU.set((bdl_base >> 32) as u32);
+        regs.SDxBDPL.set(bdl_base as u32);
+
+        let mut format = SDxFMT::CHAN.val(config.channels as u16 - 1);
+
+        match config.sample_format {
+            SampleFormat::S8 => format += SDxFMT::BITS::Bits8,
+            SampleFormat::S16Le => format += SDxFMT::BITS::Bits16,
+        }
+        match config.sample_rate {
+            SampleRate::Rate8000 => format += SDxFMT::DIV.val(5),
+            SampleRate::Rate11025 => format += SDxFMT::DIV.val(3) + SDxFMT::BASE::SET,
+            SampleRate::Rate16000 => format += SDxFMT::DIV.val(2),
+            SampleRate::Rate22050 => format += SDxFMT::DIV.val(1) + SDxFMT::BASE::SET,
+            SampleRate::Rate32000 => format += SDxFMT::DIV.val(2) + SDxFMT::MULT.val(1),
+            SampleRate::Rate44100 => format += SDxFMT::BASE::SET,
+            SampleRate::Rate48000 => (),
+            SampleRate::Rate88200 => format += SDxFMT::MULT.val(1) + SDxFMT::BASE::SET,
+            SampleRate::Rate96000 => format += SDxFMT::MULT.val(1),
+            SampleRate::Rate176400 => format += SDxFMT::MULT.val(3) + SDxFMT::BASE::SET,
+            SampleRate::Rate192000 => format += SDxFMT::MULT.val(3),
+            SampleRate::Rate384000 => todo!(),
+        }
+
+        regs.SDxFMT.write(format);
+
+        regs.SDxCTL0.modify(SDxCTL0::IOCE::SET);
+
+        Ok(())
+    }
+}

kernel/driver/sound/intel-hda/src/ring.rs

@@ -0,0 +1,396 @@
+use core::{future::poll_fn, task::Poll};
+
+use device_api::dma::DmaAllocator;
+use futures_util::task::AtomicWaker;
+use libk::{
+    dma::{BusAddress, DmaBuffer},
+    error::Error,
+};
+use libk_util::{sync::IrqSafeSpinlock, waker::QueueWaker};
+use tock_registers::register_bitfields;
+use yggdrasil_abi::primitive_enum;
+
+// 4 bit address
+const MAX_CODEC: usize = 16;
+
+// Combines CORB and RIRB
+struct Inner {
+    corb: DmaBuffer<[u32]>,
+    rirb: DmaBuffer<[(u32, u32)]>,
+
+    codec_responses: [Option<u32>; MAX_CODEC],
+
+    corb_head: u32,
+    corb_tail: u32,
+    rirb_tail: u32,
+}
+
+pub struct CommandRing {
+    inner: IrqSafeSpinlock<Inner>,
+
+    // 16 possible codec addresses
+    codec_notify: [AtomicWaker; MAX_CODEC],
+    // Unsolicited response notify
+    unsol_notify: QueueWaker,
+}
+
+primitive_enum! {
+    pub enum Verb: u32 {
+        GetParameter = 0xF00,
+        GetConnectionListEntry = 0xF02,
+        SetSelectedInput = 0x701,
+        SetPowerState = 0x705,
+        SetConverterStreamChannel = 0x706,
+        SetPinWidgetControl = 0x707,
+        GetPinWidgetControl = 0xF07,
+        GetPinWidgetSense = 0xF09,
+        SetUnsolicitedResponse = 0x708,
+        GetUnsolicitedResponse = 0xF08,
+        SetEapdBtl = 0x70C,
+        GetPinWidgetDefaultConfig = 0xF1C,
+        SetOutputConverterChannelCount = 0x72D,
+        AudioFunctionNodeReset = 0x7FF,
+        SetAmplifierGain = 0x003,
+        SetStreamConverterFormat = 0x002,
+        SetStreamFormat = 0x200,
+    }
+}
+
+primitive_enum! {
+    pub enum NodeParameterNumber: u32 {
+        DeviceId = 0x00,
+        RevisionId = 0x02,
+        NodeCount = 0x04,
+        FunctionGroupType = 0x05,
+        AudioGroupCapabilities = 0x08,
+        AudioWidgetCapabilities = 0x09,
+        SupportedPcmRates = 0x0A,
+        SupportedFormats = 0x0B,
+        PinCapabilities = 0x0C,
+        InputAmplifierCapabilities = 0x0D,
+        OutputAmplifierCapabilities = 0x12,
+        ConnectionListLength = 0x0E,
+        SupportedPowerStates = 0x0F,
+        ProcessingCapabilities = 0x10,
+        GpioCount = 0x11,
+        VolumeCapabilities = 0x13,
+    }
+}
+
+register_bitfields! {
+    u32,
+    pub PinControl [
+        VRefEn      OFFSET(0)   NUMBITS(3)  [],
+        InEnable    OFFSET(5)   NUMBITS(1)  [],
+        OutEnable   OFFSET(6)   NUMBITS(1)  [],
+        HPhnEnable  OFFSET(7)   NUMBITS(1)  [],
+    ],
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum PinColor {
+    Black,
+    Gray,
+    Blue,
+    Green,
+    Red,
+    Orange,
+    Yellow,
+    Purple,
+    Pink,
+    White,
+    Other,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum PinType {
+    I1_8,
+    I1_4,
+    AtapiInternal,
+    Rca,
+    Optical,
+    OtherDigital,
+    OtherAnalog,
+    MultichannelAnalog,
+    Xlr,
+    Rj11,
+    Combination,
+    Other,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum PinDevice {
+    LineOut,
+    Speaker,
+    HeadphoneOut,
+    Cd,
+    SpdifOut,
+    OtherDigitalOut,
+    ModemLineSide,
+    ModemHandsetSide,
+    LineIn,
+    Aux,
+    MicIn,
+    Telephony,
+    SpdifIn,
+    OtherDigitalIn,
+    Other,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum PinConnectivity {
+    None,
+    Jack,
+    Fixed,
+    Both,
+}
+
+#[derive(Debug)]
+pub struct PinDefaultConfig {
+    pub sequence: u8,
+    pub default_association: u8,
+    pub color: PinColor,
+    pub connection_type: PinType,
+    pub default_device: PinDevice,
+    pub connectivity: PinConnectivity,
+}
+
+#[derive(Clone, Copy)]
+pub struct Command(u32);
+
+impl Inner {
+    // Submit a command, returns new head and command token
+    pub fn push(&mut self, command: Command) -> Option<u8> {
+        if (self.corb_head + 1) % (self.corb.len() as u32) == self.corb_tail {
+            return None;
+        }
+        self.corb_head = (self.corb_head + 1) % self.corb.len() as u32;
+        let index = self.corb_head as usize % self.corb.len();
+        self.corb[index] = command.0;
+        self.corb.cache_flush_element(index, true);
+        Some(index as u8)
+    }
+
+    pub fn process_responses<U: Fn(u8, u32), C: Fn(u8)>(
+        &mut self,
+        unsol_handler: U,
+        codec_handler: C,
+        corb_tail: u32,
+        rirb_head: u32,
+    ) -> usize {
+        self.corb_tail = corb_tail;
+
+        let mut count = 0;
+        while self.rirb_tail != rirb_head {
+            self.rirb_tail = (self.rirb_tail + 1) % self.rirb.len() as u32;
+
+            let index = self.rirb_tail as usize;
+            self.rirb.cache_flush_element(index, false);
+            let (w0, w1) = self.rirb[index];
+            let codec = (w1 & 0xF) as u8;
+            if w1 & (1 << 4) == 0 {
+                // Solicited
+                self.codec_responses[codec as usize] = Some(w0);
+                codec_handler(codec);
+            } else {
+                // Unsolicited
+                unsol_handler(codec, w0);
+            }
+
+            count += 1;
+        }
+        count
+    }
+
+    pub fn take_codec_response(&mut self, codec: u8) -> Option<u32> {
+        self.codec_responses[codec as usize].take()
+    }
+}
+
+impl CommandRing {
+    pub fn with_capacity(
+        dma: &dyn DmaAllocator,
+        corb_size: usize,
+        rirb_size: usize,
+    ) -> Result<(Self, BusAddress, BusAddress), Error> {
+        let corb = DmaBuffer::new_slice(dma, 0, corb_size)?;
+        let rirb = DmaBuffer::new_slice(dma, (0, 0), rirb_size)?;
+        let corb_base = corb.bus_address();
+        let rirb_base = rirb.bus_address();
+        Ok((
+            Self {
+                inner: IrqSafeSpinlock::new(Inner {
+                    corb,
+                    rirb,
+                    corb_head: 0,
+                    corb_tail: 0,
+                    rirb_tail: 0,
+
+                    codec_responses: [None; MAX_CODEC],
+                }),
+
+                codec_notify: [const { AtomicWaker::new() }; 16],
+                unsol_notify: QueueWaker::new(),
+            },
+            corb_base,
+            rirb_base,
+        ))
+    }
+
+    // Wait for a specific codec to send its response
+    pub async fn wait_codec(&self, codec: u8) -> u32 {
+        poll_fn(|cx| {
+            if let Some(response) = self.inner.lock().take_codec_response(codec) {
+                Poll::Ready(response)
+            } else {
+                self.codec_notify[codec as usize].register(cx.waker());
+                Poll::Pending
+            }
+        })
+        .await
+    }
+
+    pub async fn submit_command(&self, command: Command) -> Result<u8, Error> {
+        // TODO block and wait for free slots in CORB
+        let mut inner = self.inner.lock();
+        // Clear previous codec response
+        let head = inner.push(command).ok_or(Error::WouldBlock)?;
+        inner.codec_responses[command.codec() as usize] = None;
+        Ok(head)
+    }
+
+    pub(super) fn process_completions(&self, corb_tail: u8, rirb_head: u8) -> usize {
+        let mut inner = self.inner.lock();
+        inner.process_responses(
+            |codec, message| {
+                log::info!("Unsolicited message: {message:#x} from codec {codec:#x}");
+            },
+            |codec| self.codec_notify[codec as usize].wake(),
+            corb_tail as u32,
+            rirb_head as u32,
+        )
+    }
+}
+
+impl Command {
+    pub const fn new(codec: u8, node: u8, verb: Verb, payload: u32) -> Self {
+        Self(((codec as u32) << 28) | ((node as u32) << 20) | ((verb as u32) << 8) | payload)
+    }
+
+    pub const fn get_parameter(codec: u8, node: u8, parameter: NodeParameterNumber) -> Self {
+        Self::new(codec, node, Verb::GetParameter, parameter as u32)
+    }
+
+    pub const fn get_connection_list_entry(codec: u8, node: u8, index: usize) -> Self {
+        Self::new(codec, node, Verb::GetConnectionListEntry, index as u32)
+    }
+
+    pub const fn set_stream_number(codec: u8, node: u8, stream: usize) -> Self {
+        Self::new(codec, node, Verb::SetConverterStreamChannel, stream as u32)
+    }
+
+    pub fn codec(&self) -> u8 {
+        ((self.0 >> 28) & 0xF) as u8
+    }
+}
+
+impl From<u32> for PinConnectivity {
+    fn from(value: u32) -> Self {
+        match value {
+            0b00 => Self::Jack,
+            0b10 => Self::Fixed,
+            0b11 => Self::Both,
+            _ => Self::None,
+        }
+    }
+}
+
+impl From<u32> for PinColor {
+    fn from(value: u32) -> Self {
+        match value {
+            1 => Self::Black,
+            2 => Self::Gray,
+            3 => Self::Blue,
+            4 => Self::Green,
+            5 => Self::Red,
+            6 => Self::Orange,
+            7 => Self::Yellow,
+            8 => Self::Purple,
+            9 => Self::Pink,
+            14 => Self::White,
+            _ => Self::Other,
+        }
+    }
+}
+
+impl From<u32> for PinType {
+    fn from(value: u32) -> Self {
+        match value {
+            1 => Self::I1_8,
+            2 => Self::I1_4,
+            3 => Self::AtapiInternal,
+            4 => Self::Rca,
+            5 => Self::Optical,
+            6 => Self::OtherDigital,
+            7 => Self::OtherAnalog,
+            8 => Self::MultichannelAnalog,
+            9 => Self::Xlr,
+            10 => Self::Rj11,
+            11 => Self::Combination,
+            _ => Self::Other,
+        }
+    }
+}
+
+impl PinDevice {
+    pub fn output_score(&self) -> u32 {
+        match self {
+            Self::HeadphoneOut => 10,
+            Self::Speaker => 9,
+            Self::LineOut => 8,
+            Self::SpdifOut => 7,
+            _ => 0,
+        }
+    }
+}
+impl From<u32> for PinDevice {
+    fn from(value: u32) -> Self {
+        match value {
+            0 => Self::LineOut,
+            1 => Self::Speaker,
+            2 => Self::HeadphoneOut,
+            3 => Self::Cd,
+            4 => Self::SpdifOut,
+            5 => Self::OtherDigitalOut,
+            6 => Self::ModemLineSide,
+            7 => Self::ModemHandsetSide,
+            8 => Self::LineIn,
+            9 => Self::Aux,
+            10 => Self::MicIn,
+            11 => Self::Telephony,
+            12 => Self::SpdifIn,
+            13 => Self::OtherDigitalIn,
+            _ => Self::Other,
+        }
+    }
+}
+
+impl From<u32> for PinDefaultConfig {
+    fn from(value: u32) -> Self {
+        let sequence = (value & 0xF) as u8;
+        let default_association = ((value >> 4) & 0xF) as u8;
+        let color = PinColor::from((value >> 12) & 0xF);
+        let connection_type = PinType::from((value >> 16) & 0xF);
+        let default_device = PinDevice::from((value >> 20) & 0xF);
+        let connectivity = PinConnectivity::from(value >> 30);
+
+        Self {
+            sequence,
+            color,
+            connectivity,
+            connection_type,
+            default_device,
+            default_association,
+        }
+    }
+}

kernel/driver/sound/intel-hda/src/sink.rs

@@ -0,0 +1,171 @@
+use core::{
+    sync::atomic::{AtomicBool, Ordering},
+    task::{Context, Poll},
+    time::Duration,
+};
+
+use alloc::{boxed::Box, sync::Arc, vec::Vec};
+use async_trait::async_trait;
+use device_api::device::Device;
+use futures_util::task::AtomicWaker;
+use libk::{device::char::CharDevice, error::Error, fs::devfs, task::runtime, vfs::FileReadiness};
+use libk_mm::PageBox;
+use libk_util::{event::BoolEvent, sync::spin_rwlock::IrqSafeRwLock};
+use tock_registers::interfaces::ReadWriteable;
+use ygg_driver_sound_core::{
+    AudioSink, SampleFormat, SampleRate, SinkFormat, SinkSupportedFormats,
+};
+use yggdrasil_abi::io::FileMode;
+
+use crate::{
+    codec::{Codec, Node},
+    regs::SDxCTL0,
+    ring::Verb,
+    stream::OutputStream,
+    HdAudio,
+};
+
+pub struct HdAudioSink {
+    hda: Arc<HdAudio>,
+    codec: Arc<Codec>,
+    audio_widget: Node,
+    pub index: usize,
+    stream_tag: u8,
+    stream: OutputStream,
+    notify: BoolEvent,
+    config: IrqSafeRwLock<SinkFormat>,
+    playing: AtomicBool,
+}
+
+impl HdAudioSink {
+    pub async fn create(
+        hda: Arc<HdAudio>,
+        index: usize,
+        stream_tag: u8,
+        codec: Arc<Codec>,
+        audio_widget: Node,
+    ) -> Result<Arc<Self>, Error> {
+        // TODO use default/current stream format (if PCM), don't always set 48KHz S16 x2
+        log::info!("hda: create sink #{index}, tag {stream_tag}");
+        let config = SinkFormat {
+            sample_rate: SampleRate::Rate48000,
+            sample_format: SampleFormat::S16Le,
+            channels: 2,
+        };
+        let one_millisecond =
+            (config.sample_rate as usize * config.sample_format.sample_size() * config.channels)
+                / 1000;
+        let bdl_capacity = 32;
+        log::info!("buffer_count = {bdl_capacity}, buffer_size = {one_millisecond}");
+        let (stream, bdl_address) = OutputStream::new(hda.clone(), one_millisecond, bdl_capacity)?;
+
+        // TODO check if the widget actually supports such format
+        audio_widget
+            .perform_command(&*hda, Verb::SetStreamFormat, (1 << 4) | 1)
+            .await?;
+
+        {
+            let regs = hda.regs.lock();
+            regs.configure_stream(
+                index,
+                &config,
+                bdl_address,
+                one_millisecond,
+                bdl_capacity,
+                stream_tag,
+            )?;
+            regs.enable_stream_interrupts(index);
+        }
+
+        audio_widget.set_stream(&*hda, stream_tag, 0).await?;
+
+        let this = Arc::new(Self {
+            hda,
+            codec,
+            audio_widget,
+            index,
+            stream_tag,
+            stream,
+            notify: BoolEvent::new(),
+            playing: AtomicBool::new(false),
+            config: IrqSafeRwLock::new(config),
+        });
+
+        ygg_driver_sound_core::register_audio_sink(this.clone());
+
+        Ok(this)
+    }
+
+    pub async fn handle_softirq(&self, position: u32) {
+        self.stream.update_tail(position);
+        self.notify.signal_saturating();
+    }
+
+    async fn write_blocking(&self, data: &[u8]) -> Result<(), Error> {
+        let mut position = 0;
+        while position != data.len() {
+            let written = self.stream.write(&data[position..]);
+            position += written;
+            if written == 0 {
+                self.notify.wait_reset().await;
+            }
+        }
+        Ok(())
+    }
+}
+
+#[async_trait]
+impl AudioSink for HdAudioSink {
+    async fn set_format(&self, format: SinkFormat) -> Result<(), Error> {
+        todo!()
+    }
+    async fn current_format(&self) -> Result<SinkFormat, Error> {
+        todo!()
+    }
+    async fn supported_formats(&self) -> Result<SinkSupportedFormats, Error> {
+        todo!()
+    }
+
+    async fn write(&self, data: &[u8]) -> Result<usize, Error> {
+        let amount = data.len().min(48000 * 2);
+        if !self.playing.swap(true, Ordering::Acquire) {
+            self.start().await?;
+        }
+        self.write_blocking(&data[..amount]).await?;
+        Ok(data.len())
+    }
+    async fn start(&self) -> Result<(), Error> {
+        self.playing.store(true, Ordering::Release);
+        let regs = self.hda.regs.lock();
+        log::info!("hda: start stream #{}", self.index);
+        self.stream.reset();
+        regs.STREAMS[self.index].SDxCTL0.modify(SDxCTL0::RUN::SET);
+        Ok(())
+    }
+    fn stop(&self) -> Result<(), Error> {
+        self.playing.store(false, Ordering::Release);
+        let regs = self.hda.regs.lock();
+        log::info!("hda: stop stream #{}", self.index);
+        regs.STREAMS[self.index].SDxCTL0.modify(SDxCTL0::RUN::CLEAR);
+        Ok(())
+    }
+}
+
+#[async_trait]
+impl CharDevice for HdAudioSink {
+    async fn write(&self, buffer: &[u8]) -> Result<usize, Error> {
+        AudioSink::write(self, buffer).await
+    }
+}
+
+impl FileReadiness for HdAudioSink {
+    fn poll_read(&self, cx: &mut Context<'_>) -> Poll<Result<(), Error>> {
+        todo!()
+    }
+}
+
+impl Device for HdAudioSink {
+    fn display_name(&self) -> &str {
+        "HD Audio Sink"
+    }
+}

kernel/driver/sound/intel-hda/src/stream.rs

@@ -0,0 +1,135 @@
+use alloc::{sync::Arc, vec::Vec};
+use device_api::dma::DmaAllocator;
+use libk::{
+    dma::{BusAddress, DmaBuffer},
+    error::Error,
+};
+use libk_util::{
+    event::BoolEvent,
+    sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock},
+};
+use ygg_driver_sound_core::SinkFormat;
+
+use crate::HdAudio;
+
+#[derive(Debug)]
+#[repr(C)]
+pub struct BufferDescriptor {
+    address: BusAddress,
+    length: u32,
+    flags: u32,
+}
+
+pub struct BufferDescriptorList {
+    entries: DmaBuffer<[BufferDescriptor]>,
+    periods: Vec<DmaBuffer<[u8]>>,
+    wr: usize,
+    rd: usize,
+    buffer_size: usize,
+}
+
+pub enum StreamKind {
+    Input,
+    Output,
+    Bidi,
+}
+
+pub struct OutputStream {
+    pub hda: Arc<HdAudio>,
+    pub bdl: IrqSafeSpinlock<BufferDescriptorList>,
+}
+
+impl BufferDescriptorList {
+    pub fn with_capacity(
+        dma: &dyn DmaAllocator,
+        capacity: usize,
+        frame_size: usize,
+    ) -> Result<Self, Error> {
+        let periods: Vec<DmaBuffer<[u8]>> = (0..capacity)
+            .map(|_| DmaBuffer::new_slice(dma, 0, frame_size))
+            .collect::<Result<_, _>>()?;
+        let entries =
+            DmaBuffer::new_slice_with(dma, |i| BufferDescriptor::new(&periods[i]), capacity)?;
+        Ok(Self {
+            entries,
+            wr: 0,
+            rd: 0,
+            periods,
+            buffer_size: frame_size,
+        })
+    }
+
+    pub fn write(&mut self, data: &[u8]) -> usize {
+        let capacity = self.periods.len() * self.buffer_size;
+
+        let mut position = 0;
+        while (self.wr + 1) % capacity != self.rd && position != data.len() {
+            let p = self.wr / self.buffer_size;
+            let o = self.wr % self.buffer_size;
+
+            self.periods[p][o] = data[position];
+
+            position += 1;
+            self.wr = (self.wr + 1) % capacity;
+        }
+
+        position
+    }
+
+    pub fn update_tail(&mut self, tail: u32) {
+        self.rd = tail as usize;
+    }
+
+    pub fn reset(&mut self) {
+        self.wr = self.rd;
+    }
+}
+
+impl BufferDescriptor {
+    pub fn new(buffer: &DmaBuffer<[u8]>) -> Self {
+        Self {
+            address: buffer.bus_address(),
+            length: buffer.len() as _,
+            flags: 1,
+        }
+    }
+}
+
+impl OutputStream {
+    pub fn new(
+        hda: Arc<HdAudio>,
+        buffer_size: usize,
+        bdl_capacity: usize,
+    ) -> Result<(Self, BusAddress), Error> {
+        let bdl = BufferDescriptorList::with_capacity(&*hda.dma, bdl_capacity, buffer_size)?;
+        let bdl_base = bdl.entries.bus_address();
+        Ok((
+            Self {
+                hda,
+                bdl: IrqSafeSpinlock::new(bdl),
+            },
+            bdl_base,
+        ))
+    }
+
+    pub fn params(&self) -> (BusAddress, usize, usize) {
+        let bdl = self.bdl.lock();
+        (
+            bdl.entries.bus_address(),
+            bdl.buffer_size,
+            bdl.entries.len(),
+        )
+    }
+
+    pub fn reset(&self) {
+        self.bdl.lock().reset()
+    }
+
+    pub fn write(&self, bytes: &[u8]) -> usize {
+        self.bdl.lock().write(bytes)
+    }
+
+    pub fn update_tail(&self, tail: u32) {
+        self.bdl.lock().update_tail(tail)
+    }
+}

kernel/driver/usb/xhci/src/controller.rs

@@ -25,11 +25,10 @@ use tock_registers::{
 };
 use ygg_driver_pci::{device::PciDeviceInfo, PciConfigurationSpace};
 use ygg_driver_usb::{
-    address::UsbBusAddress,
-    bus::{UsbBusManager, UsbBusWrapper},
-    descriptor,
-    device::{UsbDeviceAccess, UsbSpeed},
+    bus::UsbBusManager,
+    device::{UsbBusAddress, UsbDeviceAccess, UsbSpeed},
     error::UsbError,
+    info::UsbVersion,
     pipe::control::UsbControlPipeAccess,
     UsbHostController,
 };
@@ -68,7 +67,7 @@ struct ScratchpadArray {
 }
 
 struct RootHubPort {
-    version: u16,
+    version: UsbVersion,
     slot_type: u8,
 }
 
@@ -91,7 +90,6 @@ pub struct Xhci {
     pub(crate) slots: Vec<IrqSafeRwLock<Option<Arc<XhciBusDevice>>>>,
     pub(crate) port_slot_map: Vec<AtomicU8>,
     bus_index: OneTimeInit<u16>,
-    bus: OneTimeInit<Arc<UsbBusWrapper>>,
     port_event_map: EventBitmap,
 }
 
@@ -148,7 +146,9 @@ impl Xhci {
         for cap in regs.extended_capabilities.iter() {
             match cap {
                 ExtendedCapability::ProtocolSupport(support) => {
-                    let version = support.usb_revision();
+                    let Some(version) = support.usb_revision() else {
+                        continue;
+                    };
 
                     for port in support.port_range() {
                         log::info!("* Port {port}: {version}");
@@ -195,7 +195,6 @@ impl Xhci {
 
             root_hub_ports,
             bus_index: OneTimeInit::new(),
-            bus: OneTimeInit::new(),
             endpoints: IrqSafeRwLock::new(BTreeMap::new()),
             slots,
             port_slot_map,
@@ -283,7 +282,7 @@ impl Xhci {
             .as_ref()
             .ok_or(UsbError::PortInitFailed)?;
 
-        let need_reset = !descriptor::is_version_3(root_hub_port.version);
+        let need_reset = !root_hub_port.version.is_version_3();
 
         if need_reset {
             self.reset_port(regs).await?;
@@ -342,8 +341,7 @@ impl Xhci {
             device: bus_address,
         });
 
-        let bus = self.bus.get();
-        let device = UsbDeviceAccess::setup(bus.clone(), slot).await?;
+        let device = UsbDeviceAccess::setup(slot).await?;
         UsbBusManager::register_device(device.into());
 
         Ok(())
@@ -525,9 +523,8 @@ impl Device for Xhci {
 
         op.wait_usbsts_bit(USBSTS::CNR::CLEAR, 100000000)?;
 
-        let (bus_index, bus) = UsbBusManager::register_bus(self.clone());
-        self.bus_index.init(bus_index);
-        self.bus.init(bus);
+        let bus = UsbBusManager::register_bus(self.clone());
+        self.bus_index.init(bus);
 
         runtime::spawn(self.clone().port_handler_task()).ok();
 

kernel/driver/usb/xhci/src/device.rs

@@ -6,9 +6,8 @@ use libk_util::{
 };
 use xhci_lib::context;
 use ygg_driver_usb::{
-    address::UsbBusAddress,
     communication::UsbDirection,
-    device::{UsbDevice, UsbDeviceDetachHandler, UsbSpeed},
+    device::{UsbBusAddress, UsbDevice, UsbDeviceDetachHandler, UsbSpeed},
     error::UsbError,
     info::UsbEndpointType,
     pipe::{
@@ -64,8 +63,8 @@ impl UsbDevice for XhciBusDevice {
         *self.detach_handler.lock() = Some(handler);
     }
 
-    fn host_controller(&self) -> Arc<dyn UsbHostController> {
-        self.xhci.clone()
+    fn controller_ref(&self) -> &dyn UsbHostController {
+        self.xhci.as_ref()
     }
 
     fn debug(&self) {}

kernel/driver/usb/xhci/src/lib.rs

@@ -9,7 +9,7 @@ use controller::Xhci;
 use device_api::{device::Device, dma::DmaAllocator, interrupt::InterruptAffinity};
 use regs::Regs;
 use ygg_driver_pci::{
-    capability::{DevicePowerState, PowerManagementCapability},
+    capability::{power::DevicePowerState, PowerManagementCapability},
     device::{PciDeviceInfo, PreferredInterruptMode},
     macros::pci_driver,
     PciCommandRegister, PciConfigurationSpace,

kernel/driver/usb/xhci/src/regs/extended.rs

@@ -7,7 +7,7 @@ use alloc::vec::Vec;
 use libk::error::Error;
 use libk_mm::{address::PhysicalAddress, device::DeviceMemoryIo};
 use libk_util::sync::spin_rwlock::IrqSafeRwLock;
-use ygg_driver_usb::error::UsbError;
+use ygg_driver_usb::{error::UsbError, info::UsbVersion};
 
 pub struct ProtocolSupport {
     words: [u32; 4],
@@ -69,8 +69,8 @@ impl ExtendedCapability {
 }
 
 impl ProtocolSupport {
-    pub fn usb_revision(&self) -> u16 {
-        (self.words[0] >> 16) as u16
+    pub fn usb_revision(&self) -> Option<UsbVersion> {
+        UsbVersion::from_bcd_usb((self.words[0] >> 16) as u16)
     }
 
     pub fn slot_type(&self) -> u8 {

kernel/driver/usb/xhci/src/ring/transfer.rs

@@ -94,7 +94,7 @@ impl TransferRing {
         })
     }
 
-    pub fn transaction_builder(self: &Arc<Self>) -> Result<TransactionBuilder<'_>, UsbError> {
+    pub fn transaction_builder(self: &Arc<Self>) -> Result<TransactionBuilder, UsbError> {
         if self.shutdown.load(Ordering::Acquire) {
             return Err(UsbError::DeviceDisconnected);
         }

kernel/driver/virtio/core/src/transport/pci.rs

@@ -5,7 +5,7 @@ use tock_registers::{
 };
 use ygg_driver_pci::{
     capability::{
-        VirtioCapabilityData, VirtioCommonConfigCapability, VirtioDeviceConfigCapability,
+        virtio::VirtioCapabilityData, VirtioCommonConfigCapability, VirtioDeviceConfigCapability,
         VirtioInterruptStatusCapability, VirtioNotifyConfigCapability,
     },
     PciCommandRegister, PciConfigurationSpace,
@@ -40,7 +40,7 @@ impl Transport for PciTransport {
         true
     }
 
-    fn device_cfg(&self) -> Option<&DeviceMemoryIo<'_, [u8]>> {
+    fn device_cfg(&self) -> Option<&DeviceMemoryIo<[u8]>> {
         Some(&self.device_cfg)
     }
 

kernel/driver/virtio/gpu/src/lib.rs

@@ -161,7 +161,7 @@ impl<T: Transport + 'static> VirtioGpu<T> {
         Ok(())
     }
 
-    fn begin_command(&self) -> CommandExecution<'_, T> {
+    fn begin_command(&self) -> CommandExecution<T> {
         CommandExecution {
             transport: &self.transport,
             control: &self.queues.get().control,

kernel/lib/device-tree/src/tree.rs

@@ -78,7 +78,7 @@ impl<'a> DeviceTree<'a> {
     }
 
     /// Returns the root node of this device tree
-    pub fn root(&self) -> TNode<'_> {
+    pub fn root(&self) -> TNode {
         self.index.root()
     }
 
@@ -172,12 +172,12 @@ impl<'a> DeviceTree<'a> {
     }
 
     /// Returns an iterator over the memory regions specified by this device tree
-    pub fn memory_regions(&self) -> DeviceTreeMemoryRegionIter<'_> {
+    pub fn memory_regions(&self) -> DeviceTreeMemoryRegionIter {
         DeviceTreeMemoryRegionIter::new(self)
     }
 
     /// Returns an iterator over the reserved memory regions specified by this device tree
-    pub fn reserved_regions(&self) -> DeviceTreeReservedRegionIter<'_> {
+    pub fn reserved_regions(&self) -> DeviceTreeReservedRegionIter {
         DeviceTreeReservedRegionIter::new(self)
     }
 

kernel/lib/memtables/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "memtables"
+version = "0.1.0"
+edition = "2021"
+authors = ["Mark Poliakov <mark@alnyan.me>"]
+
+[dependencies]
+bitflags.workspace = true
+bytemuck.workspace = true
+
+[features]
+default = []
+all = []

kernel/lib/memtables/src/aarch64.rs

@@ -0,0 +1,26 @@
+use bytemuck::{Pod, Zeroable};
+
+use crate::RawTable;
+
+pub const KERNEL_L3_COUNT: usize = 8;
+
+#[derive(Clone, Copy, Pod, Zeroable)]
+#[repr(C)]
+pub struct FixedTables {
+    // 1GiB entries
+    pub l1: RawTable,
+
+    // 2MiB entries
+    pub l2: RawTable,
+    pub l3s: [RawTable; KERNEL_L3_COUNT],
+}
+
+impl FixedTables {
+    pub const fn zeroed() -> Self {
+        Self {
+            l1: RawTable::zeroed(),
+            l2: RawTable::zeroed(),
+            l3s: [RawTable::zeroed(); KERNEL_L3_COUNT],
+        }
+    }
+}

kernel/lib/memtables/src/any.rs

@@ -0,0 +1,35 @@
+use crate::{aarch64, riscv64, x86_64};
+
+pub enum AnyTables {
+    X86_64(x86_64::FixedTables),
+    AArch64(aarch64::FixedTables),
+    Riscv64(riscv64::FixedTables),
+}
+
+impl AnyTables {
+    pub fn as_bytes(&self) -> &[u8] {
+        match self {
+            Self::X86_64(tables) => bytemuck::bytes_of(tables),
+            Self::AArch64(tables) => bytemuck::bytes_of(tables),
+            Self::Riscv64(tables) => bytemuck::bytes_of(tables),
+        }
+    }
+}
+
+impl From<x86_64::FixedTables> for AnyTables {
+    fn from(value: x86_64::FixedTables) -> Self {
+        Self::X86_64(value)
+    }
+}
+
+impl From<aarch64::FixedTables> for AnyTables {
+    fn from(value: aarch64::FixedTables) -> Self {
+        Self::AArch64(value)
+    }
+}
+
+impl From<riscv64::FixedTables> for AnyTables {
+    fn from(value: riscv64::FixedTables) -> Self {
+        Self::Riscv64(value)
+    }
+}

kernel/lib/memtables/src/lib.rs

@@ -0,0 +1,36 @@
+#![no_std]
+
+use bytemuck::{Pod, Zeroable};
+
+// AArch64
+#[cfg(any(feature = "all", target_arch = "aarch64"))]
+pub mod aarch64;
+#[cfg(all(not(feature = "all"), target_arch = "aarch64"))]
+pub use aarch64::FixedTables;
+
+// x86-64
+#[cfg(any(feature = "all", target_arch = "x86_64"))]
+pub mod x86_64;
+#[cfg(all(not(feature = "all"), target_arch = "x86_64"))]
+pub use x86_64::FixedTables;
+
+// RISC-V 64-bit
+#[cfg(any(feature = "all", target_arch = "riscv64"))]
+pub mod riscv64;
+#[cfg(all(not(feature = "all"), target_arch = "riscv64"))]
+pub use riscv64::FixedTables;
+
+#[cfg(feature = "all")]
+pub mod any;
+
+#[derive(Clone, Copy, Pod, Zeroable)]
+#[repr(C, align(0x1000))]
+pub struct RawTable {
+    pub data: [u64; 512],
+}
+
+impl RawTable {
+    pub const fn zeroed() -> Self {
+        Self { data: [0; 512] }
+    }
+}

kernel/lib/memtables/src/riscv64.rs

@@ -0,0 +1,84 @@
+use core::fmt;
+
+use bitflags::bitflags;
+use bytemuck::{Pod, Zeroable};
+
+use crate::RawTable;
+
+pub const KERNEL_L3_COUNT: usize = 8;
+
+bitflags! {
+    #[derive(Clone, Copy, PartialEq, Eq, Debug)]
+    pub struct PageAttributes: u64 {
+        const N = 1 << 63;
+        /// Software-tracked dirty bit (RSW[0])
+        const SW_DIRTY = 1 << 9;
+        /// Dirty bit
+        const D = 1 << 7;
+        /// Access bit
+        const A = 1 << 6;
+        /// Global mapping bit, implies all lower levels are also global
+        const G = 1 << 5;
+        /// U-mode access permission
+        const U = 1 << 4;
+        /// Execute permission
+        const X = 1 << 3;
+        /// Write permission
+        const W = 1 << 2;
+        /// Read-permission
+        const R = 1 << 1;
+        /// Valid bit
+        const V = 1 << 0;
+    }
+
+    // X W R    Meaning
+    // 0 0 0    Pointer to next level of page table
+    // 0 0 1    Read-only page
+    // 0 1 0    ---
+    // 0 1 1    Read-write page
+    // 1 0 0    Execute only
+    // 1 0 1    Read-execute page
+    // 1 1 0    ---
+    // 1 1 1    Read-write-execute page
+}
+
+#[derive(Clone, Copy, Pod, Zeroable)]
+#[repr(C)]
+pub struct FixedTables {
+    pub l1: RawTable,
+    pub kernel_l2: RawTable,
+    pub kernel_l3s: [RawTable; KERNEL_L3_COUNT],
+}
+
+impl FixedTables {
+    pub const fn zeroed() -> Self {
+        Self {
+            l1: RawTable::zeroed(),
+            kernel_l2: RawTable::zeroed(),
+            kernel_l3s: [RawTable::zeroed(); KERNEL_L3_COUNT],
+        }
+    }
+}
+
+impl fmt::Display for PageAttributes {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        use fmt::Write;
+
+        macro_rules! bit {
+            ($self:ident, $field:expr, $letter:literal) => {
+                if $self.contains($field) {
+                    f.write_char($letter)
+                } else {
+                    f.write_char('-')
+                }
+            };
+        }
+
+        bit!(self, Self::R, 'r')?;
+        bit!(self, Self::W, 'w')?;
+        bit!(self, Self::X, 'x')?;
+        bit!(self, Self::U, 'u')?;
+
+        Ok(())
+    }
+}

kernel/lib/memtables/src/x86_64.rs

@@ -0,0 +1,27 @@
+use bytemuck::{Pod, Zeroable};
+
+use crate::RawTable;
+
+pub const KERNEL_L3_COUNT: usize = 16;
+
+#[derive(Clone, Copy, Pod, Zeroable)]
+#[repr(C)]
+pub struct FixedTables {
+    pub l0: RawTable,
+
+    pub kernel_l1: RawTable,
+    pub kernel_l2: RawTable,
+    pub kernel_l3s: [RawTable; KERNEL_L3_COUNT],
+}
+
+impl FixedTables {
+    pub const fn zeroed() -> Self {
+        Self {
+            l0: RawTable::zeroed(),
+
+            kernel_l1: RawTable::zeroed(),
+            kernel_l2: RawTable::zeroed(),
+            kernel_l3s: [RawTable::zeroed(); KERNEL_L3_COUNT],
+        }
+    }
+}

kernel/libk/libk-mm/interface/src/table.rs

@@ -1,9 +1,6 @@
 use core::ops::{Deref, DerefMut, Range};
 
 use bitflags::bitflags;
-use kernel_arch_interface::mem::{
-    DeviceMemoryAttributes, KernelTableManager, RawDeviceMemoryMapping,
-};
 use yggdrasil_abi::error::Error;
 
 use super::address::PhysicalAddress;
@@ -187,124 +184,3 @@ impl<T: AddressLike> EntryLevelExt for T {
         self.page_offset::<L>() == 0
     }
 }
-
-pub trait DevicePageManagerLevel {
-    type Level: EntryLevel;
-    const INDEX_RANGE: Range<usize>;
-    const VIRTUAL_BASE: usize;
-
-    fn map_page(&mut self, index: usize, physical: PhysicalAddress, attrs: &DeviceMemoryAttributes);
-    fn unmap_page(&mut self, index: usize);
-    fn is_mapped(&self, index: usize) -> bool;
-    fn flush_range(range: Range<usize>);
-
-    fn allocate_mapping(
-        &mut self,
-        base: PhysicalAddress,
-        page_count: usize,
-        attrs: &DeviceMemoryAttributes,
-    ) -> Option<usize> {
-        'l0: for i in Self::INDEX_RANGE {
-            for j in 0..page_count {
-                if self.is_mapped(i + j) {
-                    continue 'l0;
-                }
-            }
-
-            for j in 0..page_count {
-                self.map_page(i + j, base.add(j * Self::Level::SIZE), attrs);
-            }
-
-            let start = Self::VIRTUAL_BASE + i * Self::Level::SIZE;
-            Self::flush_range(i..i + page_count);
-            return Some(start);
-        }
-        None
-    }
-
-    unsafe fn remove_mapping(&mut self, base: usize, page_count: usize) {
-        let base_index = (base - Self::VIRTUAL_BASE) / Self::Level::SIZE;
-        for i in 0..page_count {
-            self.unmap_page(base_index + i);
-        }
-    }
-}
-
-pub struct DevicePageManager<N: DevicePageManagerLevel, L: DevicePageManagerLevel> {
-    pub normal: N,
-    pub large: L,
-}
-
-impl<N: DevicePageManagerLevel, L: DevicePageManagerLevel> DevicePageManager<N, L> {
-    pub const fn new(normal: N, large: L) -> Self {
-        Self { normal, large }
-    }
-
-    pub unsafe fn map_device_pages<A: KernelTableManager>(
-        &mut self,
-        base: PhysicalAddress,
-        size: usize,
-        attrs: DeviceMemoryAttributes,
-    ) -> Result<RawDeviceMemoryMapping<A>, Error> {
-        let small_aligned_base = base.page_align_down::<N::Level>();
-        let small_aligned_end = base.add(size).page_align_up::<N::Level>();
-        let small_offset = base - small_aligned_base;
-        let small_page_count = (small_aligned_end - small_aligned_base).page_count::<N::Level>();
-
-        if small_page_count > 128 {
-            // Allocate from large page pool
-            let large_aligned_base = base.page_align_down::<L::Level>();
-            let large_aligned_end = base.add(size).page_align_up::<L::Level>();
-            let large_offset = base - large_aligned_base;
-            let large_page_count =
-                (large_aligned_end - large_aligned_base).page_count::<L::Level>();
-
-            let mapped_base = self
-                .large
-                .allocate_mapping(large_aligned_base, large_page_count, &attrs)
-                .ok_or(Error::OutOfMemory)?;
-            let mapped_address = mapped_base + large_offset;
-
-            Ok(RawDeviceMemoryMapping::from_raw_parts(
-                large_aligned_base.into_u64(),
-                mapped_address,
-                mapped_base,
-                large_page_count,
-                L::Level::SIZE,
-            ))
-        } else {
-            // Allocate from small page pool
-            let mapped_base = self
-                .normal
-                .allocate_mapping(small_aligned_base, small_page_count, &attrs)
-                .ok_or(Error::OutOfMemory)?;
-            let mapped_address = mapped_base + small_offset;
-
-            Ok(RawDeviceMemoryMapping::from_raw_parts(
-                small_aligned_base.into_u64(),
-                mapped_address,
-                mapped_base,
-                small_page_count,
-                N::Level::SIZE,
-            ))
-        }
-    }
-
-    pub unsafe fn unmap_device_pages<A: KernelTableManager>(
-        &mut self,
-        mapping: &RawDeviceMemoryMapping<A>,
-    ) {
-        if mapping.page_size == N::Level::SIZE {
-            self.normal
-                .remove_mapping(mapping.base_address, mapping.page_count);
-        } else if mapping.page_size == L::Level::SIZE {
-            self.large
-                .remove_mapping(mapping.base_address, mapping.page_count);
-        } else {
-            unreachable!(
-                "Invalid device memory mapping with page size {:#x}",
-                mapping.page_size
-            )
-        }
-    }
-}

kernel/libk/libk-mm/src/phys/mod.rs

@@ -1,8 +1,6 @@
 use core::ops::Range;
 
-use kernel_arch::{
-    mem::PhysicalMemoryAllocator, Architecture, ArchitectureImpl, KERNEL_VIRT_OFFSET,
-};
+use kernel_arch::{mem::PhysicalMemoryAllocator, Architecture, ArchitectureImpl};
 use libk_mm_interface::address::PhysicalAddress;
 use libk_util::{sync::IrqSafeSpinlock, OneTimeInit};
 use yggdrasil_abi::{error::Error, system::SystemMemoryStats};
@@ -180,14 +178,20 @@ pub fn find_contiguous_region<I: Iterator<Item = PhysicalMemoryRegion>>(
 ///
 /// The caller must ensure this function has not been called before and that the regions
 /// are valid and actually available.
-pub unsafe fn init_from_iter<I: Iterator<Item = PhysicalMemoryRegion> + Clone>(
+pub unsafe fn init_from_iter<
+    I: Iterator<Item = PhysicalMemoryRegion> + Clone,
+    Map: FnOnce(I, PhysicalAddress, PhysicalAddress) -> Result<(), Error>,
+>(
     it: I,
+    map_physical_memory: Map,
 ) -> Result<(), Error> {
     // Map the physical memory
     let (phys_start, phys_end) = physical_memory_range(it.clone()).unwrap();
 
     reserve_region("kernel", kernel_physical_memory_region());
 
+    map_physical_memory(it.clone(), phys_start, phys_end)?;
+
     let total_count = (phys_end - phys_start) / L3_PAGE_SIZE;
     let page_bitmap_size = (total_count + BITMAP_WORD_SIZE - 1) / (BITMAP_WORD_SIZE / 8);
     let page_bitmap_page_count = page_bitmap_size.div_ceil(L3_PAGE_SIZE);
@@ -241,16 +245,18 @@ pub unsafe fn init_from_iter<I: Iterator<Item = PhysicalMemoryRegion> + Clone>(
 }
 
 fn kernel_physical_memory_region() -> PhysicalMemoryRegion {
+    use core::ptr::addr_of;
+
     extern "C" {
         static __kernel_start: u8;
         static __kernel_end: u8;
     }
 
-    let start = (&raw const __kernel_start).addr();
-    let end = (&raw const __kernel_end).addr();
+    let start = addr_of!(__kernel_start);
+    let end = addr_of!(__kernel_end);
 
-    let base = PhysicalAddress::from_usize(start - KERNEL_VIRT_OFFSET);
-    let size = end - start;
+    let base = PhysicalAddress::from_usize(start.addr() - kernel_arch::KERNEL_VIRT_OFFSET);
+    let size = end.addr() - start.addr();
 
     PhysicalMemoryRegion { base, size }
 }

kernel/libk/libk-mm/src/process.rs

@@ -524,7 +524,7 @@ impl<TA: TableAllocator> ProcessAddressSpace<TA> {
 
     /// Same as [ProcessAddressSpace::translate], except the lock on the address space is held
     /// until the resulting [TranslateGuard] is dropped.
-    pub fn translate_lock(&self, address: usize) -> Result<TranslateGuard<'_, TA>, Error> {
+    pub fn translate_lock(&self, address: usize) -> Result<TranslateGuard<TA>, Error> {
         let guard = self.inner.lock();
         let address = guard.table.translate(address).map(|e| e.0)?;
         Ok(TranslateGuard {

kernel/libk/libk-util/src/event.rs

@@ -35,6 +35,11 @@ pub struct OneTimeEvent<T> {
     notify: QueueWaker,
 }
 
+pub struct CounterEvent<N: EventNotify> {
+    counter: AtomicU64,
+    notify: N,
+}
+
 pub struct BitmapEvent<N: EventNotify> {
     value: AtomicU64,
     notify: N,
@@ -218,3 +223,39 @@ impl<N: EventNotify> BitmapEvent<N> {
         .await
     }
 }
+
+impl<N: EventNotify> CounterEvent<N> {
+    pub const fn new(notify: N) -> Self {
+        Self {
+            counter: AtomicU64::new(0),
+            notify,
+        }
+    }
+
+    pub fn signal(&self) {
+        self.counter.fetch_add(1, Ordering::Release);
+        self.notify.notify_all();
+    }
+
+    pub fn try_take(&self) -> Option<u64> {
+        let value = self.counter.swap(0, Ordering::Acquire);
+        if value > 0 {
+            Some(value)
+        } else {
+            None
+        }
+    }
+
+    pub async fn take(&self) -> u64 {
+        poll_fn(|cx| {
+            if let Some(value) = self.try_take() {
+                self.notify.unsubscribe(cx.waker());
+                Poll::Ready(value)
+            } else {
+                self.notify.subscribe(cx.waker());
+                Poll::Pending
+            }
+        })
+        .await
+    }
+}

kernel/libk/libk-util/src/lru_hash_table.rs

@@ -51,7 +51,7 @@ impl<K, V> LruCacheBucket<K, V> {
 }
 
 impl<K: Eq, V> LruCacheBucket<K, V> {
-    pub fn entry_mut(&mut self, key: &K) -> EntryMut<'_, K, V> {
+    pub fn entry_mut(&mut self, key: &K) -> EntryMut<K, V> {
         let mut cursor = self.data.cursor_front_mut();
 
         loop {
@@ -125,7 +125,7 @@ impl<K, V, H: BuildHasher> LruCache<K, V, H> {
             .find_map(|bucket| bucket.pop_front())
     }
 
-    pub fn flush(&mut self) -> FlushIter<'_, K, V, H> {
+    pub fn flush(&mut self) -> FlushIter<K, V, H> {
         FlushIter { cache: self }
     }
 }

kernel/libk/libk-util/src/ring.rs

@@ -203,7 +203,7 @@ impl<T: Copy> LossyRingQueue<T> {
     }
 
     #[inline]
-    pub fn try_read_lock(&self) -> Option<IrqSafeSpinlockGuard<'_, RingBuffer<T>>> {
+    pub fn try_read_lock(&self) -> Option<IrqSafeSpinlockGuard<RingBuffer<T>>> {
         let lock = self.ring.lock();
         lock.is_readable().then_some(lock)
     }
@@ -214,12 +214,12 @@ impl<T: Copy> LossyRingQueue<T> {
         unsafe { lock.read_single_unchecked() }
     }
 
-    pub async fn read_lock(&self) -> IrqSafeSpinlockGuard<'_, RingBuffer<T>> {
+    pub async fn read_lock(&self) -> IrqSafeSpinlockGuard<RingBuffer<T>> {
         poll_fn(|cx| self.poll_lock(cx)).await
     }
 
     #[inline]
-    pub fn poll_lock(&self, cx: &mut Context<'_>) -> Poll<IrqSafeSpinlockGuard<'_, RingBuffer<T>>> {
+    pub fn poll_lock(&self, cx: &mut Context<'_>) -> Poll<IrqSafeSpinlockGuard<RingBuffer<T>>> {
         self.read_notify.register(cx.waker());
         if let Some(lock) = self.try_read_lock() {
             self.read_notify.remove(cx.waker());
@@ -284,7 +284,7 @@ impl<T: Copy> BlockingRingQueue<T> {
         true
     }
 
-    pub async fn read_lock(&self) -> IrqSafeSpinlockGuard<'_, RingBuffer<T>> {
+    pub async fn read_lock(&self) -> IrqSafeSpinlockGuard<RingBuffer<T>> {
         poll_fn(|cx| self.poll_read_lock(cx)).await
     }
 
@@ -292,7 +292,7 @@ impl<T: Copy> BlockingRingQueue<T> {
     pub fn poll_read_lock(
         &self,
         cx: &mut Context<'_>,
-    ) -> Poll<IrqSafeSpinlockGuard<'_, RingBuffer<T>>> {
+    ) -> Poll<IrqSafeSpinlockGuard<RingBuffer<T>>> {
         if let Some(lock) = self.try_read_lock() {
             self.read_notify.remove(cx.waker());
             Poll::Ready(lock)
@@ -303,7 +303,7 @@ impl<T: Copy> BlockingRingQueue<T> {
     }
 
     #[inline]
-    pub fn try_read_lock(&self) -> Option<IrqSafeSpinlockGuard<'_, RingBuffer<T>>> {
+    pub fn try_read_lock(&self) -> Option<IrqSafeSpinlockGuard<RingBuffer<T>>> {
         let lock = self.ring.lock();
         lock.is_readable().then_some(lock)
     }

kernel/libk/libk-util/src/string.rs

@@ -28,7 +28,7 @@ impl Utf16LeStr {
         core::mem::transmute(raw)
     }
 
-    pub fn chars(&self) -> Utf16LeIter<'_> {
+    pub fn chars(&self) -> Utf16LeIter {
         Utf16LeIter {
             chars: self.raw.chunks_exact(size_of::<u16>()),
         }

kernel/libk/libk-util/src/sync/spin_rwlock.rs

@@ -131,7 +131,7 @@ impl<T> IrqSafeRwLock<T> {
         }
     }
 
-    pub fn read(&self) -> IrqSafeRwLockReadGuard<'_, T> {
+    pub fn read(&self) -> IrqSafeRwLockReadGuard<T> {
         let guard = IrqGuard::acquire();
         self.inner.acquire_read();
         IrqSafeRwLockReadGuard {
@@ -140,7 +140,7 @@ impl<T> IrqSafeRwLock<T> {
         }
     }
 
-    pub fn write(&self) -> IrqSafeRwLockWriteGuard<'_, T> {
+    pub fn write(&self) -> IrqSafeRwLockWriteGuard<T> {
         let guard = IrqGuard::acquire();
         self.inner.acquire_write();
         IrqSafeRwLockWriteGuard {