WIP usb: better driver structure, hid mouse driver

Cargo.lock

@@ -532,12 +532,6 @@ dependencies = [
  "hashbrown 0.14.5",
 ]
 
-[[package]]
-name = "elf"
-version = "0.7.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4445909572dbd556c457c849c4ca58623d84b27c8fff1e74b0b4227d8b90d17b"
-
 [[package]]
 name = "encode_unicode"
 version = "1.0.0"
@@ -741,18 +735,6 @@ dependencies = [
  "pin-utils",
 ]
 
-[[package]]
-name = "gentables"
-version = "0.1.0"
-dependencies = [
- "bitflags 2.8.0",
- "bytemuck",
- "clap",
- "elf 0.7.4",
- "memtables",
- "thiserror",
-]
-
 [[package]]
 name = "getrandom"
 version = "0.2.15"
@@ -1135,7 +1117,7 @@ dependencies = [
  "device-api",
  "kernel-arch-interface",
  "libk-mm-interface",
- "memtables",
+ "log",
  "static_assertions",
  "tock-registers",
  "yggdrasil-abi",
@@ -1184,7 +1166,6 @@ dependencies = [
  "kernel-arch-interface",
  "libk-mm-interface",
  "log",
- "memtables",
  "static_assertions",
  "tock-registers",
  "yggdrasil-abi",
@@ -1212,7 +1193,6 @@ dependencies = [
  "kernel-arch-x86",
  "libk-mm-interface",
  "log",
- "memtables",
  "static_assertions",
  "tock-registers",
  "yggdrasil-abi",
@@ -1270,7 +1250,7 @@ dependencies = [
  "cfg-if",
  "crossbeam-queue",
  "device-api",
- "elf 0.7.2",
+ "elf",
  "futures-util",
  "kernel-arch",
  "libc",
@@ -1465,14 +1445,6 @@ dependencies = [
  "autocfg",
 ]
 
-[[package]]
-name = "memtables"
-version = "0.1.0"
-dependencies = [
- "bitflags 2.8.0",
- "bytemuck",
-]
-
 [[package]]
 name = "miniz_oxide"
 version = "0.8.4"
@@ -2708,24 +2680,6 @@ dependencies = [
  "yggdrasil-abi",
 ]
 
-[[package]]
-name = "ygg_driver_intel_hda"
-version = "0.1.0"
-dependencies = [
- "async-trait",
- "bytemuck",
- "device-api",
- "futures-util",
- "libk",
- "libk-mm",
- "libk-util",
- "log",
- "tock-registers",
- "ygg_driver_pci",
- "ygg_driver_sound_core",
- "yggdrasil-abi",
-]
-
 [[package]]
 name = "ygg_driver_net_core"
 version = "0.1.0"
@@ -2854,20 +2808,6 @@ dependencies = [
  "yggdrasil-abi",
 ]
 
-[[package]]
-name = "ygg_driver_sound_core"
-version = "0.1.0"
-dependencies = [
- "async-trait",
- "device-api",
- "futures-util",
- "libk",
- "libk-mm",
- "libk-util",
- "log",
- "yggdrasil-abi",
-]
-
 [[package]]
 name = "ygg_driver_usb"
 version = "0.1.0"
@@ -3006,7 +2946,7 @@ dependencies = [
  "device-api",
  "device-api-macros",
  "device-tree",
- "elf 0.7.2",
+ "elf",
  "ext2",
  "futures-util",
  "git-version",
@@ -3022,7 +2962,6 @@ dependencies = [
  "libk-util",
  "log",
  "memfs",
- "memtables",
  "prettyplease",
  "static_assertions",
  "tock-registers",
@@ -3032,7 +2971,6 @@ dependencies = [
  "ygg_driver_ahci",
  "ygg_driver_fat32",
  "ygg_driver_input",
- "ygg_driver_intel_hda",
  "ygg_driver_net_core",
  "ygg_driver_net_igbe",
  "ygg_driver_net_loopback",
@@ -3040,7 +2978,6 @@ dependencies = [
  "ygg_driver_net_stmmac",
  "ygg_driver_nvme",
  "ygg_driver_pci",
- "ygg_driver_sound_core",
  "ygg_driver_usb",
  "ygg_driver_usb_xhci",
  "ygg_driver_virtio_blk",

Cargo.toml

@@ -11,7 +11,6 @@ exclude = [
 ]
 members = [
     "xtask",
-    "kernel/tools/gentables",
     "kernel",
     "lib/abi",
     "lib/libyalloc",
@@ -67,7 +66,6 @@ libk-util.path = "kernel/libk/libk-util"
 libk-mm.path = "kernel/libk/libk-mm"
 libk-mm-interface.path = "kernel/libk/libk-mm/interface"
 libk-device.path = "kernel/libk/libk-device"
-memtables.path = "kernel/lib/memtables"
 vmalloc.path = "kernel/lib/vmalloc"
 device-api-macros.path = "kernel/lib/device-api/macros"
 device-tree.path = "kernel/lib/device-tree"

boot/yboot/src/elf.rs

@@ -1,7 +1,8 @@
 use core::mem::size_of;
 
 use bytemuck::Zeroable;
-use log::{debug, error, info};
+use log::{error, info};
+use types::{Rela, SHT_RELA};
 // TODO use 'elf' crate
 use uefi::{
     prelude::BootServices,
@@ -23,14 +24,18 @@ mod types {
     pub type Half = u16;
     pub type Word = u32;
     pub type XWord = u64;
+    pub type SXWord = i64;
 
     pub const PT_LOAD: Word = 1;
 
     pub const SHT_PROGBITS: Word = 1;
+    pub const SHT_RELA: Word = 4;
 
     pub const SHF_WRITE: XWord = 1 << 0;
     pub const SHF_ALLOC: XWord = 1 << 1;
 
+    pub const R_X86_64_RELATIVE: u32 = 8;
+
     #[derive(Clone, Copy, Zeroable, Pod)]
     #[repr(C)]
     pub struct Ehdr {
@@ -77,6 +82,20 @@ mod types {
         pub memsz: XWord,
         pub align: XWord,
     }
+
+    #[derive(Clone, Copy, Zeroable, Pod)]
+    #[repr(C)]
+    pub struct Rela {
+        pub offset: Addr,
+        pub info: XWord,
+        pub addend: SXWord,
+    }
+
+    impl Rela {
+        pub fn r_type(&self) -> u32 {
+            self.info as u32
+        }
+    }
 }
 
 // Maximum address this loader can map in the target kernel
@@ -92,6 +111,8 @@ pub struct LoadedObject {
     pub image_start: u64,
     pub image_end: u64,
 
+    pub load_address: u64,
+
     pub entry: u64,
 
     pub protocol_struct_paddr: u64,
@@ -105,6 +126,12 @@ struct LocatedProtocol {
     size: usize,
 }
 
+struct RelaSection {
+    offset: u64,
+    entry_count: usize,
+    entry_size: usize,
+}
+
 trait ReadExact {
     fn read_exact(&mut self, buf: &mut [u8]) -> Result<(), Error>;
 }
@@ -119,6 +146,23 @@ impl ReadExact for RegularFile {
     }
 }
 
+impl RelaSection {
+    pub fn from_shdr(shdr: &Shdr) -> Option<Self> {
+        if shdr.type_ != SHT_RELA {
+            return None;
+        }
+
+        let entry_size = shdr.entsize as usize;
+        let entry_count = shdr.size as usize / entry_size;
+
+        Some(Self {
+            offset: shdr.offset,
+            entry_size,
+            entry_count,
+        })
+    }
+}
+
 impl Object {
     pub fn open<D: File>(root: &mut D, path: &CStr16) -> Result<Self, Error> {
         let file = root.open(path, FileMode::Read, FileAttribute::empty())?;
@@ -140,11 +184,11 @@ impl Object {
             return Err(Error::new(Status::LOAD_ERROR, ()));
         }
 
-        // Check that the entry point is set
-        if ehdr.entry == 0 {
-            error!("Image does not have a valid entry point");
-            return Err(Error::new(Status::LOAD_ERROR, ()));
-        }
+        // // Check that the entry point is set
+        // if ehdr.entry == 0 {
+        //     error!("Image does not have a valid entry point");
+        //     return Err(Error::new(Status::LOAD_ERROR, ()));
+        // }
 
         Ok(Self { file, ehdr })
     }
@@ -170,11 +214,6 @@ impl Object {
         self.file
             .read_exact(bytemuck::bytes_of_mut(&mut proto_data))?;
 
-        info!(
-            "Kernel is virtually mapped at {:#x}",
-            proto_data.kernel_virt_offset
-        );
-
         // 2. Find the kernel's range and check that the loaded physical addresses are actually
         //    usable from UEFI
         let mut image_start = u64::MAX;
@@ -209,16 +248,22 @@ impl Object {
         assert_eq!(image_start & 0xFFF, 0);
         assert_eq!(image_end & 0xFFF, 0);
 
-        info!("Image start: {:#x}, end: {:#x}", image_start, image_end);
-        // Reserve the kernel memory
-        let reserved_addr = bs
+        // Allocate memory to load the kernel into
+        let kernel_load_address = bs
             .allocate_pages(
-                AllocateType::Address(image_start),
+                AllocateType::MaxAddress(0xFFFFFFFF),
                 MemoryType::LOADER_DATA,
                 (image_end - image_start) as usize / 0x1000,
             )
-            .expect("Could not allocate memory for kernel image");
-        assert_eq!(reserved_addr, image_start);
+            .expect("Could not allocate memory for the kernel");
+
+        // Print info
+        info!("Image start: {:#x}, end: {:#x}", image_start, image_end);
+        info!(
+            "Kernel virtual offset: {:#x}",
+            proto_data.kernel_virt_offset
+        );
+        info!("Kernel load address: {kernel_load_address:#x}");
 
         // 3. Load the segments
         for i in 0..self.ehdr.phnum {
@@ -228,53 +273,80 @@ impl Object {
                 continue;
             }
 
+            let segment_load_base = phdr.paddr + kernel_load_address;
+
             info!(
-                "Load segment {}: {:#x?}",
-                i,
-                phdr.paddr..phdr.paddr + phdr.memsz
+                "[{i}] Load {:#x?}",
+                segment_load_base..segment_load_base + phdr.memsz
             );
 
             if phdr.filesz > 0 {
-                // The section has load data
-                let dst = unsafe {
-                    core::slice::from_raw_parts_mut(phdr.paddr as *mut u8, phdr.filesz as usize)
+                let dst_slice = unsafe {
+                    core::slice::from_raw_parts_mut(
+                        segment_load_base as *mut u8,
+                        phdr.filesz as usize,
+                    )
                 };
 
-                debug!(
-                    "Load {:#x?} from ELF offset {:#x}",
-                    phdr.paddr..phdr.paddr + phdr.filesz,
-                    phdr.offset
-                );
                 self.file.set_position(phdr.offset)?;
-                self.file.read_exact(dst)?;
+                self.file.read_exact(dst_slice)?;
             }
 
             if phdr.memsz > phdr.filesz {
-                let dst = unsafe {
+                let dst_slice = unsafe {
                     core::slice::from_raw_parts_mut(
-                        (phdr.paddr + phdr.filesz) as *mut u8,
+                        (segment_load_base + phdr.filesz) as *mut u8,
                         (phdr.memsz - phdr.filesz) as usize,
                     )
                 };
 
-                debug!(
-                    "Zero data {:#x?}",
-                    phdr.paddr + phdr.filesz..phdr.paddr + phdr.memsz
-                );
+                dst_slice.fill(0);
+            }
+        }
 
-                dst.fill(0);
+        // 4. Perform kernel relocation
+        let mut rela_section = None;
+        for i in 0..self.ehdr.shnum as usize {
+            let shdr = self.read_shdr(i)?;
+            if let Some(rela) = RelaSection::from_shdr(&shdr) {
+                rela_section = Some(rela);
+                break;
+            }
+        }
+
+        if let Some(rela_section) = rela_section {
+            info!("Relocating kernel: {image_start:#x} -> {kernel_load_address:#x}");
+            info!("({} relocations)", rela_section.entry_count);
+
+            let b = (kernel_load_address + proto_data.kernel_virt_offset) as i64;
+
+            for i in 0..rela_section.entry_count {
+                let mut rela = Rela::zeroed();
+                self.file
+                    .set_position(rela_section.offset + (i * rela_section.entry_size) as u64)?;
+                self.file.read_exact(bytemuck::bytes_of_mut(&mut rela))?;
+
+                match rela.r_type() {
+                    types::R_X86_64_RELATIVE => {
+                        let qword = (rela.offset + kernel_load_address) as *mut i64;
+                        let value = rela.addend + b;
+                        unsafe { qword.write_volatile(value) };
+                    }
+                    other => todo!("Unsupported relocation type: {other}"),
+                }
             }
         }
 
         // Now that the image is in memory, protocol structure can be written in the further steps
-        let protocol_struct_paddr = (loc_proto.address as u64) - proto_data.kernel_virt_offset;
+        let protocol_struct_paddr = loc_proto.address as u64 + kernel_load_address; // (loc_proto.address as u64) - proto_data.kernel_virt_offset;
         let protocol_version = proto_data.header.version;
 
-        let entry = self.ehdr.entry;
+        let entry = self.ehdr.entry + kernel_load_address;
 
         Ok(LoadedObject {
             image_start,
             image_end,
+            load_address: kernel_load_address,
             entry,
             protocol_struct_paddr,
             protocol_version,

boot/yboot/src/main.rs

@@ -16,7 +16,7 @@ use log::{debug, error, info};
 use uefi::{
     prelude::*,
     proto::{
-        console::gop::{GraphicsOutput, PixelFormat},
+        console::gop::{self, GraphicsOutput, PixelFormat},
         device_path::DevicePath,
         loaded_image::LoadedImage,
         media::{file::Directory, fs::SimpleFileSystem},
@@ -32,23 +32,31 @@ use yboot_proto::{
     LoadProtocolV1, LOADER_MAGIC,
 };
 
-use crate::{mem::MemoryDescriptorExt, protocol_ext::GraphicsOutputExt};
+use crate::mem::MemoryDescriptorExt;
+
+fn mode_score(mode: &gop::Mode) -> usize {
+    let (w, h) = mode.info().resolution();
+    let mut size_score = w * h;
+    if w > 1920 || h > 1080 {
+        // Don't pick too large sizes
+        size_score = 0;
+    }
+    size_score
+}
 
 fn setup_framebuffer(bs: &BootServices, fb: &mut FramebufferOption) -> Result<(), Error> {
     let gop_handle = bs.get_handle_for_protocol::<GraphicsOutput>()?;
     let mut gop = bs.open_protocol_exclusive::<GraphicsOutput>(gop_handle)?;
 
-    // Find the requested mode
-    let mode = gop.match_mode(fb.req_width, fb.req_height).ok_or_else(|| {
-        error!(
-            "Requested mode is not supported: {}x{}",
-            fb.req_width, fb.req_height
-        );
+    let mode = gop.modes().max_by_key(mode_score).ok_or_else(|| {
+        error!("No mode found");
         Error::new(Status::INVALID_PARAMETER, ())
     })?;
 
     gop.set_mode(&mode)?;
 
+    let (res_width, res_height) = mode.info().resolution();
+
     let mut result = gop.frame_buffer();
 
     let format = match mode.info().pixel_format() {
@@ -57,8 +65,8 @@ fn setup_framebuffer(bs: &BootServices, fb: &mut FramebufferOption) -> Result<()
         _ => 0,
     };
 
-    fb.res_width = fb.req_width;
-    fb.res_height = fb.req_height;
+    fb.res_width = res_width as _;
+    fb.res_height = res_height as _;
     fb.res_address = result.as_mut_ptr() as _;
     fb.res_stride = mode.info().stride() as u64 * 4;
     fb.res_size = result.size() as _;
@@ -85,7 +93,7 @@ fn locate_rsdp(st: &SystemTable<Boot>) -> Option<u64> {
 fn boot_partition(
     image: Handle,
     bs: &BootServices,
-) -> Result<ScopedProtocol<SimpleFileSystem>, Error> {
+) -> Result<ScopedProtocol<'_, SimpleFileSystem>, Error> {
     let loaded_image = bs.open_protocol_exclusive::<LoadedImage>(image)?;
     let device_handle = loaded_image.device();
 
@@ -105,7 +113,7 @@ fn load_kernel<'a>(
     config: &Config,
     root: &mut Directory,
     st: &SystemTable<Boot>,
-) -> Result<(u64, u64, &'a mut LoadProtocolV1), Error> {
+) -> Result<(u64, u64, u64, &'a mut LoadProtocolV1), Error> {
     let bs = st.boot_services();
 
     let mut kernel_obj = Object::open(root, cstr16!("kernel.elf"))?;
@@ -175,13 +183,14 @@ fn load_kernel<'a>(
 
     let entry = loaded_obj.entry + proto_data.kernel_virt_offset;
 
-    Ok((entry, mmap_memory, proto_data))
+    Ok((entry, loaded_obj.load_address, mmap_memory, proto_data))
 }
 
 unsafe fn map_and_enter_kernel(
     st: SystemTable<Boot>,
     proto_data: &mut LoadProtocolV1,
     mmap_memory: u64,
+    load_base: u64,
     entry: u64,
 ) -> ! {
     let (_, mmap) = st.exit_boot_services();
@@ -208,7 +217,7 @@ unsafe fn map_and_enter_kernel(
     let cr3 = mem::map_image();
     asm!("cli; wbinvd; mov {0}, %cr3", in(reg) cr3, options(att_syntax));
 
-    asm!("jmp *{0}", in(reg) entry, in("eax") LOADER_MAGIC, options(noreturn, att_syntax));
+    asm!("jmp *{0}", in(reg) entry, in("eax") LOADER_MAGIC, in("ecx") load_base, options(noreturn, att_syntax));
 }
 
 #[entry]
@@ -235,15 +244,16 @@ fn efi_main(image_handle: Handle, mut system_table: SystemTable<Boot>) -> Status
         }
     };
 
-    let (entry, mmap_memory, proto_data) = match load_kernel(&config, &mut root, &system_table) {
-        Ok(e) => e,
-        Err(error) => {
-            error!("Failed to load the kernel/initrd: {error:?}");
-            return Status::LOAD_ERROR;
-        }
-    };
+    let (entry, load_base, mmap_memory, proto_data) =
+        match load_kernel(&config, &mut root, &system_table) {
+            Ok(e) => e,
+            Err(error) => {
+                error!("Failed to load the kernel/initrd: {error:?}");
+                return Status::LOAD_ERROR;
+            }
+        };
 
     unsafe {
-        map_and_enter_kernel(system_table, proto_data, mmap_memory, entry);
+        map_and_enter_kernel(system_table, proto_data, mmap_memory, load_base, entry);
     }
 }

boot/yboot/src/mem.rs

@@ -66,7 +66,7 @@ impl MemoryDescriptorExt for MemoryDescriptor {
     }
 }
 
-pub fn memory_map(bs: &BootServices) -> Result<MemoryMap, Error> {
+pub fn memory_map(bs: &BootServices) -> Result<MemoryMap<'_>, Error> {
     bs.memory_map(unsafe { &mut MMAP_BUFFER.data })
 }
 

etc/aarch64-unknown-none.json

@@ -13,6 +13,7 @@
   "panic-strategy": "abort",
   "dynamic-linking": true,
   "relocation-model": "pic",
+  "position-independent-executables": true,
   "eh-frame-header": false,
 
   "linker": "rust-lld",

etc/gdb/aarch64-virt.gdb

@@ -0,0 +1,4 @@
+# Lower-half
+symbol-file -o 0x40080000 target/aarch64-unknown-none/release/yggdrasil-kernel
+symbol-file -o 0xFFFFFF8040080000 target/aarch64-unknown-none/release/yggdrasil-kernel
+target remote :1234

etc/ld/arm/aarch64-unknown-none.ld

@@ -0,0 +1,67 @@
+ENTRY(__aarch64_entry);
+
+/* KERNEL_PHYS_BASE = 0x40080000; */
+KERNEL_VIRT_OFFSET = 0xFFFFFF8000000000;
+
+SECTIONS {
+    /* . = KERNEL_PHYS_BASE; */
+    . = 0x0;
+
+    PROVIDE(__kernel_start = .);
+
+    /* .text.entry : {
+        *(.text.entry)
+    }
+
+    . = ALIGN(16);
+    . = . + KERNEL_VIRT_OFFSET; */
+
+    .text : {
+        KEEP(*(.text.entry));
+        KEEP(*(.text.vectors));
+        *(.text*)
+    }
+
+    . = ALIGN(4K);
+    .rodata : {
+        *(.rodata*)
+        *(.eh_frame*)
+
+        . = ALIGN(16);
+        PROVIDE(__init_array_start = .);
+        KEEP(*(.init_array*))
+        PROVIDE(__init_array_end = .);
+    }
+
+    . = ALIGN(4K);
+    PROVIDE(__rela_start = .);
+    .rela : {
+        *(.rela*)
+    }
+    PROVIDE(__rela_end = .);
+
+    /* . = ALIGN(4K);            */
+    /* .data.tables : {          */
+    /*     KEEP(*(.data.tables)) */
+    /* }                         */
+
+    . = ALIGN(4K);
+    .data : {
+        *(.data*)
+        *(.got*)
+    }
+
+    . = ALIGN(4K);
+    PROVIDE(__bss_start = .);
+    /* PROVIDE(__bss_start_phys = . - KERNEL_VIRT_OFFSET); */
+    .bss : {
+        *(COMMON)
+        *(.bss*)
+    }
+    . = ALIGN(4K);
+    PROVIDE(__bss_end = .);
+    /* PROVIDE(__bss_end_phys = . - KERNEL_VIRT_OFFSET); */
+    /* PROVIDE(__bss_size = __bss_end_phys - __bss_start_phys); */
+
+    PROVIDE(__kernel_end = .);
+};

etc/ld/arm/aarch64-unknown-qemu.ld

@@ -1,55 +0,0 @@
-ENTRY(__aarch64_entry);
-
-KERNEL_PHYS_BASE = 0x40080000;
-KERNEL_VIRT_OFFSET = 0xFFFFFF8000000000;
-
-SECTIONS {
-    . = KERNEL_PHYS_BASE;
-    PROVIDE(__kernel_start = . + KERNEL_VIRT_OFFSET);
-
-    .text.entry : {
-        *(.text.entry)
-    }
-
-    . = ALIGN(16);
-    . = . + KERNEL_VIRT_OFFSET;
-
-    .text : AT(. - KERNEL_VIRT_OFFSET) {
-        KEEP(*(.text.vectors));
-        *(.text*)
-    }
-
-    . = ALIGN(4K);
-    .rodata : AT(. - KERNEL_VIRT_OFFSET) {
-        *(.rodata*)
-        *(.eh_frame*)
-
-        . = ALIGN(16);
-        PROVIDE(__init_array_start = .);
-        KEEP(*(.init_array*))
-        PROVIDE(__init_array_end = .);
-    }
-
-    . = ALIGN(4K);
-    .data.tables : AT (. - KERNEL_VIRT_OFFSET) {
-        KEEP(*(.data.tables))
-    }
-
-    . = ALIGN(4K);
-    .data : AT(. - KERNEL_VIRT_OFFSET) {
-        *(.data*)
-        *(.got*)
-    }
-
-    . = ALIGN(4K);
-    PROVIDE(__bss_start_phys = . - KERNEL_VIRT_OFFSET);
-    .bss : AT(. - KERNEL_VIRT_OFFSET) {
-        *(COMMON)
-        *(.bss*)
-    }
-    . = ALIGN(4K);
-    PROVIDE(__bss_end_phys = . - KERNEL_VIRT_OFFSET);
-    PROVIDE(__bss_size = __bss_end_phys - __bss_start_phys);
-
-    PROVIDE(__kernel_end = .);
-};

etc/ld/arm/aarch64-unknown-raspi4b.ld

@@ -1,55 +0,0 @@
-ENTRY(__aarch64_entry);
-
-KERNEL_PHYS_BASE = 0x80000;
-KERNEL_VIRT_OFFSET = 0xFFFFFF8000000000;
-
-SECTIONS {
-    . = KERNEL_PHYS_BASE;
-    PROVIDE(__kernel_start = . + KERNEL_VIRT_OFFSET);
-
-    .text.entry : {
-        *(.text.entry)
-    }
-
-    . = ALIGN(16);
-    . = . + KERNEL_VIRT_OFFSET;
-
-    .text : AT(. - KERNEL_VIRT_OFFSET) {
-        KEEP(*(.text.vectors));
-        *(.text*)
-    }
-
-    . = ALIGN(4K);
-    .rodata : AT(. - KERNEL_VIRT_OFFSET) {
-        *(.rodata*)
-        *(.eh_frame*)
-
-        . = ALIGN(16);
-        PROVIDE(__init_array_start = .);
-        KEEP(*(.init_array*))
-        PROVIDE(__init_array_end = .);
-    }
-
-    . = ALIGN(4K);
-    .data.tables : AT (. - KERNEL_VIRT_OFFSET) {
-        KEEP(*(.data.tables))
-    }
-
-    . = ALIGN(4K);
-    .data : AT(. - KERNEL_VIRT_OFFSET) {
-        *(.data*)
-        *(.got*)
-    }
-
-    . = ALIGN(4K);
-    PROVIDE(__bss_start_phys = . - KERNEL_VIRT_OFFSET);
-    .bss : AT(. - KERNEL_VIRT_OFFSET) {
-        *(COMMON)
-        *(.bss*)
-    }
-    . = ALIGN(4K);
-    PROVIDE(__bss_end_phys = . - KERNEL_VIRT_OFFSET);
-    PROVIDE(__bss_size = __bss_end_phys - __bss_start_phys);
-
-    PROVIDE(__kernel_end = .);
-};

etc/ld/riscv/riscv64-unknown-none.ld

@@ -1,37 +1,32 @@
 ENTRY(__rv64_entry);
 
-KERNEL_PHYS_BASE = 0x40200000;
 KERNEL_VIRT_OFFSET = 0xFFFFFFF000000000;
 
 SECTIONS {
-    . = KERNEL_PHYS_BASE;
-    PROVIDE(__kernel_start = . + KERNEL_VIRT_OFFSET);
+    . = 0;
+    PROVIDE(__kernel_start = .);
 
-    .text.entry : {
-        *(.text.entry)
-    }
-
-    . = ALIGN(16);
-    . = . + KERNEL_VIRT_OFFSET;
-
-    .text : AT(. - KERNEL_VIRT_OFFSET) {
+    .text : {
+        KEEP(*(.text.entry));
         KEEP(*(.text.vectors));
         *(.text*)
     }
 
     . = ALIGN(4K);
-    .rodata : AT(. - KERNEL_VIRT_OFFSET) {
+    .rodata : {
         *(.rodata*)
         *(.eh_frame*)
     }
 
     . = ALIGN(4K);
-    .data.tables : AT (. - KERNEL_VIRT_OFFSET) {
-        KEEP(*(.data.tables))
+    PROVIDE(__rela_start = .);
+    .rela : {
+        *(.rela*)
     }
+    PROVIDE(__rela_end = .);
 
     . = ALIGN(4K);
-    .data : AT(. - KERNEL_VIRT_OFFSET) {
+    .data : {
         *(.data*)
         . = ALIGN(8);
         /* PROVIDE(__global_pointer = . + 0x800 - KERNEL_VIRT_OFFSET); */
@@ -45,14 +40,13 @@ SECTIONS {
     }
 
     . = ALIGN(4K);
-    PROVIDE(__bss_start_phys = . - KERNEL_VIRT_OFFSET);
-    .bss : AT(. - KERNEL_VIRT_OFFSET) {
+    PROVIDE(__bss_start = .);
+    .bss : {
         *(COMMON)
         *(.bss*)
     }
     . = ALIGN(4K);
-    PROVIDE(__bss_end_phys = . - KERNEL_VIRT_OFFSET);
-    PROVIDE(__bss_size = __bss_end_phys - __bss_start_phys);
+    PROVIDE(__bss_end = .);
 
     PROVIDE(__kernel_end = .);
 };

etc/ld/riscv/riscv64-unknown-qemu.ld

@@ -1,58 +0,0 @@
-ENTRY(__rv64_entry);
-
-KERNEL_PHYS_BASE = 0x80200000;
-KERNEL_VIRT_OFFSET = 0xFFFFFFF000000000;
-
-SECTIONS {
-    . = KERNEL_PHYS_BASE;
-    PROVIDE(__kernel_start = . + KERNEL_VIRT_OFFSET);
-
-    .text.entry : {
-        *(.text.entry)
-    }
-
-    . = ALIGN(16);
-    . = . + KERNEL_VIRT_OFFSET;
-
-    .text : AT(. - KERNEL_VIRT_OFFSET) {
-        KEEP(*(.text.vectors));
-        *(.text*)
-    }
-
-    . = ALIGN(4K);
-    .rodata : AT(. - KERNEL_VIRT_OFFSET) {
-        *(.rodata*)
-        *(.eh_frame*)
-    }
-
-    . = ALIGN(4K);
-    .data.tables : AT (. - KERNEL_VIRT_OFFSET) {
-        KEEP(*(.data.tables))
-    }
-
-    . = ALIGN(4K);
-    .data : AT(. - KERNEL_VIRT_OFFSET) {
-        *(.data*)
-        . = ALIGN(8);
-        /* PROVIDE(__global_pointer = . + 0x800 - KERNEL_VIRT_OFFSET); */
-
-        . = ALIGN(16);
-        PROVIDE(__init_array_start = .);
-        KEEP(*(.init_array*))
-        PROVIDE(__init_array_end = .);
-
-        *(.got*)
-    }
-
-    . = ALIGN(4K);
-    PROVIDE(__bss_start_phys = . - KERNEL_VIRT_OFFSET);
-    .bss : AT(. - KERNEL_VIRT_OFFSET) {
-        *(COMMON)
-        *(.bss*)
-    }
-    . = ALIGN(4K);
-    PROVIDE(__bss_end_phys = . - KERNEL_VIRT_OFFSET);
-    PROVIDE(__bss_size = __bss_end_phys - __bss_start_phys);
-
-    PROVIDE(__kernel_end = .);
-};

etc/ld/x86/x86_64-unknown-none.ld

@@ -1,52 +1,52 @@
 ENTRY(__x86_64_entry);
 
-KERNEL_PHYS_BASE = 0x200000;
+/* KERNEL_PHYS_BASE = 0x200000; */
 KERNEL_VIRT_OFFSET = 0xFFFFFF8000000000;
 
 SECTIONS {
-    . = KERNEL_PHYS_BASE;
-    PROVIDE(__kernel_start = . + KERNEL_VIRT_OFFSET);
+    /* . = KERNEL_PHYS_BASE; */
+    /* PROVIDE(__kernel_start = . + KERNEL_VIRT_OFFSET); */
 
-    .text.entry : {
+    . = 0x0;
+    PROVIDE(__kernel_start = .);
+
+    .text : {
         *(.text.entry)
-    }
-
-    . = ALIGN(16);
-    . = . + KERNEL_VIRT_OFFSET;
-
-    .text : AT(. - KERNEL_VIRT_OFFSET) {
         *(.text*)
     }
 
-    .export.text : AT(. - KERNEL_VIRT_OFFSET) {
+    .export.text : {
         KEEP(*(.export.text*))
     }
 
     . = ALIGN(4K);
-    .rodata : AT(. - KERNEL_VIRT_OFFSET) {
+    .rodata : {
         *(.eh_frame*)
         *(.rodata*)
     }
 
     . = ALIGN(4K);
-    .data.tables : AT (. - KERNEL_VIRT_OFFSET) {
-        KEEP(*(.data.tables))
+    PROVIDE(__rela_start = .);
+    .rela : {
+        *(.rela*)
     }
+    PROVIDE(__rela_end = .);
 
-    .data : AT(. - KERNEL_VIRT_OFFSET) {
+    . = ALIGN(4K);
+    .data : {
         KEEP(*(.data.yboot))
         *(.data*)
         *(.got*)
     }
 
     . = ALIGN(4K);
-    PROVIDE(__bss_start_phys = . - KERNEL_VIRT_OFFSET);
-    .bss : AT(. - KERNEL_VIRT_OFFSET) {
+    PROVIDE(__bss_start = .);
+    .bss : {
         *(COMMON)
         *(.bss*)
     }
     . = ALIGN(4K);
-    PROVIDE(__bss_end_phys = . - KERNEL_VIRT_OFFSET);
+    PROVIDE(__bss_end = .);
 
     PROVIDE(__kernel_end = .);
 };

etc/riscv64-unknown-none.json

@@ -14,6 +14,7 @@
   "panic-strategy": "abort",
   "dynamic-linking": true,
   "relocation-model": "pic",
+  "position-independent-executables": true,
   "code-model": "medium",
   "eh-frame-header": false,
 

etc/x86_64-unknown-none.json

@@ -15,6 +15,7 @@
   "panic-strategy": "abort",
   "dynamic-linking": true,
   "relocation-model": "pic",
+  "position-independent-executables": true,
 
   "has-thread-local": false,
 

kernel/Cargo.toml

@@ -19,7 +19,6 @@ chrono.workspace = true
 device-api = { workspace = true, features = ["derive"] }
 device-api-macros.workspace = true
 
-memtables.workspace = true
 vmalloc.workspace = true
 kernel-arch.workspace = true
 
@@ -36,7 +35,6 @@ ygg_driver_ahci = { path = "driver/block/ahci" }
 ygg_driver_input = { path = "driver/input" }
 ygg_driver_usb_xhci.path = "driver/usb/xhci"
 ygg_driver_net_rtl81xx.path = "driver/net/rtl81xx"
-ygg_driver_sound_core.path = "driver/sound/core"
 
 memfs = { path = "driver/fs/memfs" }
 ext2 = { path = "driver/fs/ext2" }
@@ -71,7 +69,6 @@ kernel-arch-x86.workspace = true
 
 ygg_driver_acpi.path = "driver/acpi"
 ygg_driver_net_igbe.path = "driver/net/igbe"
-ygg_driver_intel_hda.path = "driver/sound/intel-hda"
 
 acpi.workspace = true
 
@@ -101,12 +98,5 @@ ygg_driver_net_stmmac.path = "driver/net/stmmac"
 default = ["fb_console"]
 fb_console = []
 
-# TODO replace this with a better configuration mechanism
-aarch64_board_virt = ["kernel-arch-aarch64/aarch64_board_virt"]
-aarch64_board_raspi4b = ["kernel-arch-aarch64/aarch64_board_raspi4b"]
-
-riscv64_board_virt = ["kernel-arch-riscv64/riscv64_board_virt"]
-riscv64_board_jh7110 = ["kernel-arch-riscv64/riscv64_board_jh7110"]
-
 [lints]
 workspace = true

kernel/arch/aarch64/Cargo.toml

@@ -7,13 +7,13 @@ edition = "2021"
 yggdrasil-abi.workspace = true
 kernel-arch-interface.workspace = true
 libk-mm-interface.workspace = true
-memtables.workspace = true
 device-api = { workspace = true, features = ["derive"] }
 
 bitflags.workspace = true
 static_assertions.workspace = true
 aarch64-cpu.workspace = true
 tock-registers.workspace = true
+log.workspace = true
 
 [build-dependencies]
 cc = "1.0"

kernel/arch/aarch64/src/lib.rs

@@ -1,5 +1,5 @@
 #![no_std]
-#![feature(naked_functions, decl_macro)]
+#![feature(decl_macro)]
 #![allow(clippy::new_without_default)]
 
 extern crate alloc;
@@ -40,11 +40,9 @@ impl CpuData for PerCpuData {}
 static IPI_QUEUES: OneTimeInit<Vec<IpiQueue<ArchitectureImpl>>> = OneTimeInit::new();
 pub static CPU_COUNT: AtomicUsize = AtomicUsize::new(1);
 
-#[naked]
+#[unsafe(naked)]
 extern "C" fn idle_task(_: usize) -> ! {
-    unsafe {
-        core::arch::naked_asm!("1: nop; b 1b");
-    }
+    core::arch::naked_asm!("1: nop; b 1b");
 }
 
 impl ArchitectureImpl {

kernel/arch/aarch64/src/mem/device.rs

@@ -0,0 +1,95 @@
+use core::ops::Range;
+
+use kernel_arch_interface::mem::DeviceMemoryAttributes;
+use libk_mm_interface::{
+    address::PhysicalAddress,
+    table::{DevicePageManager, DevicePageManagerLevel, EntryLevel},
+};
+
+use crate::mem::table::PageEntry;
+
+use super::{
+    table::{PageTable, L2, L3},
+    tlb_flush_range_va, DEVICE_MAPPING_OFFSET, DEVICE_MEMORY_L3_COUNT,
+};
+
+#[repr(transparent)]
+pub struct L2DeviceMemory(pub PageTable<L2>);
+#[repr(transparent)]
+pub struct L3DeviceMemory(pub [PageTable<L3>; DEVICE_MEMORY_L3_COUNT]);
+
+pub(super) static mut DEVICE_MEMORY: DevicePageManager<L3DeviceMemory, L2DeviceMemory> =
+    DevicePageManager::new(
+        L3DeviceMemory([PageTable::zeroed(); DEVICE_MEMORY_L3_COUNT]),
+        L2DeviceMemory(PageTable::zeroed()),
+    );
+
+impl DevicePageManagerLevel for L2DeviceMemory {
+    type Level = L2;
+    const VIRTUAL_BASE: usize = DEVICE_MAPPING_OFFSET;
+    const INDEX_RANGE: Range<usize> = DEVICE_MEMORY_L3_COUNT..512;
+
+    fn map_page(
+        &mut self,
+        index: usize,
+        physical: PhysicalAddress,
+        attrs: &DeviceMemoryAttributes,
+    ) {
+        let _ = index;
+        let _ = physical;
+        let _ = attrs;
+        todo!()
+    }
+
+    fn unmap_page(&mut self, index: usize) {
+        let _ = index;
+        todo!()
+    }
+
+    fn is_mapped(&self, index: usize) -> bool {
+        let _ = index;
+        todo!()
+    }
+
+    fn flush_range(range: Range<usize>) {
+        let start = range.start * L2::SIZE + Self::VIRTUAL_BASE;
+        let size = (range.end - range.start) * L2::SIZE;
+        tlb_flush_range_va(start, size);
+    }
+}
+
+impl DevicePageManagerLevel for L3DeviceMemory {
+    type Level = L3;
+    const VIRTUAL_BASE: usize = DEVICE_MAPPING_OFFSET;
+    const INDEX_RANGE: Range<usize> = 0..512 * DEVICE_MEMORY_L3_COUNT;
+
+    fn map_page(
+        &mut self,
+        index: usize,
+        physical: PhysicalAddress,
+        attrs: &DeviceMemoryAttributes,
+    ) {
+        let _ = attrs;
+        let l2i = index / 512;
+        let l3i = index % 512;
+        self.0[l2i][l3i] = PageEntry::device_page(physical);
+    }
+
+    // TODO
+    fn unmap_page(&mut self, index: usize) {
+        let _ = index;
+        todo!()
+    }
+
+    fn is_mapped(&self, index: usize) -> bool {
+        let l2i = index / 512;
+        let l3i = index % 512;
+        self.0[l2i][l3i].is_present()
+    }
+
+    fn flush_range(range: Range<usize>) {
+        let start = range.start * L3::SIZE + Self::VIRTUAL_BASE;
+        let size = (range.end - range.start) * L3::SIZE;
+        tlb_flush_range_va(start, size);
+    }
+}

kernel/arch/aarch64/src/mem/intrinsics.rs

@@ -0,0 +1,93 @@
+use core::sync::atomic::{self, Ordering};
+
+use aarch64_cpu::{asm::barrier, registers::PAR_EL1};
+use libk_mm_interface::table::{EntryLevel, EntryLevelExt};
+use tock_registers::interfaces::Readable;
+
+use super::table::L3;
+
+#[inline]
+pub fn tlb_flush_asid(asid: u8) {
+    barrier::dsb(barrier::ISHST);
+    let value = (asid as u64) << 48;
+    unsafe {
+        core::arch::asm!("tlbi aside1, {value}", value = in(reg) value);
+    }
+    barrier::dsb(barrier::ISH);
+    barrier::isb(barrier::SY);
+}
+
+#[inline]
+pub fn tlb_flush_all() {
+    barrier::dsb(barrier::ISHST);
+    unsafe {
+        core::arch::asm!("tlbi vmalle1is");
+    }
+    barrier::dsb(barrier::ISH);
+    barrier::isb(barrier::SY);
+}
+
+#[inline]
+pub fn tlb_flush_vaae1(page: usize) {
+    barrier::dsb(barrier::ISHST);
+    let argument = page >> 12;
+    unsafe {
+        core::arch::asm!("tlbi vaae1, {argument}", argument = in(reg) argument);
+    }
+    barrier::dsb(barrier::ISH);
+    barrier::isb(barrier::SY);
+}
+
+pub fn tlb_flush_range_va(base: usize, size: usize) {
+    let end = (base + size).page_align_up::<L3>();
+    let base = base.page_align_down::<L3>();
+    let count = (end - base).page_count::<L3>();
+    for i in 0..count {
+        tlb_flush_vaae1(base + i * L3::SIZE);
+    }
+}
+
+pub fn at_s1e0r(input: usize) -> Option<u64> {
+    barrier::dsb(barrier::ISHST);
+    unsafe {
+        core::arch::asm!("at s1e0r, {address}", address = in(reg) input);
+    }
+    barrier::dsb(barrier::ISH);
+    barrier::isb(barrier::SY);
+    if PAR_EL1.matches_all(PAR_EL1::F::TranslationSuccessfull) {
+        Some(PAR_EL1.read(PAR_EL1::PA))
+    } else {
+        None
+    }
+}
+
+pub fn at_s1e1r(input: usize) -> Option<u64> {
+    barrier::dsb(barrier::ISHST);
+    unsafe {
+        core::arch::asm!("at s1e1r, {address}", address = in(reg) input);
+    }
+    barrier::dsb(barrier::ISH);
+    barrier::isb(barrier::SY);
+    if PAR_EL1.matches_all(PAR_EL1::F::TranslationSuccessfull) {
+        Some(PAR_EL1.read(PAR_EL1::PA))
+    } else {
+        None
+    }
+}
+
+pub fn ic_iallu() {
+    atomic::compiler_fence(Ordering::SeqCst);
+    barrier::dsb(barrier::ISH);
+    barrier::isb(barrier::SY);
+    unsafe {
+        core::arch::asm!("ic iallu");
+    }
+    barrier::isb(barrier::SY);
+}
+
+pub fn dc_cvac(input: usize) {
+    barrier::dsb(barrier::ISHST);
+    unsafe {
+        core::arch::asm!("dc cvac, {address}", address = in(reg) input);
+    }
+}

kernel/arch/aarch64/src/mem/mod.rs

@@ -1,108 +1,43 @@
-use core::{
-    alloc::Layout,
-    ops::{Deref, DerefMut},
-    sync::atomic::{self, AtomicUsize, Ordering},
-};
-
 use aarch64_cpu::{
     asm::barrier,
-    registers::{MAIR_EL1, PAR_EL1, SCTLR_EL1, TTBR0_EL1, TTBR1_EL1},
+    registers::{MAIR_EL1, SCTLR_EL1, TCR_EL1, TTBR0_EL1, TTBR1_EL1},
 };
 use kernel_arch_interface::{
     mem::{DeviceMemoryAttributes, KernelTableManager, RawDeviceMemoryMapping},
-    split_spinlock, Architecture, KERNEL_VIRT_OFFSET,
+    sync::IrqSafeSpinlock,
+    KERNEL_VIRT_OFFSET,
 };
-use libk_mm_interface::{
-    address::PhysicalAddress,
-    table::{page_index, EntryLevel, EntryLevelExt},
-};
-use memtables::aarch64::{FixedTables, KERNEL_L3_COUNT};
-use static_assertions::const_assert_eq;
-use tock_registers::interfaces::{ReadWriteable, Readable, Writeable};
+use libk_mm_interface::{address::PhysicalAddress, table::EntryLevel};
+use table::{PageAttributes, PageEntry, PageTable, L1};
+use tock_registers::interfaces::{ReadWriteable, Writeable};
+
+mod intrinsics;
+
+pub mod device;
+pub mod process;
+pub mod table;
+
+pub use intrinsics::*;
 use yggdrasil_abi::error::Error;
 
 use crate::ArchitectureImpl;
 
-use self::table::{PageAttributes, PageEntry, PageTable, L1, L2, L3};
-
-pub mod process;
-pub mod table;
-
 #[derive(Debug)]
 pub struct KernelTableManagerImpl;
 
-// TODO eliminate this requirement by using precomputed indices
-const MAPPING_OFFSET: usize = KERNEL_VIRT_OFFSET;
-
-#[cfg(any(feature = "aarch64_board_virt", rust_analyzer))]
-const KERNEL_PHYS_BASE: usize = 0x40080000;
-#[cfg(any(feature = "aarch64_board_raspi4b", rust_analyzer))]
-const KERNEL_PHYS_BASE: usize = 0x80000;
-
-// Precomputed mappings
-const KERNEL_L1_INDEX: usize = page_index::<L1>(KERNEL_VIRT_OFFSET + KERNEL_PHYS_BASE);
-const KERNEL_START_L2_INDEX: usize = page_index::<L2>(KERNEL_VIRT_OFFSET + KERNEL_PHYS_BASE);
-const KERNEL_END_L2_INDEX: usize = KERNEL_START_L2_INDEX + KERNEL_L3_COUNT;
-
-// Must not be zero, should be at 4MiB
-const_assert_eq!(KERNEL_START_L2_INDEX, 0);
-// From static mapping
-#[cfg(any(feature = "aarch64_board_raspi4b", rust_analyzer))]
-const_assert_eq!(KERNEL_L1_INDEX, 0);
-#[cfg(any(feature = "aarch64_board_virt", rust_analyzer))]
-const_assert_eq!(KERNEL_L1_INDEX, 1);
-
-// Runtime mappings
-// 2MiB max
-const EARLY_MAPPING_L2I: usize = KERNEL_END_L2_INDEX + 1;
-// 1GiB max
-const DEVICE_MAPPING_L1I: usize = KERNEL_L1_INDEX + 2;
-const DEVICE_MAPPING_L3_COUNT: usize = 4;
-// 16GiB max
-const RAM_MAPPING_START_L1I: usize = KERNEL_L1_INDEX + 3;
-pub const RAM_MAPPING_L1_COUNT: usize = 16;
-
-// 2MiB for early mappings
-const EARLY_MAPPING_OFFSET: usize =
-    MAPPING_OFFSET | (KERNEL_L1_INDEX * L1::SIZE) | (EARLY_MAPPING_L2I * L2::SIZE);
-static mut EARLY_MAPPING_L3: PageTable<L3> = PageTable::zeroed();
-// 1GiB for device MMIO mapping
-const DEVICE_MAPPING_OFFSET: usize = MAPPING_OFFSET | (DEVICE_MAPPING_L1I * L1::SIZE);
-static mut DEVICE_MAPPING_L2: PageTable<L2> = PageTable::zeroed();
-static mut DEVICE_MAPPING_L3S: [PageTable<L3>; DEVICE_MAPPING_L3_COUNT] =
-    [PageTable::zeroed(); DEVICE_MAPPING_L3_COUNT];
-// 16GiB for RAM mapping
-pub const RAM_MAPPING_OFFSET: usize = MAPPING_OFFSET | (RAM_MAPPING_START_L1I * L1::SIZE);
-pub static MEMORY_LIMIT: AtomicUsize = AtomicUsize::new(0);
-
-split_spinlock! {
-    use crate::ArchitectureImpl;
-    use crate::mem::FixedTables;
-    use libk_mm_interface::KernelImageObject;
-
-    #[link_section = ".data.tables"]
-    static KERNEL_TABLES: KernelImageObject<FixedTables> =
-        unsafe { KernelImageObject::new(FixedTables::zeroed()) };
-}
-
 impl KernelTableManager for KernelTableManagerImpl {
-    fn virtualize(address: u64) -> usize {
-        let address = address as usize;
-        if address < MEMORY_LIMIT.load(Ordering::Acquire) {
-            address + RAM_MAPPING_OFFSET
-        } else {
-            panic!("Invalid physical address: {:#x}", address);
+    fn virtualize(phys: u64) -> usize {
+        if phys as usize >= IDENTITY_SIZE {
+            unreachable!("Invalid physical address to virtualize: {phys:#x}");
         }
+        phys as usize + KERNEL_VIRT_OFFSET
     }
 
-    fn physicalize(address: usize) -> u64 {
-        if address < RAM_MAPPING_OFFSET
-            || address - RAM_MAPPING_OFFSET >= MEMORY_LIMIT.load(Ordering::Acquire)
-        {
-            panic!("Not a virtualized physical address: {:#x}", address);
+    fn physicalize(virt: usize) -> u64 {
+        if virt < KERNEL_VIRT_OFFSET || virt - KERNEL_VIRT_OFFSET >= IDENTITY_SIZE {
+            unreachable!("Invalid virtualized address: {virt:#x}");
         }
-
-        (address - RAM_MAPPING_OFFSET) as _
+        (virt - KERNEL_VIRT_OFFSET) as _
     }
 
     unsafe fn map_device_pages(
@@ -110,390 +45,71 @@ impl KernelTableManager for KernelTableManagerImpl {
         count: usize,
         attrs: DeviceMemoryAttributes,
     ) -> Result<RawDeviceMemoryMapping<Self>, Error> {
-        map_device_memory(PhysicalAddress::from_u64(base), count, attrs)
-    }
-
-    unsafe fn unmap_device_pages(mapping: &RawDeviceMemoryMapping<Self>) {
-        unmap_device_memory(mapping)
-    }
-}
-
-/// Memory mapping which may be used for performing early kernel initialization
-pub struct EarlyMapping<'a, T: ?Sized> {
-    value: &'a mut T,
-    page_count: usize,
-}
-
-impl<'a, T: Sized> EarlyMapping<'a, T> {
-    /// # Safety
-    ///
-    /// `physical` address provided must be a valid non-NULL address actually containing `T`.
-    pub unsafe fn map_slice(
-        physical: PhysicalAddress,
-        len: usize,
-    ) -> Result<EarlyMapping<'a, [T]>, Error> {
-        let layout = Layout::array::<T>(len).unwrap();
-        let aligned = physical.page_align_down::<L3>();
-        let offset = physical.page_offset::<L3>();
-        let page_count = (offset + layout.size()).div_ceil(L3::SIZE);
-
-        let virt = map_early_pages(aligned, page_count)?;
-        let value = core::slice::from_raw_parts_mut((virt + offset) as *mut T, len);
-
-        Ok(EarlyMapping { value, page_count })
-    }
-}
-
-impl<T: ?Sized> Deref for EarlyMapping<'_, T> {
-    type Target = T;
-
-    fn deref(&self) -> &Self::Target {
-        self.value
-    }
-}
-
-impl<T: ?Sized> DerefMut for EarlyMapping<'_, T> {
-    fn deref_mut(&mut self) -> &mut Self::Target {
-        self.value
-    }
-}
-
-impl<T: ?Sized> Drop for EarlyMapping<'_, T> {
-    fn drop(&mut self) {
-        let address = (self.value as *mut T).addr() & !(L3::SIZE - 1);
-
-        for i in 0..self.page_count {
-            let page = address + i * L3::SIZE;
-
-            unsafe {
-                unmap_early_page(page);
-            }
+        let _guard = DEVICE_MEMORY_LOCK.lock();
+        #[allow(static_mut_refs)]
+        unsafe {
+            device::DEVICE_MEMORY.map_device_pages(PhysicalAddress::from_u64(base), count, attrs)
         }
     }
-}
 
-fn kernel_table_flags() -> PageAttributes {
-    PageAttributes::TABLE
-        | PageAttributes::ACCESS
-        | PageAttributes::SH_INNER
-        | PageAttributes::PAGE_ATTR_NORMAL
-        | PageAttributes::PRESENT
-}
-
-fn ram_block_flags() -> PageAttributes {
-    // TODO UXN, PXN
-    PageAttributes::BLOCK
-        | PageAttributes::ACCESS
-        | PageAttributes::SH_OUTER
-        | PageAttributes::PAGE_ATTR_NORMAL
-        | PageAttributes::PRESENT
-}
-
-// Early mappings
-unsafe fn map_early_pages(physical: PhysicalAddress, count: usize) -> Result<usize, Error> {
-    for l3i in 0..512 {
-        let mut taken = false;
-        for i in 0..count {
-            if EARLY_MAPPING_L3[i + l3i].is_present() {
-                taken = true;
-                break;
-            }
-        }
-
-        if taken {
-            continue;
-        }
-
-        for i in 0..count {
-            let page = physical.add(i * L3::SIZE);
-            // TODO NX, NC
-            EARLY_MAPPING_L3[i + l3i] = PageEntry::normal_page(page, PageAttributes::empty());
-            tlb_flush_vaae1(EARLY_MAPPING_OFFSET + (l3i + i) * L3::SIZE);
-        }
-
-        return Ok(EARLY_MAPPING_OFFSET + l3i * L3::SIZE);
-    }
-
-    Err(Error::OutOfMemory)
-}
-
-unsafe fn unmap_early_page(address: usize) {
-    if !(EARLY_MAPPING_OFFSET..EARLY_MAPPING_OFFSET + L2::SIZE).contains(&address) {
-        panic!("Tried to unmap invalid early mapping: {:#x}", address);
-    }
-
-    let l3i = (address - EARLY_MAPPING_OFFSET).page_index::<L3>();
-
-    assert!(EARLY_MAPPING_L3[l3i].is_present());
-    EARLY_MAPPING_L3[l3i] = PageEntry::INVALID;
-}
-
-/// # Safety
-///
-/// Only meant to be used by the architecture initialization functions.
-pub unsafe fn map_ram_l1(index: usize) {
-    if index >= RAM_MAPPING_L1_COUNT {
-        ArchitectureImpl::halt();
-    }
-    let mut tables = KERNEL_TABLES.lock();
-    let table_index = index + RAM_MAPPING_START_L1I;
-
-    if tables.l1.data[table_index] != 0 {
-        ArchitectureImpl::halt();
-    }
-
-    tables.l1.data[table_index] = ((index * L1::SIZE) as u64) | ram_block_flags().bits();
-    tlb_flush_vaae1(RAM_MAPPING_OFFSET + index * L1::SIZE);
-}
-
-// Device mappings
-unsafe fn map_device_memory_l3(
-    base: PhysicalAddress,
-    count: usize,
-    _attrs: DeviceMemoryAttributes,
-) -> Result<usize, Error> {
-    // TODO don't map pages if already mapped
-
-    'l0: for i in 0..DEVICE_MAPPING_L3_COUNT * 512 {
-        for j in 0..count {
-            let l2i = (i + j) / 512;
-            let l3i = (i + j) % 512;
-
-            if DEVICE_MAPPING_L3S[l2i][l3i].is_present() {
-                continue 'l0;
-            }
-        }
-
-        for j in 0..count {
-            let l2i = (i + j) / 512;
-            let l3i = (i + j) % 512;
-
-            // TODO NX, NC
-            DEVICE_MAPPING_L3S[l2i][l3i] = PageEntry::device_page(base.add(j * L3::SIZE));
-            tlb_flush_vaae1(DEVICE_MAPPING_OFFSET + l2i * L2::SIZE + l3i * L3::SIZE);
-        }
-
-        return Ok(DEVICE_MAPPING_OFFSET + i * L3::SIZE);
-    }
-
-    Err(Error::OutOfMemory)
-}
-
-#[allow(unused)]
-unsafe fn map_device_memory_l2(
-    base: PhysicalAddress,
-    count: usize,
-    _attrs: DeviceMemoryAttributes,
-) -> Result<usize, Error> {
-    'l0: for i in DEVICE_MAPPING_L3_COUNT..512 {
-        for j in 0..count {
-            if DEVICE_MAPPING_L2[i + j].is_present() {
-                continue 'l0;
-            }
-        }
-
-        for j in 0..count {
-            DEVICE_MAPPING_L2[i + j] = PageEntry::<L2>::device_block(base.add(j * L2::SIZE));
-            tlb_flush_vaae1(DEVICE_MAPPING_OFFSET + (i + j) * L2::SIZE);
-        }
-
-        return Ok(DEVICE_MAPPING_OFFSET + i * L2::SIZE);
-    }
-
-    Err(Error::OutOfMemory)
-}
-
-pub(crate) unsafe fn map_device_memory(
-    base: PhysicalAddress,
-    size: usize,
-    attrs: DeviceMemoryAttributes,
-) -> Result<RawDeviceMemoryMapping<KernelTableManagerImpl>, Error> {
-    let l3_aligned = base.page_align_down::<L3>();
-    let l3_offset = base.page_offset::<L3>();
-    let page_count = (l3_offset + size).page_count::<L3>();
-
-    if page_count > 256 {
-        // Large mapping, use L2 mapping instead
-        let l2_aligned = base.page_align_down::<L2>();
-        let l2_offset = base.page_offset::<L2>();
-        let page_count = (l2_offset + size).page_count::<L2>();
-
-        let base_address = map_device_memory_l2(l2_aligned, page_count, attrs)?;
-        let address = base_address + l2_offset;
-
-        Ok(RawDeviceMemoryMapping::from_raw_parts(
-            l2_aligned.into_u64(),
-            address,
-            base_address,
-            page_count,
-            L2::SIZE,
-        ))
-    } else {
-        // Just map the pages directly
-        let base_address = map_device_memory_l3(l3_aligned, page_count, attrs)?;
-        let address = base_address + l3_offset;
-
-        Ok(RawDeviceMemoryMapping::from_raw_parts(
-            l3_aligned.into_u64(),
-            address,
-            base_address,
-            page_count,
-            L3::SIZE,
-        ))
+    unsafe fn unmap_device_pages(_mapping: &RawDeviceMemoryMapping<Self>) {
+        // TODO
     }
 }
 
-pub(crate) unsafe fn unmap_device_memory(map: &RawDeviceMemoryMapping<KernelTableManagerImpl>) {
-    match map.page_size {
-        L3::SIZE => {
-            for i in 0..map.page_count {
-                let page = map.base_address + i * L3::SIZE;
-                let l2i = page.page_index::<L2>();
-                let l3i = page.page_index::<L3>();
-                assert!(DEVICE_MAPPING_L3S[l2i][l3i].is_present());
-                DEVICE_MAPPING_L3S[l2i][l3i] = PageEntry::INVALID;
+// 64GiB of identity-mapped memory
+pub const IDENTITY_SIZE: usize = 64 * L1::SIZE;
+pub const IDENTITY_L1_START: usize = (KERNEL_VIRT_OFFSET >> L1::SHIFT) & 0x1FF;
+// 1GiB of memory-mapped devices
+pub const DEVICE_MEMORY_L1: usize = IDENTITY_L1_START + IDENTITY_SIZE / L1::SIZE;
+pub const DEVICE_MEMORY_L3_COUNT: usize = 32;
+pub const DEVICE_MAPPING_OFFSET: usize = KERNEL_VIRT_OFFSET + DEVICE_MEMORY_L1 * L1::SIZE;
+static DEVICE_MEMORY_LOCK: IrqSafeSpinlock<ArchitectureImpl, ()> = IrqSafeSpinlock::new(());
 
-                tlb_flush_vaae1(page);
-            }
-        }
-        L2::SIZE => todo!(),
-        _ => unimplemented!(),
+static mut FIXED_L1: PageTable<L1> = const {
+    if IDENTITY_L1_START != 0 {
+        panic!("Invalid KERNEL_VIRT_OFFSET");
     }
-}
-
-#[inline]
-pub fn tlb_flush_asid(asid: u8) {
-    barrier::dsb(barrier::ISHST);
-    let value = (asid as u64) << 48;
-    unsafe {
-        core::arch::asm!("tlbi aside1, {value}", value = in(reg) value);
+    if IDENTITY_L1_START + IDENTITY_SIZE / L1::SIZE >= 500 {
+        panic!("Invalid KERNEL_VIRT_OFFSET + IDENTITY_SIZE");
     }
-    barrier::dsb(barrier::ISH);
-    barrier::isb(barrier::SY);
-}
 
-#[inline]
-pub fn tlb_flush_all() {
-    barrier::dsb(barrier::ISHST);
-    unsafe {
-        core::arch::asm!("tlbi vmalle1is");
+    let mut table = PageTable::<L1>::zeroed();
+    let mut i = 0;
+
+    while i < IDENTITY_SIZE / L1::SIZE {
+        let phys = PhysicalAddress::from_usize(i * L1::SIZE);
+        table.entries[i + IDENTITY_L1_START] =
+            PageEntry::normal_block(phys, PageAttributes::empty());
+        i += 1;
     }
-    barrier::dsb(barrier::ISH);
-    barrier::isb(barrier::SY);
-}
 
-#[inline]
-pub fn tlb_flush_vaae1(page: usize) {
-    barrier::dsb(barrier::ISHST);
-    let argument = page >> 12;
-    unsafe {
-        core::arch::asm!("tlbi vaae1, {argument}", argument = in(reg) argument);
-    }
-    barrier::dsb(barrier::ISH);
-    barrier::isb(barrier::SY);
-}
+    table
+};
 
-pub fn at_s1e0r(input: usize) -> Option<u64> {
-    barrier::dsb(barrier::ISHST);
-    unsafe {
-        core::arch::asm!("at s1e0r, {address}", address = in(reg) input);
-    }
-    barrier::dsb(barrier::ISH);
-    barrier::isb(barrier::SY);
-    if PAR_EL1.matches_all(PAR_EL1::F::TranslationSuccessfull) {
-        Some(PAR_EL1.read(PAR_EL1::PA))
-    } else {
-        None
-    }
-}
-
-pub fn at_s1e1r(input: usize) -> Option<u64> {
-    barrier::dsb(barrier::ISHST);
-    unsafe {
-        core::arch::asm!("at s1e1r, {address}", address = in(reg) input);
-    }
-    barrier::dsb(barrier::ISH);
-    barrier::isb(barrier::SY);
-    if PAR_EL1.matches_all(PAR_EL1::F::TranslationSuccessfull) {
-        Some(PAR_EL1.read(PAR_EL1::PA))
-    } else {
-        None
-    }
-}
-
-pub fn ic_iallu() {
-    atomic::compiler_fence(Ordering::SeqCst);
-    barrier::dsb(barrier::ISH);
-    barrier::isb(barrier::SY);
-    unsafe {
-        core::arch::asm!("ic iallu");
-    }
-    barrier::isb(barrier::SY);
-}
-
-pub fn dc_cvac(input: usize) {
-    barrier::dsb(barrier::ISHST);
-    unsafe {
-        core::arch::asm!("dc cvac, {address}", address = in(reg) input);
-    }
-}
-
-fn auto_address<T>(value: *const T) -> usize {
-    let addr = value.addr();
-    if addr < KERNEL_VIRT_OFFSET {
-        // Called from lower half
-        addr
-    } else {
-        // Called from higher-half
-        addr - KERNEL_VIRT_OFFSET
-    }
-}
-
-/// (BSP-early init) loads precomputed kernel mapping tables for the kernel to jump to "higher-half"
-///
-/// # Safety
-///
-/// Unsafe, must only be called by BSP during its early init while still in "lower-half"
-pub unsafe fn load_fixed_tables() {
-    let ttbr0 = auto_address(&raw const KERNEL_TABLES) as u64;
-
-    TTBR0_EL1.set(ttbr0);
-    TTBR1_EL1.set(ttbr0);
-}
-
-/// Sets up additional translation tables for kernel usage
-///
-/// # Safety
-///
-/// Unsafe, must only be called by BSP during its early init, must already be in "higher-half"
-pub unsafe fn init_fixed_tables() {
-    // TODO this could be built in compile-time too?
-    let mut tables = KERNEL_TABLES.grab();
-    let early_mapping_l3_phys = auto_address(&raw const EARLY_MAPPING_L3);
-    let device_mapping_l2_phys = auto_address(&raw const DEVICE_MAPPING_L2);
-
-    for i in 0..DEVICE_MAPPING_L3_COUNT {
-        let device_mapping_l3_phys = PhysicalAddress::from_usize(
-            &DEVICE_MAPPING_L3S[i] as *const _ as usize - KERNEL_VIRT_OFFSET,
+pub unsafe fn setup_fixed_tables() {
+    let device_l2_physical = auto_lower_address(&raw const device::DEVICE_MEMORY.large);
+    FIXED_L1[DEVICE_MEMORY_L1] = PageEntry::table(
+        PhysicalAddress::from_usize(device_l2_physical),
+        PageAttributes::empty(),
+    );
+    for i in 0..DEVICE_MEMORY_L3_COUNT {
+        let device_l3_physical = auto_lower_address(&raw const device::DEVICE_MEMORY.normal.0[i]);
+        device::DEVICE_MEMORY.large.0[i] = PageEntry::table(
+            PhysicalAddress::from_usize(device_l3_physical),
+            PageAttributes::empty(),
         );
-        DEVICE_MAPPING_L2[i] = PageEntry::table(device_mapping_l3_phys, PageAttributes::empty());
     }
-
-    assert_eq!(tables.l2.data[EARLY_MAPPING_L2I], 0);
-    tables.l2.data[EARLY_MAPPING_L2I] =
-        (early_mapping_l3_phys as u64) | kernel_table_flags().bits();
-    tlb_flush_vaae1(EARLY_MAPPING_OFFSET);
-
-    assert_eq!(tables.l1.data[DEVICE_MAPPING_L1I], 0);
-    tables.l1.data[DEVICE_MAPPING_L1I] =
-        (device_mapping_l2_phys as u64) | kernel_table_flags().bits();
-
-    tlb_flush_all();
 }
 
-pub fn setup_memory_attributes() {
-    // TODO: Figure out why WriteBack_NonTransient_ReadWriteAlloc doesn't work on Pi 4B
+pub fn load_fixed_tables() {
+    let l1_physical = auto_lower_address(&raw const FIXED_L1);
+    TTBR0_EL1.set_baddr(l1_physical as _);
+    TTBR1_EL1.set_baddr(l1_physical as _);
+}
 
+pub fn configure_mmu() {
+    // TODO: Figure out why WriteBack_NonTransient_ReadWriteAlloc doesn't work on Pi 4B
     MAIR_EL1.write(
         //// Attribute 0 -- normal memory
         MAIR_EL1::Attr0_Normal_Inner::WriteBack_NonTransient +
@@ -504,6 +120,39 @@ pub fn setup_memory_attributes() {
         //// Attribute 2 -- device memory
         MAIR_EL1::Attr1_Device::nonGathering_nonReordering_EarlyWriteAck,
     );
+
+    TCR_EL1.write(
+        TCR_EL1::AS::ASID8Bits +
+             TCR_EL1::A1::TTBR0 +
+             TCR_EL1::HD::CLEAR +
+             // General
+             TCR_EL1::IPS::Bits_48 +
+             // TTBR0
+             TCR_EL1::TG0::KiB_4 + TCR_EL1::T0SZ.val(25) + TCR_EL1::SH0::Inner +
+             // TTBR1
+             TCR_EL1::TG1::KiB_4 + TCR_EL1::T1SZ.val(25) + TCR_EL1::SH1::Inner,
+    );
+    barrier::dsb(barrier::ISHST);
+    barrier::isb(barrier::SY);
+
+    SCTLR_EL1.modify(
+        SCTLR_EL1::E0E::LittleEndian
+            + SCTLR_EL1::EE::LittleEndian
+            + SCTLR_EL1::WXN::Disable
+            + SCTLR_EL1::SA0::Enable
+            + SCTLR_EL1::SA::Enable
+            + SCTLR_EL1::A::Enable
+            + SCTLR_EL1::I::NonCacheable
+            + SCTLR_EL1::C::NonCacheable,
+    );
+
+    barrier::dsb(barrier::ISH);
+    barrier::isb(barrier::SY);
+}
+
+pub fn enable_mmu() {
+    // Enable translation
+    SCTLR_EL1.modify(SCTLR_EL1::M::Enable);
 }
 
 /// Enables data cache.
@@ -549,3 +198,12 @@ pub unsafe fn disable_icache() {
     barrier::dsb(barrier::ISH);
     barrier::isb(barrier::SY);
 }
+
+fn auto_lower_address<T>(ptr: *const T) -> usize {
+    let address = ptr.addr();
+    if address < KERNEL_VIRT_OFFSET {
+        address
+    } else {
+        address - KERNEL_VIRT_OFFSET
+    }
+}

kernel/arch/aarch64/src/mem/table.rs

@@ -56,7 +56,7 @@ bitflags! {
 #[derive(Clone, Copy)]
 #[repr(C, align(0x1000))]
 pub struct PageTable<L: EntryLevel> {
-    entries: [PageEntry<L>; 512],
+    pub entries: [PageEntry<L>; 512],
 }
 
 #[derive(Clone, Copy)]
@@ -262,18 +262,13 @@ impl<L: NonTerminalEntryLevel> PageEntry<L> {
         )
     }
 
-    pub fn normal_block(phys: PhysicalAddress, attrs: PageAttributes) -> Self {
-        Self(
-            phys.into_u64()
-                | (PageAttributes::BLOCK
-                    | PageAttributes::PRESENT
-                    | PageAttributes::ACCESS
-                    | PageAttributes::SH_OUTER
-                    | PageAttributes::PAGE_ATTR_NORMAL
-                    | attrs)
-                    .bits(),
-            PhantomData,
-        )
+    pub const fn normal_block(phys: PhysicalAddress, attrs: PageAttributes) -> Self {
+        const ATTR: u64 = PageAttributes::BLOCK.bits()
+            | PageAttributes::PRESENT.bits()
+            | PageAttributes::ACCESS.bits()
+            | PageAttributes::SH_OUTER.bits()
+            | PageAttributes::PAGE_ATTR_NORMAL.bits();
+        Self(phys.into_u64() | attrs.bits() | ATTR, PhantomData)
     }
 
     pub fn device_block(phys: PhysicalAddress) -> Self {

kernel/arch/interface/src/sync.rs

@@ -42,7 +42,7 @@ impl<A: Architecture, T> Spinlock<A, T> {
         }
     }
 
-    pub fn lock(&self) -> SpinlockGuard<A, T> {
+    pub fn lock(&self) -> SpinlockGuard<'_, A, T> {
         // Loop until the lock can be acquired
         if LOCK_HACK.load(Ordering::Acquire) {
             return SpinlockGuard { lock: self };
@@ -103,7 +103,7 @@ impl<A: Architecture, T> IrqSafeSpinlock<A, T> {
     }
 
     /// Attempts to acquire a lock. IRQs will be disabled until the lock is released.
-    pub fn lock(&self) -> IrqSafeSpinlockGuard<A, T> {
+    pub fn lock(&self) -> IrqSafeSpinlockGuard<'_, A, T> {
         // Disable IRQs to avoid IRQ handler trying to acquire the same lock
         let irq_guard = IrqGuard::acquire();
 

kernel/arch/riscv64/Cargo.toml

@@ -7,7 +7,6 @@ edition = "2024"
 yggdrasil-abi.workspace = true
 kernel-arch-interface.workspace = true
 libk-mm-interface.workspace = true
-memtables.workspace = true
 device-api = { workspace = true, features = ["derive"] }
 
 tock-registers.workspace = true

kernel/arch/riscv64/src/context.rs

@@ -124,8 +124,7 @@ impl<K: KernelTableManager, PA: PhysicalMemoryAllocator<Address = PhysicalAddres
 
         // TODO stack is leaked
         let satp = InMemoryRegister::new(0);
-        let kernel_table_phys =
-            ((&raw const mem::KERNEL_TABLES).addr() - KERNEL_VIRT_OFFSET) as u64;
+        let kernel_table_phys = ((&raw const mem::FIXED_L1).addr() - KERNEL_VIRT_OFFSET) as u64;
         satp.write(SATP::MODE::Sv39 + SATP::ASID.val(0) + SATP::PPN.val(kernel_table_phys >> 12));
 
         Ok(Self {

kernel/arch/riscv64/src/lib.rs

@@ -1,4 +1,4 @@
-#![feature(decl_macro, naked_functions)]
+#![feature(decl_macro)]
 #![no_std]
 
 extern crate alloc;
@@ -60,11 +60,9 @@ impl CpuData for PerCpuData {
     }
 }
 
-#[naked]
+#[unsafe(naked)]
 extern "C" fn idle_task(_: usize) -> ! {
-    unsafe {
-        core::arch::naked_asm!("1: nop; j 1b");
-    }
+    core::arch::naked_asm!("1: nop; j 1b");
 }
 
 impl ArchitectureImpl {

kernel/arch/riscv64/src/mem/mod.rs

@@ -1,70 +1,38 @@
-use cfg_if::cfg_if;
-use kernel_arch_interface::{
-    mem::{DeviceMemoryAttributes, KernelTableManager, RawDeviceMemoryMapping},
-    split_spinlock,
+use kernel_arch_interface::mem::{
+    DeviceMemoryAttributes, KernelTableManager, RawDeviceMemoryMapping,
 };
 use libk_mm_interface::{
     address::PhysicalAddress,
     table::{page_index, EntryLevel, EntryLevelExt},
 };
-use memtables::riscv64::PageAttributes;
-use static_assertions::{const_assert, const_assert_eq};
-use table::{PageEntry, PageTable, L1, L2, L3};
+use table::{PageEntry, PageTable, L1, L3};
 use tock_registers::interfaces::Writeable;
 use yggdrasil_abi::error::Error;
 
-pub use memtables::riscv64::FixedTables;
-
 use crate::registers::SATP;
 
 pub mod process;
 pub mod table;
 
-split_spinlock! {
-    use crate::ArchitectureImpl;
-    use crate::mem::FixedTables;
-    use libk_mm_interface::KernelImageObject;
-
-    #[link_section = ".data.tables"]
-    #[used]
-    static KERNEL_TABLES: KernelImageObject<FixedTables> =
-        unsafe { KernelImageObject::new(FixedTables::zeroed()) };
-}
-
-cfg_if! {
-    if #[cfg(feature = "riscv64_board_virt")] {
-        pub const KERNEL_PHYS_BASE: usize = 0x80200000;
-    } else if #[cfg(feature = "riscv64_board_jh7110")] {
-        pub const KERNEL_PHYS_BASE: usize = 0x40200000;
-    } else if #[cfg(rust_analyzer)] {
-        pub const KERNEL_PHYS_BASE: usize = 0x80200000;
-    }
-}
-
 pub const KERNEL_VIRT_OFFSET: usize = kernel_arch_interface::KERNEL_VIRT_OFFSET;
 pub const SIGN_EXTEND_MASK: usize = 0xFFFFFF80_00000000;
 
-pub const KERNEL_START_L1I: usize = page_index::<L1>(KERNEL_VIRT_OFFSET + KERNEL_PHYS_BASE);
-pub const KERNEL_L2I: usize = page_index::<L2>(KERNEL_VIRT_OFFSET + KERNEL_PHYS_BASE);
-const_assert_eq!(KERNEL_L2I, 1);
+pub const IDENTITY_SIZE: usize = 64 * L1::SIZE;
+pub const IDENTITY_L1_START: usize = page_index::<L1>(KERNEL_VIRT_OFFSET);
 
-// Runtime mappings
-// 1GiB of device memory space
-const DEVICE_MAPPING_L1I: usize = KERNEL_START_L1I + 1;
-const DEVICE_MAPPING_L3_COUNT: usize = 4;
-// 32GiB of RAM space
-const RAM_MAPPING_START_L1I: usize = KERNEL_START_L1I + 2;
-const RAM_MAPPING_L1_COUNT: usize = 32;
-const_assert!(RAM_MAPPING_START_L1I + RAM_MAPPING_L1_COUNT <= 512);
-const_assert!(DEVICE_MAPPING_L1I < 512);
+pub static mut FIXED_L1: PageTable<L1> = const {
+    let mut table = PageTable::zeroed();
 
-const DEVICE_MAPPING_OFFSET: usize = (DEVICE_MAPPING_L1I << L1::SHIFT) | SIGN_EXTEND_MASK;
-const RAM_MAPPING_OFFSET: usize = (RAM_MAPPING_START_L1I << L1::SHIFT) | SIGN_EXTEND_MASK;
+    let mut i = 0;
+    while i < IDENTITY_SIZE / L1::SIZE {
+        let entry = PageEntry::identity_block(PhysicalAddress::from_usize(i * L1::SIZE));
+        table.entries[i] = entry;
+        table.entries[i + IDENTITY_L1_START] = entry;
+        i += 1;
+    }
 
-// Runtime tables
-static mut DEVICE_MAPPING_L2: PageTable<L2> = PageTable::zeroed();
-static mut DEVICE_MAPPING_L3S: [PageTable<L3>; DEVICE_MAPPING_L3_COUNT] =
-    [const { PageTable::zeroed() }; DEVICE_MAPPING_L3_COUNT];
+    table
+};
 
 /// Any VAs above this one are sign-extended
 pub const USER_BOUNDARY: usize = 0x40_00000000;
@@ -75,17 +43,17 @@ pub struct KernelTableManagerImpl;
 impl KernelTableManager for KernelTableManagerImpl {
     fn virtualize(address: u64) -> usize {
         let address = address as usize;
-        if address >= RAM_MAPPING_OFFSET {
+        if address >= IDENTITY_SIZE {
             panic!("Invalid physical address: {address:#x}");
         }
-        address + RAM_MAPPING_OFFSET
+        address + KERNEL_VIRT_OFFSET
     }
 
     fn physicalize(address: usize) -> u64 {
-        if address < RAM_MAPPING_OFFSET {
+        if address < KERNEL_VIRT_OFFSET || address - KERNEL_VIRT_OFFSET >= IDENTITY_SIZE {
             panic!("Invalid \"physicalized\" virtual address {address:#x}");
         }
-        (address - RAM_MAPPING_OFFSET) as u64
+        (address - KERNEL_VIRT_OFFSET) as u64
     }
 
     unsafe fn map_device_pages(
@@ -93,146 +61,31 @@ impl KernelTableManager for KernelTableManagerImpl {
         count: usize,
         attrs: DeviceMemoryAttributes,
     ) -> Result<RawDeviceMemoryMapping<Self>, Error> {
-        unsafe { map_device_memory(PhysicalAddress::from_u64(base), count, attrs) }
+        let _ = attrs;
+        let base = PhysicalAddress::from_u64(base);
+        let l3_aligned_base = base.page_align_down::<L3>();
+        let l3_aligned_end = base.add(count).page_align_up::<L3>();
+        let l3_offset = base - l3_aligned_base;
+        let l3_page_count = (l3_aligned_end - l3_aligned_base).page_count::<L3>();
+        let l3_aligned_virt = l3_aligned_base.add(KERNEL_VIRT_OFFSET).into_usize();
+
+        Ok(unsafe {
+            RawDeviceMemoryMapping::from_raw_parts(
+                l3_aligned_base.into_u64(),
+                l3_aligned_virt + l3_offset,
+                l3_aligned_virt,
+                l3_page_count,
+                L3::SIZE,
+            )
+        })
     }
 
     unsafe fn unmap_device_pages(mapping: &RawDeviceMemoryMapping<Self>) {
-        unsafe { unmap_device_memory(mapping) }
+        let _ = mapping;
     }
 }
 
-// Device mappings
-unsafe fn map_device_memory_l3(
-    base: PhysicalAddress,
-    count: usize,
-    _attrs: DeviceMemoryAttributes,
-) -> Result<usize, Error> {
-    // TODO don't map pages if already mapped
-
-    'l0: for i in 0..DEVICE_MAPPING_L3_COUNT * 512 {
-        for j in 0..count {
-            let l2i = (i + j) / 512;
-            let l3i = (i + j) % 512;
-
-            unsafe {
-                if DEVICE_MAPPING_L3S[l2i][l3i].is_present() {
-                    continue 'l0;
-                }
-            }
-        }
-
-        for j in 0..count {
-            let l2i = (i + j) / 512;
-            let l3i = (i + j) % 512;
-
-            unsafe {
-                DEVICE_MAPPING_L3S[l2i][l3i] =
-                    PageEntry::page(base.add(j * L3::SIZE), PageAttributes::W);
-            }
-        }
-
-        let start = DEVICE_MAPPING_OFFSET + i * L3::SIZE;
-        tlb_flush_range_va(start, count * L3::SIZE);
-        return Ok(start);
-    }
-
-    Err(Error::OutOfMemory)
-}
-
-#[allow(unused)]
-unsafe fn map_device_memory_l2(
-    base: PhysicalAddress,
-    count: usize,
-    _attrs: DeviceMemoryAttributes,
-) -> Result<usize, Error> {
-    'l0: for i in DEVICE_MAPPING_L3_COUNT..512 {
-        for j in 0..count {
-            unsafe {
-                if DEVICE_MAPPING_L2[i + j].is_present() {
-                    continue 'l0;
-                }
-            }
-        }
-
-        unsafe {
-            for j in 0..count {
-                DEVICE_MAPPING_L2[i + j] =
-                    PageEntry::<L2>::block(base.add(j * L2::SIZE), PageAttributes::W);
-            }
-        }
-
-        let start = DEVICE_MAPPING_OFFSET + i * L2::SIZE;
-        tlb_flush_range_va(start, count * L2::SIZE);
-        return Ok(start);
-    }
-
-    Err(Error::OutOfMemory)
-}
-
-pub(crate) unsafe fn map_device_memory(
-    base: PhysicalAddress,
-    size: usize,
-    attrs: DeviceMemoryAttributes,
-) -> Result<RawDeviceMemoryMapping<KernelTableManagerImpl>, Error> {
-    let l3_aligned = base.page_align_down::<L3>();
-    let l3_offset = base.page_offset::<L3>();
-    let page_count = (l3_offset + size).page_count::<L3>();
-
-    if page_count > 256 {
-        // Large mapping, use L2 mapping instead
-        let l2_aligned = base.page_align_down::<L2>();
-        let l2_offset = base.page_offset::<L2>();
-        let page_count = (l2_offset + size).page_count::<L2>();
-
-        unsafe {
-            let base_address = map_device_memory_l2(l2_aligned, page_count, attrs)?;
-            let address = base_address + l2_offset;
-
-            Ok(RawDeviceMemoryMapping::from_raw_parts(
-                l2_aligned.into_u64(),
-                address,
-                base_address,
-                page_count,
-                L2::SIZE,
-            ))
-        }
-    } else {
-        // Just map the pages directly
-        unsafe {
-            let base_address = map_device_memory_l3(l3_aligned, page_count, attrs)?;
-            let address = base_address + l3_offset;
-
-            Ok(RawDeviceMemoryMapping::from_raw_parts(
-                l3_aligned.into_u64(),
-                address,
-                base_address,
-                page_count,
-                L3::SIZE,
-            ))
-        }
-    }
-}
-
-pub(crate) unsafe fn unmap_device_memory(map: &RawDeviceMemoryMapping<KernelTableManagerImpl>) {
-    match map.page_size {
-        L3::SIZE => {
-            for i in 0..map.page_count {
-                let page = map.base_address + i * L3::SIZE;
-                let l2i = page.page_index::<L2>();
-                let l3i = page.page_index::<L3>();
-                unsafe {
-                    assert!(DEVICE_MAPPING_L3S[l2i][l3i].is_present());
-                    DEVICE_MAPPING_L3S[l2i][l3i] = PageEntry::INVALID;
-                }
-            }
-            tlb_flush_range_va(map.base_address, map.page_count * L3::SIZE);
-        }
-        L2::SIZE => todo!(),
-        _ => unimplemented!(),
-    }
-}
-
-pub fn auto_address<T>(x: *const T) -> usize {
+pub fn auto_lower_address<T>(x: *const T) -> usize {
     let x = x.addr();
     if x >= KERNEL_VIRT_OFFSET {
         x - KERNEL_VIRT_OFFSET
@@ -247,7 +100,7 @@ pub fn auto_address<T>(x: *const T) -> usize {
 ///
 /// Only meant to be called once per each HART during their early init.
 pub unsafe fn enable_mmu() {
-    let l1_phys = auto_address(&raw const KERNEL_TABLES) as u64;
+    let l1_phys = auto_lower_address(&raw const FIXED_L1) as u64;
     tlb_flush_full();
     SATP.write(SATP::PPN.val(l1_phys >> 12) + SATP::MODE::Sv39);
 }
@@ -258,49 +111,10 @@ pub unsafe fn enable_mmu() {
 ///
 /// Needs to be called once after secondary HARTs are initialized.
 pub unsafe fn unmap_lower_half() {
-    let mut tables = KERNEL_TABLES.lock();
-    let kernel_l1i_lower = page_index::<L1>(KERNEL_PHYS_BASE);
-    tables.l1.data[kernel_l1i_lower] = 0;
-    tlb_flush_range_va(0x0, L1::SIZE);
-}
-
-/// Sets up run-time kernel translation tables.
-///
-/// # Safety
-///
-/// The caller must ensure MMU is already enabled.
-pub unsafe fn setup_fixed_tables() {
-    let mut tables = KERNEL_TABLES.lock();
-
-    let device_mapping_l2_phys = auto_address(&raw const DEVICE_MAPPING_L2);
-
-    // Set up static runtime mappings
-    for i in 0..DEVICE_MAPPING_L3_COUNT {
-        unsafe {
-            let device_mapping_l3_phys = PhysicalAddress::from_usize(
-                (&raw const DEVICE_MAPPING_L3S[i]).addr() - KERNEL_VIRT_OFFSET,
-            );
-            DEVICE_MAPPING_L2[i] =
-                PageEntry::table(device_mapping_l3_phys, PageAttributes::empty());
-        }
-    }
-
-    assert_eq!(tables.l1.data[DEVICE_MAPPING_L1I], 0);
-    tables.l1.data[DEVICE_MAPPING_L1I] =
-        ((device_mapping_l2_phys as u64) >> 2) | PageAttributes::V.bits();
-
-    for l1i in 0..RAM_MAPPING_L1_COUNT {
-        let physical = (l1i as u64) << L1::SHIFT;
-        tables.l1.data[l1i + RAM_MAPPING_START_L1I] = (physical >> 2)
-            | (PageAttributes::R
-                | PageAttributes::W
-                | PageAttributes::A
-                | PageAttributes::D
-                | PageAttributes::V)
-                .bits();
-    }
-
-    tlb_flush_full();
+    // for i in 0..(IDENTITY_SIZE / L1::SIZE) {
+    //     unsafe { FIXED_L1[i] = PageEntry::INVALID };
+    // }
+    // tlb_flush_full();
 }
 
 pub fn tlb_flush_global_full() {
@@ -352,8 +166,9 @@ pub fn tlb_flush_va_asid(va: usize, asid: usize) {
 }
 
 pub fn clone_kernel_tables(dst: &mut PageTable<L1>) {
-    let tables = KERNEL_TABLES.lock();
+    // let tables = KERNEL_TABLES.lock();
     for l1i in page_index::<L1>(USER_BOUNDARY)..512 {
-        dst[l1i] = unsafe { PageEntry::from_raw(tables.l1.data[l1i]) };
+        // dst[l1i] = unsafe { PageEntry::from_raw(tables.l1.data[l1i]) };
+        dst[l1i] = unsafe { FIXED_L1[l1i] };
     }
 }

kernel/arch/riscv64/src/mem/process.rs

@@ -11,13 +11,12 @@ use libk_mm_interface::{
         EntryLevel, EntryLevelDrop, EntryLevelExt, MapAttributes, NextPageTable, TableAllocator,
     },
 };
-use memtables::riscv64::PageAttributes;
 use yggdrasil_abi::error::Error;
 
 use crate::mem::{clone_kernel_tables, table::PageEntry};
 
 use super::{
-    table::{DroppableRange, PageTable, L1, L2, L3},
+    table::{DroppableRange, PageAttributes, PageTable, L1, L2, L3},
     KernelTableManagerImpl, USER_BOUNDARY,
 };
 

kernel/arch/riscv64/src/mem/table.rs

@@ -3,6 +3,7 @@ use core::{
     ops::{Index, IndexMut, Range},
 };
 
+use bitflags::bitflags;
 use libk_mm_interface::{
     address::{AsPhysicalAddress, PhysicalAddress},
     pointer::{PhysicalRef, PhysicalRefMut},
@@ -16,7 +17,41 @@ use yggdrasil_abi::error::Error;
 
 use super::{KernelTableManagerImpl, USER_BOUNDARY};
 
-pub use memtables::riscv64::PageAttributes;
+// pub use memtables::riscv64::PageAttributes;
+bitflags! {
+    #[derive(Clone, Copy, PartialEq, Eq, Debug)]
+    pub struct PageAttributes: u64 {
+        const N = 1 << 63;
+        /// Software-tracked dirty bit (RSW[0])
+        const SW_DIRTY = 1 << 9;
+        /// Dirty bit
+        const D = 1 << 7;
+        /// Access bit
+        const A = 1 << 6;
+        /// Global mapping bit, implies all lower levels are also global
+        const G = 1 << 5;
+        /// U-mode access permission
+        const U = 1 << 4;
+        /// Execute permission
+        const X = 1 << 3;
+        /// Write permission
+        const W = 1 << 2;
+        /// Read-permission
+        const R = 1 << 1;
+        /// Valid bit
+        const V = 1 << 0;
+    }
+
+    // X W R    Meaning
+    // 0 0 0    Pointer to next level of page table
+    // 0 0 1    Read-only page
+    // 0 1 0    ---
+    // 0 1 1    Read-write page
+    // 1 0 0    Execute only
+    // 1 0 1    Read-execute page
+    // 1 1 0    ---
+    // 1 1 1    Read-write-execute page
+}
 
 /// L3 - entry is 4KiB
 #[derive(Debug, Clone, Copy)]
@@ -40,9 +75,10 @@ impl EntryLevel for L1 {
     const SHIFT: usize = 30;
 }
 
+#[derive(Clone, Copy)]
 #[repr(C, align(0x1000))]
 pub struct PageTable<L: EntryLevel> {
-    entries: [PageEntry<L>; 512],
+    pub entries: [PageEntry<L>; 512],
 }
 
 #[derive(Clone, Copy, Debug, PartialEq)]
@@ -204,6 +240,17 @@ impl<L: NonTerminalEntryLevel + 'static> NextPageTable for PageTable<L> {
 }
 
 impl<L: NonTerminalEntryLevel> PageEntry<L> {
+    pub const fn identity_block(address: PhysicalAddress) -> Self {
+        const ATTR: u64 = PageAttributes::R.bits()
+            | PageAttributes::W.bits()
+            | PageAttributes::X.bits()
+            | PageAttributes::A.bits()
+            | PageAttributes::D.bits()
+            | PageAttributes::V.bits();
+
+        Self((address.into_u64() >> 2) | ATTR, PhantomData)
+    }
+
     pub fn block(address: PhysicalAddress, attrs: PageAttributes) -> Self {
         // TODO validate address alignment
         Self(

kernel/arch/x86_64/Cargo.toml

@@ -7,7 +7,6 @@ edition = "2021"
 yggdrasil-abi.workspace = true
 kernel-arch-interface.workspace = true
 libk-mm-interface.workspace = true
-memtables.workspace = true
 device-api = { workspace = true, features = ["derive"] }
 kernel-arch-x86.workspace = true
 

kernel/arch/x86_64/src/context.rs

@@ -5,11 +5,14 @@ use kernel_arch_interface::{
     task::{ForkFrame, StackBuilder, TaskContext, TaskFrame, UserContextInfo},
 };
 use kernel_arch_x86::registers::{FpuContext, CR3, MSR_IA32_FS_BASE};
-use libk_mm_interface::address::{AsPhysicalAddress, PhysicalAddress};
+use libk_mm_interface::address::PhysicalAddress;
 use tock_registers::interfaces::Writeable;
 use yggdrasil_abi::{arch::SavedFrame, error::Error};
 
-use crate::{mem::KERNEL_TABLES, ArchitectureImpl};
+use crate::{
+    mem::{auto_lower_address, FIXED_PML4},
+    ArchitectureImpl,
+};
 
 /// Frame saved onto the stack when taking an IRQ
 #[derive(Debug)]
@@ -431,7 +434,7 @@ impl<K: KernelTableManager, PA: PhysicalMemoryAllocator<Address = PhysicalAddres
     fn kernel(entry: extern "C" fn(usize) -> !, arg: usize) -> Result<Self, Error> {
         const KERNEL_TASK_PAGES: usize = 32;
 
-        let cr3: usize = unsafe { KERNEL_TABLES.lock().as_physical_address() }.into();
+        let cr3 = auto_lower_address(&raw const FIXED_PML4);
 
         let stack_base_phys = PA::allocate_contiguous_pages(KERNEL_TASK_PAGES)?;
         let stack_base = stack_base_phys.raw_virtualize::<K>();

kernel/arch/x86_64/src/lib.rs

@@ -1,6 +1,5 @@
 #![no_std]
 #![allow(clippy::new_without_default)]
-#![feature(naked_functions)]
 
 extern crate alloc;
 
@@ -69,18 +68,16 @@ impl PerCpuData {
 static IPI_QUEUES: OneTimeInit<Vec<IpiQueue<ArchitectureImpl>>> = OneTimeInit::new();
 pub static CPU_COUNT: AtomicUsize = AtomicUsize::new(1);
 
-#[naked]
+#[unsafe(naked)]
 extern "C" fn idle_task(_: usize) -> ! {
-    unsafe {
-        core::arch::naked_asm!(
-            r#"
+    core::arch::naked_asm!(
+        r#"
             1:
                 nop
                 jmp 1b
             "#,
-            options(att_syntax)
-        );
-    }
+        options(att_syntax)
+    );
 }
 
 impl ArchitectureImpl {

kernel/arch/x86_64/src/mem/device.rs

@@ -0,0 +1,87 @@
+use core::ops::Range;
+
+use kernel_arch_interface::mem::DeviceMemoryAttributes;
+use libk_mm_interface::{
+    address::PhysicalAddress,
+    table::{DevicePageManager, DevicePageManagerLevel},
+};
+
+use crate::mem::table::PageEntry;
+
+use super::{
+    table::{PageAttributes, PageTable, L2, L3},
+    DEVICE_MAPPING_OFFSET, DEVICE_MEMORY_L3_COUNT,
+};
+
+#[repr(transparent)]
+pub struct L2DeviceMemory(pub PageTable<L2>);
+#[repr(transparent)]
+pub struct L3DeviceMemory(pub [PageTable<L3>; DEVICE_MEMORY_L3_COUNT]);
+
+pub(super) static mut DEVICE_MEMORY: DevicePageManager<L3DeviceMemory, L2DeviceMemory> =
+    DevicePageManager::new(
+        L3DeviceMemory([PageTable::zeroed(); DEVICE_MEMORY_L3_COUNT]),
+        L2DeviceMemory(PageTable::zeroed()),
+    );
+
+impl DevicePageManagerLevel for L2DeviceMemory {
+    type Level = L2;
+    const VIRTUAL_BASE: usize = DEVICE_MAPPING_OFFSET;
+    const INDEX_RANGE: Range<usize> = DEVICE_MEMORY_L3_COUNT..512;
+
+    fn map_page(
+        &mut self,
+        index: usize,
+        physical: PhysicalAddress,
+        attrs: &DeviceMemoryAttributes,
+    ) {
+        let _ = attrs;
+        self.0[index] = PageEntry::<L2>::block(physical, PageAttributes::WRITABLE);
+    }
+
+    fn unmap_page(&mut self, index: usize) {
+        self.0[index] = PageEntry::INVALID;
+    }
+
+    fn is_mapped(&self, index: usize) -> bool {
+        self.0[index].is_present()
+    }
+
+    fn flush_range(range: Range<usize>) {
+        let _ = range;
+    }
+}
+
+impl DevicePageManagerLevel for L3DeviceMemory {
+    type Level = L3;
+    const VIRTUAL_BASE: usize = DEVICE_MAPPING_OFFSET;
+    const INDEX_RANGE: Range<usize> = 0..512 * DEVICE_MEMORY_L3_COUNT;
+
+    fn map_page(
+        &mut self,
+        index: usize,
+        physical: PhysicalAddress,
+        attrs: &DeviceMemoryAttributes,
+    ) {
+        let _ = attrs;
+        let l2i = index / 512;
+        let l3i = index % 512;
+        self.0[l2i][l3i] = PageEntry::page(physical, PageAttributes::WRITABLE);
+    }
+
+    fn unmap_page(&mut self, index: usize) {
+        let l2i = index / 512;
+        let l3i = index % 512;
+        self.0[l2i][l3i] = PageEntry::INVALID;
+    }
+
+    fn is_mapped(&self, index: usize) -> bool {
+        let l2i = index / 512;
+        let l3i = index % 512;
+        self.0[l2i][l3i].is_present()
+    }
+
+    fn flush_range(range: Range<usize>) {
+        let _ = range;
+    }
+}

kernel/arch/x86_64/src/mem/mod.rs

@@ -1,98 +1,39 @@
-use core::{
-    alloc::Layout,
-    ops::{Deref, DerefMut},
-    sync::atomic::{AtomicUsize, Ordering},
-};
-
 use kernel_arch_interface::{
     mem::{DeviceMemoryAttributes, KernelTableManager, RawDeviceMemoryMapping},
-    split_spinlock,
+    sync::IrqSafeSpinlock,
 };
 use kernel_arch_x86::registers::CR3;
 use libk_mm_interface::{
     address::PhysicalAddress,
-    table::{page_index, EntryLevel, EntryLevelExt},
+    table::{EntryLevel, EntryLevelExt},
 };
-use static_assertions::{const_assert_eq, const_assert_ne};
 use yggdrasil_abi::error::Error;
 
-use crate::KERNEL_VIRT_OFFSET;
+use crate::{ArchitectureImpl, KERNEL_VIRT_OFFSET};
 
-use self::table::{PageAttributes, PageEntry, PageTable, L0, L1, L2, L3};
+use self::table::{PageAttributes, PageEntry, PageTable, L0, L1, L2};
 
+pub mod device;
 pub mod process;
 pub mod table;
 
 #[derive(Debug)]
 pub struct KernelTableManagerImpl;
 
-const CANONICAL_ADDRESS_MASK: usize = 0xFFFF000000000000;
-const KERNEL_PHYS_BASE: usize = 0x200000;
-
-// Mapped at compile time
-const KERNEL_MAPPING_BASE: usize = KERNEL_VIRT_OFFSET + KERNEL_PHYS_BASE;
-const KERNEL_L0_INDEX: usize = page_index::<L0>(KERNEL_MAPPING_BASE);
-const KERNEL_L1_INDEX: usize = page_index::<L1>(KERNEL_MAPPING_BASE);
-const KERNEL_START_L2_INDEX: usize = page_index::<L2>(KERNEL_MAPPING_BASE);
-
-// Must not be zero, should be at 4MiB
-const_assert_ne!(KERNEL_START_L2_INDEX, 0);
-// From static mapping
-const_assert_eq!(KERNEL_L0_INDEX, 511);
-const_assert_eq!(KERNEL_L1_INDEX, 0);
-
-// Mapped at boot
-const EARLY_MAPPING_L2I: usize = KERNEL_START_L2_INDEX - 1;
-const DEVICE_MAPPING_L1I: usize = KERNEL_L1_INDEX + 2;
-const RAM_MAPPING_L0I: usize = KERNEL_L0_INDEX - 1;
-
-const DEVICE_MAPPING_L3_COUNT: usize = 4;
-
-split_spinlock! {
-    use libk_mm_interface::KernelImageObject;
-    use memtables::x86_64::FixedTables;
-    use crate::ArchitectureImpl;
-
-    #[link_section = ".data.tables"]
-    static KERNEL_TABLES: KernelImageObject<FixedTables> =
-        unsafe { KernelImageObject::new(FixedTables::zeroed()) };
-}
-
-// 2MiB for early mappings
-const EARLY_MAPPING_OFFSET: usize = CANONICAL_ADDRESS_MASK
-    | (KERNEL_L0_INDEX * L0::SIZE)
-    | (KERNEL_L1_INDEX * L1::SIZE)
-    | (EARLY_MAPPING_L2I * L2::SIZE);
-static mut EARLY_MAPPING_L3: PageTable<L3> = PageTable::zeroed();
-// 1GiB for device MMIO mapping
-const DEVICE_MAPPING_OFFSET: usize =
-    CANONICAL_ADDRESS_MASK | (KERNEL_L0_INDEX * L0::SIZE) | (DEVICE_MAPPING_L1I * L1::SIZE);
-static mut DEVICE_MAPPING_L2: PageTable<L2> = PageTable::zeroed();
-static mut DEVICE_MAPPING_L3S: [PageTable<L3>; DEVICE_MAPPING_L3_COUNT] =
-    [PageTable::zeroed(); DEVICE_MAPPING_L3_COUNT];
-// 512GiB for whole RAM mapping
-pub const RAM_MAPPING_OFFSET: usize = CANONICAL_ADDRESS_MASK | (RAM_MAPPING_L0I * L0::SIZE);
-pub static MEMORY_LIMIT: AtomicUsize = AtomicUsize::new(0);
-pub static mut RAM_MAPPING_L1: PageTable<L1> = PageTable::zeroed();
-
 impl KernelTableManager for KernelTableManagerImpl {
     fn virtualize(address: u64) -> usize {
         let address = address as usize;
-        if address < MEMORY_LIMIT.load(Ordering::Acquire) {
-            address + RAM_MAPPING_OFFSET
-        } else {
-            panic!("Invalid physical address: {:#x}", address);
+        if address >= IDENTITY_SIZE {
+            panic!("Invalid physical address to virtualize: {address:#x}");
         }
+        address + KERNEL_VIRT_OFFSET
     }
 
     fn physicalize(address: usize) -> u64 {
-        if address < RAM_MAPPING_OFFSET
-            || address - RAM_MAPPING_OFFSET >= MEMORY_LIMIT.load(Ordering::Acquire)
-        {
-            panic!("Not a virtualized physical address: {:#x}", address);
+        if address < KERNEL_VIRT_OFFSET || address - KERNEL_VIRT_OFFSET >= IDENTITY_SIZE {
+            panic!("Invalid virtualized address: {address:#x}");
         }
-
-        (address - RAM_MAPPING_OFFSET) as _
+        (address - KERNEL_VIRT_OFFSET) as _
     }
 
     unsafe fn map_device_pages(
@@ -100,245 +41,42 @@ impl KernelTableManager for KernelTableManagerImpl {
         count: usize,
         attrs: DeviceMemoryAttributes,
     ) -> Result<RawDeviceMemoryMapping<Self>, Error> {
-        map_device_memory(PhysicalAddress::from_u64(base), count, attrs)
+        let _guard = DEVICE_MEMORY_LOCK.lock();
+        #[allow(static_mut_refs)]
+        {
+            device::DEVICE_MEMORY.map_device_pages(PhysicalAddress::from_u64(base), count, attrs)
+        }
     }
 
     unsafe fn unmap_device_pages(mapping: &RawDeviceMemoryMapping<Self>) {
-        unmap_device_memory(mapping)
-    }
-}
-
-// Early mappings
-unsafe fn map_early_pages(physical: PhysicalAddress, count: usize) -> Result<usize, Error> {
-    for l3i in 0..512 {
-        let mut taken = false;
-        for i in 0..count {
-            if EARLY_MAPPING_L3[i + l3i].is_present() {
-                taken = true;
-                break;
-            }
-        }
-
-        if taken {
-            continue;
-        }
-
-        for i in 0..count {
-            // TODO NX, NC
-            EARLY_MAPPING_L3[i + l3i] =
-                PageEntry::page(physical.add(i * L3::SIZE), PageAttributes::WRITABLE);
-            flush_tlb_entry(EARLY_MAPPING_OFFSET + (i + l3i) * L3::SIZE);
-        }
-
-        return Ok(EARLY_MAPPING_OFFSET + l3i * L3::SIZE);
-    }
-
-    Err(Error::OutOfMemory)
-}
-
-unsafe fn unmap_early_page(address: usize) {
-    if !(EARLY_MAPPING_OFFSET..EARLY_MAPPING_OFFSET + L2::SIZE).contains(&address) {
-        panic!("Tried to unmap invalid early mapping: {:#x}", address);
-    }
-
-    let l3i = (address - EARLY_MAPPING_OFFSET).page_index::<L3>();
-
-    assert!(EARLY_MAPPING_L3[l3i].is_present());
-    EARLY_MAPPING_L3[l3i] = PageEntry::INVALID;
-}
-
-// Device mappings
-unsafe fn map_device_memory_l3(
-    base: PhysicalAddress,
-    count: usize,
-    _attrs: DeviceMemoryAttributes,
-) -> Result<usize, Error> {
-    // TODO don't map pages if already mapped
-
-    'l0: for i in 0..DEVICE_MAPPING_L3_COUNT * 512 {
-        for j in 0..count {
-            let l2i = (i + j) / 512;
-            let l3i = (i + j) % 512;
-
-            if DEVICE_MAPPING_L3S[l2i][l3i].is_present() {
-                continue 'l0;
-            }
-        }
-
-        for j in 0..count {
-            let l2i = (i + j) / 512;
-            let l3i = (i + j) % 512;
-
-            // TODO NX, NC
-            DEVICE_MAPPING_L3S[l2i][l3i] =
-                PageEntry::page(base.add(j * L3::SIZE), PageAttributes::WRITABLE);
-        }
-
-        return Ok(DEVICE_MAPPING_OFFSET + i * L3::SIZE);
-    }
-
-    Err(Error::OutOfMemory)
-}
-
-unsafe fn map_device_memory_l2(
-    base: PhysicalAddress,
-    count: usize,
-    _attrs: DeviceMemoryAttributes,
-) -> Result<usize, Error> {
-    'l0: for i in DEVICE_MAPPING_L3_COUNT..512 {
-        for j in 0..count {
-            if DEVICE_MAPPING_L2[i + j].is_present() {
-                continue 'l0;
-            }
-        }
-
-        for j in 0..count {
-            DEVICE_MAPPING_L2[i + j] =
-                PageEntry::<L2>::block(base.add(j * L2::SIZE), PageAttributes::WRITABLE);
-        }
-
-        return Ok(DEVICE_MAPPING_OFFSET + i * L2::SIZE);
-    }
-
-    Err(Error::OutOfMemory)
-}
-
-unsafe fn map_device_memory(
-    base: PhysicalAddress,
-    size: usize,
-    attrs: DeviceMemoryAttributes,
-) -> Result<RawDeviceMemoryMapping<KernelTableManagerImpl>, Error> {
-    let l3_aligned = base.page_align_down::<L3>();
-    let l3_offset = base.page_offset::<L3>();
-    let page_count = (l3_offset + size).page_count::<L3>();
-
-    if page_count > 256 {
-        // Large mapping, use L2 mapping instead
-        let l2_aligned = base.page_align_down::<L2>();
-        let l2_offset = base.page_offset::<L2>();
-        let page_count = (l2_offset + size).page_count::<L2>();
-
-        let base_address = map_device_memory_l2(l2_aligned, page_count, attrs)?;
-        let address = base_address + l2_offset;
-
-        Ok(RawDeviceMemoryMapping::from_raw_parts(
-            l2_aligned.into_u64(),
-            address,
-            base_address,
-            page_count,
-            L2::SIZE,
-        ))
-    } else {
-        // Just map the pages directly
-        let base_address = map_device_memory_l3(l3_aligned, page_count, attrs)?;
-        let address = base_address + l3_offset;
-
-        Ok(RawDeviceMemoryMapping::from_raw_parts(
-            l3_aligned.into_u64(),
-            address,
-            base_address,
-            page_count,
-            L3::SIZE,
-        ))
-    }
-}
-
-unsafe fn unmap_device_memory(map: &RawDeviceMemoryMapping<KernelTableManagerImpl>) {
-    match map.page_size {
-        L3::SIZE => {
-            for i in 0..map.page_count {
-                let page = map.base_address + i * L3::SIZE;
-                let l2i = page.page_index::<L2>();
-                let l3i = page.page_index::<L3>();
-                assert!(DEVICE_MAPPING_L3S[l2i][l3i].is_present());
-                DEVICE_MAPPING_L3S[l2i][l3i] = PageEntry::INVALID;
-                flush_tlb_entry(page);
-            }
-        }
-        L2::SIZE => todo!(),
-        _ => unimplemented!(),
-    }
-}
-
-/// Memory mapping which may be used for performing early kernel initialization
-pub struct EarlyMapping<'a, T: ?Sized> {
-    value: &'a mut T,
-    page_count: usize,
-}
-
-impl<'a, T: Sized> EarlyMapping<'a, T> {
-    /// # Safety
-    ///
-    /// `physical` address provided must be a valid non-NULL address actually containing `T`.
-    pub unsafe fn map(physical: PhysicalAddress) -> Result<EarlyMapping<'a, T>, Error> {
-        let layout = Layout::new::<T>();
-        let aligned = physical.page_align_down::<L3>();
-        let offset = physical.page_offset::<L3>();
-        let page_count = (offset + layout.size()).div_ceil(L3::SIZE);
-
-        let virt = map_early_pages(aligned, page_count)?;
-        let value = &mut *((virt + offset) as *mut T);
-
-        Ok(EarlyMapping { value, page_count })
-    }
-
-    /// # Safety
-    ///
-    /// `physical` address provided must be a valid non-NULL address actually containing a `T`
-    /// slice of given `len`.
-    pub unsafe fn map_slice(
-        physical: PhysicalAddress,
-        len: usize,
-    ) -> Result<EarlyMapping<'a, [T]>, Error> {
-        let layout = Layout::array::<T>(len).unwrap();
-        let aligned = physical.page_align_down::<L3>();
-        let offset = physical.page_offset::<L3>();
-        let page_count = (offset + layout.size()).div_ceil(L3::SIZE);
-
-        let virt = map_early_pages(aligned, page_count)?;
-        let value = core::slice::from_raw_parts_mut((virt + offset) as *mut T, len);
-
-        Ok(EarlyMapping { value, page_count })
-    }
-}
-
-impl<T: ?Sized> Deref for EarlyMapping<'_, T> {
-    type Target = T;
-
-    fn deref(&self) -> &Self::Target {
-        self.value
-    }
-}
-
-impl<T: ?Sized> DerefMut for EarlyMapping<'_, T> {
-    fn deref_mut(&mut self) -> &mut Self::Target {
-        self.value
-    }
-}
-
-impl<T: ?Sized> Drop for EarlyMapping<'_, T> {
-    fn drop(&mut self) {
-        let address = (self.value as *mut T).addr() & !(L3::SIZE - 1);
-
-        for i in 0..self.page_count {
-            let page = address + i * L3::SIZE;
-
-            unsafe {
-                unmap_early_page(page);
-            }
+        let _guard = DEVICE_MEMORY_LOCK.lock();
+        #[allow(static_mut_refs)]
+        {
+            device::DEVICE_MEMORY.unmap_device_pages(mapping);
         }
     }
 }
 
 pub fn clone_kernel_tables(dst: &mut PageTable<L0>) {
-    let tables = KERNEL_TABLES.lock();
     unsafe {
-        dst[KERNEL_L0_INDEX] = PageEntry::from_raw(tables.l0.data[KERNEL_L0_INDEX]);
-        dst[RAM_MAPPING_L0I] = PageEntry::from_raw(tables.l0.data[RAM_MAPPING_L0I]);
+        let kernel_l0i = KERNEL_VIRT_OFFSET.page_index::<L0>();
+        dst[kernel_l0i] = FIXED_PML4[kernel_l0i];
     }
 }
 
-pub fn auto_address<T>(pointer: *const T) -> usize {
+const FIXED_PD_COUNT: usize = 32;
+const IDENTITY_SIZE: usize = FIXED_PD_COUNT * L1::SIZE;
+
+const DEVICE_MAPPING_L1I: usize = FIXED_PD_COUNT;
+const DEVICE_MEMORY_L3_COUNT: usize = 16;
+const DEVICE_MAPPING_OFFSET: usize = KERNEL_VIRT_OFFSET + IDENTITY_SIZE;
+
+static DEVICE_MEMORY_LOCK: IrqSafeSpinlock<ArchitectureImpl, ()> = IrqSafeSpinlock::new(());
+static mut FIXED_PDS: [PageTable<L2>; FIXED_PD_COUNT] = [PageTable::zeroed(); FIXED_PD_COUNT];
+static mut FIXED_PDPT: PageTable<L1> = PageTable::zeroed();
+pub static mut FIXED_PML4: PageTable<L0> = PageTable::zeroed();
+
+pub fn auto_lower_address<T>(pointer: *const T) -> usize {
     let address = pointer.addr();
     if address < KERNEL_VIRT_OFFSET {
         address
@@ -347,53 +85,80 @@ pub fn auto_address<T>(pointer: *const T) -> usize {
     }
 }
 
-/// Sets up the following memory map:
-/// ...: KERNEL_TABLES.l0:
-///     * 0xFFFFFF0000000000 .. 0xFFFFFFFF8000000000        : RAM_MAPPING_L1
-///     * 0xFFFFFF8000000000 .. ...                         : KERNEL_TABLES.kernel_l1:
-///         * 0xFFFFFF8000000000 .. 0xFFFFFF8040000000      : KERNEL_TABLES.kernel_l2
-///             * 0xFFFFFF8000000000 .. 0xFFFFFF8000200000  : ---
-///             * 0xFFFFFF8000200000 .. 0xFFFFFF8000400000  : EARLY_MAPPING_L3
-///             * 0xFFFFFF8000400000 .. ...                 : KERNEL_TABLES.kernel_l3s
-///         * 0xFFFFFF8040000000 .. 0xFFFFFF8080000000      : ---
-///         * 0xFFFFFF8080000000 .. 0xFFFFFF8100000000      : DEVICE_MAPPING_L2
-///             * 0xFFFFFF8080000000 .. 0xFFFFFF8080800000  : DEVICE_MAPPING_L3S
-///             * 0xFFFFFF8080800000 .. 0xFFFFFF8100000000  : ...
-///
-/// # Safety
-///
-/// Unsafe, must only be called by BSP during its early init, must already be in "higher-half"
-pub unsafe fn init_fixed_tables() {
-    let mut tables = KERNEL_TABLES.lock();
-
-    // TODO this could be built in compile-time too?
-    let early_mapping_l3_phys = auto_address(&raw const EARLY_MAPPING_L3);
-    let device_mapping_l2_phys = auto_address(&raw const DEVICE_MAPPING_L2);
-    let ram_mapping_l1_phys = auto_address(&raw const RAM_MAPPING_L1);
-
-    for i in 0..DEVICE_MAPPING_L3_COUNT {
-        let device_mapping_l3_phys =
-            PhysicalAddress::from_usize(auto_address(&raw const DEVICE_MAPPING_L3S[i]));
-        DEVICE_MAPPING_L2[i] = PageEntry::table(device_mapping_l3_phys, PageAttributes::WRITABLE);
+pub unsafe fn init_fixed_tables(have_1gib_pages: bool) {
+    if have_1gib_pages {
+        for i in 0..IDENTITY_SIZE / L1::SIZE {
+            FIXED_PDPT[i] = PageEntry::<L1>::block(
+                PhysicalAddress::from_usize(i * L1::SIZE),
+                PageAttributes::WRITABLE,
+            );
+        }
+    } else {
+        for i in 0..IDENTITY_SIZE / L1::SIZE {
+            for j in 0..512 {
+                FIXED_PDS[i][j] = PageEntry::<L2>::block(
+                    PhysicalAddress::from_usize(i * L1::SIZE + j * L2::SIZE),
+                    PageAttributes::WRITABLE,
+                );
+            }
+            let pd_physical =
+                PhysicalAddress::from_usize(auto_lower_address(&raw const FIXED_PDS[i]));
+            FIXED_PDPT[i] = PageEntry::table(pd_physical, PageAttributes::WRITABLE);
+        }
     }
 
-    assert_eq!(tables.kernel_l2.data[EARLY_MAPPING_L2I], 0);
-    tables.kernel_l2.data[EARLY_MAPPING_L2I] = (early_mapping_l3_phys as u64)
-        | (PageAttributes::WRITABLE | PageAttributes::PRESENT).bits();
+    // Device memory
+    let device_pd_physical =
+        PhysicalAddress::from_usize(auto_lower_address(&raw const device::DEVICE_MEMORY.large));
 
-    assert_eq!(tables.kernel_l1.data[DEVICE_MAPPING_L1I], 0);
-    tables.kernel_l1.data[DEVICE_MAPPING_L1I] = (device_mapping_l2_phys as u64)
-        | (PageAttributes::WRITABLE | PageAttributes::PRESENT).bits();
+    for i in 0..DEVICE_MEMORY_L3_COUNT {
+        let device_pt_physical = PhysicalAddress::from_usize(auto_lower_address(
+            &raw const device::DEVICE_MEMORY.normal.0[i],
+        ));
+        device::DEVICE_MEMORY.large.0[i] =
+            PageEntry::table(device_pt_physical, PageAttributes::WRITABLE);
+    }
 
-    assert_eq!(tables.l0.data[RAM_MAPPING_L0I], 0);
-    tables.l0.data[RAM_MAPPING_L0I] =
-        (ram_mapping_l1_phys as u64) | (PageAttributes::WRITABLE | PageAttributes::PRESENT).bits();
+    FIXED_PDPT[DEVICE_MAPPING_L1I] =
+        PageEntry::<L1>::table(device_pd_physical, PageAttributes::WRITABLE);
 
-    // TODO ENABLE EFER.NXE
-    let cr3 = auto_address(&raw const tables.l0);
-    CR3.set_address(cr3);
+    let pdpt_physical = PhysicalAddress::from_usize(auto_lower_address(&raw const FIXED_PDPT));
+
+    FIXED_PML4[KERNEL_VIRT_OFFSET.page_index::<L0>()] =
+        PageEntry::table(pdpt_physical, PageAttributes::WRITABLE);
+
+    let pml4_physical = auto_lower_address(&raw const FIXED_PML4);
+    CR3.set_address(pml4_physical);
 }
-
+//     let mut tables = KERNEL_TABLES.lock();
+//
+//     // TODO this could be built in compile-time too?
+//     let early_mapping_l3_phys = auto_address(&raw const EARLY_MAPPING_L3);
+//     let device_mapping_l2_phys = auto_address(&raw const DEVICE_MAPPING_L2);
+//     let ram_mapping_l1_phys = auto_address(&raw const RAM_MAPPING_L1);
+//
+//     for i in 0..DEVICE_MAPPING_L3_COUNT {
+//         let device_mapping_l3_phys =
+//             PhysicalAddress::from_usize(auto_address(&raw const DEVICE_MAPPING_L3S[i]));
+//         DEVICE_MAPPING_L2[i] = PageEntry::table(device_mapping_l3_phys, PageAttributes::WRITABLE);
+//     }
+//
+//     assert_eq!(tables.kernel_l2.data[EARLY_MAPPING_L2I], 0);
+//     tables.kernel_l2.data[EARLY_MAPPING_L2I] = (early_mapping_l3_phys as u64)
+//         | (PageAttributes::WRITABLE | PageAttributes::PRESENT).bits();
+//
+//     assert_eq!(tables.kernel_l1.data[DEVICE_MAPPING_L1I], 0);
+//     tables.kernel_l1.data[DEVICE_MAPPING_L1I] = (device_mapping_l2_phys as u64)
+//         | (PageAttributes::WRITABLE | PageAttributes::PRESENT).bits();
+//
+//     assert_eq!(tables.l0.data[RAM_MAPPING_L0I], 0);
+//     tables.l0.data[RAM_MAPPING_L0I] =
+//         (ram_mapping_l1_phys as u64) | (PageAttributes::WRITABLE | PageAttributes::PRESENT).bits();
+//
+//     // TODO ENABLE EFER.NXE
+//     let cr3 = auto_address(&raw const tables.l0);
+// }
+//
 /// # Safety
 ///
 /// `address` must be page-aligned.

kernel/driver/block/ahci/src/port.rs

@@ -235,7 +235,7 @@ impl AhciPort {
         .await
     }
 
-    async fn submit<C: AtaCommand>(&self, command: &C) -> Result<SubmittedCommand, AhciError> {
+    async fn submit<C: AtaCommand>(&self, command: &C) -> Result<SubmittedCommand<'_>, AhciError> {
         if command.prd_count() > 2 {
             log::warn!("TODO: AHCI doesn't like 3+ PRD transfers");
             return Err(AhciError::RegionTooLarge);

kernel/driver/bus/pci/src/capability.rs

@@ -1,21 +1,91 @@
-use alloc::{sync::Arc, vec, vec::Vec};
+//! PCI capability structures and queries
 
+use alloc::{sync::Arc, vec, vec::Vec};
+use bitflags::bitflags;
 use device_api::interrupt::{
     InterruptAffinity, InterruptHandler, MessageInterruptController, MsiInfo,
 };
-use libk::error::Error;
 use libk_mm::{address::PhysicalAddress, device::DeviceMemoryIoMut};
 use tock_registers::{
     interfaces::{Readable, Writeable},
     registers::{ReadWrite, WriteOnly},
 };
+use yggdrasil_abi::error::Error;
+
+use crate::PciBaseAddress;
+
+use super::{PciCapability, PciCapabilityId, PciConfigurationSpace};
+
+bitflags! {
+    pub struct PcieLinkControl: u16 {
+        const ASPM_DISABLE = 0 << 0;
+        // Active state power management control
+        const ASPM_MASK = 0x3 << 0;
+        // Enable clock power management
+        const ECPM = 1 << 8;
+    }
+}
 
 #[cfg(any(target_arch = "x86", target_arch = "x86_64", rust_analyzer))]
 use core::mem::offset_of;
 #[cfg(any(target_arch = "x86", target_arch = "x86_64", rust_analyzer))]
 use kernel_arch_x86::intrinsics;
 
-use crate::{PciBaseAddress, PciCapability, PciCapabilityId, PciConfigurationSpace};
+pub trait VirtioCapabilityData<'s, S: PciConfigurationSpace + ?Sized + 's>: Sized {
+    fn from_space_offset(space: &'s S, offset: usize) -> Self;
+
+    fn space(&self) -> &'s S;
+    fn offset(&self) -> usize;
+
+    fn bar_index(&self) -> Option<usize> {
+        let value = self.space().read_u8(self.offset() + 4);
+        (value <= 0x5).then_some(value as _)
+    }
+
+    fn bar_offset(&self) -> usize {
+        let value = self.space().read_u32(self.offset() + 8);
+        value as _
+    }
+
+    fn length(&self) -> usize {
+        let value = self.space().read_u32(self.offset() + 12);
+        value as _
+    }
+}
+
+pub trait VirtioCapability {
+    const CFG_TYPE: u8;
+    const MIN_LEN: usize = 0;
+    type Output<'a, S: PciConfigurationSpace + ?Sized + 'a>: VirtioCapabilityData<'a, S>;
+}
+
+/// Power management capability entry
+pub struct PowerManagementCapability;
+/// MSI-X capability query
+pub struct MsiXCapability;
+/// MSI capability query
+pub struct MsiCapability;
+/// PCIe capability
+pub struct PciExpressCapability;
+
+// VirtIO-over-PCI capabilities
+/// VirtIO PCI configuration access
+pub struct VirtioDeviceConfigCapability;
+/// VirtIO common configuration
+pub struct VirtioCommonConfigCapability;
+/// VirtIO notify configuration
+pub struct VirtioNotifyConfigCapability;
+/// VirtIO interrupt status
+pub struct VirtioInterruptStatusCapability;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum DevicePowerState {
+    D0,
+    D1,
+    D2,
+    D3Cold,
+    D3Hot,
+}
 
 /// Represents an entry in MSI-X vector table
 #[repr(C)]
@@ -39,10 +109,11 @@ pub struct MsiXVectorTable<'a> {
     len: usize,
 }
 
-/// MSI-X capability query
-pub struct MsiXCapability;
-/// MSI capability query
-pub struct MsiCapability;
+/// PCI Power Management capability data structure
+pub struct PowerManagementData<'s, S: PciConfigurationSpace + ?Sized + 's> {
+    space: &'s S,
+    offset: usize,
+}
 
 /// MSI-X capability data structure
 pub struct MsiXData<'s, S: PciConfigurationSpace + ?Sized + 's> {
@@ -56,6 +127,63 @@ pub struct MsiData<'s, S: PciConfigurationSpace + ?Sized + 's> {
     offset: usize,
 }
 
+/// PCI Express capability data structure
+pub struct PcieData<'s, S: PciConfigurationSpace + ?Sized + 's> {
+    space: &'s S,
+    offset: usize,
+}
+
+pub struct VirtioDeviceConfigData<'s, S: PciConfigurationSpace + ?Sized + 's> {
+    space: &'s S,
+    offset: usize,
+}
+
+pub struct VirtioCommonConfigData<'s, S: PciConfigurationSpace + ?Sized + 's> {
+    space: &'s S,
+    offset: usize,
+}
+
+pub struct VirtioNotifyConfigData<'s, S: PciConfigurationSpace + ?Sized + 's> {
+    space: &'s S,
+    offset: usize,
+}
+
+pub struct VirtioInterruptStatusData<'s, S: PciConfigurationSpace + ?Sized + 's> {
+    space: &'s S,
+    offset: usize,
+}
+
+impl<T: VirtioCapability> PciCapability for T {
+    const ID: PciCapabilityId = PciCapabilityId::VendorSpecific;
+    type CapabilityData<'a, S: PciConfigurationSpace + ?Sized + 'a> = T::Output<'a, S>;
+
+    fn check<S: PciConfigurationSpace + ?Sized>(space: &S, offset: usize, len: usize) -> bool {
+        let cfg_type = space.read_u8(offset + 3);
+        cfg_type == T::CFG_TYPE && len >= T::MIN_LEN
+    }
+
+    fn data<'s, S: PciConfigurationSpace + ?Sized + 's>(
+        space: &'s S,
+        offset: usize,
+        _len: usize,
+    ) -> Self::CapabilityData<'s, S> {
+        T::Output::from_space_offset(space, offset)
+    }
+}
+
+impl PciCapability for PowerManagementCapability {
+    const ID: PciCapabilityId = PciCapabilityId::PowerManagement;
+    type CapabilityData<'a, S: PciConfigurationSpace + ?Sized + 'a> = PowerManagementData<'a, S>;
+
+    fn data<'s, S: PciConfigurationSpace + ?Sized + 's>(
+        space: &'s S,
+        offset: usize,
+        _len: usize,
+    ) -> Self::CapabilityData<'s, S> {
+        PowerManagementData { space, offset }
+    }
+}
+
 impl PciCapability for MsiXCapability {
     const ID: PciCapabilityId = PciCapabilityId::MsiX;
     type CapabilityData<'a, S: PciConfigurationSpace + ?Sized + 'a> = MsiXData<'a, S>;
@@ -82,6 +210,167 @@ impl PciCapability for MsiCapability {
     }
 }
 
+impl PciCapability for PciExpressCapability {
+    const ID: PciCapabilityId = PciCapabilityId::PciExpress;
+    type CapabilityData<'a, S: PciConfigurationSpace + ?Sized + 'a> = PcieData<'a, S>;
+
+    fn data<'s, S: PciConfigurationSpace + ?Sized + 's>(
+        space: &'s S,
+        offset: usize,
+        _len: usize,
+    ) -> Self::CapabilityData<'s, S> {
+        PcieData { space, offset }
+    }
+}
+
+impl VirtioCapability for VirtioDeviceConfigCapability {
+    const CFG_TYPE: u8 = 0x04;
+    type Output<'a, S: PciConfigurationSpace + ?Sized + 'a> = VirtioDeviceConfigData<'a, S>;
+}
+
+impl<'s, S: PciConfigurationSpace + ?Sized + 's> VirtioCapabilityData<'s, S>
+    for VirtioDeviceConfigData<'s, S>
+{
+    fn from_space_offset(space: &'s S, offset: usize) -> Self {
+        Self { space, offset }
+    }
+
+    fn space(&self) -> &'s S {
+        self.space
+    }
+
+    fn offset(&self) -> usize {
+        self.offset
+    }
+}
+
+impl VirtioCapability for VirtioCommonConfigCapability {
+    const CFG_TYPE: u8 = 0x01;
+    type Output<'a, S: PciConfigurationSpace + ?Sized + 'a> = VirtioCommonConfigData<'a, S>;
+}
+
+impl<'s, S: PciConfigurationSpace + ?Sized + 's> VirtioCapabilityData<'s, S>
+    for VirtioCommonConfigData<'s, S>
+{
+    fn from_space_offset(space: &'s S, offset: usize) -> Self {
+        Self { space, offset }
+    }
+
+    fn space(&self) -> &'s S {
+        self.space
+    }
+
+    fn offset(&self) -> usize {
+        self.offset
+    }
+}
+
+impl VirtioCapability for VirtioNotifyConfigCapability {
+    const CFG_TYPE: u8 = 0x02;
+    const MIN_LEN: usize = 0x14;
+    type Output<'a, S: PciConfigurationSpace + ?Sized + 'a> = VirtioNotifyConfigData<'a, S>;
+}
+
+impl<'s, S: PciConfigurationSpace + ?Sized + 's> VirtioNotifyConfigData<'s, S> {
+    pub fn offset_multiplier(&self) -> usize {
+        self.space.read_u32(self.offset + 16) as usize
+    }
+}
+
+impl<'s, S: PciConfigurationSpace + ?Sized + 's> VirtioCapabilityData<'s, S>
+    for VirtioNotifyConfigData<'s, S>
+{
+    fn from_space_offset(space: &'s S, offset: usize) -> Self {
+        Self { space, offset }
+    }
+
+    fn space(&self) -> &'s S {
+        self.space
+    }
+
+    fn offset(&self) -> usize {
+        self.offset
+    }
+}
+
+impl VirtioCapability for VirtioInterruptStatusCapability {
+    const CFG_TYPE: u8 = 0x03;
+    const MIN_LEN: usize = 1;
+    type Output<'a, S: PciConfigurationSpace + ?Sized + 'a> = VirtioInterruptStatusData<'a, S>;
+}
+
+impl<'s, S: PciConfigurationSpace + ?Sized + 's> VirtioInterruptStatusData<'s, S> {
+    pub fn read_status(&self) -> (bool, bool) {
+        todo!()
+    }
+}
+
+impl<'s, S: PciConfigurationSpace + ?Sized + 's> VirtioCapabilityData<'s, S>
+    for VirtioInterruptStatusData<'s, S>
+{
+    fn from_space_offset(space: &'s S, offset: usize) -> Self {
+        Self { space, offset }
+    }
+
+    fn space(&self) -> &'s S {
+        self.space
+    }
+
+    fn offset(&self) -> usize {
+        self.offset
+    }
+}
+
+impl<'s, S: PciConfigurationSpace + ?Sized + 's> PowerManagementData<'s, S> {
+    pub fn set_device_power_state(&self, state: DevicePowerState) {
+        let pmcsr = self.space.read_u16(self.offset + 4) & !0x3;
+        let current = self.get_device_power_state();
+
+        if state == current {
+            return;
+        }
+
+        log::info!("Set device power state: {state:?}");
+
+        match state {
+            DevicePowerState::D0 => {
+                // power = 0b00 | PME_EN
+                self.space.write_u16(self.offset + 4, pmcsr);
+            }
+            _ => {
+                log::warn!("TODO: {state:?} power state");
+            }
+        }
+    }
+
+    pub fn set_pme_en(&self, state: bool) {
+        let pmcsr = self.space.read_u16(self.offset + 4);
+        let new = if state {
+            pmcsr | (1 << 8)
+        } else {
+            pmcsr & !(1 << 8)
+        };
+        if pmcsr == new {
+            return;
+        }
+
+        log::info!("Set PMCSR.PME_En = {state}");
+
+        self.space.write_u16(self.offset + 4, new);
+    }
+
+    pub fn get_device_power_state(&self) -> DevicePowerState {
+        let pmcsr = self.space.read_u16(self.offset + 4);
+        match pmcsr & 0x3 {
+            0b00 => DevicePowerState::D0,
+            0b01 => DevicePowerState::D1,
+            0b10 => DevicePowerState::D2,
+            0b11 => DevicePowerState::D3Hot,
+            _ => unreachable!(),
+        }
+    }
+}
+
 impl<'s, S: PciConfigurationSpace + ?Sized + 's> MsiXData<'s, S> {
     // TODO use pending bits as well
     /// Maps and returns the vector table associated with the device's MSI-X capability
@@ -312,3 +601,13 @@ impl<'s, S: PciConfigurationSpace + ?Sized + 's> MsiData<'s, S> {
         Ok(info)
     }
 }
+
+impl<'s, S: PciConfigurationSpace + ?Sized + 's> PcieData<'s, S> {
+    pub fn link_control(&self) -> PcieLinkControl {
+        PcieLinkControl::from_bits_retain(self.space.read_u16(self.offset + 0x10))
+    }
+
+    pub fn set_link_control(&mut self, value: PcieLinkControl) {
+        self.space.write_u16(self.offset + 0x10, value.bits());
+    }
+}

kernel/driver/bus/pci/src/capability/express.rs

@@ -1,177 +0,0 @@
-use core::time::Duration;
-
-use libk::{error::Error, task::runtime::psleep};
-use tock_registers::{fields::FieldValue, register_bitfields, LocalRegisterCopy};
-
-use crate::{PciCapability, PciCapabilityId, PciConfigurationSpace};
-
-// bitflags! {
-//     pub struct PcieLinkControl: u16 {
-//         const ASPM_DISABLE = 0 << 0;
-//         // Active state power management control
-//         const ASPM_MASK = 0x3 << 0;
-//         // Enable clock power management
-//         const ECPM = 1 << 8;
-//     }
-// }
-
-register_bitfields! {
-    u32,
-    pub DeviceCapabilities [
-        MAX_PAYLOAD_SIZE            OFFSET(0)   NUMBITS(3)  [],
-        PHANTOM_FUNCTIONS           OFFSET(3)   NUMBITS(2)  [],
-        L0S_ACCEPTABLE_LATENCY      OFFSET(6)   NUMBITS(3)  [],
-        L1_ACCEPTABLE_LATENCY       OFFSET(9)   NUMBITS(3)  [],
-        ROLE_ERROR_REPORTING        OFFSET(15)  NUMBITS(1)  [],
-        CAPTURED_SLOT_POWER_LIMIT   OFFSET(18)  NUMBITS(8)  [],
-        CAPTURED_SLOT_POWER_SCALE   OFFSET(26)  NUMBITS(3)  [],
-        FUNCTION_LEVEL_RESET        OFFSET(28)  NUMBITS(1)  [],
-    ],
-    pub LinkCapabilities [
-        MAX_LINK_SPEED              OFFSET(0)   NUMBITS(4)  [],
-        MAX_LINK_WIDTH              OFFSET(4)   NUMBITS(6)  [],
-        ASPM                        OFFSET(10)  NUMBITS(2)  [],
-        L0S_EXIT_LATENCY            OFFSET(12)  NUMBITS(3)  [],
-        L1_EXIT_LATENCY             OFFSET(15)  NUMBITS(3)  [],
-        CLOCK_PM                    OFFSET(18)  NUMBITS(1)  [],
-        SURPRISE_DOWN_ERROR         OFFSET(19)  NUMBITS(1)  [],
-        DATA_LINK_ACTIVE_REPORTING  OFFSET(20)  NUMBITS(1)  [],
-        LINK_BANDWIDTH_NOTIFICATION OFFSET(21)  NUMBITS(1)  [],
-        ASPM_COMPLIANCE             OFFSET(22)  NUMBITS(1)  [],
-        PORT_NUMBER                 OFFSET(24)  NUMBITS(8)  [],
-    ],
-}
-
-register_bitfields! {
-    u16,
-    pub DeviceControl [
-        CORRECTABLE_ERROR_REPORTING OFFSET(0)   NUMBITS(1)  [],
-        NONFATAL_ERROR_REPORTING    OFFSET(1)   NUMBITS(1)  [],
-        FATAL_ERROR_REPORTING       OFFSET(2)   NUMBITS(1)  [],
-        UNSUPPORTED_REQ_REPORTING   OFFSET(3)   NUMBITS(1)  [],
-        RELAXED_ORDERING            OFFSET(4)   NUMBITS(1)  [],
-        MAX_PAYLOAD_SIZE            OFFSET(5)   NUMBITS(3)  [],
-        EXTENDED_TAG_FIELD          OFFSET(8)   NUMBITS(1)  [],
-        PHANTOM_FUNCTIONS           OFFSET(9)   NUMBITS(1)  [],
-        AUX_POWER_PM_ENABLE         OFFSET(10)  NUMBITS(1)  [],
-        NO_SNOOP                    OFFSET(11)  NUMBITS(1)  [],
-        MAX_READ_REQ_SIZE           OFFSET(12)  NUMBITS(3)  [],
-        FUNCTION_LEVEL_RESET        OFFSET(15)  NUMBITS(1)  [],
-    ],
-    pub LinkControl [
-        ASPM                        OFFSET(0)   NUMBITS(2)  [
-            Disabled = 0,
-            L0sEntryEnabled = 1,
-            L1EntryEnabled = 2,
-            L0sL1EntryEnabled = 3,
-        ],
-        READ_COMPLETION_BOUNDARY    OFFSET(3)   NUMBITS(1)  [],
-        LINK_DISABLE                OFFSET(4)   NUMBITS(1)  [],
-        RETRAIN_LINK                OFFSET(5)   NUMBITS(1)  [],
-        COMMON_CLOCK_CONFIG         OFFSET(6)   NUMBITS(1)  [],
-        EXTENDED_SYNCH              OFFSET(7)   NUMBITS(1)  [],
-        CLOCK_PM                    OFFSET(8)   NUMBITS(1)  [],
-        HARDWARE_ABW_DISABLE        OFFSET(9)   NUMBITS(1)  [],
-        LINK_BW_MANAGEMENT_IRQ      OFFSET(10)  NUMBITS(1)  [],
-        LINK_ABW_IRQ                OFFSET(11)  NUMBITS(1)  [],
-    ],
-}
-
-/// PCIe capability
-pub struct PciExpressCapability;
-
-/// PCI Express capability data structure
-pub struct PciExpressData<'s, S: PciConfigurationSpace + ?Sized + 's> {
-    space: &'s S,
-    offset: usize,
-}
-
-impl PciCapability for PciExpressCapability {
-    const ID: PciCapabilityId = PciCapabilityId::PciExpress;
-    type CapabilityData<'a, S: PciConfigurationSpace + ?Sized + 'a> = PciExpressData<'a, S>;
-
-    fn data<'s, S: PciConfigurationSpace + ?Sized + 's>(
-        space: &'s S,
-        offset: usize,
-        _len: usize,
-    ) -> Self::CapabilityData<'s, S> {
-        PciExpressData { space, offset }
-    }
-}
-
-macro_rules! reg_read {
-    ($self:expr, $offset:literal, u32) => {
-        $self.space.read_u32($self.offset + $offset)
-    };
-    ($self:expr, $offset:literal, u16) => {
-        $self.space.read_u16($self.offset + $offset)
-    };
-}
-macro_rules! reg_write {
-    ($self:expr, $offset:literal, u32, $value:expr) => {
-        $self.space.write_u32($self.offset + $offset, $value)
-    };
-    ($self:expr, $offset:literal, u16, $value:expr) => {
-        $self.space.write_u16($self.offset + $offset, $value)
-    };
-}
-macro_rules! make_register {
-    (
-        $reg:ident : $ty:ident @ $offset:literal {
-            $get:ident
-            $(, $set:ident, $modify:ident)?
-            $(,)?
-        }
-    ) => {
-        pub fn $get(&self) -> LocalRegisterCopy<$ty, $reg::Register> {
-            LocalRegisterCopy::new(reg_read!(self, $offset, $ty))
-        }
-
-        $(
-            pub fn $set(&mut self, value: $ty) {
-                reg_write!(self, $offset, $ty, value)
-            }
-
-            pub fn $modify(&mut self, field: FieldValue<$ty, $reg::Register>) {
-                let mut value = self.$get();
-                value.modify(field);
-                self.$set(value.get());
-            }
-        )?
-    };
-}
-
-impl<'s, S: PciConfigurationSpace + ?Sized + 's> PciExpressData<'s, S> {
-    make_register!(DeviceCapabilities : u32 @ 0x04 { device_capabilities });
-    make_register!(DeviceControl : u16 @ 0x08 {
-        device_control,
-        set_device_control,
-        modify_device_control,
-    });
-    make_register!(LinkCapabilities : u32 @ 0x0C { link_capabilities });
-    make_register!(LinkControl : u16 @ 0x10 {
-        link_control,
-        set_link_control,
-        modify_link_control,
-    });
-
-    pub fn function_level_reset(&mut self) -> Result<(), Error> {
-        if self
-            .device_capabilities()
-            .matches_all(DeviceCapabilities::FUNCTION_LEVEL_RESET::SET)
-        {
-            self.modify_device_control(DeviceControl::FUNCTION_LEVEL_RESET::SET);
-            psleep(Duration::from_millis(10));
-            Ok(())
-        } else {
-            Err(Error::NotImplemented)
-        }
-    }
-
-    pub fn hot_link_reset(&mut self) {
-        self.modify_link_control(LinkControl::LINK_DISABLE::SET);
-        psleep(Duration::from_millis(10));
-        self.modify_link_control(LinkControl::LINK_DISABLE::CLEAR);
-        psleep(Duration::from_millis(100));
-    }
-}

kernel/driver/bus/pci/src/capability/mod.rs

@@ -1,14 +0,0 @@
-//! PCI capability structures and queries
-
-pub mod express;
-pub mod msi;
-pub mod power;
-pub mod virtio;
-
-pub use express::PciExpressCapability;
-pub use msi::{MsiCapability, MsiXCapability};
-pub use power::PowerManagementCapability;
-pub use virtio::{
-    VirtioCapability, VirtioCommonConfigCapability, VirtioDeviceConfigCapability,
-    VirtioInterruptStatusCapability, VirtioNotifyConfigCapability,
-};

kernel/driver/bus/pci/src/capability/power.rs

@@ -1,82 +0,0 @@
-use crate::{PciCapability, PciCapabilityId, PciConfigurationSpace};
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum DevicePowerState {
-    D0,
-    D1,
-    D2,
-    D3Cold,
-    D3Hot,
-}
-
-/// Power management capability entry
-pub struct PowerManagementCapability;
-
-/// PCI Power Management capability data structure
-pub struct PowerManagementData<'s, S: PciConfigurationSpace + ?Sized + 's> {
-    space: &'s S,
-    offset: usize,
-}
-
-impl PciCapability for PowerManagementCapability {
-    const ID: PciCapabilityId = PciCapabilityId::PowerManagement;
-    type CapabilityData<'a, S: PciConfigurationSpace + ?Sized + 'a> = PowerManagementData<'a, S>;
-
-    fn data<'s, S: PciConfigurationSpace + ?Sized + 's>(
-        space: &'s S,
-        offset: usize,
-        _len: usize,
-    ) -> Self::CapabilityData<'s, S> {
-        PowerManagementData { space, offset }
-    }
-}
-
-impl<'s, S: PciConfigurationSpace + ?Sized + 's> PowerManagementData<'s, S> {
-    pub fn set_device_power_state(&self, state: DevicePowerState) {
-        let pmcsr = self.space.read_u16(self.offset + 4) & !0x3;
-        let current = self.get_device_power_state();
-
-        if state == current {
-            return;
-        }
-
-        log::info!("Set device power state: {state:?}");
-
-        match state {
-            DevicePowerState::D0 => {
-                // power = 0b00 | PME_EN
-                self.space.write_u16(self.offset + 4, pmcsr);
-            }
-            _ => {
-                log::warn!("TODO: {state:?} power state");
-            }
-        }
-    }
-
-    pub fn set_pme_en(&self, state: bool) {
-        let pmcsr = self.space.read_u16(self.offset + 4);
-        let new = if state {
-            pmcsr | (1 << 8)
-        } else {
-            pmcsr & !(1 << 8)
-        };
-        if pmcsr == new {
-            return;
-        }
-
-        log::info!("Set PMCSR.PME_En = {state}");
-
-        self.space.write_u16(self.offset + 4, new);
-    }
-
-    pub fn get_device_power_state(&self) -> DevicePowerState {
-        let pmcsr = self.space.read_u16(self.offset + 4);
-        match pmcsr & 0x3 {
-            0b00 => DevicePowerState::D0,
-            0b01 => DevicePowerState::D1,
-            0b10 => DevicePowerState::D2,
-            0b11 => DevicePowerState::D3Hot,
-            _ => unreachable!(),
-        }
-    }
-}

kernel/driver/bus/pci/src/capability/virtio.rs

@@ -1,175 +0,0 @@
-use crate::{PciCapability, PciCapabilityId, PciConfigurationSpace};
-
-pub trait VirtioCapabilityData<'s, S: PciConfigurationSpace + ?Sized + 's>: Sized {
-    fn from_space_offset(space: &'s S, offset: usize) -> Self;
-
-    fn space(&self) -> &'s S;
-    fn offset(&self) -> usize;
-
-    fn bar_index(&self) -> Option<usize> {
-        let value = self.space().read_u8(self.offset() + 4);
-        (value <= 0x5).then_some(value as _)
-    }
-
-    fn bar_offset(&self) -> usize {
-        let value = self.space().read_u32(self.offset() + 8);
-        value as _
-    }
-
-    fn length(&self) -> usize {
-        let value = self.space().read_u32(self.offset() + 12);
-        value as _
-    }
-}
-
-pub trait VirtioCapability {
-    const CFG_TYPE: u8;
-    const MIN_LEN: usize = 0;
-    type Output<'a, S: PciConfigurationSpace + ?Sized + 'a>: VirtioCapabilityData<'a, S>;
-}
-
-// VirtIO-over-PCI capabilities
-/// VirtIO PCI configuration access
-pub struct VirtioDeviceConfigCapability;
-/// VirtIO common configuration
-pub struct VirtioCommonConfigCapability;
-/// VirtIO notify configuration
-pub struct VirtioNotifyConfigCapability;
-/// VirtIO interrupt status
-pub struct VirtioInterruptStatusCapability;
-
-pub struct VirtioDeviceConfigData<'s, S: PciConfigurationSpace + ?Sized + 's> {
-    space: &'s S,
-    offset: usize,
-}
-
-pub struct VirtioCommonConfigData<'s, S: PciConfigurationSpace + ?Sized + 's> {
-    space: &'s S,
-    offset: usize,
-}
-
-pub struct VirtioNotifyConfigData<'s, S: PciConfigurationSpace + ?Sized + 's> {
-    space: &'s S,
-    offset: usize,
-}
-
-pub struct VirtioInterruptStatusData<'s, S: PciConfigurationSpace + ?Sized + 's> {
-    space: &'s S,
-    offset: usize,
-}
-
-impl<T: VirtioCapability> PciCapability for T {
-    const ID: PciCapabilityId = PciCapabilityId::VendorSpecific;
-    type CapabilityData<'a, S: PciConfigurationSpace + ?Sized + 'a> = T::Output<'a, S>;
-
-    fn check<S: PciConfigurationSpace + ?Sized>(space: &S, offset: usize, len: usize) -> bool {
-        let cfg_type = space.read_u8(offset + 3);
-        cfg_type == T::CFG_TYPE && len >= T::MIN_LEN
-    }
-
-    fn data<'s, S: PciConfigurationSpace + ?Sized + 's>(
-        space: &'s S,
-        offset: usize,
-        _len: usize,
-    ) -> Self::CapabilityData<'s, S> {
-        T::Output::from_space_offset(space, offset)
-    }
-}
-
-impl VirtioCapability for VirtioDeviceConfigCapability {
-    const CFG_TYPE: u8 = 0x04;
-    type Output<'a, S: PciConfigurationSpace + ?Sized + 'a> = VirtioDeviceConfigData<'a, S>;
-}
-
-impl<'s, S: PciConfigurationSpace + ?Sized + 's> VirtioCapabilityData<'s, S>
-    for VirtioDeviceConfigData<'s, S>
-{
-    fn from_space_offset(space: &'s S, offset: usize) -> Self {
-        Self { space, offset }
-    }
-
-    fn space(&self) -> &'s S {
-        self.space
-    }
-
-    fn offset(&self) -> usize {
-        self.offset
-    }
-}
-
-impl VirtioCapability for VirtioCommonConfigCapability {
-    const CFG_TYPE: u8 = 0x01;
-    type Output<'a, S: PciConfigurationSpace + ?Sized + 'a> = VirtioCommonConfigData<'a, S>;
-}
-
-impl<'s, S: PciConfigurationSpace + ?Sized + 's> VirtioCapabilityData<'s, S>
-    for VirtioCommonConfigData<'s, S>
-{
-    fn from_space_offset(space: &'s S, offset: usize) -> Self {
-        Self { space, offset }
-    }
-
-    fn space(&self) -> &'s S {
-        self.space
-    }
-
-    fn offset(&self) -> usize {
-        self.offset
-    }
-}
-
-impl VirtioCapability for VirtioNotifyConfigCapability {
-    const CFG_TYPE: u8 = 0x02;
-    const MIN_LEN: usize = 0x14;
-    type Output<'a, S: PciConfigurationSpace + ?Sized + 'a> = VirtioNotifyConfigData<'a, S>;
-}
-
-impl<'s, S: PciConfigurationSpace + ?Sized + 's> VirtioNotifyConfigData<'s, S> {
-    pub fn offset_multiplier(&self) -> usize {
-        self.space.read_u32(self.offset + 16) as usize
-    }
-}
-
-impl<'s, S: PciConfigurationSpace + ?Sized + 's> VirtioCapabilityData<'s, S>
-    for VirtioNotifyConfigData<'s, S>
-{
-    fn from_space_offset(space: &'s S, offset: usize) -> Self {
-        Self { space, offset }
-    }
-
-    fn space(&self) -> &'s S {
-        self.space
-    }
-
-    fn offset(&self) -> usize {
-        self.offset
-    }
-}
-
-impl VirtioCapability for VirtioInterruptStatusCapability {
-    const CFG_TYPE: u8 = 0x03;
-    const MIN_LEN: usize = 1;
-    type Output<'a, S: PciConfigurationSpace + ?Sized + 'a> = VirtioInterruptStatusData<'a, S>;
-}
-
-impl<'s, S: PciConfigurationSpace + ?Sized + 's> VirtioInterruptStatusData<'s, S> {
-    pub fn read_status(&self) -> (bool, bool) {
-        todo!()
-    }
-}
-
-impl<'s, S: PciConfigurationSpace + ?Sized + 's> VirtioCapabilityData<'s, S>
-    for VirtioInterruptStatusData<'s, S>
-{
-    fn from_space_offset(space: &'s S, offset: usize) -> Self {
-        Self { space, offset }
-    }
-
-    fn space(&self) -> &'s S {
-        self.space
-    }
-
-    fn offset(&self) -> usize {
-        self.offset
-    }
-}

kernel/driver/bus/pci/src/device.rs

@@ -13,7 +13,7 @@ use libk_util::{sync::spin_rwlock::IrqSafeRwLock, OneTimeInit};
 use yggdrasil_abi::error::Error;
 
 use crate::{
-    capability::{msi::MsiXVectorTable, MsiCapability, MsiXCapability, PciExpressCapability},
+    capability::{MsiCapability, MsiXCapability, MsiXVectorTable},
     driver::PciDriver,
     PciAddress, PciCommandRegister, PciConfigSpace, PciConfigurationSpace, PciSegmentInfo,
 };
@@ -142,23 +142,6 @@ impl PciDeviceInfo {
         }
     }
 
-    pub fn function_level_reset(&self) -> Result<(), Error> {
-        if let Some(mut pcie) = self.config_space.capability::<PciExpressCapability>() {
-            pcie.function_level_reset()
-        } else {
-            Err(Error::NotImplemented)
-        }
-    }
-
-    pub fn hot_link_reset(&self) -> Result<(), Error> {
-        if let Some(mut pcie) = self.config_space.capability::<PciExpressCapability>() {
-            pcie.hot_link_reset();
-            Ok(())
-        } else {
-            Err(Error::NotImplemented)
-        }
-    }
-
     pub fn init_interrupts(&self, preferred_mode: PreferredInterruptMode) -> Result<(), Error> {
         self.interrupt_config
             .try_init_with(|| {

kernel/driver/bus/pci/src/space/mod.rs

@@ -349,7 +349,7 @@ pub trait PciConfigurationSpace {
     }
 
     /// Returns an iterator over the PCI capabilities
-    fn capability_iter(&self) -> CapabilityIterator<Self> {
+    fn capability_iter(&self) -> CapabilityIterator<'_, Self> {
         let status = PciStatusRegister::from_bits_retain(self.status());
 
         let current = if status.contains(PciStatusRegister::CAPABILITIES_LIST) {

kernel/driver/bus/usb/src/address.rs

@@ -0,0 +1,58 @@
+use core::fmt;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
+pub struct UsbRoute {
+    bus: u16,
+    ports: [u8; 8],
+    len: u8,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
+pub struct UsbBusAddress {
+    pub bus: u16,
+    pub device: u8,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
+pub struct UsbInterfaceAddress {
+    pub device: UsbBusAddress,
+    pub interface: u8,
+}
+
+impl UsbBusAddress {
+    pub fn with_interface(self, interface: u8) -> UsbInterfaceAddress {
+        UsbInterfaceAddress {
+            device: self,
+            interface,
+        }
+    }
+}
+
+impl fmt::Display for UsbBusAddress {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "<Bus {} Device {}>", self.bus, self.device)
+    }
+}
+
+impl fmt::Display for UsbInterfaceAddress {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(
+            f,
+            "<Bus {} Device {} Interface {}>",
+            self.device.bus, self.device.device, self.interface
+        )
+    }
+}
+
+impl fmt::Display for UsbRoute {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}-", self.bus)?;
+        for (i, &port) in self.ports[..self.len as usize].iter().enumerate() {
+            if i != 0 {
+                write!(f, ".")?;
+            }
+            write!(f, "{port}")?;
+        }
+        Ok(())
+    }
+}

kernel/driver/bus/usb/src/bus.rs

@@ -1,34 +1,56 @@
 use core::sync::atomic::{AtomicU16, Ordering};
 
-use alloc::{collections::BTreeMap, sync::Arc};
-use libk_util::{queue::UnboundedMpmcQueue, sync::spin_rwlock::IrqSafeRwLock};
+use alloc::{collections::BTreeMap, format, sync::Arc};
+use libk_util::{queue::UnboundedMpmcQueue, sync::spin_rwlock::IrqSafeRwLock, OneTimeInit};
 
 use crate::{
+    address::UsbBusAddress,
     class_driver,
-    device::{UsbBusAddress, UsbDeviceAccess},
+    device::{UsbDevice, UsbDeviceAccess},
+    sysfs::{self, UsbBusKObject},
     UsbHostController,
 };
 
+pub struct UsbBusWrapper {
+    pub(crate) hc: Arc<dyn UsbHostController>,
+    pub(crate) index: u16,
+    kobject: OneTimeInit<UsbBusKObject>,
+}
+
 pub struct UsbBusManager {
-    busses: IrqSafeRwLock<BTreeMap<u16, Arc<dyn UsbHostController>>>,
+    busses: IrqSafeRwLock<BTreeMap<u16, Arc<UsbBusWrapper>>>,
     devices: IrqSafeRwLock<BTreeMap<UsbBusAddress, Arc<UsbDeviceAccess>>>,
 
     last_bus_address: AtomicU16,
 }
 
+impl UsbBusWrapper {
+    pub fn kobject(&self) -> &UsbBusKObject {
+        self.kobject.get()
+    }
+}
+
 impl UsbBusManager {
-    pub fn register_bus(hc: Arc<dyn UsbHostController>) -> u16 {
+    pub fn register_bus(hc: Arc<dyn UsbHostController>) -> (u16, Arc<UsbBusWrapper>) {
         let i = BUS_MANAGER.last_bus_address.fetch_add(1, Ordering::AcqRel);
-        BUS_MANAGER.busses.write().insert(i, hc);
-        i
+        let wrapper = Arc::new(UsbBusWrapper {
+            hc,
+            index: i,
+            kobject: OneTimeInit::new(),
+        });
+        BUS_MANAGER.busses.write().insert(i, wrapper.clone());
+        wrapper.kobject.init(sysfs::register_bus_kobject(&wrapper));
+        (i, wrapper)
     }
 
     pub fn register_device(device: Arc<UsbDeviceAccess>) {
         log::info!("usb: register device {}", device.bus_address());
+
         BUS_MANAGER
             .devices
             .write()
             .insert(device.bus_address(), device.clone());
+        device.kobject.init(sysfs::register_device_kobject(&device));
 
         QUEUE.push_back(device);
     }
@@ -51,7 +73,11 @@ pub async fn bus_handler() {
             new_device.bus_address()
         );
 
-        class_driver::spawn_driver(new_device).await.ok();
+        let address = new_device.bus_address();
+        if let Err(error) = class_driver::setup_device(new_device).await {
+            log::warn!("USB device {address} setup error: {error:?}",);
+        }
+        // class_driver::spawn_driver(new_device).await.ok();
     }
 }
 

kernel/driver/bus/usb/src/class_driver/hid_keyboard.rs

@@ -4,9 +4,12 @@ use alloc::{boxed::Box, sync::Arc};
 use async_trait::async_trait;
 use yggdrasil_abi::io::{KeyboardKey, KeyboardKeyEvent};
 
-use crate::{device::UsbDeviceAccess, error::UsbError, info::UsbDeviceClass};
-
-use super::{UsbClassInfo, UsbDriver};
+use crate::{
+    class_driver::{UsbInterfaceClass, UsbInterfaceDriver},
+    device::UsbDeviceAccess,
+    error::UsbError,
+    info::UsbInterfaceInfo,
+};
 
 pub struct UsbHidKeyboardDriver;
 
@@ -125,10 +128,14 @@ impl KeyboardState {
 }
 
 #[async_trait]
-impl UsbDriver for UsbHidKeyboardDriver {
-    async fn run(self: Arc<Self>, device: Arc<UsbDeviceAccess>) -> Result<(), UsbError> {
+impl UsbInterfaceDriver for UsbHidKeyboardDriver {
+    async fn run(
+        self: Arc<Self>,
+        device: Arc<UsbDeviceAccess>,
+        interface: UsbInterfaceInfo,
+    ) -> Result<(), UsbError> {
         // TODO not sure whether to use boot protocol (easy) or GetReport
-        let config = device.select_configuration(|_| true).await?.unwrap();
+        let config = device.current_configuration().unwrap();
 
         log::info!("Setup HID keyboard");
         let pipe = device
@@ -156,7 +163,7 @@ impl UsbDriver for UsbHidKeyboardDriver {
 
             for &event in events {
                 log::trace!("Generic Keyboard: {:?}", event);
-                ygg_driver_input::send_event(event);
+                ygg_driver_input::send_keyboard_event(event);
             }
         }
     }
@@ -165,12 +172,18 @@ impl UsbDriver for UsbHidKeyboardDriver {
         "USB HID Keyboard"
     }
 
-    fn probe(&self, class: &UsbClassInfo, _device: &UsbDeviceAccess) -> bool {
-        log::info!(
-            "class = {:?}, subclass = {:02x}",
-            class.class,
-            class.subclass
-        );
-        class.class == UsbDeviceClass::Hid && (class.subclass == 0x00 || class.subclass == 0x01)
+    fn probe(
+        &self,
+        device: &UsbDeviceAccess,
+        interface: &UsbInterfaceInfo,
+        class: UsbInterfaceClass,
+    ) -> bool {
+        let _ = (device, interface);
+        class
+            == UsbInterfaceClass {
+                class: 3,
+                subclass: 1,
+                protocol: 1,
+            }
     }
 }

kernel/driver/bus/usb/src/class_driver/hid_mouse.rs

@@ -0,0 +1,64 @@
+use alloc::{boxed::Box, sync::Arc};
+use async_trait::async_trait;
+use yggdrasil_abi::io::{ButtonMask, MouseEvent};
+
+use crate::{
+    class_driver::{UsbInterfaceClass, UsbInterfaceDriver},
+    device::UsbDeviceAccess,
+    error::UsbError,
+    info::UsbInterfaceInfo,
+};
+
+pub struct UsbHidMouseDriver;
+
+#[async_trait]
+impl UsbInterfaceDriver for UsbHidMouseDriver {
+    async fn run(
+        self: Arc<Self>,
+        device: Arc<UsbDeviceAccess>,
+        interface: UsbInterfaceInfo,
+    ) -> Result<(), UsbError> {
+        let config = device.current_configuration().unwrap();
+
+        log::info!("Setup HID mouse");
+        let pipe = device
+            .open_interrupt_in_pipe(1, config.endpoints[0].max_packet_size as u16)
+            .await?;
+
+        let mut buffer = [0; 16];
+        let mut button_state = 0;
+
+        loop {
+            let len = pipe.read(&mut buffer).await?;
+            if len < 4 {
+                continue;
+            }
+
+            let event = MouseEvent {
+                buttons: ButtonMask(buffer[0]),
+                dx: (buffer[1] as i8) as i32,
+                dy: (buffer[2] as i8) as i32,
+            };
+
+            ygg_driver_input::send_mouse_event(event);
+        }
+    }
+
+    fn name(&self) -> &'static str {
+        "USB HID Mouse"
+    }
+
+    fn probe(
+        &self,
+        device: &UsbDeviceAccess,
+        interface: &UsbInterfaceInfo,
+        class: UsbInterfaceClass,
+    ) -> bool {
+        class
+            == UsbInterfaceClass {
+                class: 3,
+                subclass: 1,
+                protocol: 2,
+            }
+    }
+}

kernel/driver/bus/usb/src/class_driver/mass_storage.rs

@@ -1,273 +1,273 @@
-use core::mem::MaybeUninit;
-
-use alloc::{boxed::Box, sync::Arc};
-use async_trait::async_trait;
-use bytemuck::{Pod, Zeroable};
-use libk::{
-    dma::{DmaBuffer, DmaSliceMut},
-    error::Error,
-};
-use ygg_driver_scsi::{transport::ScsiTransport, ScsiEnclosure};
-
-use crate::{
-    communication::UsbDirection,
-    device::{UsbDeviceAccess, UsbDeviceDetachHandler},
-    error::UsbError,
-    info::{UsbDeviceClass, UsbEndpointType},
-    pipe::{
-        control::{ControlTransferSetup, UsbClassSpecificRequest},
-        normal::{UsbBulkInPipeAccess, UsbBulkOutPipeAccess},
-    },
-};
-
-use super::{UsbClassInfo, UsbDriver};
-
-pub struct UsbMassStorageDriverBulkOnly;
-
-#[derive(Debug, Clone, Copy, Zeroable, Pod)]
-#[repr(C)]
-struct Cbw {
-    signature: u32,       // 0x00
-    tag: u32,             // 0x04
-    transfer_length: u32, // 0x08
-    flags: u8,            // 0x0C
-    lun: u8,              // 0x0D
-    cb_length: u8,        // 0x0E
-    cb_data: [u8; 16],    // 0x0F
-    // Not sent
-    _0: u8,
-}
-
-#[derive(Debug, Clone, Copy, Zeroable, Pod)]
-#[repr(C)]
-struct Csw {
-    signature: u32,
-    tag: u32,
-    data_residue: u32,
-    status: u8,
-    _0: [u8; 3],
-}
-
-struct Bbb {
-    #[allow(unused)]
-    device: Arc<UsbDeviceAccess>,
-    in_pipe: UsbBulkInPipeAccess,
-    out_pipe: UsbBulkOutPipeAccess,
-    last_tag: u32,
-}
-
-struct DetachHandler(Arc<ScsiEnclosure>);
-
-impl Bbb {
-    pub fn new(
-        device: Arc<UsbDeviceAccess>,
-        in_pipe: UsbBulkInPipeAccess,
-        out_pipe: UsbBulkOutPipeAccess,
-    ) -> Result<Self, UsbError> {
-        Ok(Self {
-            device,
-            in_pipe,
-            out_pipe,
-            last_tag: 0,
-        })
-    }
-}
-
-impl Bbb {
-    async fn send_cbw(
-        &mut self,
-        lun: u8,
-        host_to_dev: bool,
-        command: &[u8],
-        response_len: usize,
-    ) -> Result<u32, Error> {
-        self.last_tag = self.last_tag.wrapping_add(1);
-
-        let flags = if !host_to_dev { 1 << 7 } else { 0 };
-        let tag = self.last_tag;
-        let mut cbw_bytes = [0; 32];
-        let cbw = bytemuck::from_bytes_mut::<Cbw>(&mut cbw_bytes);
-
-        cbw.signature = 0x43425355;
-        cbw.transfer_length = response_len as u32;
-        cbw.flags = flags;
-        cbw.tag = tag;
-        cbw.lun = lun;
-        cbw.cb_length = command.len() as u8;
-        cbw.cb_data[..command.len()].copy_from_slice(command);
-
-        self.out_pipe
-            .write(&cbw_bytes[..31])
-            .await
-            .inspect_err(|error| log::error!("msc: CBW send error: {error:?}"))?;
-
-        Ok(tag)
-    }
-
-    async fn read_csw(&mut self, tag: u32) -> Result<(), Error> {
-        let mut csw_bytes = [0; 16];
-        self.in_pipe
-            .read_exact(&mut csw_bytes[..13])
-            .await
-            .inspect_err(|error| log::error!("msc: CSW receive error: {error:?}"))?;
-        let csw = bytemuck::from_bytes::<Csw>(&csw_bytes);
-
-        if csw.signature != 0x53425355 {
-            log::warn!("msc: invalid csw signature");
-            return Err(Error::InvalidArgument);
-        }
-        if csw.tag != tag {
-            let csw_tag = csw.tag;
-            log::warn!("msc: invalid csw tag (got {}, expected {tag})", csw_tag);
-            return Err(Error::InvalidArgument);
-        }
-        if csw.status != 0x00 {
-            return Err(Error::InvalidArgument);
-        }
-        Ok(())
-    }
-
-    async fn read_response_data(
-        &mut self,
-        buffer: DmaSliceMut<'_, MaybeUninit<u8>>,
-    ) -> Result<usize, Error> {
-        if buffer.len() == 0 {
-            return Ok(0);
-        }
-        let len = self
-            .in_pipe
-            .read_dma(buffer)
-            .await
-            .inspect_err(|error| log::error!("msc: DMA read error: {error:?}"))?;
-        Ok(len)
-    }
-}
-
-#[async_trait]
-impl ScsiTransport for Bbb {
-    fn allocate_buffer(&self, size: usize) -> Result<DmaBuffer<[MaybeUninit<u8>]>, Error> {
-        Ok(self.in_pipe.allocate_dma_buffer(size)?)
-    }
-
-    async fn perform_request_raw(
-        &mut self,
-        lun: u8,
-        request_data: &[u8],
-        response_buffer: DmaSliceMut<'_, MaybeUninit<u8>>,
-    ) -> Result<usize, Error> {
-        if request_data.len() > 16 || response_buffer.len() > self.max_bytes_per_request() {
-            return Err(Error::InvalidArgument);
-        }
-
-        let tag = self
-            .send_cbw(lun, false, request_data, response_buffer.len())
-            .await?;
-        let response_len = self.read_response_data(response_buffer).await?;
-        self.read_csw(tag).await?;
-        Ok(response_len)
-    }
-
-    fn max_bytes_per_request(&self) -> usize {
-        32768
-    }
-}
-
-impl UsbDeviceDetachHandler for DetachHandler {
-    fn handle_device_detach(&self) {
-        log::info!("Mass storage detached");
-        self.0.detach();
-    }
-}
-
-#[derive(Debug, Pod, Zeroable, Clone, Copy)]
-#[repr(C)]
-pub struct BulkOnlyMassStorageReset;
-
-#[derive(Debug, Pod, Zeroable, Clone, Copy)]
-#[repr(C)]
-pub struct GetMaxLun;
-
-impl UsbClassSpecificRequest for BulkOnlyMassStorageReset {
-    const BM_REQUEST_TYPE: u8 = 0b00100001;
-    const B_REQUEST: u8 = 0b11111111;
-}
-
-impl UsbClassSpecificRequest for GetMaxLun {
-    const BM_REQUEST_TYPE: u8 = 0b10100001;
-    const B_REQUEST: u8 = 0b11111110;
-}
-
-#[async_trait]
-impl UsbDriver for UsbMassStorageDriverBulkOnly {
-    async fn run(self: Arc<Self>, device: Arc<UsbDeviceAccess>) -> Result<(), UsbError> {
-        // TODO filter to only accept BBB config
-        let config = device.select_configuration(|_| true).await?.unwrap();
-        // Bulk-in, bulk-out
-        assert_eq!(config.endpoints.len(), 2);
-        let control_pipe = device.control_pipe();
-        let (in_index, in_info) = config
-            .find_endpoint(|ep| ep.is(UsbEndpointType::Bulk, UsbDirection::In))
-            .ok_or(UsbError::InvalidConfiguration)?;
-        let (out_index, out_info) = config
-            .find_endpoint(|ep| ep.is(UsbEndpointType::Bulk, UsbDirection::Out))
-            .ok_or(UsbError::InvalidConfiguration)?;
-        let in_pipe = device
-            .open_bulk_in_pipe(in_index, in_info.max_packet_size as u16)
-            .await?;
-        let out_pipe = device
-            .open_bulk_out_pipe(out_index, out_info.max_packet_size as u16)
-            .await?;
-
-        // Perform a Bulk-Only Mass Storage Reset
-        // TODO interface id?
-        control_pipe
-            .control_transfer(ControlTransferSetup {
-                bm_request_type: BulkOnlyMassStorageReset::BM_REQUEST_TYPE,
-                b_request: BulkOnlyMassStorageReset::B_REQUEST,
-                w_value: 0,
-                w_index: 0,
-                w_length: 0,
-            })
-            .await?;
-
-        // Get max LUN
-        // TODO on devices which do not support multiple LUNs, this command may STALL
-        let mut buffer = [MaybeUninit::uninit()];
-        let len = control_pipe
-            .control_transfer_in(
-                ControlTransferSetup {
-                    bm_request_type: GetMaxLun::BM_REQUEST_TYPE,
-                    b_request: GetMaxLun::B_REQUEST,
-                    w_value: 0,
-                    w_index: 0,
-                    w_length: 1,
-                },
-                &mut buffer,
-            )
-            .await?;
-        let max_lun = if len < 1 {
-            0
-        } else {
-            unsafe { buffer[0].assume_init() }
-        };
-
-        let bbb = Bbb::new(device.clone(), in_pipe, out_pipe)?;
-        let scsi = ScsiEnclosure::setup(Box::new(bbb), max_lun as usize + 1)
-            .await
-            .inspect_err(|error| log::error!("msc: scsi error {error:?}"))
-            .map_err(|_| UsbError::DriverError)?;
-        let detach = DetachHandler(scsi.clone());
-        device.set_detach_handler(Arc::new(detach));
-
-        Ok(())
-    }
-
-    fn name(&self) -> &'static str {
-        "USB Mass Storage"
-    }
-
-    fn probe(&self, class: &UsbClassInfo, _device: &UsbDeviceAccess) -> bool {
-        // TODO support other protocols
-        class.class == UsbDeviceClass::MassStorage && class.interface_protocol_number == 0x50
-    }
-}
+// use core::mem::MaybeUninit;
+//
+// use alloc::{boxed::Box, sync::Arc};
+// use async_trait::async_trait;
+// use bytemuck::{Pod, Zeroable};
+// use libk::{
+//     dma::{DmaBuffer, DmaSliceMut},
+//     error::Error,
+// };
+// use ygg_driver_scsi::{transport::ScsiTransport, ScsiEnclosure};
+//
+// use crate::{
+//     communication::UsbDirection,
+//     device::{UsbDeviceAccess, UsbDeviceDetachHandler},
+//     error::UsbError,
+//     info::{UsbDeviceClass, UsbEndpointType},
+//     pipe::{
+//         control::{ControlTransferSetup, UsbClassSpecificRequest},
+//         normal::{UsbBulkInPipeAccess, UsbBulkOutPipeAccess},
+//     },
+// };
+//
+// use super::{UsbClassInfo, UsbDriver};
+//
+// pub struct UsbMassStorageDriverBulkOnly;
+//
+// #[derive(Debug, Clone, Copy, Zeroable, Pod)]
+// #[repr(C)]
+// struct Cbw {
+//     signature: u32,       // 0x00
+//     tag: u32,             // 0x04
+//     transfer_length: u32, // 0x08
+//     flags: u8,            // 0x0C
+//     lun: u8,              // 0x0D
+//     cb_length: u8,        // 0x0E
+//     cb_data: [u8; 16],    // 0x0F
+//     // Not sent
+//     _0: u8,
+// }
+//
+// #[derive(Debug, Clone, Copy, Zeroable, Pod)]
+// #[repr(C)]
+// struct Csw {
+//     signature: u32,
+//     tag: u32,
+//     data_residue: u32,
+//     status: u8,
+//     _0: [u8; 3],
+// }
+//
+// struct Bbb {
+//     #[allow(unused)]
+//     device: Arc<UsbDeviceAccess>,
+//     in_pipe: UsbBulkInPipeAccess,
+//     out_pipe: UsbBulkOutPipeAccess,
+//     last_tag: u32,
+// }
+//
+// struct DetachHandler(Arc<ScsiEnclosure>);
+//
+// impl Bbb {
+//     pub fn new(
+//         device: Arc<UsbDeviceAccess>,
+//         in_pipe: UsbBulkInPipeAccess,
+//         out_pipe: UsbBulkOutPipeAccess,
+//     ) -> Result<Self, UsbError> {
+//         Ok(Self {
+//             device,
+//             in_pipe,
+//             out_pipe,
+//             last_tag: 0,
+//         })
+//     }
+// }
+//
+// impl Bbb {
+//     async fn send_cbw(
+//         &mut self,
+//         lun: u8,
+//         host_to_dev: bool,
+//         command: &[u8],
+//         response_len: usize,
+//     ) -> Result<u32, Error> {
+//         self.last_tag = self.last_tag.wrapping_add(1);
+//
+//         let flags = if !host_to_dev { 1 << 7 } else { 0 };
+//         let tag = self.last_tag;
+//         let mut cbw_bytes = [0; 32];
+//         let cbw = bytemuck::from_bytes_mut::<Cbw>(&mut cbw_bytes);
+//
+//         cbw.signature = 0x43425355;
+//         cbw.transfer_length = response_len as u32;
+//         cbw.flags = flags;
+//         cbw.tag = tag;
+//         cbw.lun = lun;
+//         cbw.cb_length = command.len() as u8;
+//         cbw.cb_data[..command.len()].copy_from_slice(command);
+//
+//         self.out_pipe
+//             .write(&cbw_bytes[..31])
+//             .await
+//             .inspect_err(|error| log::error!("msc: CBW send error: {error:?}"))?;
+//
+//         Ok(tag)
+//     }
+//
+//     async fn read_csw(&mut self, tag: u32) -> Result<(), Error> {
+//         let mut csw_bytes = [0; 16];
+//         self.in_pipe
+//             .read_exact(&mut csw_bytes[..13])
+//             .await
+//             .inspect_err(|error| log::error!("msc: CSW receive error: {error:?}"))?;
+//         let csw = bytemuck::from_bytes::<Csw>(&csw_bytes);
+//
+//         if csw.signature != 0x53425355 {
+//             log::warn!("msc: invalid csw signature");
+//             return Err(Error::InvalidArgument);
+//         }
+//         if csw.tag != tag {
+//             let csw_tag = csw.tag;
+//             log::warn!("msc: invalid csw tag (got {}, expected {tag})", csw_tag);
+//             return Err(Error::InvalidArgument);
+//         }
+//         if csw.status != 0x00 {
+//             return Err(Error::InvalidArgument);
+//         }
+//         Ok(())
+//     }
+//
+//     async fn read_response_data(
+//         &mut self,
+//         buffer: DmaSliceMut<'_, MaybeUninit<u8>>,
+//     ) -> Result<usize, Error> {
+//         if buffer.len() == 0 {
+//             return Ok(0);
+//         }
+//         let len = self
+//             .in_pipe
+//             .read_dma(buffer)
+//             .await
+//             .inspect_err(|error| log::error!("msc: DMA read error: {error:?}"))?;
+//         Ok(len)
+//     }
+// }
+//
+// #[async_trait]
+// impl ScsiTransport for Bbb {
+//     fn allocate_buffer(&self, size: usize) -> Result<DmaBuffer<[MaybeUninit<u8>]>, Error> {
+//         Ok(self.in_pipe.allocate_dma_buffer(size)?)
+//     }
+//
+//     async fn perform_request_raw(
+//         &mut self,
+//         lun: u8,
+//         request_data: &[u8],
+//         response_buffer: DmaSliceMut<'_, MaybeUninit<u8>>,
+//     ) -> Result<usize, Error> {
+//         if request_data.len() > 16 || response_buffer.len() > self.max_bytes_per_request() {
+//             return Err(Error::InvalidArgument);
+//         }
+//
+//         let tag = self
+//             .send_cbw(lun, false, request_data, response_buffer.len())
+//             .await?;
+//         let response_len = self.read_response_data(response_buffer).await?;
+//         self.read_csw(tag).await?;
+//         Ok(response_len)
+//     }
+//
+//     fn max_bytes_per_request(&self) -> usize {
+//         32768
+//     }
+// }
+//
+// impl UsbDeviceDetachHandler for DetachHandler {
+//     fn handle_device_detach(&self) {
+//         log::info!("Mass storage detached");
+//         self.0.detach();
+//     }
+// }
+//
+// #[derive(Debug, Pod, Zeroable, Clone, Copy)]
+// #[repr(C)]
+// pub struct BulkOnlyMassStorageReset;
+//
+// #[derive(Debug, Pod, Zeroable, Clone, Copy)]
+// #[repr(C)]
+// pub struct GetMaxLun;
+//
+// impl UsbClassSpecificRequest for BulkOnlyMassStorageReset {
+//     const BM_REQUEST_TYPE: u8 = 0b00100001;
+//     const B_REQUEST: u8 = 0b11111111;
+// }
+//
+// impl UsbClassSpecificRequest for GetMaxLun {
+//     const BM_REQUEST_TYPE: u8 = 0b10100001;
+//     const B_REQUEST: u8 = 0b11111110;
+// }
+//
+// #[async_trait]
+// impl UsbDriver for UsbMassStorageDriverBulkOnly {
+//     async fn run(self: Arc<Self>, device: Arc<UsbDeviceAccess>) -> Result<(), UsbError> {
+//         // TODO filter to only accept BBB config
+//         let config = device.select_configuration(|_| true).await?.unwrap();
+//         // Bulk-in, bulk-out
+//         assert_eq!(config.endpoints.len(), 2);
+//         let control_pipe = device.control_pipe();
+//         let (in_index, in_info) = config
+//             .find_endpoint(|ep| ep.is(UsbEndpointType::Bulk, UsbDirection::In))
+//             .ok_or(UsbError::InvalidConfiguration)?;
+//         let (out_index, out_info) = config
+//             .find_endpoint(|ep| ep.is(UsbEndpointType::Bulk, UsbDirection::Out))
+//             .ok_or(UsbError::InvalidConfiguration)?;
+//         let in_pipe = device
+//             .open_bulk_in_pipe(in_index, in_info.max_packet_size as u16)
+//             .await?;
+//         let out_pipe = device
+//             .open_bulk_out_pipe(out_index, out_info.max_packet_size as u16)
+//             .await?;
+//
+//         // Perform a Bulk-Only Mass Storage Reset
+//         // TODO interface id?
+//         control_pipe
+//             .control_transfer(ControlTransferSetup {
+//                 bm_request_type: BulkOnlyMassStorageReset::BM_REQUEST_TYPE,
+//                 b_request: BulkOnlyMassStorageReset::B_REQUEST,
+//                 w_value: 0,
+//                 w_index: 0,
+//                 w_length: 0,
+//             })
+//             .await?;
+//
+//         // Get max LUN
+//         // TODO on devices which do not support multiple LUNs, this command may STALL
+//         let mut buffer = [MaybeUninit::uninit()];
+//         let len = control_pipe
+//             .control_transfer_in(
+//                 ControlTransferSetup {
+//                     bm_request_type: GetMaxLun::BM_REQUEST_TYPE,
+//                     b_request: GetMaxLun::B_REQUEST,
+//                     w_value: 0,
+//                     w_index: 0,
+//                     w_length: 1,
+//                 },
+//                 &mut buffer,
+//             )
+//             .await?;
+//         let max_lun = if len < 1 {
+//             0
+//         } else {
+//             unsafe { buffer[0].assume_init() }
+//         };
+//
+//         let bbb = Bbb::new(device.clone(), in_pipe, out_pipe)?;
+//         let scsi = ScsiEnclosure::setup(Box::new(bbb), max_lun as usize + 1)
+//             .await
+//             .inspect_err(|error| log::error!("msc: scsi error {error:?}"))
+//             .map_err(|_| UsbError::DriverError)?;
+//         let detach = DetachHandler(scsi.clone());
+//         device.set_detach_handler(Arc::new(detach));
+//
+//         Ok(())
+//     }
+//
+//     fn name(&self) -> &'static str {
+//         "USB Mass Storage"
+//     }
+//
+//     fn probe(&self, class: &UsbClassInfo, _device: &UsbDeviceAccess) -> bool {
+//         // TODO support other protocols
+//         class.class == UsbDeviceClass::MassStorage && class.interface_protocol_number == 0x50
+//     }
+// }

kernel/driver/bus/usb/src/class_driver/mod.rs

@@ -1,117 +1,208 @@
+use core::mem::MaybeUninit;
+
 use alloc::{boxed::Box, sync::Arc, vec::Vec};
 use async_trait::async_trait;
 use libk::task::runtime;
 use libk_util::sync::spin_rwlock::IrqSafeRwLock;
 
 use crate::{
+    address::UsbInterfaceAddress,
     device::UsbDeviceAccess,
     error::UsbError,
-    info::{UsbDeviceClass, UsbDeviceProtocol},
+    info::{UsbInterfaceInfo, CLASS_FROM_INTERFACE},
+    pipe::control::{ControlTransferSetup, UsbClassSpecificRequest},
 };
 
+// use alloc::{boxed::Box, sync::Arc, vec::Vec};
+// use async_trait::async_trait;
+// use libk::task::runtime;
+// use libk_util::sync::spin_rwlock::IrqSafeRwLock;
+//
+// use crate::{
+//     device::UsbDeviceAccess,
+//     error::UsbError,
+//     info::{UsbDeviceClass, UsbDeviceProtocol},
+// };
+//
 pub mod hid_keyboard;
-pub mod mass_storage;
-
-#[derive(Debug)]
-pub struct UsbClassInfo {
-    pub class: UsbDeviceClass,
+pub mod hid_mouse;
+// pub mod mass_storage;
+//
+// #[derive(Debug)]
+// pub struct UsbClassInfo {
+//     pub class: UsbDeviceClass,
+//     pub subclass: u8,
+//     pub protocol: UsbDeviceProtocol,
+//     pub device_protocol_number: u8,
+//     pub interface_protocol_number: u8,
+// }
+//
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct UsbInterfaceClass {
+    pub class: u8,
     pub subclass: u8,
-    pub protocol: UsbDeviceProtocol,
-    pub device_protocol_number: u8,
-    pub interface_protocol_number: u8,
+    pub protocol: u8,
 }
 
 #[async_trait]
-pub trait UsbDriver: Send + Sync {
-    async fn run(self: Arc<Self>, device: Arc<UsbDeviceAccess>) -> Result<(), UsbError>;
+pub trait UsbInterfaceDriver: Send + Sync {
+    async fn run(
+        self: Arc<Self>,
+        device: Arc<UsbDeviceAccess>,
+        interface: UsbInterfaceInfo,
+    ) -> Result<(), UsbError>;
 
     fn name(&self) -> &'static str;
-    fn probe(&self, class: &UsbClassInfo, device: &UsbDeviceAccess) -> bool;
+    fn probe(
+        &self,
+        device: &UsbDeviceAccess,
+        interface: &UsbInterfaceInfo,
+        class: UsbInterfaceClass,
+    ) -> bool;
 }
 
-async fn extract_class_info(device: &UsbDeviceAccess) -> Result<Option<UsbClassInfo>, UsbError> {
-    if device.info.num_configurations != 1 {
-        return Ok(None);
-    }
-    let device_info = &device.info;
-    let config_info = device.query_configuration_info(0).await?;
+// async fn extract_class_info(device: &UsbDeviceAccess) -> Result<Option<UsbClassInfo>, UsbError> {
+//     if device.info.num_configurations != 1 {
+//         return Ok(None);
+//     }
+//     let device_info = &device.info;
+//     let config_info = device.query_configuration_info(0).await?;
+//
+//     if config_info.interfaces.len() >= 1 {
+//         let if_info = &config_info.interfaces[0];
+//
+//         let class = if device_info.device_class == UsbDeviceClass::FromInterface {
+//             if_info.interface_class
+//         } else {
+//             device_info.device_class
+//         };
+//         let subclass = if device_info.device_subclass == 0 {
+//             if_info.interface_subclass
+//         } else {
+//             device_info.device_subclass
+//         };
+//         let protocol = if device_info.device_protocol == UsbDeviceProtocol::FromInterface {
+//             if_info.interface_protocol
+//         } else {
+//             device_info.device_protocol
+//         };
+//
+//         Ok(Some(UsbClassInfo {
+//             class,
+//             subclass,
+//             protocol,
+//             interface_protocol_number: if_info.interface_protocol_number,
+//             device_protocol_number: device_info.device_protocol_number,
+//         }))
+//     } else {
+//         Ok(None)
+//     }
+// }
+//
+// async fn pick_driver(
+//     device: &UsbDeviceAccess,
+// ) -> Result<Option<Arc<dyn UsbDriver + 'static>>, UsbError> {
+//     let Some(class) = extract_class_info(device).await? else {
+//         return Ok(None);
+//     };
+//
+//     for driver in USB_DEVICE_DRIVERS.read().iter() {
+//         if driver.probe(&class, device) {
+//             return Ok(Some(driver.clone()));
+//         }
+//     }
+//     Ok(None)
+// }
+//
+// pub async fn spawn_driver(device: Arc<UsbDeviceAccess>) -> Result<(), UsbError> {
+//     // if let Some(driver) = pick_driver(&device).await? {
+//     //     runtime::spawn(async move {
+//     //         let name = driver.name();
+//     //         match driver.run(device).await {
+//     //             e @ Err(UsbError::DeviceDisconnected) => {
+//     //                 log::warn!(
+//     //                     "Driver {:?} did not exit cleanly: device disconnected",
+//     //                     name,
+//     //                 );
+//
+//     //                 e
+//     //             }
+//     //             e => e,
+//     //         }
+//     //     })
+//     //     .map_err(UsbError::SystemError)?;
+//     // }
+//     Ok(())
+// }
 
-    if config_info.interfaces.len() >= 1 {
-        let if_info = &config_info.interfaces[0];
-
-        let class = if device_info.device_class == UsbDeviceClass::FromInterface {
-            if_info.interface_class
-        } else {
-            device_info.device_class
-        };
-        let subclass = if device_info.device_subclass == 0 {
-            if_info.interface_subclass
-        } else {
-            device_info.device_subclass
-        };
-        let protocol = if device_info.device_protocol == UsbDeviceProtocol::FromInterface {
-            if_info.interface_protocol
-        } else {
-            device_info.device_protocol
-        };
-
-        Ok(Some(UsbClassInfo {
-            class,
-            subclass,
-            protocol,
-            interface_protocol_number: if_info.interface_protocol_number,
-            device_protocol_number: device_info.device_protocol_number,
-        }))
-    } else {
-        Ok(None)
-    }
-}
-
-async fn pick_driver(
-    device: &UsbDeviceAccess,
-) -> Result<Option<Arc<dyn UsbDriver + 'static>>, UsbError> {
-    let Some(class) = extract_class_info(device).await? else {
-        return Ok(None);
+async fn setup_interface(
+    device: &Arc<UsbDeviceAccess>,
+    address: UsbInterfaceAddress,
+    interface: &UsbInterfaceInfo,
+) -> Result<(), UsbError> {
+    let class = UsbInterfaceClass {
+        class: interface.interface_class,
+        subclass: interface.interface_subclass,
+        protocol: interface.interface_protocol,
     };
-
-    for driver in USB_DEVICE_DRIVERS.read().iter() {
-        if driver.probe(&class, device) {
-            return Ok(Some(driver.clone()));
-        }
-    }
-    Ok(None)
-}
-
-pub async fn spawn_driver(device: Arc<UsbDeviceAccess>) -> Result<(), UsbError> {
-    if let Some(driver) = pick_driver(&device).await? {
-        runtime::spawn(async move {
-            let name = driver.name();
-            match driver.run(device).await {
-                e @ Err(UsbError::DeviceDisconnected) => {
-                    log::warn!(
-                        "Driver {:?} did not exit cleanly: device disconnected",
-                        name,
-                    );
-
-                    e
+    let drivers = USB_INTERFACE_DRIVERS.read();
+    for driver in drivers.iter() {
+        if driver.probe(device, interface, class) {
+            let driver = driver.clone();
+            let device = device.clone();
+            let interface = interface.clone();
+            runtime::spawn(async move {
+                let name = driver.name();
+                match driver.run(device, interface).await {
+                    e @ Err(UsbError::DeviceDisconnected) => {
+                        log::warn!("{address} did not exit cleanly: device disconnected ({name})");
+                        e
+                    }
+                    e => e,
                 }
-                e => e,
-            }
-        })
-        .map_err(UsbError::SystemError)?;
+            })
+            .map_err(UsbError::SystemError)?;
+            break;
+        }
     }
     Ok(())
 }
 
-pub fn register_driver(driver: Arc<dyn UsbDriver + 'static>) {
+pub async fn setup_device(device: Arc<UsbDeviceAccess>) -> Result<(), UsbError> {
+    // If device has only one configuration available, use it
+    // TODO support devices with multiple configurations
+    let address = device.bus_address();
+    log::info!("Setup USB device @ {address}");
+
+    let Some(config_info) = device.use_default_configuration().await? else {
+        log::warn!("{address} has multiple configurations, not supported yet",);
+        return Ok(());
+    };
+
+    // Setup drivers for interfaces
+    log::info!("{address}: {config_info:#?}");
+
+    // TODO device-level drivers
+    for interface in config_info.interfaces.iter() {
+        let address = address.with_interface(interface.number);
+        if let Err(error) = setup_interface(&device, address, interface).await {
+            log::error!("{}: {:?}", address, error);
+        }
+    }
+
+    Ok(())
+}
+
+pub fn register_driver(driver: Arc<dyn UsbInterfaceDriver + 'static>) {
     // TODO check for duplicates
-    USB_DEVICE_DRIVERS.write().push(driver);
+    USB_INTERFACE_DRIVERS.write().push(driver);
 }
 
 pub fn register_default_class_drivers() {
     register_driver(Arc::new(hid_keyboard::UsbHidKeyboardDriver));
-    register_driver(Arc::new(mass_storage::UsbMassStorageDriverBulkOnly));
+    register_driver(Arc::new(hid_mouse::UsbHidMouseDriver));
+    // register_driver(Arc::new(mass_storage::UsbMassStorageDriverBulkOnly));
 }
 
-static USB_DEVICE_DRIVERS: IrqSafeRwLock<Vec<Arc<dyn UsbDriver + 'static>>> =
+static USB_INTERFACE_DRIVERS: IrqSafeRwLock<Vec<Arc<dyn UsbInterfaceDriver + 'static>>> =
     IrqSafeRwLock::new(Vec::new());

kernel/driver/bus/usb/src/descriptor.rs

@@ -2,9 +2,9 @@ use bytemuck::{Pod, Zeroable};
 
 use crate::{
     communication::UsbDirection,
-    device::UsbSpeed,
+    device::{self, UsbSpeed},
     error::UsbError,
-    info::{UsbDeviceClass, UsbDeviceProtocol, UsbEndpointType, UsbVersion},
+    info::UsbEndpointType,
 };
 
 #[derive(Clone, Copy, Debug, Default, Pod, Zeroable)]
@@ -91,15 +91,15 @@ pub struct UsbOtherSpeedConfiguration {
     pub max_power: u8,
 }
 
-impl UsbInterfaceDescriptor {
-    pub fn class(&self) -> UsbDeviceClass {
-        UsbDeviceClass::try_from(self.interface_class).unwrap_or(UsbDeviceClass::Unknown)
-    }
-
-    pub fn protocol(&self) -> UsbDeviceProtocol {
-        UsbDeviceProtocol::try_from(self.interface_protocol).unwrap_or(UsbDeviceProtocol::Unknown)
-    }
-}
+// impl UsbInterfaceDescriptor {
+//     pub fn class(&self) -> UsbDeviceClass {
+//         UsbDeviceClass::try_from(self.interface_class).unwrap_or(UsbDeviceClass::Unknown)
+//     }
+//
+//     pub fn protocol(&self) -> UsbDeviceProtocol {
+//         UsbDeviceProtocol::try_from(self.interface_protocol).unwrap_or(UsbDeviceProtocol::Unknown)
+//     }
+// }
 
 impl UsbEndpointDescriptor {
     pub fn direction(&self) -> UsbDirection {
@@ -127,16 +127,16 @@ impl UsbEndpointDescriptor {
 }
 
 impl UsbDeviceDescriptor {
-    pub fn class(&self) -> UsbDeviceClass {
-        UsbDeviceClass::try_from(self.device_class).unwrap_or(UsbDeviceClass::Unknown)
-    }
+    // pub fn class(&self) -> UsbDeviceClass {
+    //     UsbDeviceClass::try_from(self.device_class).unwrap_or(UsbDeviceClass::Unknown)
+    // }
 
-    pub fn protocol(&self) -> UsbDeviceProtocol {
-        UsbDeviceProtocol::try_from(self.device_protocol).unwrap_or(UsbDeviceProtocol::Unknown)
-    }
+    // pub fn protocol(&self) -> UsbDeviceProtocol {
+    //     UsbDeviceProtocol::try_from(self.device_protocol).unwrap_or(UsbDeviceProtocol::Unknown)
+    // }
 
-    pub fn max_packet_size(&self, version: UsbVersion, speed: UsbSpeed) -> Result<usize, UsbError> {
-        match (version.is_version_3(), speed, self.max_packet_size_0) {
+    pub fn max_packet_size(&self, version: u16, speed: UsbSpeed) -> Result<usize, UsbError> {
+        match (is_version_3(version), speed, self.max_packet_size_0) {
             (true, UsbSpeed::Super, 9) => Ok(1 << 9),
             (true, _, _) => todo!("Non-GenX speed USB3+ maxpacketsize0"),
             (false, _, 8) => Ok(8),
@@ -147,3 +147,7 @@ impl UsbDeviceDescriptor {
         }
     }
 }
+
+pub fn is_version_3(version: u16) -> bool {
+    version & 0xFF00 == 0x300
+}

kernel/driver/bus/usb/src/device.rs

@@ -1,14 +1,28 @@
-use core::{fmt, ops::Deref};
+use core::{
+    fmt,
+    ops::{Deref, Sub},
+};
 
-use alloc::{boxed::Box, sync::Arc, vec::Vec};
+use alloc::{
+    boxed::Box,
+    format,
+    string::{String, ToString},
+    sync::Arc,
+    vec::Vec,
+};
 use async_trait::async_trait;
-use libk_util::sync::spin_rwlock::{IrqSafeRwLock, IrqSafeRwLockReadGuard};
+use libk::error::Error;
+use libk_util::{
+    sync::spin_rwlock::{IrqSafeRwLock, IrqSafeRwLockReadGuard},
+    OneTimeInit,
+};
 
 use crate::{
+    address::UsbBusAddress,
+    bus::UsbBusWrapper,
     error::UsbError,
     info::{
         UsbConfigurationInfo, UsbDeviceInfo, UsbEndpointInfo, UsbEndpointType, UsbInterfaceInfo,
-        UsbVersion,
     },
     pipe::{
         control::{ConfigurationDescriptorEntry, UsbControlPipeAccess},
@@ -17,21 +31,26 @@ use crate::{
             UsbNormalPipeOut,
         },
     },
+    sysfs::UsbDeviceKObject,
     UsbHostController,
 };
 
 // High-level structures for info provided through descriptors
 
-#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)]
-pub struct UsbBusAddress {
-    pub bus: u16,
-    pub device: u8,
-}
+// #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)]
+// pub struct UsbBusAddress {
+//     pub bus: u16,
+//     pub device: u8,
+// }
 
 pub struct UsbDeviceAccess {
     pub device: Arc<dyn UsbDevice>,
+    pub bus: Arc<UsbBusWrapper>,
     pub info: UsbDeviceInfo,
-    pub current_configuration: IrqSafeRwLock<Option<UsbConfigurationInfo>>,
+    pub configurations: Vec<UsbConfigurationInfo>,
+    current_configuration: IrqSafeRwLock<Option<usize>>,
+
+    pub(crate) kobject: OneTimeInit<UsbDeviceKObject>,
 }
 
 #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)]
@@ -69,7 +88,7 @@ pub trait UsbDevice: Send + Sync {
     fn port_number(&self) -> u8;
     fn bus_address(&self) -> UsbBusAddress;
     fn speed(&self) -> UsbSpeed;
-    fn controller_ref(&self) -> &dyn UsbHostController;
+    fn host_controller(&self) -> Arc<dyn UsbHostController>;
 
     fn set_detach_handler(&self, handler: Arc<dyn UsbDeviceDetachHandler>);
     fn handle_detach(&self);
@@ -84,47 +103,43 @@ impl UsbDeviceAccess {
     /// * Device is not yet configured
     /// * Control pipe for the device has been properly set up
     /// * Device has been assigned a bus address
-    pub async fn setup(raw: Arc<dyn UsbDevice>) -> Result<Self, UsbError> {
+    pub async fn setup(bus: Arc<UsbBusWrapper>, raw: Arc<dyn UsbDevice>) -> Result<Self, UsbError> {
         let control = raw.control_pipe();
 
         let device_desc = control.query_device_descriptor().await?;
 
-        let bcd_usb = device_desc.bcd_usb;
-        let usb_version = UsbVersion::from_bcd_usb(device_desc.bcd_usb)
-            .ok_or(UsbError::InvalidDescriptorField)
-            .inspect_err(|_| {
-                log::error!(
-                    "{}: unsupported/invalid USB version: {:#x}",
-                    raw.bus_address(),
-                    bcd_usb
-                )
-            })?;
-
         let manufacturer = control.query_string(device_desc.manufacturer_str).await?;
         let product = control.query_string(device_desc.product_str).await?;
 
+        // Query device
         let info = UsbDeviceInfo {
             manufacturer,
             product,
-            usb_version,
+            usb_version: device_desc.bcd_usb,
 
             id_vendor: device_desc.id_vendor,
             id_product: device_desc.id_product,
 
-            device_class: device_desc.class(),
+            device_class: device_desc.device_class,
             device_subclass: device_desc.device_subclass,
-            device_protocol: device_desc.protocol(),
-            device_protocol_number: device_desc.device_protocol,
+            device_protocol: device_desc.device_protocol,
 
             num_configurations: device_desc.num_configurations,
 
-            max_packet_size: device_desc.max_packet_size(usb_version, raw.speed())?,
+            max_packet_size: device_desc.max_packet_size(device_desc.bcd_usb, raw.speed())?,
         };
 
+        let configurations =
+            Self::query_configurations(control, device_desc.num_configurations).await?;
+
         Ok(Self {
             device: raw,
+            bus,
             info,
             current_configuration: IrqSafeRwLock::new(None),
+            configurations,
+
+            kobject: OneTimeInit::new(),
         })
     }
 
@@ -164,45 +179,54 @@ impl UsbDeviceAccess {
         Ok(UsbBulkOutPipeAccess(pipe))
     }
 
-    pub fn read_current_configuration(
-        &self,
-    ) -> IrqSafeRwLockReadGuard<Option<UsbConfigurationInfo>> {
-        self.current_configuration.read()
+    pub fn current_configuration(&self) -> Option<&UsbConfigurationInfo> {
+        let index = (*self.current_configuration.read())?;
+        Some(&self.configurations[index])
     }
 
-    pub async fn select_configuration<F: Fn(&UsbConfigurationInfo) -> bool>(
+    pub async fn use_default_configuration(
         &self,
-        predicate: F,
     ) -> Result<Option<UsbConfigurationInfo>, UsbError> {
-        let mut current_config = self.current_configuration.write();
-        let control_pipe = self.control_pipe();
-
-        for i in 0..self.info.num_configurations {
-            let info = self.query_configuration_info(i).await?;
-
-            if predicate(&info) {
-                log::debug!("Selected configuration: {:#?}", info);
-                let config = current_config.insert(info);
-
-                control_pipe
-                    .set_configuration(config.config_value as _)
-                    .await?;
-
-                return Ok(Some(config.clone()));
-            }
+        if self.configurations.len() != 1 {
+            return Ok(None);
         }
 
-        Ok(None)
+        self.set_configuration(0).await.map(Some)
     }
 
-    pub async fn query_configuration_info(
-        &self,
-        index: u8,
-    ) -> Result<UsbConfigurationInfo, UsbError> {
-        if index >= self.info.num_configurations {
+    pub async fn set_configuration(&self, index: usize) -> Result<UsbConfigurationInfo, UsbError> {
+        if index >= self.configurations.len() {
             return Err(UsbError::InvalidConfiguration);
         }
+
+        let mut current = self.current_configuration.write();
         let control_pipe = self.control_pipe();
+        let info = self.configurations[index].clone();
+
+        control_pipe
+            .set_configuration(info.config_value as _)
+            .await?;
+        *current = Some(index);
+
+        Ok(info)
+    }
+
+    async fn query_configurations(
+        control_pipe: &UsbControlPipeAccess,
+        num_configurations: u8,
+    ) -> Result<Vec<UsbConfigurationInfo>, UsbError> {
+        let mut configurations = Vec::new();
+        for i in 0..num_configurations {
+            let configuration = Self::query_configuration(control_pipe, i).await?;
+            configurations.push(configuration);
+        }
+        Ok(configurations)
+    }
+
+    async fn query_configuration(
+        control_pipe: &UsbControlPipeAccess,
+        index: u8,
+    ) -> Result<UsbConfigurationInfo, UsbError> {
         let query = control_pipe.query_configuration_descriptor(index).await?;
 
         let configuration_name = control_pipe
@@ -228,10 +252,9 @@ impl UsbDeviceAccess {
                         name,
                         number: iface.interface_number,
 
-                        interface_class: iface.class(),
+                        interface_class: iface.interface_class,
                         interface_subclass: iface.interface_subclass,
-                        interface_protocol: iface.protocol(),
-                        interface_protocol_number: iface.interface_protocol,
+                        interface_protocol: iface.interface_protocol,
                     });
                 }
                 _ => (),
@@ -248,6 +271,13 @@ impl UsbDeviceAccess {
         Ok(info)
     }
 
+    // pub async fn query_configuration_info(
+    //     &self,
+    //     index: u8,
+    // ) -> Result<UsbConfigurationInfo, UsbError> {
+    //     let control_pipe = self.control_pipe();
+    // }
+
     pub fn set_detach_handler(&self, handler: Arc<dyn UsbDeviceDetachHandler>) {
         self.device.set_detach_handler(handler);
     }
@@ -260,9 +290,3 @@ impl Deref for UsbDeviceAccess {
         &*self.device
     }
 }
-
-impl fmt::Display for UsbBusAddress {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "{}:{}", self.bus, self.device)
-    }
-}

kernel/driver/bus/usb/src/info.rs

@@ -29,41 +29,51 @@ pub enum UsbUsageType {
     Reserved,
 }
 
-#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)]
-pub enum UsbVersion {
-    Usb11,
-    Usb20,
-    Usb21,
-    Usb30,
-    Usb31,
-    Usb32,
-}
+// #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)]
+// pub enum UsbVersion {
+//     Usb11,
+//     Usb20,
+//     Usb21,
+//     Usb30,
+//     Usb31,
+//     Usb32,
+// }
 
-primitive_enum! {
-    pub enum UsbDeviceClass: u8 {
-        FromInterface = 0x00,
-        Hid = 0x03,
-        MassStorage = 0x08,
-        Unknown = 0xFF,
-    }
-}
+pub const CLASS_FROM_INTERFACE: u8 = 0x00;
+pub const CLASS_HID: u8 = 0x03;
+pub const CLASS_MASS_STORAGE: u8 = 0x08;
 
-primitive_enum! {
-    pub enum UsbDeviceProtocol: u8 {
-        FromInterface = 0x00,
-        Unknown = 0xFF,
-    }
-}
+// primitive_enum! {
+//     pub enum UsbDeviceClass: u8 {
+//         FromInterface = 0x00,
+//         Hid = 0x03,
+//         MassStorage = 0x08,
+//         Unknown = 0xFF,
+//     }
+// }
+//
+// primitive_enum! {
+//     pub enum UsbDeviceProtocol: u8 {
+//         FromInterface = 0x00,
+//         Unknown = 0xFF,
+//     }
+// }
 
 #[derive(Debug, Clone)]
 pub struct UsbInterfaceInfo {
     pub name: String,
     pub number: u8,
 
-    pub interface_class: UsbDeviceClass,
+    pub interface_class: u8,
     pub interface_subclass: u8,
-    pub interface_protocol: UsbDeviceProtocol,
-    pub interface_protocol_number: u8,
+    pub interface_protocol: u8,
+    // pub name: String,
+    // pub number: u8,
+
+    // pub interface_class: UsbDeviceClass,
+    // pub interface_subclass: u8,
+    // pub interface_protocol: UsbDeviceProtocol,
+    // pub interface_protocol_number: u8,
 }
 
 #[derive(Debug, Clone)]
@@ -87,15 +97,14 @@ pub struct UsbDeviceInfo {
     pub manufacturer: String,
     pub product: String,
 
-    pub usb_version: UsbVersion,
+    pub usb_version: u16,
 
     pub id_vendor: u16,
     pub id_product: u16,
 
-    pub device_class: UsbDeviceClass,
+    pub device_class: u8,
     pub device_subclass: u8,
-    pub device_protocol: UsbDeviceProtocol,
-    pub device_protocol_number: u8,
+    pub device_protocol: u8,
 
     /// Max packet size for endpoint zero
     pub max_packet_size: usize,
@@ -103,37 +112,37 @@ pub struct UsbDeviceInfo {
     pub num_configurations: u8,
 }
 
-impl UsbVersion {
-    pub fn is_version_3(&self) -> bool {
-        matches!(self, Self::Usb30 | Self::Usb31 | Self::Usb32)
-    }
-
-    pub fn from_bcd_usb(value: u16) -> Option<Self> {
-        match value {
-            0x110 => Some(UsbVersion::Usb11),
-            0x200..=0x20F => Some(UsbVersion::Usb20),
-            0x210..=0x21F => Some(UsbVersion::Usb21),
-            0x300 => Some(UsbVersion::Usb30),
-            0x310 => Some(UsbVersion::Usb31),
-            0x320 => Some(UsbVersion::Usb32),
-            _ => None,
-        }
-    }
-}
-
-impl fmt::Display for UsbVersion {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        let string = match self {
-            Self::Usb11 => "USB1.1",
-            Self::Usb20 => "USB2.0",
-            Self::Usb21 => "USB2.1",
-            Self::Usb30 => "USB3.0",
-            Self::Usb31 => "USB3.1",
-            Self::Usb32 => "USB3.2",
-        };
-        f.write_str(string)
-    }
-}
+// impl UsbVersion {
+//     pub fn is_version_3(&self) -> bool {
+//         matches!(self, Self::Usb30 | Self::Usb31 | Self::Usb32)
+//     }
+//
+//     pub fn from_bcd_usb(value: u16) -> Option<Self> {
+//         match value {
+//             0x110 => Some(UsbVersion::Usb11),
+//             0x200..=0x20F => Some(UsbVersion::Usb20),
+//             0x210..=0x21F => Some(UsbVersion::Usb21),
+//             0x300 => Some(UsbVersion::Usb30),
+//             0x310 => Some(UsbVersion::Usb31),
+//             0x320 => Some(UsbVersion::Usb32),
+//             _ => None,
+//         }
+//     }
+// }
+//
+// impl fmt::Display for UsbVersion {
+//     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+//         let string = match self {
+//             Self::Usb11 => "USB1.1",
+//             Self::Usb20 => "USB2.0",
+//             Self::Usb21 => "USB2.1",
+//             Self::Usb30 => "USB3.0",
+//             Self::Usb31 => "USB3.1",
+//             Self::Usb32 => "USB3.2",
+//         };
+//         f.write_str(string)
+//     }
+// }
 
 impl UsbEndpointInfo {
     pub fn is(&self, ty: UsbEndpointType, dir: UsbDirection) -> bool {

kernel/driver/bus/usb/src/interface.rs

@@ -0,0 +1 @@
+pub struct UsbInterface {}

kernel/driver/bus/usb/src/lib.rs

@@ -8,15 +8,20 @@
     maybe_uninit_fill
 )]
 
+use crate::sysfs::UsbBusKObject;
+
 extern crate alloc;
 
+pub mod address;
 pub mod bus;
 pub mod communication;
 pub mod descriptor;
 pub mod device;
 pub mod error;
 pub mod info;
+pub mod interface;
 pub mod pipe;
+pub mod sysfs;
 pub mod util;
 
 pub mod class_driver;
@@ -25,4 +30,8 @@ pub mod class_driver;
 
 pub trait UsbEndpoint: Sync {}
 
-pub trait UsbHostController: Sync + Send {}
+pub trait UsbHostController: Sync + Send {
+    fn register_sysfs_properties(&self, kobject: &UsbBusKObject) {
+        let _ = kobject;
+    }
+}

kernel/driver/bus/usb/src/pipe/control.rs

@@ -37,6 +37,7 @@ pub trait UsbDeviceRequest: Sized + Pod {
 pub trait UsbClassSpecificRequest: Sized + Pod {
     const BM_REQUEST_TYPE: u8;
     const B_REQUEST: u8;
+    const W_VALUE: u16 = 0;
 }
 
 pub trait UsbDescriptorRequest: UsbDeviceRequest {

kernel/driver/bus/usb/src/sysfs.rs

@@ -0,0 +1,129 @@
+use alloc::{format, sync::Arc};
+use libk::{
+    error::Error,
+    fs::sysfs::{
+        self,
+        attribute::{IntegerAttribute, IntegerAttributeFormat, IntegerAttributeOps},
+        object::KObject,
+    },
+};
+use libk_util::OneTimeInit;
+
+use crate::{bus::UsbBusWrapper, device::UsbDeviceAccess};
+
+pub type UsbBusKObject = Arc<KObject<Arc<UsbBusWrapper>>>;
+pub type UsbDeviceKObject = Arc<KObject<Arc<UsbDeviceAccess>>>;
+
+pub(crate) fn register_bus_kobject(bus: &Arc<UsbBusWrapper>) -> UsbBusKObject {
+    let root = sysfs_usb_root();
+    let bus_kobject = KObject::new(bus.clone());
+    bus.hc.register_sysfs_properties(&bus_kobject);
+    root.add_object(format!("{}", bus.index), bus_kobject.clone())
+        .ok();
+    bus_kobject
+}
+
+pub(crate) fn register_device_kobject(device: &Arc<UsbDeviceAccess>) -> UsbDeviceKObject {
+    struct Class;
+    struct Subclass;
+    struct Protocol;
+    struct Version;
+    struct IdVendor;
+    struct IdProduct;
+
+    impl IntegerAttributeOps<u8> for Class {
+        type Data = Arc<UsbDeviceAccess>;
+        const NAME: &'static str = "class";
+        const FORMAT: IntegerAttributeFormat = IntegerAttributeFormat::Hex;
+
+        fn read(state: &Self::Data) -> Result<u8, Error> {
+            Ok(state.info.device_class)
+        }
+    }
+    impl IntegerAttributeOps<u8> for Subclass {
+        type Data = Arc<UsbDeviceAccess>;
+        const NAME: &'static str = "subclass";
+        const FORMAT: IntegerAttributeFormat = IntegerAttributeFormat::Hex;
+
+        fn read(state: &Self::Data) -> Result<u8, Error> {
+            Ok(state.info.device_subclass)
+        }
+    }
+    impl IntegerAttributeOps<u8> for Protocol {
+        type Data = Arc<UsbDeviceAccess>;
+        const NAME: &'static str = "protocol";
+        const FORMAT: IntegerAttributeFormat = IntegerAttributeFormat::Hex;
+
+        fn read(state: &Self::Data) -> Result<u8, Error> {
+            Ok(state.info.device_protocol)
+        }
+    }
+    impl IntegerAttributeOps<u16> for Version {
+        type Data = Arc<UsbDeviceAccess>;
+        const NAME: &'static str = "version";
+        const FORMAT: IntegerAttributeFormat = IntegerAttributeFormat::Hex;
+
+        fn read(state: &Self::Data) -> Result<u16, Error> {
+            Ok(state.info.usb_version)
+        }
+    }
+    impl IntegerAttributeOps<u16> for IdVendor {
+        type Data = Arc<UsbDeviceAccess>;
+        const NAME: &'static str = "vendor";
+        const FORMAT: IntegerAttributeFormat = IntegerAttributeFormat::Hex;
+
+        fn read(state: &Self::Data) -> Result<u16, Error> {
+            Ok(state.info.id_vendor)
+        }
+    }
+    impl IntegerAttributeOps<u16> for IdProduct {
+        type Data = Arc<UsbDeviceAccess>;
+        const NAME: &'static str = "product";
+        const FORMAT: IntegerAttributeFormat = IntegerAttributeFormat::Hex;
+
+        fn read(state: &Self::Data) -> Result<u16, Error> {
+            Ok(state.info.id_product)
+        }
+    }
+
+    let bus_kobject = device.bus.kobject();
+    let device_kobject = KObject::new(device.clone());
+
+    device_kobject
+        .add_attribute(IntegerAttribute::from(Class))
+        .ok();
+    device_kobject
+        .add_attribute(IntegerAttribute::from(Subclass))
+        .ok();
+    device_kobject
+        .add_attribute(IntegerAttribute::from(Protocol))
+        .ok();
+    device_kobject
+        .add_attribute(IntegerAttribute::from(Version))
+        .ok();
+    device_kobject
+        .add_attribute(IntegerAttribute::from(IdVendor))
+        .ok();
+    device_kobject
+        .add_attribute(IntegerAttribute::from(IdProduct))
+        .ok();
+
+    let address = device.bus_address();
+
+    bus_kobject
+        .add_object(format!("{}", address.device), device_kobject.clone())
+        .ok();
+
+    device_kobject
+}
+
+fn sysfs_usb_root() -> &'static Arc<KObject<()>> {
+    static USB_ROOT: OneTimeInit<Arc<KObject<()>>> = OneTimeInit::new();
+
+    USB_ROOT.or_init_with(|| {
+        let bus_object = sysfs::bus().expect("bus object");
+        let usb_object = KObject::new(());
+        bus_object.add_object("usb", usb_object.clone()).ok();
+        usb_object
+    })
+}

kernel/driver/fs/ext2/src/dir/walk.rs

@@ -127,7 +127,7 @@ impl<'a> DirentIter<'a> {
         self.offset
     }
 
-    pub fn next_record(&mut self) -> Option<Record<&[u8]>> {
+    pub fn next_record(&mut self) -> Option<Record<'_, &[u8]>> {
         if self.offset + size_of::<Dirent>() > self.block.len() {
             return None;
         }
@@ -183,7 +183,7 @@ impl<'a> DirentIterMut<'a> {
         Self { fs, block, offset }
     }
 
-    pub fn next_record(&mut self) -> Option<Record<&mut [u8]>> {
+    pub fn next_record(&mut self) -> Option<Record<'_, &mut [u8]>> {
         if self.offset + size_of::<Dirent>() > self.block.len() {
             return None;
         }

kernel/driver/fs/ext2/src/lib.rs

@@ -1,6 +1,6 @@
 #![feature(if_let_guard, async_drop, impl_trait_in_assoc_type)]
 #![cfg_attr(not(test), no_std)]
-#![allow(clippy::new_ret_no_self)]
+#![allow(clippy::new_ret_no_self, incomplete_features)]
 
 extern crate alloc;
 

kernel/driver/input/src/lib.rs

@@ -9,14 +9,20 @@ use async_trait::async_trait;
 use device_api::device::Device;
 use libk::{device::char::CharDevice, vfs::FileReadiness};
 use libk_util::{ring::LossyRingQueue, OneTimeInit};
-use yggdrasil_abi::{error::Error, io::KeyboardKeyEvent};
+use yggdrasil_abi::{
+    abi_serde::wire,
+    error::Error,
+    io::{KeyboardKeyEvent, MouseEvent},
+};
 
 #[derive(Clone, Copy)]
 pub struct KeyboardDevice;
+#[derive(Clone, Copy)]
+pub struct MouseDevice;
 
 impl FileReadiness for KeyboardDevice {
     fn poll_read(&self, cx: &mut Context<'_>) -> Poll<Result<(), Error>> {
-        INPUT_QUEUE.poll_readable(cx).map(Ok)
+        KEYBOARD_INPUT_QUEUE.poll_readable(cx).map(Ok)
     }
 }
 
@@ -33,7 +39,7 @@ impl CharDevice for KeyboardDevice {
             return Ok(0);
         }
 
-        let ev = INPUT_QUEUE.read().await;
+        let ev = KEYBOARD_INPUT_QUEUE.read().await;
 
         buf[..4].copy_from_slice(&ev.as_bytes());
 
@@ -45,7 +51,7 @@ impl CharDevice for KeyboardDevice {
             return Ok(0);
         }
 
-        let ev = INPUT_QUEUE.try_read().ok_or(Error::WouldBlock)?;
+        let ev = KEYBOARD_INPUT_QUEUE.try_read().ok_or(Error::WouldBlock)?;
 
         buf[..4].copy_from_slice(&ev.as_bytes());
 
@@ -68,15 +74,68 @@ impl CharDevice for KeyboardDevice {
     }
 }
 
-static INPUT_QUEUE: LossyRingQueue<KeyboardKeyEvent> = LossyRingQueue::with_capacity(32);
+impl FileReadiness for MouseDevice {
+    fn poll_read(&self, cx: &mut Context<'_>) -> Poll<Result<(), Error>> {
+        MOUSE_INPUT_QUEUE.poll_readable(cx).map(Ok)
+    }
+}
+
+impl Device for MouseDevice {
+    fn display_name(&self) -> &str {
+        "Mouse input pseudo-device"
+    }
+}
+
+#[async_trait]
+impl CharDevice for MouseDevice {
+    async fn read(&self, buf: &mut [u8]) -> Result<usize, Error> {
+        let ev = MOUSE_INPUT_QUEUE.read().await;
+        let len = wire::to_slice(&ev, buf)?;
+        Ok(len)
+    }
+
+    fn read_nonblocking(&self, buf: &mut [u8]) -> Result<usize, Error> {
+        let ev = MOUSE_INPUT_QUEUE.try_read().ok_or(Error::WouldBlock)?;
+        let len = wire::to_slice(&ev, buf)?;
+        Ok(len)
+    }
+
+    fn is_writeable(&self) -> bool {
+        false
+    }
+
+    fn device_request(&self, option: u32, buffer: &mut [u8], len: usize) -> Result<usize, Error> {
+        let _ = option;
+        let _ = buffer;
+        let _ = len;
+        Err(Error::InvalidOperation)
+    }
+
+    fn is_terminal(&self) -> bool {
+        false
+    }
+}
+
+static KEYBOARD_INPUT_QUEUE: LossyRingQueue<KeyboardKeyEvent> = LossyRingQueue::with_capacity(32);
 static KEYBOARD_DEVICE: OneTimeInit<Arc<KeyboardDevice>> = OneTimeInit::new();
 
-pub fn setup() -> Arc<KeyboardDevice> {
+static MOUSE_INPUT_QUEUE: LossyRingQueue<MouseEvent> = LossyRingQueue::with_capacity(32);
+static MOUSE_DEVICE: OneTimeInit<Arc<MouseDevice>> = OneTimeInit::new();
+
+pub fn setup_keyboard() -> Arc<KeyboardDevice> {
     KEYBOARD_DEVICE
         .or_init_with(|| Arc::new(KeyboardDevice))
         .clone()
 }
 
-pub fn send_event(ev: KeyboardKeyEvent) {
-    INPUT_QUEUE.write(ev);
+pub fn setup_mouse() -> Arc<MouseDevice> {
+    MOUSE_DEVICE.or_init_with(|| Arc::new(MouseDevice)).clone()
+}
+
+pub fn send_keyboard_event(ev: KeyboardKeyEvent) {
+    KEYBOARD_INPUT_QUEUE.write(ev);
+}
+
+pub fn send_mouse_event(ev: MouseEvent) {
+    MOUSE_INPUT_QUEUE.write(ev);
 }

kernel/driver/net/core/src/ephy.rs

@@ -125,24 +125,31 @@ impl<'a, M: MdioBus> PhyAccess<'a, M> {
         })
     }
 
-    pub fn setup_link(&self, have_pause: bool, force_gbesr: GBESR) -> Result<(), Error> {
+    pub fn setup_link(&self, have_pause: bool, force_gbesr: Option<GBESR>) -> Result<(), Error> {
         let bmsr = BMSR::from(self.read_reg(REG_BMSR)?);
-        let mut gbesr = if bmsr.contains(BMSR::EXT_STATUS_1000BASET) {
-            GBESR::from(self.read_reg(REG_GBESR)?)
+        let gbesr = if let Some(force_gbesr) = force_gbesr {
+            let mut gbesr = if bmsr.contains(BMSR::EXT_STATUS_1000BASET) {
+                GBESR::from(self.read_reg(REG_GBESR)?)
+            } else {
+                GBESR::empty()
+            };
+            gbesr |= force_gbesr;
+            Some(gbesr)
         } else {
-            GBESR::empty()
+            None
         };
-        gbesr |= force_gbesr;
         let mut anar = ANAR::from_capabilities(bmsr);
         if have_pause {
             anar |= ANAR::HAVE_PAUSE | ANAR::ASM_DIR;
         }
         let mut gbcr = GBCR::empty();
-        if gbesr.contains(GBESR::HAVE_1000BASET_HALF) {
-            gbcr |= GBCR::HAVE_1000BASET_HALF;
-        }
-        if gbesr.contains(GBESR::HAVE_1000BASET_FULL) {
-            gbcr |= GBCR::HAVE_1000BASET_FULL;
+        if let Some(gbesr) = gbesr {
+            if gbesr.contains(GBESR::HAVE_1000BASET_HALF) {
+                gbcr |= GBCR::HAVE_1000BASET_HALF;
+            }
+            if gbesr.contains(GBESR::HAVE_1000BASET_FULL) {
+                gbcr |= GBCR::HAVE_1000BASET_FULL;
+            }
         }
 
         self.write_reg(REG_ANAR, anar.bits())?;

kernel/driver/net/core/src/lib.rs

@@ -1,4 +1,4 @@
-#![feature(map_try_insert, let_chains, result_flattening)]
+#![feature(map_try_insert, let_chains)]
 #![allow(clippy::type_complexity, clippy::new_without_default)]
 #![no_std]
 

kernel/driver/net/core/src/socket/local/mod.rs

@@ -15,7 +15,7 @@ enum OwnedAddress {
     Anonymous(u64),
 }
 
-pub fn load_address(bytes: &[u8]) -> Result<LocalSocketAddress, Error> {
+pub fn load_address(bytes: &[u8]) -> Result<LocalSocketAddress<'_>, Error> {
     Ok(wire::from_slice(bytes)?)
 }
 
@@ -76,7 +76,7 @@ pub fn write_ancillary(
 }
 
 impl OwnedAddress {
-    fn to_borrowed(&self) -> LocalSocketAddress {
+    fn to_borrowed(&self) -> LocalSocketAddress<'_> {
         match self {
             Self::Path(path) => LocalSocketAddress::Path(path.as_ref()),
             Self::Anonymous(anon) => LocalSocketAddress::Anonymous(*anon),

kernel/driver/net/core/src/socket/tcp/listener.rs

@@ -70,7 +70,7 @@ impl TcpListener {
     pub(super) fn poll_accept(
         &self,
         cx: &mut Context<'_>,
-    ) -> Poll<IrqSafeSpinlockGuard<Vec<Arc<TcpStream>>>> {
+    ) -> Poll<IrqSafeSpinlockGuard<'_, Vec<Arc<TcpStream>>>> {
         let lock = self.pending_accept.lock();
         self.accept_notify.register(cx.waker());
         if !lock.is_empty() {

kernel/driver/net/core/src/socket/tcp/stream.rs

@@ -214,7 +214,7 @@ impl TcpStream {
     pub(super) fn poll_receive(
         &self,
         cx: &mut Context<'_>,
-    ) -> Poll<Result<IrqSafeRwLockWriteGuard<TcpConnection>, Error>> {
+    ) -> Poll<Result<IrqSafeRwLockWriteGuard<'_, TcpConnection>, Error>> {
         let lock = self.connection.write();
         match lock.poll_receive(cx) {
             Poll::Ready(Ok(())) => Poll::Ready(Ok(lock)),

kernel/driver/net/igbe/src/lib.rs

@@ -26,12 +26,15 @@ use ygg_driver_pci::{
 };
 use yggdrasil_abi::net::{link::LinkState, MacAddress};
 
+use crate::regs::Revision;
+
 extern crate alloc;
 
 mod regs;
 mod ring;
 
 struct Igbe {
+    chip: Revision,
     regs: IrqSafeSpinlock<Regs>,
     dma: Arc<dyn DmaAllocator>,
     pci: PciDeviceInfo,
@@ -43,8 +46,9 @@ struct Igbe {
 }
 
 impl Igbe {
-    pub fn new(dma: Arc<dyn DmaAllocator>, regs: Regs, pci: PciDeviceInfo) -> Self {
+    pub fn new(dma: Arc<dyn DmaAllocator>, regs: Regs, chip: Revision, pci: PciDeviceInfo) -> Self {
         Self {
+            chip,
             dma,
             pci,
             mac: OneTimeInit::new(),
@@ -74,7 +78,7 @@ impl Device for Igbe {
         regs.reset(Duration::from_millis(200))?;
         // Intel 8257x manuals say an additional interrupt disable is needed after a global reset
         regs.disable_interrupts();
-        regs.set_link_up()?;
+        regs.set_link_up(self.chip)?;
 
         // Initialize Rx
         regs.initialize_receiver(&rx_ring);
@@ -175,6 +179,10 @@ impl NetworkDevice for Igbe {
 
 pci_driver! {
     matches: [
+        device (0x8086:0x100E), // 82540EM (E1000)
+        device (0x8086:0x100C), // 82544GC (E1000)
+        device (0x8086:0x100F), // 82545EM (E1000)
+        device (0x8086:0x10D3), // 82574L (E1000E) [[BROKEN]]
         device (0x8086:0x10C9), // 82576 GbE
         device (0x8086:0x1502), // 82579LM GbE (Lewisville)
     ],
@@ -197,11 +205,22 @@ pci_driver! {
                 }
             };
 
+            let chip = match info.device_id {
+                0x100E | 0x100C | 0x100F => Revision::I8254x,
+                0x10D3 => Revision::I82574L,
+                0x10C9 => Revision::I82576,
+                0x1502 => Revision::I82579LM,
+                id => {
+                    log::error!("Invalid igbe chip variant: {id:#04x}");
+                    return Err(Error::InvalidOperation)
+                },
+            };
+
             info.init_interrupts(PreferredInterruptMode::Msi(true))?;
             info.set_command(true, use_mmio, !use_mmio, true);
 
             let regs = unsafe { Regs::map(base) }?;
-            let device = Igbe::new(dma.clone(), regs, info.clone());
+            let device = Igbe::new(dma.clone(), regs, chip, info.clone());
             Ok(Arc::new(device))
         }
 

kernel/driver/net/igbe/src/regs.rs

@@ -42,6 +42,14 @@ pub trait Reg {
     const OFFSET: u16;
 }
 
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub enum Revision {
+    I8254x,
+    I82574L,
+    I82576,
+    I82579LM,
+}
+
 register_bitfields! {
     u32,
     pub CTRL [
@@ -329,6 +337,7 @@ impl MdioBus for Regs {
 
         let mdic = self.inner.extract();
         if mdic.matches_all(MDIC::E::SET) {
+            log::warn!("MDIO read error: phyaddr={phyaddr:#x}, regaddr={regaddr:#x}");
             return Err(Error::InvalidOperation);
         }
 
@@ -350,6 +359,9 @@ impl MdioBus for Regs {
         )?;
 
         if self.inner.matches_all(MDIC::E::SET) {
+            log::warn!(
+                "MDIO write error: phyaddr={phyaddr:#x}, regaddr={regaddr:#x}, value={value:#x}"
+            );
             return Err(Error::InvalidOperation);
         }
 
@@ -403,7 +415,7 @@ impl Regs {
         })
     }
 
-    pub fn set_link_up(&mut self) -> Result<(), Error> {
+    pub fn set_link_up(&mut self, chip: Revision) -> Result<(), Error> {
         self.inner
             .modify(CTRL::SLU::SET + CTRL::RFCE::SET + CTRL::TFCE::SET);
 
@@ -412,8 +424,14 @@ impl Regs {
         let (id0, id1) = phy.id()?;
         log::info!("PHY {:04x}:{:04x}", id0, id1);
 
-        phy.reset(Duration::from_millis(200))?;
-        phy.setup_link(true, GBESR::empty())?;
+        phy.reset(Duration::from_millis(200))
+            .inspect_err(|e| log::error!("PHY reset error {e:?}"))?;
+        let force_gbesr = match chip {
+            Revision::I82576 | Revision::I82579LM => Some(GBESR::empty()),
+            _ => None,
+        };
+        phy.setup_link(true, force_gbesr)
+            .inspect_err(|e| log::error!("PHY setup error: {e:?}"))?;
 
         Ok(())
     }

kernel/driver/net/rtl81xx/src/lib.rs

@@ -7,8 +7,7 @@ use rtl8139::Rtl8139;
 use rtl8168::Rtl8168;
 use ygg_driver_pci::{
     capability::{
-        express::LinkControl, power::DevicePowerState, PciExpressCapability,
-        PowerManagementCapability,
+        DevicePowerState, PciExpressCapability, PcieLinkControl, PowerManagementCapability,
     },
     device::{PciDeviceInfo, PreferredInterruptMode},
     macros::pci_driver,
@@ -31,7 +30,9 @@ pci_driver! {
             info.init_interrupts(PreferredInterruptMode::Msi(false))?;
 
             if let Some(mut pcie) = info.config_space.capability::<PciExpressCapability>() {
-                pcie.modify_link_control(LinkControl::ASPM::CLEAR + LinkControl::CLOCK_PM::CLEAR);
+                let mut lcr = pcie.link_control();
+                lcr.remove(PcieLinkControl::ASPM_MASK | PcieLinkControl::ECPM);
+                pcie.set_link_control(lcr);
             }
 
             // Enable MMIO + interrupts + bus mastering
@@ -75,7 +76,9 @@ pci_driver! {
             }
 
             if let Some(mut pcie) = info.config_space.capability::<PciExpressCapability>() {
-                pcie.modify_link_control(LinkControl::ASPM::CLEAR + LinkControl::CLOCK_PM::CLEAR);
+                let mut lcr = pcie.link_control();
+                lcr.remove(PcieLinkControl::ASPM_MASK | PcieLinkControl::ECPM);
+                pcie.set_link_control(lcr);
             }
 
             let device = Rtl8168::new(dma.clone(), base, info.clone())?;

kernel/driver/net/rtl81xx/src/rtl8168.rs

@@ -498,7 +498,7 @@ impl Regs {
         phy.write_reg(0x0E, 0x00)?;
 
         phy.reset(timeout)?;
-        phy.setup_link(true, GBESR::empty())?;
+        phy.setup_link(true, Some(GBESR::empty()))?;
 
         psleep(Duration::from_millis(100));
 

kernel/driver/net/stmmac/src/lib.rs

@@ -337,7 +337,7 @@ impl Device for Stmmac {
         let (id0, id1) = phy.id()?;
         log::info!("stmmac: PHY {id0:04x}:{id1:04x}");
         phy.reset(Duration::from_millis(100))?;
-        phy.setup_link(true, GBESR::empty())?;
+        phy.setup_link(true, Some(GBESR::empty()))?;
 
         self.inner.init(Inner {
             regs: IrqSafeSpinlock::new(regs),

kernel/driver/sound/core/Cargo.toml

@@ -1,15 +0,0 @@
-[package]
-name = "ygg_driver_sound_core"
-version = "0.1.0"
-edition = "2024"
-
-[dependencies]
-device-api.workspace = true
-yggdrasil-abi.workspace = true
-libk-mm.workspace = true
-libk-util.workspace = true
-libk.workspace = true
-
-log.workspace = true
-async-trait.workspace = true
-futures-util.workspace = true

kernel/driver/sound/core/src/lib.rs

@@ -1,122 +0,0 @@
-#![no_std]
-
-use core::{
-    sync::atomic::{AtomicU32, Ordering},
-    task::{Context, Poll},
-};
-
-use alloc::{boxed::Box, collections::btree_map::BTreeMap, format, sync::Arc, vec::Vec};
-use async_trait::async_trait;
-use device_api::device::Device;
-use libk::{block, device::char::CharDevice, error::Error, fs::devfs, vfs::FileReadiness};
-use libk_util::sync::spin_rwlock::IrqSafeRwLock;
-use yggdrasil_abi::io::FileMode;
-
-extern crate alloc;
-
-#[derive(Debug, Clone)]
-pub enum SampleFormat {
-    S8,
-    S16Le,
-}
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-#[repr(usize)]
-pub enum SampleRate {
-    Rate8000 = 8000,
-    Rate11025 = 11025,
-    Rate16000 = 16000,
-    Rate22050 = 22050,
-    Rate32000 = 32000,
-    Rate44100 = 44100,
-    Rate48000 = 48000,
-    Rate88200 = 88200,
-    Rate96000 = 96000,
-    Rate176400 = 176400,
-    Rate192000 = 192000,
-    Rate384000 = 384000,
-}
-
-#[derive(Debug, Clone)]
-pub struct SinkFormat {
-    pub sample_rate: SampleRate,
-    pub sample_format: SampleFormat,
-    pub channels: usize,
-}
-
-#[derive(Debug, Clone)]
-pub struct SinkSupportedFormats {
-    pub rates: Vec<usize>,
-    pub formats: Vec<SampleFormat>,
-    pub channels: usize,
-}
-
-#[async_trait]
-pub trait AudioSink: Sync + Send {
-    async fn set_format(&self, format: SinkFormat) -> Result<(), Error>;
-    async fn current_format(&self) -> Result<SinkFormat, Error>;
-    async fn supported_formats(&self) -> Result<SinkSupportedFormats, Error>;
-
-    async fn write(&self, data: &[u8]) -> Result<usize, Error>;
-
-    async fn start(&self) -> Result<(), Error>;
-    fn stop(&self) -> Result<(), Error>;
-
-    fn display_name(&self) -> &str {
-        "Audio Sink"
-    }
-}
-
-pub struct AudioSinkWrapper(Arc<dyn AudioSink>);
-
-impl Device for AudioSinkWrapper {
-    fn display_name(&self) -> &str {
-        self.0.display_name()
-    }
-}
-
-impl FileReadiness for AudioSinkWrapper {
-    fn poll_read(&self, cx: &mut Context<'_>) -> Poll<Result<(), Error>> {
-        todo!()
-    }
-}
-
-#[async_trait]
-impl CharDevice for AudioSinkWrapper {
-    fn close(&self) -> Result<(), Error> {
-        self.0.stop()
-    }
-
-    async fn write(&self, buffer: &[u8]) -> Result<usize, Error> {
-        self.0.write(buffer).await
-    }
-
-    fn device_request(&self, option: u32, buffer: &mut [u8], len: usize) -> Result<usize, Error> {
-        let _ = option;
-        let _ = buffer;
-        let _ = len;
-        log::warn!("device_request unimplemented: {option:#x}");
-        Err(Error::InvalidOperation)
-    }
-}
-
-static SINKS: IrqSafeRwLock<BTreeMap<u32, Arc<AudioSinkWrapper>>> =
-    IrqSafeRwLock::new(BTreeMap::new());
-static LAST_SINK_ID: AtomicU32 = AtomicU32::new(0);
-
-pub fn register_audio_sink(sink: Arc<dyn AudioSink>) {
-    let id = LAST_SINK_ID.fetch_add(1, Ordering::Relaxed);
-    let name = format!("snd{id}");
-    let sink = Arc::new(AudioSinkWrapper(sink));
-    SINKS.write().insert(id, sink.clone());
-    devfs::add_named_char_device(sink, &name, FileMode::new(0o222)).ok();
-}
-
-impl SampleFormat {
-    pub fn sample_size(&self) -> usize {
-        match self {
-            Self::S8 => 1,
-            Self::S16Le => 2,
-        }
-    }
-}

kernel/driver/sound/intel-hda/Cargo.toml

@@ -1,20 +0,0 @@
-[package]
-name = "ygg_driver_intel_hda"
-version = "0.1.0"
-edition = "2024"
-
-[dependencies]
-device-api.workspace = true
-yggdrasil-abi.workspace = true
-libk-mm.workspace = true
-libk-util.workspace = true
-libk.workspace = true
-
-ygg_driver_pci.path = "../../bus/pci"
-ygg_driver_sound_core.path = "../core"
-
-log.workspace = true
-tock-registers.workspace = true
-futures-util.workspace = true
-async-trait.workspace = true
-bytemuck.workspace = true

kernel/driver/sound/intel-hda/src/codec/mod.rs

@@ -1,527 +0,0 @@
-use core::fmt;
-
-use alloc::{sync::Arc, vec::Vec};
-use libk::error::Error;
-use tock_registers::{fields::FieldValue, register_bitfields, LocalRegisterCopy};
-use yggdrasil_abi::bitflags;
-
-use crate::{
-    ring::{Command, PinControl, PinDefaultConfig, PinDevice, Verb},
-    HdAudio,
-};
-
-pub mod parameter;
-pub use parameter::*;
-
-#[derive(Debug, Clone, Copy)]
-pub struct Node {
-    pub codec: u8,
-    pub nid: u8,
-}
-
-pub struct WidgetConnectionChain<'a> {
-    afg: &'a AudioNode,
-    current: Option<u8>,
-}
-
-#[derive(Debug)]
-pub struct AudioOutputWidget {
-    channel_count: usize,
-}
-#[derive(Debug)]
-pub struct AudioInputWidget {
-    channel_count: usize,
-}
-#[derive(Debug)]
-pub struct PinComplexWidget {
-    default_config: PinDefaultConfig,
-}
-#[derive(Debug)]
-pub struct MixerWidget {}
-#[derive(Debug)]
-pub enum AudioWidget {
-    AudioOutput(AudioOutputWidget),
-    AudioInput(AudioInputWidget),
-    PinComplex(PinComplexWidget),
-    Mixer(MixerWidget),
-    Other,
-}
-
-#[derive(Debug)]
-pub enum AudioFormat {
-    Pcm { rates: Vec<usize>, bits: Vec<u8> },
-    Other,
-}
-
-#[derive(Debug)]
-pub struct AudioWidgetNode {
-    parent_nid: u8,
-    node: Node,
-
-    output_amp: Option<LocalRegisterCopy<u32, AmpCapabilities::Register>>,
-    input_amp: Option<LocalRegisterCopy<u32, AmpCapabilities::Register>>,
-    connection: Option<u8>,
-    formats: Vec<AudioFormat>,
-
-    widget: AudioWidget,
-}
-
-pub struct AudioNode {
-    node: Node,
-    widget_connection: Vec<(u8, u8)>,
-    widgets: Vec<AudioWidgetNode>,
-}
-
-pub struct Codec {
-    hda: Arc<HdAudio>,
-
-    root: Node,
-    nodes: Vec<AudioNode>,
-}
-
-impl AudioFormat {
-    async fn query_from_node(hda: &HdAudio, node: Node) -> Result<Vec<Self>, Error> {
-        let supported_formats = node
-            .get_parameter::<SupportedStreamFormats::Register>(hda)
-            .await?;
-
-        let mut formats = Vec::new();
-
-        if supported_formats.matches_all(SupportedStreamFormats::PCM::SET) {
-            formats.push(Self::query_pcm(hda, node).await?);
-        }
-        if supported_formats.matches_all(SupportedStreamFormats::F32::SET) {
-            formats.push(Self::Other);
-        }
-        if supported_formats.matches_all(SupportedStreamFormats::AC3::SET) {
-            formats.push(Self::Other);
-        }
-
-        Ok(formats)
-    }
-
-    async fn query_pcm(hda: &HdAudio, node: Node) -> Result<Self, Error> {
-        // Conversion tables
-        const RATES: &[(usize, FieldValue<u32, SupportedPcmFormats::Register>)] = &[
-            (8000, SupportedPcmFormats::RATE_8::SET),
-            (11024, SupportedPcmFormats::RATE_11_025::SET),
-            (16000, SupportedPcmFormats::RATE_16::SET),
-            (22050, SupportedPcmFormats::RATE_22_05::SET),
-            (32000, SupportedPcmFormats::RATE_32::SET),
-            (44100, SupportedPcmFormats::RATE_44_1::SET),
-            (48000, SupportedPcmFormats::RATE_48::SET),
-            (88200, SupportedPcmFormats::RATE_88_2::SET),
-            (96000, SupportedPcmFormats::RATE_96::SET),
-            (176400, SupportedPcmFormats::RATE_176_4::SET),
-            (192000, SupportedPcmFormats::RATE_192::SET),
-            (384000, SupportedPcmFormats::RATE_384::SET),
-        ];
-        const BITS: &[(u8, FieldValue<u32, SupportedPcmFormats::Register>)] = &[
-            (8, SupportedPcmFormats::BITS_8::SET),
-            (16, SupportedPcmFormats::BITS_16::SET),
-            (20, SupportedPcmFormats::BITS_20::SET),
-            (24, SupportedPcmFormats::BITS_24::SET),
-            (32, SupportedPcmFormats::BITS_32::SET),
-        ];
-
-        let supported_pcm_formats = node
-            .get_parameter::<SupportedPcmFormats::Register>(hda)
-            .await?;
-
-        let mut supported_rates = Vec::new();
-        let mut supported_bits = Vec::new();
-
-        for &(rate, bit) in RATES {
-            if supported_pcm_formats.matches_all(bit) {
-                supported_rates.push(rate);
-            }
-        }
-        for &(bits, bit) in BITS {
-            if supported_pcm_formats.matches_all(bit) {
-                supported_bits.push(bits);
-            }
-        }
-
-        Ok(Self::Pcm {
-            rates: supported_rates,
-            bits: supported_bits,
-        })
-    }
-}
-
-impl Node {
-    pub async fn perform_command(
-        &self,
-        hda: &HdAudio,
-        verb: Verb,
-        parameter: u32,
-    ) -> Result<u32, Error> {
-        hda.perform_command(Command::new(self.codec, self.nid, verb, parameter))
-            .await
-    }
-
-    pub async fn get_parameter<P: NodeParameter>(&self, hda: &HdAudio) -> Result<P::Value, Error> {
-        self.perform_command(hda, Verb::GetParameter, P::NUMBER as u32)
-            .await
-            .map(P::from_response)
-    }
-
-    pub async fn get_connection_list_entry(
-        &self,
-        hda: &HdAudio,
-        index: usize,
-    ) -> Result<u32, Error> {
-        self.perform_command(hda, Verb::GetConnectionListEntry, index as u32)
-            .await
-    }
-
-    pub async fn set_stream(&self, hda: &HdAudio, stream: u8, channel: u8) -> Result<(), Error> {
-        self.perform_command(
-            hda,
-            Verb::SetConverterStreamChannel,
-            ((stream as u32) << 4) | (channel as u32),
-        )
-        .await?;
-        Ok(())
-    }
-
-    pub async fn get_pin_default_config(&self, hda: &HdAudio) -> Result<PinDefaultConfig, Error> {
-        self.perform_command(hda, Verb::GetPinWidgetDefaultConfig, 0)
-            .await
-            .map(PinDefaultConfig::from)
-    }
-
-    pub async fn set_power_state(&self, hda: &HdAudio, power: bool) -> Result<(), Error> {
-        let ps_set = if power { 0x0 } else { 0x4 };
-        self.perform_command(hda, Verb::SetPowerState, ps_set)
-            .await?;
-        Ok(())
-    }
-
-    pub async fn set_unsolicited_response(
-        &self,
-        hda: &HdAudio,
-        tag: Option<u8>,
-    ) -> Result<(), Error> {
-        let value = match tag {
-            Some(tag) => (tag as u32) | (1 << 7),
-            None => 0,
-        };
-        self.perform_command(hda, Verb::SetUnsolicitedResponse, value)
-            .await?;
-        Ok(())
-    }
-
-    pub async fn set_pin_control(&self, hda: &HdAudio, pin_control: u32) -> Result<(), Error> {
-        self.perform_command(hda, Verb::SetPinWidgetControl, pin_control & 0xFF)
-            .await?;
-        Ok(())
-    }
-
-    pub async fn get_pin_control(
-        &self,
-        hda: &HdAudio,
-    ) -> Result<LocalRegisterCopy<u32, PinControl::Register>, Error> {
-        self.perform_command(hda, Verb::GetPinWidgetControl, 0)
-            .await
-            .map(LocalRegisterCopy::new)
-    }
-
-    pub async fn modify_pin_control(
-        &self,
-        hda: &HdAudio,
-        change: FieldValue<u32, PinControl::Register>,
-    ) -> Result<(), Error> {
-        let mut value = self.get_pin_control(hda).await?;
-        value.modify(change);
-        self.set_pin_control(hda, value.get()).await
-    }
-}
-
-impl AudioWidgetNode {
-    pub async fn probe(hda: &HdAudio, parent_nid: u8, node: Node) -> Result<Self, Error> {
-        let capabilities = node
-            .get_parameter::<AudioWidgetCapabilities::Register>(hda)
-            .await?;
-        let channel_count = ((capabilities.read(AudioWidgetCapabilities::EXT_CHANNEL_COUNT) << 1)
-            | capabilities.read(AudioWidgetCapabilities::CHANNEL_COUNT))
-            + 1;
-        let in_amp = capabilities.matches_all(AudioWidgetCapabilities::IN_AMP::SET);
-        let out_amp = capabilities.matches_all(AudioWidgetCapabilities::OUT_AMP::SET);
-
-        let in_amp =
-            if in_amp && capabilities.matches_all(AudioWidgetCapabilities::AMP_OVERRIDE::SET) {
-                node.get_parameter::<InputAmpCapabilities>(hda).await?
-            } else {
-                None
-            };
-        let out_amp =
-            if out_amp && capabilities.matches_all(AudioWidgetCapabilities::AMP_OVERRIDE::SET) {
-                node.get_parameter::<OutputAmpCapabilities>(hda).await?
-            } else {
-                None
-            };
-
-        let connection = if capabilities.matches_all(AudioWidgetCapabilities::CONN_LIST::SET) {
-            let (length, long) = node.get_parameter::<ConnectionListLength>(hda).await?;
-            // if length > 1 || long {
-            //     todo!()
-            // }
-            let connection = node.get_connection_list_entry(hda, 0).await?;
-            Some(connection as u8)
-        } else {
-            None
-        };
-
-        let formats = if capabilities.matches_all(AudioWidgetCapabilities::FORMAT_OVERRIDE::SET) {
-            AudioFormat::query_from_node(hda, node).await?
-        } else {
-            Vec::new()
-        };
-
-        let widget = match capabilities.read_as_enum(AudioWidgetCapabilities::TYPE) {
-            Some(AudioWidgetCapabilities::TYPE::Value::AudioOutput) => {
-                // Disable audio outputs immediately
-                node.set_stream(hda, 0x00, 0x00).await?;
-
-                AudioWidget::AudioOutput(AudioOutputWidget {
-                    channel_count: channel_count as usize,
-                })
-            }
-            Some(AudioWidgetCapabilities::TYPE::Value::AudioInput) => {
-                AudioWidget::AudioInput(AudioInputWidget {
-                    channel_count: channel_count as usize,
-                })
-            }
-            Some(AudioWidgetCapabilities::TYPE::Value::PinComplex) => {
-                let default_config = node.get_pin_default_config(hda).await?;
-                AudioWidget::PinComplex(PinComplexWidget { default_config })
-            }
-            Some(AudioWidgetCapabilities::TYPE::Value::AudioMixer) => {
-                AudioWidget::Mixer(MixerWidget {})
-            }
-            v => {
-                log::warn!("hda: unknown audio widget: {v:?}");
-                AudioWidget::Other
-            }
-        };
-
-        Ok(Self {
-            parent_nid,
-            node,
-            widget,
-
-            output_amp: out_amp,
-            input_amp: in_amp,
-            formats,
-            connection,
-        })
-    }
-}
-
-impl AudioNode {
-    pub async fn probe(hda: &HdAudio, node: Node) -> Result<Self, Error> {
-        // Query widgets
-        let subnodes = node.get_parameter::<SubNodeInfo>(hda).await?;
-
-        let mut widgets = Vec::new();
-        for wid in subnodes.iter() {
-            let widget_node = Node {
-                codec: node.codec,
-                nid: wid,
-            };
-            match AudioWidgetNode::probe(hda, node.nid, widget_node).await {
-                Ok(widget) => {
-                    widgets.push(widget);
-                }
-                Err(error) => {
-                    log::warn!(
-                        "hda: could not enumerate {}.{}.{}: {error:?}",
-                        node.codec,
-                        node.nid,
-                        wid
-                    );
-                }
-            }
-        }
-
-        let mut widget_connection = Vec::new();
-        for widget in &widgets {
-            if let Some(connection) = widget.connection {
-                widget_connection.push((widget.node.nid, connection));
-            }
-        }
-
-        Ok(Self {
-            node,
-            widgets,
-            widget_connection,
-        })
-    }
-
-    pub fn widget(&self, wid: u8) -> Option<&AudioWidgetNode> {
-        self.widgets.iter().find(|w| w.node.nid == wid)
-    }
-
-    pub fn widget_supported_formats<'a>(&self, widget: &'a AudioWidgetNode) -> &'a [AudioFormat] {
-        if !widget.formats.is_empty() {
-            &widget.formats
-        } else {
-            // TODO node supported formats
-            &[]
-        }
-    }
-
-    // Returns (pin, kind) pair
-    pub fn default_output_pin(&self) -> Option<(Node, PinDevice)> {
-        let mut best: Option<(Node, PinDevice)> = None;
-        for widget in &self.widgets {
-            if let AudioWidget::PinComplex(pin) = &widget.widget {
-                let Some(connection) = widget.connection else {
-                    continue;
-                };
-                let dev = pin.default_config.default_device;
-
-                let better = match best {
-                    None => true,
-                    Some((_, other)) => dev.output_score() > other.output_score(),
-                };
-
-                if better {
-                    best = Some((widget.node, dev));
-                }
-            }
-        }
-
-        best
-    }
-
-    pub fn connection_chain(&self, source: u8) -> WidgetConnectionChain<'_> {
-        WidgetConnectionChain {
-            afg: self,
-            current: Some(source),
-        }
-    }
-}
-
-impl fmt::Debug for AudioNode {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        f.debug_struct("AudioNode")
-            .field("id", &self.node.nid)
-            .field("widgets", &self.widgets)
-            .finish()
-    }
-}
-
-impl Codec {
-    pub async fn probe(hda: Arc<HdAudio>, codec: u8) -> Result<Self, Error> {
-        let root_node = Node { codec, nid: 0 };
-        let id = root_node.get_parameter::<DeviceId>(&*hda).await?;
-        if (id.vendor_id == 0 && id.device_id == 0)
-            || (id.vendor_id == 0xFFFF && id.device_id == 0xFFFF)
-        {
-            return Err(Error::DoesNotExist);
-        }
-
-        let root_subnodes = root_node.get_parameter::<SubNodeInfo>(&*hda).await?;
-
-        let mut nodes = Vec::new();
-
-        for nid in root_subnodes.iter() {
-            let node = Node { codec, nid };
-            let node_type = node.get_parameter::<NodeType>(&*hda).await?;
-
-            // Ignore non-audio nodes
-            if node_type != 0x01 {
-                continue;
-            }
-
-            match AudioNode::probe(&*hda, node).await {
-                Ok(node) => nodes.push(node),
-                Err(error) => {
-                    log::warn!("hda: could not enumerate {}.{}: {error:?}", codec, nid);
-                }
-            }
-        }
-
-        Ok(Self {
-            hda,
-            root: root_node,
-            nodes,
-        })
-    }
-
-    pub fn node(&self, nid: u8) -> Option<&AudioNode> {
-        self.nodes.iter().find(|n| n.node.nid == nid)
-    }
-
-    pub async fn setup_default_output(&self) -> Result<Node, Error> {
-        let (pin_id, dev, node_id) = self
-            .nodes
-            .iter()
-            .find_map(|node| {
-                node.default_output_pin()
-                    .map(|(pin, dev)| (pin, dev, node.node))
-            })
-            .ok_or(Error::DoesNotExist)?;
-
-        log::info!(
-            "hda#{}: default output {pin_id:?}, {dev:?}",
-            self.root.codec
-        );
-        let audio_node = self.node(node_id.nid).ok_or(Error::DoesNotExist)?;
-        let mut audio_widget = None;
-
-        for widget in audio_node.connection_chain(pin_id.nid) {
-            let wid = widget.node;
-
-            match &widget.widget {
-                AudioWidget::PinComplex(_) => {
-                    log::info!("  Configure pin @ {wid:?}");
-                    wid.set_power_state(&*self.hda, true).await?;
-                    // TODO only set for jacks
-                    wid.set_unsolicited_response(&*self.hda, Some(wid.nid))
-                        .await?;
-                    wid.modify_pin_control(
-                        &*self.hda,
-                        PinControl::OutEnable::SET + PinControl::HPhnEnable::SET,
-                    )
-                    .await?;
-                }
-                AudioWidget::AudioOutput(out) => {
-                    log::info!("  Configure audio output @ {wid:?}");
-                    audio_widget = Some(widget.node);
-                    wid.set_power_state(&*self.hda, true).await?;
-                    wid.set_unsolicited_response(&*self.hda, None).await?;
-                }
-                AudioWidget::Mixer(_) => {
-                    log::info!("  Configure mixer @ {wid:?}");
-                    wid.set_power_state(&*self.hda, true).await?;
-                }
-                _ => (),
-            }
-        }
-
-        Ok(audio_widget.unwrap())
-    }
-}
-
-impl fmt::Debug for Codec {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        f.debug_struct("Codec")
-            .field("id", &self.root.codec)
-            .field("nodes", &self.nodes)
-            .finish()
-    }
-}
-
-impl<'a> Iterator for WidgetConnectionChain<'a> {
-    type Item = &'a AudioWidgetNode;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        let current = self.current?;
-        let widget = self.afg.widget(current)?;
-        self.current = widget.connection;
-        Some(widget)
-    }
-}

kernel/driver/sound/intel-hda/src/codec/parameter.rs

@@ -1,198 +0,0 @@
-use libk::error::Error;
-use tock_registers::{register_bitfields, LocalRegisterCopy};
-
-use crate::ring::NodeParameterNumber;
-
-pub trait NodeParameter {
-    const NUMBER: NodeParameterNumber;
-    type Value;
-
-    fn data(&self) -> u32 {
-        0
-    }
-    fn from_response(response: u32) -> Self::Value;
-}
-
-#[derive(Debug)]
-pub struct DeviceId {
-    pub vendor_id: u16,
-    pub device_id: u16,
-}
-#[derive(Debug)]
-pub struct SubNodeInfo {
-    pub start_number: u8,
-    pub total_number: u8,
-}
-pub struct NodeType;
-register_bitfields! {
-    u32,
-    pub AudioWidgetCapabilities [
-        CHANNEL_COUNT       OFFSET(0)   NUMBITS(1)  [],
-        IN_AMP              OFFSET(1)   NUMBITS(1)  [],
-        OUT_AMP             OFFSET(2)   NUMBITS(1)  [],
-        AMP_OVERRIDE        OFFSET(3)   NUMBITS(1)  [],
-        FORMAT_OVERRIDE     OFFSET(4)   NUMBITS(1)  [],
-        STRIPE              OFFSET(5)   NUMBITS(1)  [],
-        PROC_WIDGET         OFFSET(6)   NUMBITS(1)  [],
-        UNSOL_CAPABLE       OFFSET(7)   NUMBITS(1)  [],
-        CONN_LIST           OFFSET(8)   NUMBITS(1)  [],
-        DIGITAL             OFFSET(9)   NUMBITS(1)  [],
-        POWER_CONTROL       OFFSET(10)  NUMBITS(1)  [],
-        LR_SWAP             OFFSET(11)  NUMBITS(1)  [],
-        CP_CAPS             OFFSET(12)  NUMBITS(1)  [],
-        EXT_CHANNEL_COUNT   OFFSET(13)  NUMBITS(3)  [],
-        DELAY               OFFSET(16)  NUMBITS(4)  [],
-        TYPE                OFFSET(20)  NUMBITS(4)  [
-            AudioOutput     = 0x0,
-            AudioInput      = 0x1,
-            AudioMixer      = 0x2,
-            AudioSelector   = 0x3,
-            PinComplex      = 0x4,
-            Power           = 0x5,
-            VolumeKnob      = 0x6,
-            BeepGenerator   = 0x7,
-            VendorDefined   = 0xF,
-        ],
-    ],
-}
-register_bitfields! {
-    u32,
-    pub AmpCapabilities [
-        OFFSET          OFFSET(0)   NUMBITS(7)  [],
-        NUM_STEPS       OFFSET(8)   NUMBITS(7)  [],
-        STEP_SIZE       OFFSET(16)  NUMBITS(7)  [],
-        MUTE_CAPABLE    OFFSET(31)  NUMBITS(1)  [],
-    ],
-}
-pub struct InputAmpCapabilities;
-pub struct OutputAmpCapabilities;
-pub struct ConnectionListLength;
-register_bitfields! {
-    u32,
-    pub SupportedStreamFormats [
-        PCM OFFSET(0)   NUMBITS(1)  [],
-        F32 OFFSET(1)   NUMBITS(1)  [],
-        AC3 OFFSET(2)   NUMBITS(1)  [],
-    ],
-}
-register_bitfields! {
-    u32,
-    pub SupportedPcmFormats [
-        RATE_8    OFFSET(0)   NUMBITS(1)  [],
-        RATE_11_025 OFFSET(1)   NUMBITS(1)  [],
-        RATE_16   OFFSET(2)   NUMBITS(1)  [],
-        RATE_22_05  OFFSET(3)   NUMBITS(1)  [],
-        RATE_32     OFFSET(4)   NUMBITS(1)  [],
-        RATE_44_1   OFFSET(5)   NUMBITS(1)  [],
-        RATE_48     OFFSET(6)   NUMBITS(1)  [],
-        RATE_88_2   OFFSET(7)   NUMBITS(1)  [],
-        RATE_96     OFFSET(8)   NUMBITS(1)  [],
-        RATE_176_4  OFFSET(9)   NUMBITS(1)  [],
-        RATE_192    OFFSET(10)  NUMBITS(1)  [],
-        RATE_384    OFFSET(11)  NUMBITS(1)  [],
-
-        BITS_8      OFFSET(16)  NUMBITS(1)  [],
-        BITS_16     OFFSET(17)  NUMBITS(1)  [],
-        BITS_20     OFFSET(18)  NUMBITS(1)  [],
-        BITS_24     OFFSET(19)  NUMBITS(1)  [],
-        BITS_32     OFFSET(20)  NUMBITS(1)  [],
-    ],
-}
-
-impl NodeParameter for DeviceId {
-    const NUMBER: NodeParameterNumber = NodeParameterNumber::DeviceId;
-    type Value = Self;
-
-    fn from_response(response: u32) -> Self::Value {
-        Self {
-            vendor_id: (response >> 16) as u16,
-            device_id: response as u16,
-        }
-    }
-}
-impl NodeParameter for SubNodeInfo {
-    const NUMBER: NodeParameterNumber = NodeParameterNumber::NodeCount;
-    type Value = Self;
-
-    fn from_response(response: u32) -> Self::Value {
-        Self {
-            start_number: (response >> 16) as u8,
-            total_number: response as u8,
-        }
-    }
-}
-impl NodeParameter for NodeType {
-    const NUMBER: NodeParameterNumber = NodeParameterNumber::FunctionGroupType;
-    type Value = u8;
-
-    fn from_response(response: u32) -> Self::Value {
-        response as u8
-    }
-}
-impl NodeParameter for AudioWidgetCapabilities::Register {
-    const NUMBER: NodeParameterNumber = NodeParameterNumber::AudioWidgetCapabilities;
-    type Value = LocalRegisterCopy<u32, Self>;
-
-    fn from_response(response: u32) -> Self::Value {
-        LocalRegisterCopy::new(response)
-    }
-}
-impl NodeParameter for InputAmpCapabilities {
-    const NUMBER: NodeParameterNumber = NodeParameterNumber::InputAmplifierCapabilities;
-    type Value = Option<LocalRegisterCopy<u32, AmpCapabilities::Register>>;
-
-    fn from_response(response: u32) -> Self::Value {
-        if response != 0 {
-            Some(LocalRegisterCopy::new(response))
-        } else {
-            None
-        }
-    }
-}
-impl NodeParameter for OutputAmpCapabilities {
-    const NUMBER: NodeParameterNumber = NodeParameterNumber::OutputAmplifierCapabilities;
-    type Value = Option<LocalRegisterCopy<u32, AmpCapabilities::Register>>;
-
-    fn from_response(response: u32) -> Self::Value {
-        if response != 0 {
-            Some(LocalRegisterCopy::new(response))
-        } else {
-            None
-        }
-    }
-}
-impl NodeParameter for ConnectionListLength {
-    const NUMBER: NodeParameterNumber = NodeParameterNumber::ConnectionListLength;
-    type Value = (usize, bool);
-
-    fn from_response(response: u32) -> Self::Value {
-        let long_form = response & (1 << 7) != 0;
-        let length = (response & 0x3F) as usize;
-        (length, long_form)
-    }
-}
-impl NodeParameter for SupportedStreamFormats::Register {
-    const NUMBER: NodeParameterNumber = NodeParameterNumber::SupportedFormats;
-    type Value = LocalRegisterCopy<u32, Self>;
-
-    fn from_response(response: u32) -> Self::Value {
-        LocalRegisterCopy::new(response)
-    }
-}
-impl NodeParameter for SupportedPcmFormats::Register {
-    const NUMBER: NodeParameterNumber = NodeParameterNumber::SupportedPcmRates;
-    type Value = LocalRegisterCopy<u32, Self>;
-
-    fn from_response(response: u32) -> Self::Value {
-        LocalRegisterCopy::new(response)
-    }
-}
-
-impl SubNodeInfo {
-    pub fn iter(&self) -> impl Iterator<Item = u8> {
-        // Exclude root
-        let start_subnode = self.start_number.max(1);
-        let end_subnode = self.start_number.saturating_add(self.total_number);
-        start_subnode..end_subnode
-    }
-}

kernel/driver/sound/intel-hda/src/lib.rs

@@ -1,344 +0,0 @@
-#![no_std]
-#![feature(let_chains)]
-
-use core::{
-    sync::atomic::{AtomicU8, Ordering},
-    time::Duration,
-};
-
-use alloc::{boxed::Box, sync::Arc, vec::Vec};
-use async_trait::async_trait;
-use codec::{AudioFormat, Codec, Node};
-use device_api::{
-    device::{Device, DeviceInitContext},
-    dma::DmaAllocator,
-    interrupt::{InterruptAffinity, InterruptHandler, IrqVector},
-};
-use futures_util::task::AtomicWaker;
-use libk::{
-    dma::DmaBuffer,
-    error::Error,
-    task::runtime::{self, psleep, with_timeout},
-    time::monotonic_time,
-};
-use libk_mm::device::DeviceMemoryIo;
-use libk_util::{
-    event::{BitmapEvent, BoolEvent, CounterEvent},
-    sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock},
-    OneTimeInit,
-};
-use regs::{Regs, SDxCTL0, CORBRP, GCAP, RIRBWP};
-use ring::{Command, CommandRing, NodeParameterNumber, Verb};
-use sink::HdAudioSink;
-use stream::{BufferDescriptorList, OutputStream};
-use tock_registers::interfaces::{ReadWriteable, Readable, Writeable};
-use ygg_driver_pci::{
-    capability::{power::DevicePowerState, PowerManagementCapability},
-    device::{PciDeviceInfo, PreferredInterruptMode},
-    macros::pci_driver,
-    PciBaseAddress, PciConfigurationSpace,
-};
-
-extern crate alloc;
-
-mod codec;
-mod regs;
-mod ring;
-mod sink;
-mod stream;
-
-struct HdAudio {
-    regs: IrqSafeSpinlock<DeviceMemoryIo<'static, Regs>>,
-    dma: Arc<dyn DmaAllocator>,
-    pci: PciDeviceInfo,
-
-    input_stream_count: usize,
-    output_stream_count: usize,
-    bidi_stream_count: usize,
-
-    sinks: Vec<IrqSafeSpinlock<Option<Arc<HdAudioSink>>>>,
-    last_stream_tag: AtomicU8,
-
-    softirq_event: BitmapEvent<AtomicWaker>,
-
-    cring: OneTimeInit<CommandRing>,
-}
-
-impl HdAudio {
-    fn new(
-        pci: PciDeviceInfo,
-        dma: Arc<dyn DmaAllocator>,
-        regs: DeviceMemoryIo<'static, Regs>,
-        input_stream_count: usize,
-        output_stream_count: usize,
-        bidi_stream_count: usize,
-    ) -> Self {
-        Self {
-            pci,
-            dma,
-            regs: IrqSafeSpinlock::new(regs),
-
-            input_stream_count,
-            output_stream_count,
-            bidi_stream_count,
-
-            sinks: (0..output_stream_count)
-                .map(|_| IrqSafeSpinlock::new(None))
-                .collect(),
-            last_stream_tag: AtomicU8::new(1),
-
-            softirq_event: BitmapEvent::new(AtomicWaker::new()),
-
-            cring: OneTimeInit::new(),
-        }
-    }
-}
-
-impl HdAudio {
-    async fn late_init(self: &Arc<Self>) -> Result<(), Error> {
-        // Interrogate present codecs
-        let codecs = self.regs.lock().take_attached_codecs();
-
-        for index in (0..16).filter(|&i| codecs & (1 << i) != 0) {
-            match Codec::probe(self.clone(), index).await {
-                Ok(codec) => {
-                    let codec = Arc::new(codec);
-                    log::info!("{codec:#?}");
-
-                    match codec.setup_default_output().await {
-                        Ok(id) => match self.add_sink(codec, id).await {
-                            Ok(_) => {
-                                log::info!("hda#{}: sink added", id.codec);
-                            }
-                            Err(error) => {
-                                log::info!("hda#{}: could not add sink: {error:?}", id.codec);
-                            }
-                        },
-                        Err(error) => {
-                            log::error!("Default output setup error: {error:?}");
-                        }
-                    }
-                }
-                Err(error) => {
-                    log::error!("hda: codec #{index} setup error: {error:?}");
-                }
-            }
-        }
-
-        loop {
-            runtime::sleep(Duration::from_secs(1)).await;
-        }
-    }
-
-    async fn add_sink(
-        self: &Arc<Self>,
-        codec: Arc<Codec>,
-        audio_widget: Node,
-    ) -> Result<Arc<HdAudioSink>, Error> {
-        for (index, slot) in self.sinks.iter().enumerate() {
-            let mut slot = slot.lock();
-            if slot.is_none() {
-                log::info!("hda: add sink #{index}");
-                let stream_tag = self.last_stream_tag.fetch_add(1, Ordering::Relaxed);
-                let sink = HdAudioSink::create(
-                    self.clone(),
-                    index + self.input_stream_count,
-                    stream_tag,
-                    codec.clone(),
-                    audio_widget,
-                )
-                .await?;
-
-                *slot = Some(sink.clone());
-                return Ok(sink);
-            }
-        }
-        log::warn!("hda: too many sinks");
-        Err(Error::InvalidOperation)
-    }
-
-    async fn softirq(&self) -> Result<(), Error> {
-        let cring = self.cring.get();
-
-        loop {
-            match with_timeout(self.softirq_event.wait(), Duration::from_millis(100)).await {
-                // IRQs happened
-                Ok(events) => {
-                    log::info!("softirq()");
-                    for sink_index in
-                        self.input_stream_count..self.input_stream_count + self.output_stream_count
-                    {
-                        // Handle sink events
-                        if events & (1 << sink_index) != 0
-                            && let Some(sink) = self.sink(sink_index - self.input_stream_count)
-                        {
-                            let position = self.regs.lock().STREAMS[sink_index].SDxLPIB.get();
-                            sink.handle_softirq(position).await;
-                        }
-                    }
-                    //
-                }
-                // Just poll
-                Err(_) => {}
-            }
-
-            let regs = self.regs.lock();
-            let rirb_head = regs.RIRBWP.read(RIRBWP::RIRBWP) as u8;
-            let corb_tail = regs.CORBRP.read(CORBRP::CORBRP) as u8;
-
-            cring.process_completions(corb_tail, rirb_head);
-        }
-    }
-
-    async fn perform_command(&self, command: Command) -> Result<u32, Error> {
-        let cring = self.cring.get();
-        let codec = command.codec();
-        let head = cring.submit_command(command).await?;
-
-        {
-            let regs = self.regs.lock();
-            regs.CORBWP.set(head as u16);
-        }
-
-        with_timeout(cring.wait_codec(codec), Duration::from_secs(3)).await
-    }
-
-    fn sink(&self, index: usize) -> Option<Arc<HdAudioSink>> {
-        self.sinks[index].lock().clone()
-    }
-}
-
-impl InterruptHandler for HdAudio {
-    fn handle_irq(self: Arc<Self>, _vector: IrqVector) -> bool {
-        let regs = self.regs.lock();
-        let rirbsts = regs.RIRBSTS.extract();
-        let status = regs.INTSTS.get() & 0x3FFFFFFF;
-
-        if rirbsts.get() != 0 {
-            regs.RIRBSTS.set(rirbsts.get());
-            self.softirq_event.signal(1 << 30);
-        }
-
-        if status != 0 {
-            for stream in 0..30 {
-                if status & (1 << stream) != 0 {
-                    let stream_status = regs.STREAMS[stream].SDxSTS.extract();
-                    regs.STREAMS[stream].SDxSTS.set(stream_status.get());
-                }
-            }
-            self.softirq_event.signal(status as u64);
-        }
-
-        true
-    }
-}
-
-impl Device for HdAudio {
-    unsafe fn init(self: Arc<Self>, _cx: DeviceInitContext) -> Result<(), Error> {
-        // FLR if capable
-        self.pci.function_level_reset().ok();
-        self.pci
-            .map_interrupt(InterruptAffinity::Any, self.clone())?;
-
-        let regs = self.regs.lock();
-
-        let vmaj = regs.VMAJ.get();
-        let vmin = regs.VMIN.get();
-
-        log::info!(
-            "hda: version={}.{}, iss={}, oss={}, bss={}",
-            vmaj,
-            vmin,
-            self.input_stream_count,
-            self.output_stream_count,
-            self.bidi_stream_count
-        );
-
-        regs.reset(
-            Duration::from_millis(100),
-            self.input_stream_count + self.output_stream_count + self.bidi_stream_count,
-        )?;
-
-        regs.disable_interrupts();
-        regs.disable_control();
-        let corb_size = regs.set_max_corb_size()?;
-        let rirb_size = regs.set_max_rirb_size()?;
-        log::info!("hda: corb size = {corb_size}, rirb size = {rirb_size}");
-
-        let (cring, corb_base, rirb_base) =
-            CommandRing::with_capacity(&*self.dma, corb_size, rirb_size)?;
-        regs.initialize_corb_rirb(corb_base, rirb_base)?;
-        regs.start_corb_rirb();
-        regs.enable_interrupts();
-
-        self.cring.init(cring);
-
-        drop(regs);
-
-        // Spawn softirq worker
-        let this = self.clone();
-        runtime::spawn(async move { this.softirq().await })?;
-
-        // Do the main initialization in background
-        let this = self.clone();
-        runtime::spawn(async move { this.late_init().await })?;
-
-        Ok(())
-    }
-
-    fn display_name(&self) -> &str {
-        "High Definition Audio"
-    }
-}
-
-pci_driver! {
-    matches: [
-        device (0x8086:0x2668), // Intel ICH6 82801FB/FBM/FR/FW/FRW HDA
-        device (0x8086:0x293E), // Intel ICH9 82801I HDA
-
-        device (0x1022:0x1487), // AMD Starship/Matisse HDA
-    ],
-    driver: {
-        fn probe(
-            &self,
-            info: &PciDeviceInfo,
-            dma: &Arc<dyn DmaAllocator>,
-        ) -> Result<Arc<dyn Device>, Error> {
-            info.set_command(true, true, false, true);
-
-            let base = info
-                .config_space
-                .bar(0)
-                .and_then(PciBaseAddress::as_memory)
-                .ok_or(Error::InvalidArgument)?;
-
-            info.init_interrupts(PreferredInterruptMode::Msi(true))?;
-
-            let regs = unsafe { DeviceMemoryIo::<Regs>::map(base, Default::default()) }?;
-
-            let gcap = regs.GCAP.extract();
-            if gcap.matches_all(GCAP::OK64::CLEAR) {
-                log::error!("Non 64-bit HD Audio not supported yet");
-                return Err(Error::NotImplemented);
-            }
-            let input_stream_count = gcap.read(GCAP::ISS) as usize;
-            let output_stream_count = gcap.read(GCAP::OSS).min(30) as usize;
-            let bidi_stream_count = gcap.read(GCAP::BSS) as usize;
-
-            let hda = Arc::new(HdAudio::new(
-                info.clone(),
-                dma.clone(),
-                regs,
-                input_stream_count,
-                output_stream_count,
-                bidi_stream_count,
-            ));
-
-            Ok(hda)
-        }
-
-        fn driver_name(&self) -> &str {
-            "intel-hda"
-        }
-    }
-}

kernel/driver/sound/intel-hda/src/regs.rs

@@ -1,453 +0,0 @@
-use core::time::Duration;
-
-use libk::{
-    dma::BusAddress,
-    error::Error,
-    task::runtime::{psleep, pwait},
-};
-use tock_registers::{
-    fields::FieldValue,
-    interfaces::{ReadWriteable, Readable, Writeable},
-    register_bitfields, register_structs,
-    registers::{ReadOnly, ReadWrite},
-};
-use ygg_driver_sound_core::{SampleFormat, SampleRate, SinkFormat};
-
-use crate::stream::OutputStream;
-
-register_bitfields! {
-    u8,
-    pub CORBCTL [
-        /// Enable CORB DMA engine
-        CORBRUN OFFSET(1)   NUMBITS(1)  [],
-        /// CORB memory error interrupt enable
-        CMEIE   OFFSET(0)   NUMBITS(1)  [],
-    ],
-    pub CORBSTS [
-        /// CORB memory error
-        CMEI    OFFSET(0)   NUMBITS(1)  [],
-    ],
-    pub CORBSIZE [
-        CORBSZCAP   OFFSET(4)   NUMBITS(4)  [],
-        CORBSIZE    OFFSET(0)   NUMBITS(2)  [
-            Size2   = 0b00,
-            Size16  = 0b01,
-            Size256 = 0b10,
-        ],
-    ],
-    pub RIRBCTL [
-        /// RIRB overrun interrupt control
-        RURBOIC     OFFSET(2)   NUMBITS(1)  [],
-        /// Enable RIRB DMA engine
-        RIRBDMAEN   OFFSET(1)   NUMBITS(1)  [],
-        /// Response interrupt control
-        RINTCTL     OFFSET(0)   NUMBITS(1)  [],
-    ],
-    pub RIRBSTS [
-        /// RIRB overrun interrupt
-        RIRBOIS     OFFSET(2)   NUMBITS(1)  [],
-        /// Response interrupt
-        RINTFL      OFFSET(0)   NUMBITS(1)  [],
-    ],
-    pub RIRBSIZE [
-        /// RIRB size capability
-        RIRBSZCAP   OFFSET(4)   NUMBITS(4)  [],
-        RIRBSIZE    OFFSET(0)   NUMBITS(2)  [
-            Size2   = 0b00,
-            Size16  = 0b01,
-            Size256 = 0b10,
-        ],
-    ],
-
-    pub SDxCTL0 [
-        SRST    OFFSET(0)   NUMBITS(1)  [],
-        RUN     OFFSET(1)   NUMBITS(1)  [],
-        IOCE    OFFSET(2)   NUMBITS(1)  [],
-        FEIE    OFFSET(3)   NUMBITS(1)  [],
-        DEIE    OFFSET(4)   NUMBITS(1)  [],
-    ],
-    pub SDxCTL2 [
-        STRIPE  OFFSET(0)   NUMBITS(2)  [],
-        TP      OFFSET(2)   NUMBITS(1)  [],
-        DIR     OFFSET(3)   NUMBITS(1)  [],
-        STRM    OFFSET(4)   NUMBITS(4)  [],
-    ],
-    pub SDxSTS [
-        BCIS    OFFSET(2)   NUMBITS(1)  [],
-        FIFOE   OFFSET(3)   NUMBITS(1)  [],
-        DESE    OFFSET(4)   NUMBITS(1)  [],
-        FIFORDY OFFSET(5)   NUMBITS(1)  [],
-    ],
-}
-
-register_bitfields! {
-    u16,
-    pub GCAP [
-        /// Number of output streams supported
-        OSS     OFFSET(12)  NUMBITS(4)  [],
-        /// Number of input streams supported
-        ISS     OFFSET(8)   NUMBITS(4)  [],
-        /// Number of bidirectional streams supported
-        BSS     OFFSET(3)   NUMBITS(5)  [],
-        /// Number of serial data out signals
-        NSDO    OFFSET(1)   NUMBITS(2)  [],
-        /// 64-bit addresses supported
-        OK64    OFFSET(0)   NUMBITS(1)  [],
-    ],
-    pub GSTS [
-        /// Flush status bit
-        FSTS    OFFSET(1)   NUMBITS(1)  [],
-    ],
-    pub CORBRP [
-        /// CORB read pointer reset
-        CORBRPRST   OFFSET(15)  NUMBITS(1)  [],
-        /// CORB read pointer
-        CORBRP      OFFSET(0)   NUMBITS(8)  [],
-    ],
-    pub RIRBWP [
-        /// RIRB write pointer reset
-        RIRBWPRST   OFFSET(15)  NUMBITS(1)  [],
-        /// RIRB write pointer
-        RIRBWP      OFFSET(0)   NUMBITS(8)  [],
-    ],
-
-    pub SDxFMT [
-        CHAN        OFFSET(0)   NUMBITS(4)  [],
-        BITS        OFFSET(4)   NUMBITS(3)  [
-            Bits8   = 0b000,
-            Bits16  = 0b001,
-            Bits20  = 0b010,
-            Bits24  = 0b011,
-            Bits32  = 0b100,
-        ],
-        DIV         OFFSET(8)   NUMBITS(3)  [],
-        MULT        OFFSET(11)  NUMBITS(3)  [],
-        BASE        OFFSET(14)  NUMBITS(1)  [],
-    ],
-}
-
-register_bitfields! {
-    u32,
-    pub GCTL [
-        /// Accept unsolicited response enable
-        UNSOL   OFFSET(8)   NUMBITS(1)  [],
-        /// Flush control
-        FCNTRL  OFFSET(1)   NUMBITS(1)  [],
-        /// Controller reset
-        CRST    OFFSET(0)   NUMBITS(1)  [],
-    ],
-    pub INTCTL [
-        /// Global interrupt enable
-        GIE     OFFSET(31)  NUMBITS(1)  [],
-        /// Controller interrupt enable
-        CIE     OFFSET(30)  NUMBITS(1)  [],
-        // Stream interrupt enable bits are based on stream counts
-    ],
-    pub INTSTS [
-        /// Global interrupt status
-        GIS     OFFSET(31)  NUMBITS(1)  [],
-        /// Controller interrupt status
-        CIS     OFFSET(30)  NUMBITS(1)  [],
-        // Stream interrupt status bits are based on stream counts
-    ],
-}
-
-register_structs! {
-    #[allow(non_snake_case)]
-    pub Regs {
-        (0x0000 => pub GCAP: ReadOnly<u16, GCAP::Register>),
-        (0x0002 => pub VMIN: ReadOnly<u8>),
-        (0x0003 => pub VMAJ: ReadOnly<u8>),
-        (0x0004 => pub OUTPAY: ReadOnly<u16>),
-        (0x0006 => pub INPAY: ReadOnly<u16>),
-        (0x0008 => pub GCTL: ReadWrite<u32, GCTL::Register>),
-        (0x000C => pub WAKEEN: ReadWrite<u16>),
-        (0x000E => pub WAKESTS: ReadWrite<u16>),
-        (0x0010 => pub GSTS: ReadWrite<u16, GSTS::Register>),
-        (0x0012 => _0),
-        (0x0018 => pub OUTSTRMPAY: ReadOnly<u16>),
-        (0x001A => pub INSTRMPAY: ReadOnly<u16>),
-        (0x001C => _1),
-        (0x0020 => pub INTCTL: ReadWrite<u32, INTCTL::Register>),
-        (0x0024 => pub INTSTS: ReadOnly<u32, INTSTS::Register>),
-        (0x0028 => _2),
-        (0x0030 => pub WALCLK: ReadOnly<u32>),
-        (0x0034 => _3),
-        (0x0038 => pub SSYNC: ReadWrite<u32>),
-        (0x003C => _4),
-        (0x0040 => pub CORBLBASE: ReadWrite<u32>),
-        (0x0044 => pub CORBUBASE: ReadWrite<u32>),
-        (0x0048 => pub CORBWP: ReadWrite<u16>),
-        (0x004A => pub CORBRP: ReadWrite<u16, CORBRP::Register>),
-        (0x004C => pub CORBCTL: ReadWrite<u8, CORBCTL::Register>),
-        (0x004D => pub CORBSTS: ReadWrite<u8, CORBSTS::Register>),
-        (0x004E => pub CORBSIZE: ReadWrite<u8, CORBSIZE::Register>),
-        (0x004F => _5),
-        (0x0050 => pub RIRBLBASE: ReadWrite<u32>),
-        (0x0054 => pub RIRBUBASE: ReadWrite<u32>),
-        (0x0058 => pub RIRBWP: ReadWrite<u16, RIRBWP::Register>),
-        (0x005A => pub RINTCNT: ReadWrite<u16>),
-        (0x005C => pub RIRBCTL: ReadWrite<u8, RIRBCTL::Register>),
-        (0x005D => pub RIRBSTS: ReadWrite<u8, RIRBSTS::Register>),
-        (0x005E => pub RIRBSIZE: ReadWrite<u8, RIRBSIZE::Register>),
-        (0x005F => _6),
-        (0x0060 => pub ICOI: ReadWrite<u32>),
-        (0x0064 => pub ICII: ReadWrite<u32>),
-        (0x0068 => pub ICIS: ReadOnly<u16>),
-        (0x006A => _7),
-        (0x0070 => pub DPLBASE: ReadWrite<u32>),
-        (0x0074 => pub DPUBASE: ReadWrite<u32>),
-        (0x0078 => _8),
-        (0x0080 => pub STREAMS: [StreamRegs; 128]),
-        (0x1080 => _9),
-        (0x2030 => pub WALCLKA: ReadOnly<u32>),
-        (0x2034 => _10),
-        (0x2080 => pub SDxLPIBA: [StreamLinkRegs; 128]),
-        (0x3080 => _11),
-        (0x4000 => @END),
-    }
-}
-
-register_structs! {
-    #[allow(non_snake_case)]
-    pub StreamRegs {
-        (0x00 => pub SDxCTL0: ReadWrite<u8, SDxCTL0::Register>),
-        (0x01 => pub SDxCTL1: ReadWrite<u8>),
-        (0x02 => pub SDxCTL2: ReadWrite<u8, SDxCTL2::Register>),
-        (0x03 => pub SDxSTS: ReadWrite<u8, SDxSTS::Register>),
-        (0x04 => pub SDxLPIB: ReadWrite<u32>),
-        (0x08 => pub SDxCBL: ReadWrite<u32>),
-        (0x0C => pub SDxLVI: ReadWrite<u16>),
-        (0x0E => _0),
-        (0x10 => pub SDxFIFOD: ReadWrite<u16>),
-        (0x12 => pub SDxFMT: ReadWrite<u16, SDxFMT::Register>),
-        (0x14 => _1),
-        (0x18 => pub SDxBDPL: ReadWrite<u32>),
-        (0x1C => pub SDxBDPU: ReadWrite<u32>),
-        (0x20 => @END),
-    }
-}
-
-register_structs! {
-    #[allow(non_snake_case)]
-    pub StreamLinkRegs {
-        (0x00 => _0),
-        (0x04 => pub SDxLPIBA: ReadOnly<u32>),
-        (0x08 => _1),
-        (0x20 => @END),
-    }
-}
-
-impl Regs {
-    pub fn reset(&self, timeout: Duration, stream_count: usize) -> Result<(), Error> {
-        // Go through all the streams, disable their DMA
-        for i in 0..stream_count {
-            let stream = &self.STREAMS[i];
-            stream
-                .SDxCTL0
-                .write(SDxCTL0::RUN::CLEAR + SDxCTL0::SRST::SET);
-        }
-
-        self.CORBCTL.write(CORBCTL::CORBRUN::CLEAR);
-        self.RIRBCTL.write(RIRBCTL::RIRBDMAEN::CLEAR);
-
-        // Flush the FIFO
-        self.GCTL.modify(GCTL::FCNTRL::SET);
-        psleep(Duration::from_millis(10));
-        pwait(timeout, Duration::from_millis(10), || {
-            self.GSTS.matches_all(GSTS::FSTS::SET)
-        })?;
-        self.GCTL.modify(GCTL::FCNTRL::CLEAR);
-        self.GSTS.write(GSTS::FSTS::SET);
-
-        // TODO does FIFO need to be flushed here as well?
-        // Begin controller reset
-        self.GCTL.write(GCTL::CRST::CLEAR);
-        psleep(Duration::from_millis(10));
-        // End controller reset
-        self.GCTL.write(GCTL::CRST::SET);
-        psleep(Duration::from_millis(10));
-        pwait(timeout, Duration::from_millis(10), || {
-            self.GCTL.matches_all(GCTL::CRST::SET)
-        })?;
-
-        Ok(())
-    }
-
-    pub fn disable_interrupts(&self) {
-        self.INTCTL.set(0);
-    }
-
-    pub fn enable_interrupts(&self) {
-        // Also accept unsolicited responses from codecs
-        self.GCTL.modify(GCTL::UNSOL::SET);
-        self.INTCTL.write(INTCTL::GIE::SET + INTCTL::CIE::SET);
-    }
-
-    pub fn enable_stream_interrupts(&self, stream: usize) {
-        self.INTCTL.set(self.INTCTL.get() | (1 << stream));
-    }
-
-    pub fn disable_control(&self) {
-        self.DPLBASE.set(0);
-        self.DPUBASE.set(0);
-
-        self.CORBCTL.write(CORBCTL::CORBRUN::CLEAR);
-        self.RIRBCTL.write(RIRBCTL::RIRBDMAEN::CLEAR);
-    }
-
-    pub fn set_max_corb_size(&self) -> Result<usize, Error> {
-        const SIZES: &[(usize, u8, FieldValue<u8, CORBSIZE::Register>)] = &[
-            (256, 0b0100, CORBSIZE::CORBSIZE::Size256),
-            (16, 0b0010, CORBSIZE::CORBSIZE::Size16),
-            (2, 0b0001, CORBSIZE::CORBSIZE::Size2),
-        ];
-
-        let corbsize = self.CORBSIZE.extract();
-        let cap = corbsize.read(CORBSIZE::CORBSZCAP);
-        for &(size, bit, cfg) in SIZES {
-            if cap & bit != 0 {
-                self.CORBSIZE.modify_no_read(corbsize, cfg);
-                return Ok(size);
-            }
-        }
-
-        log::error!("hda: no supported CORB size");
-        Err(Error::InvalidOperation)
-    }
-
-    pub fn set_max_rirb_size(&self) -> Result<usize, Error> {
-        const SIZES: &[(usize, u8, FieldValue<u8, RIRBSIZE::Register>)] = &[
-            (256, 0b0100, RIRBSIZE::RIRBSIZE::Size256),
-            (16, 0b0010, RIRBSIZE::RIRBSIZE::Size16),
-            (2, 0b0001, RIRBSIZE::RIRBSIZE::Size2),
-        ];
-
-        let rirbsize = self.RIRBSIZE.extract();
-        let cap = rirbsize.read(RIRBSIZE::RIRBSZCAP);
-        for &(size, bit, cfg) in SIZES {
-            if cap & bit != 0 {
-                self.RIRBSIZE.modify_no_read(rirbsize, cfg);
-                return Ok(size);
-            }
-        }
-
-        log::error!("hda: no supported RIRB size");
-        Err(Error::InvalidOperation)
-    }
-
-    pub fn initialize_corb_rirb(
-        &self,
-        corb_base: BusAddress,
-        rirb_base: BusAddress,
-    ) -> Result<(), Error> {
-        let corb_base = corb_base.into_u64();
-        let rirb_base = rirb_base.into_u64();
-
-        self.CORBUBASE.set((corb_base >> 32) as u32);
-        self.CORBLBASE.set(corb_base as u32);
-
-        self.RIRBUBASE.set((rirb_base >> 32) as u32);
-        self.RIRBLBASE.set(rirb_base as u32);
-
-        // Reset write/read pointers
-        self.CORBWP.set(0);
-        self.CORBRP.write(CORBRP::CORBRPRST::SET);
-        self.RIRBSTS
-            .write(RIRBSTS::RIRBOIS::SET + RIRBSTS::RINTFL::SET);
-        self.RINTCNT.set(255);
-
-        // Ensure CORBRP is reset properly
-        pwait(
-            Duration::from_millis(100),
-            Duration::from_millis(10),
-            || self.CORBRP.matches_all(CORBRP::CORBRPRST::SET),
-        )?;
-        self.CORBRP.write(CORBRP::CORBRPRST::CLEAR);
-        pwait(
-            Duration::from_millis(100),
-            Duration::from_millis(10),
-            || self.CORBRP.matches_all(CORBRP::CORBRPRST::CLEAR),
-        )?;
-
-        Ok(())
-    }
-
-    pub fn start_corb_rirb(&self) {
-        self.CORBCTL
-            .modify(CORBCTL::CORBRUN::SET + CORBCTL::CMEIE::SET);
-        self.RIRBCTL
-            .modify(RIRBCTL::RIRBDMAEN::SET + RIRBCTL::RINTCTL::SET);
-    }
-
-    pub fn take_attached_codecs(&self) -> u16 {
-        let state = self.WAKESTS.get();
-        self.WAKESTS.set(0xFFFF);
-        state
-    }
-
-    pub fn configure_stream(
-        &self,
-        index: usize,
-        config: &SinkFormat,
-        bdl_base: BusAddress,
-        buffer_size: usize,
-        buffer_count: usize,
-        number: u8,
-    ) -> Result<(), Error> {
-        log::info!("Configure stream: #{index}");
-        let bdl_base = bdl_base.into_u64();
-
-        // TODO don't use pwait in async context
-        let regs = &self.STREAMS[index];
-
-        // Stop and reset the stream
-        regs.SDxCTL0.write(SDxCTL0::SRST::SET);
-        pwait(
-            Duration::from_millis(100),
-            Duration::from_millis(10),
-            || regs.SDxCTL0.matches_all(SDxCTL0::SRST::SET),
-        )?;
-        regs.SDxCTL0.write(SDxCTL0::SRST::CLEAR);
-        pwait(
-            Duration::from_millis(100),
-            Duration::from_millis(10),
-            || regs.SDxCTL0.matches_all(SDxCTL0::SRST::CLEAR),
-        )?;
-
-        regs.SDxCTL2.write(SDxCTL2::STRM.val(number));
-        regs.SDxSTS.write(SDxSTS::BCIS::SET);
-
-        regs.SDxLVI.set(buffer_count as u16 - 1);
-        regs.SDxCBL.set((buffer_count * buffer_size) as _);
-        regs.SDxBDPU.set((bdl_base >> 32) as u32);
-        regs.SDxBDPL.set(bdl_base as u32);
-
-        let mut format = SDxFMT::CHAN.val(config.channels as u16 - 1);
-
-        match config.sample_format {
-            SampleFormat::S8 => format += SDxFMT::BITS::Bits8,
-            SampleFormat::S16Le => format += SDxFMT::BITS::Bits16,
-        }
-        match config.sample_rate {
-            SampleRate::Rate8000 => format += SDxFMT::DIV.val(5),
-            SampleRate::Rate11025 => format += SDxFMT::DIV.val(3) + SDxFMT::BASE::SET,
-            SampleRate::Rate16000 => format += SDxFMT::DIV.val(2),
-            SampleRate::Rate22050 => format += SDxFMT::DIV.val(1) + SDxFMT::BASE::SET,
-            SampleRate::Rate32000 => format += SDxFMT::DIV.val(2) + SDxFMT::MULT.val(1),
-            SampleRate::Rate44100 => format += SDxFMT::BASE::SET,
-            SampleRate::Rate48000 => (),
-            SampleRate::Rate88200 => format += SDxFMT::MULT.val(1) + SDxFMT::BASE::SET,
-            SampleRate::Rate96000 => format += SDxFMT::MULT.val(1),
-            SampleRate::Rate176400 => format += SDxFMT::MULT.val(3) + SDxFMT::BASE::SET,
-            SampleRate::Rate192000 => format += SDxFMT::MULT.val(3),
-            SampleRate::Rate384000 => todo!(),
-        }
-
-        regs.SDxFMT.write(format);
-
-        regs.SDxCTL0.modify(SDxCTL0::IOCE::SET);
-
-        Ok(())
-    }
-}

kernel/driver/sound/intel-hda/src/ring.rs

@@ -1,396 +0,0 @@
-use core::{future::poll_fn, task::Poll};
-
-use device_api::dma::DmaAllocator;
-use futures_util::task::AtomicWaker;
-use libk::{
-    dma::{BusAddress, DmaBuffer},
-    error::Error,
-};
-use libk_util::{sync::IrqSafeSpinlock, waker::QueueWaker};
-use tock_registers::register_bitfields;
-use yggdrasil_abi::primitive_enum;
-
-// 4 bit address
-const MAX_CODEC: usize = 16;
-
-// Combines CORB and RIRB
-struct Inner {
-    corb: DmaBuffer<[u32]>,
-    rirb: DmaBuffer<[(u32, u32)]>,
-
-    codec_responses: [Option<u32>; MAX_CODEC],
-
-    corb_head: u32,
-    corb_tail: u32,
-    rirb_tail: u32,
-}
-
-pub struct CommandRing {
-    inner: IrqSafeSpinlock<Inner>,
-
-    // 16 possible codec addresses
-    codec_notify: [AtomicWaker; MAX_CODEC],
-    // Unsolicited response notify
-    unsol_notify: QueueWaker,
-}
-
-primitive_enum! {
-    pub enum Verb: u32 {
-        GetParameter = 0xF00,
-        GetConnectionListEntry = 0xF02,
-        SetSelectedInput = 0x701,
-        SetPowerState = 0x705,
-        SetConverterStreamChannel = 0x706,
-        SetPinWidgetControl = 0x707,
-        GetPinWidgetControl = 0xF07,
-        GetPinWidgetSense = 0xF09,
-        SetUnsolicitedResponse = 0x708,
-        GetUnsolicitedResponse = 0xF08,
-        SetEapdBtl = 0x70C,
-        GetPinWidgetDefaultConfig = 0xF1C,
-        SetOutputConverterChannelCount = 0x72D,
-        AudioFunctionNodeReset = 0x7FF,
-        SetAmplifierGain = 0x003,
-        SetStreamConverterFormat = 0x002,
-        SetStreamFormat = 0x200,
-    }
-}
-
-primitive_enum! {
-    pub enum NodeParameterNumber: u32 {
-        DeviceId = 0x00,
-        RevisionId = 0x02,
-        NodeCount = 0x04,
-        FunctionGroupType = 0x05,
-        AudioGroupCapabilities = 0x08,
-        AudioWidgetCapabilities = 0x09,
-        SupportedPcmRates = 0x0A,
-        SupportedFormats = 0x0B,
-        PinCapabilities = 0x0C,
-        InputAmplifierCapabilities = 0x0D,
-        OutputAmplifierCapabilities = 0x12,
-        ConnectionListLength = 0x0E,
-        SupportedPowerStates = 0x0F,
-        ProcessingCapabilities = 0x10,
-        GpioCount = 0x11,
-        VolumeCapabilities = 0x13,
-    }
-}
-
-register_bitfields! {
-    u32,
-    pub PinControl [
-        VRefEn      OFFSET(0)   NUMBITS(3)  [],
-        InEnable    OFFSET(5)   NUMBITS(1)  [],
-        OutEnable   OFFSET(6)   NUMBITS(1)  [],
-        HPhnEnable  OFFSET(7)   NUMBITS(1)  [],
-    ],
-}
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum PinColor {
-    Black,
-    Gray,
-    Blue,
-    Green,
-    Red,
-    Orange,
-    Yellow,
-    Purple,
-    Pink,
-    White,
-    Other,
-}
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum PinType {
-    I1_8,
-    I1_4,
-    AtapiInternal,
-    Rca,
-    Optical,
-    OtherDigital,
-    OtherAnalog,
-    MultichannelAnalog,
-    Xlr,
-    Rj11,
-    Combination,
-    Other,
-}
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum PinDevice {
-    LineOut,
-    Speaker,
-    HeadphoneOut,
-    Cd,
-    SpdifOut,
-    OtherDigitalOut,
-    ModemLineSide,
-    ModemHandsetSide,
-    LineIn,
-    Aux,
-    MicIn,
-    Telephony,
-    SpdifIn,
-    OtherDigitalIn,
-    Other,
-}
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum PinConnectivity {
-    None,
-    Jack,
-    Fixed,
-    Both,
-}
-
-#[derive(Debug)]
-pub struct PinDefaultConfig {
-    pub sequence: u8,
-    pub default_association: u8,
-    pub color: PinColor,
-    pub connection_type: PinType,
-    pub default_device: PinDevice,
-    pub connectivity: PinConnectivity,
-}
-
-#[derive(Clone, Copy)]
-pub struct Command(u32);
-
-impl Inner {
-    // Submit a command, returns new head and command token
-    pub fn push(&mut self, command: Command) -> Option<u8> {
-        if (self.corb_head + 1) % (self.corb.len() as u32) == self.corb_tail {
-            return None;
-        }
-        self.corb_head = (self.corb_head + 1) % self.corb.len() as u32;
-        let index = self.corb_head as usize % self.corb.len();
-        self.corb[index] = command.0;
-        self.corb.cache_flush_element(index, true);
-        Some(index as u8)
-    }
-
-    pub fn process_responses<U: Fn(u8, u32), C: Fn(u8)>(
-        &mut self,
-        unsol_handler: U,
-        codec_handler: C,
-        corb_tail: u32,
-        rirb_head: u32,
-    ) -> usize {
-        self.corb_tail = corb_tail;
-
-        let mut count = 0;
-        while self.rirb_tail != rirb_head {
-            self.rirb_tail = (self.rirb_tail + 1) % self.rirb.len() as u32;
-
-            let index = self.rirb_tail as usize;
-            self.rirb.cache_flush_element(index, false);
-            let (w0, w1) = self.rirb[index];
-            let codec = (w1 & 0xF) as u8;
-            if w1 & (1 << 4) == 0 {
-                // Solicited
-                self.codec_responses[codec as usize] = Some(w0);
-                codec_handler(codec);
-            } else {
-                // Unsolicited
-                unsol_handler(codec, w0);
-            }
-
-            count += 1;
-        }
-        count
-    }
-
-    pub fn take_codec_response(&mut self, codec: u8) -> Option<u32> {
-        self.codec_responses[codec as usize].take()
-    }
-}
-
-impl CommandRing {
-    pub fn with_capacity(
-        dma: &dyn DmaAllocator,
-        corb_size: usize,
-        rirb_size: usize,
-    ) -> Result<(Self, BusAddress, BusAddress), Error> {
-        let corb = DmaBuffer::new_slice(dma, 0, corb_size)?;
-        let rirb = DmaBuffer::new_slice(dma, (0, 0), rirb_size)?;
-        let corb_base = corb.bus_address();
-        let rirb_base = rirb.bus_address();
-        Ok((
-            Self {
-                inner: IrqSafeSpinlock::new(Inner {
-                    corb,
-                    rirb,
-                    corb_head: 0,
-                    corb_tail: 0,
-                    rirb_tail: 0,
-
-                    codec_responses: [None; MAX_CODEC],
-                }),
-
-                codec_notify: [const { AtomicWaker::new() }; 16],
-                unsol_notify: QueueWaker::new(),
-            },
-            corb_base,
-            rirb_base,
-        ))
-    }
-
-    // Wait for a specific codec to send its response
-    pub async fn wait_codec(&self, codec: u8) -> u32 {
-        poll_fn(|cx| {
-            if let Some(response) = self.inner.lock().take_codec_response(codec) {
-                Poll::Ready(response)
-            } else {
-                self.codec_notify[codec as usize].register(cx.waker());
-                Poll::Pending
-            }
-        })
-        .await
-    }
-
-    pub async fn submit_command(&self, command: Command) -> Result<u8, Error> {
-        // TODO block and wait for free slots in CORB
-        let mut inner = self.inner.lock();
-        // Clear previous codec response
-        let head = inner.push(command).ok_or(Error::WouldBlock)?;
-        inner.codec_responses[command.codec() as usize] = None;
-        Ok(head)
-    }
-
-    pub(super) fn process_completions(&self, corb_tail: u8, rirb_head: u8) -> usize {
-        let mut inner = self.inner.lock();
-        inner.process_responses(
-            |codec, message| {
-                log::info!("Unsolicited message: {message:#x} from codec {codec:#x}");
-            },
-            |codec| self.codec_notify[codec as usize].wake(),
-            corb_tail as u32,
-            rirb_head as u32,
-        )
-    }
-}
-
-impl Command {
-    pub const fn new(codec: u8, node: u8, verb: Verb, payload: u32) -> Self {
-        Self(((codec as u32) << 28) | ((node as u32) << 20) | ((verb as u32) << 8) | payload)
-    }
-
-    pub const fn get_parameter(codec: u8, node: u8, parameter: NodeParameterNumber) -> Self {
-        Self::new(codec, node, Verb::GetParameter, parameter as u32)
-    }
-
-    pub const fn get_connection_list_entry(codec: u8, node: u8, index: usize) -> Self {
-        Self::new(codec, node, Verb::GetConnectionListEntry, index as u32)
-    }
-
-    pub const fn set_stream_number(codec: u8, node: u8, stream: usize) -> Self {
-        Self::new(codec, node, Verb::SetConverterStreamChannel, stream as u32)
-    }
-
-    pub fn codec(&self) -> u8 {
-        ((self.0 >> 28) & 0xF) as u8
-    }
-}
-
-impl From<u32> for PinConnectivity {
-    fn from(value: u32) -> Self {
-        match value {
-            0b00 => Self::Jack,
-            0b10 => Self::Fixed,
-            0b11 => Self::Both,
-            _ => Self::None,
-        }
-    }
-}
-
-impl From<u32> for PinColor {
-    fn from(value: u32) -> Self {
-        match value {
-            1 => Self::Black,
-            2 => Self::Gray,
-            3 => Self::Blue,
-            4 => Self::Green,
-            5 => Self::Red,
-            6 => Self::Orange,
-            7 => Self::Yellow,
-            8 => Self::Purple,
-            9 => Self::Pink,
-            14 => Self::White,
-            _ => Self::Other,
-        }
-    }
-}
-
-impl From<u32> for PinType {
-    fn from(value: u32) -> Self {
-        match value {
-            1 => Self::I1_8,
-            2 => Self::I1_4,
-            3 => Self::AtapiInternal,
-            4 => Self::Rca,
-            5 => Self::Optical,
-            6 => Self::OtherDigital,
-            7 => Self::OtherAnalog,
-            8 => Self::MultichannelAnalog,
-            9 => Self::Xlr,
-            10 => Self::Rj11,
-            11 => Self::Combination,
-            _ => Self::Other,
-        }
-    }
-}
-
-impl PinDevice {
-    pub fn output_score(&self) -> u32 {
-        match self {
-            Self::HeadphoneOut => 10,
-            Self::Speaker => 9,
-            Self::LineOut => 8,
-            Self::SpdifOut => 7,
-            _ => 0,
-        }
-    }
-}
-impl From<u32> for PinDevice {
-    fn from(value: u32) -> Self {
-        match value {
-            0 => Self::LineOut,
-            1 => Self::Speaker,
-            2 => Self::HeadphoneOut,
-            3 => Self::Cd,
-            4 => Self::SpdifOut,
-            5 => Self::OtherDigitalOut,
-            6 => Self::ModemLineSide,
-            7 => Self::ModemHandsetSide,
-            8 => Self::LineIn,
-            9 => Self::Aux,
-            10 => Self::MicIn,
-            11 => Self::Telephony,
-            12 => Self::SpdifIn,
-            13 => Self::OtherDigitalIn,
-            _ => Self::Other,
-        }
-    }
-}
-
-impl From<u32> for PinDefaultConfig {
-    fn from(value: u32) -> Self {
-        let sequence = (value & 0xF) as u8;
-        let default_association = ((value >> 4) & 0xF) as u8;
-        let color = PinColor::from((value >> 12) & 0xF);
-        let connection_type = PinType::from((value >> 16) & 0xF);
-        let default_device = PinDevice::from((value >> 20) & 0xF);
-        let connectivity = PinConnectivity::from(value >> 30);
-
-        Self {
-            sequence,
-            color,
-            connectivity,
-            connection_type,
-            default_device,
-            default_association,
-        }
-    }
-}

kernel/driver/sound/intel-hda/src/sink.rs

@@ -1,171 +0,0 @@
-use core::{
-    sync::atomic::{AtomicBool, Ordering},
-    task::{Context, Poll},
-    time::Duration,
-};
-
-use alloc::{boxed::Box, sync::Arc, vec::Vec};
-use async_trait::async_trait;
-use device_api::device::Device;
-use futures_util::task::AtomicWaker;
-use libk::{device::char::CharDevice, error::Error, fs::devfs, task::runtime, vfs::FileReadiness};
-use libk_mm::PageBox;
-use libk_util::{event::BoolEvent, sync::spin_rwlock::IrqSafeRwLock};
-use tock_registers::interfaces::ReadWriteable;
-use ygg_driver_sound_core::{
-    AudioSink, SampleFormat, SampleRate, SinkFormat, SinkSupportedFormats,
-};
-use yggdrasil_abi::io::FileMode;
-
-use crate::{
-    codec::{Codec, Node},
-    regs::SDxCTL0,
-    ring::Verb,
-    stream::OutputStream,
-    HdAudio,
-};
-
-pub struct HdAudioSink {
-    hda: Arc<HdAudio>,
-    codec: Arc<Codec>,
-    audio_widget: Node,
-    pub index: usize,
-    stream_tag: u8,
-    stream: OutputStream,
-    notify: BoolEvent,
-    config: IrqSafeRwLock<SinkFormat>,
-    playing: AtomicBool,
-}
-
-impl HdAudioSink {
-    pub async fn create(
-        hda: Arc<HdAudio>,
-        index: usize,
-        stream_tag: u8,
-        codec: Arc<Codec>,
-        audio_widget: Node,
-    ) -> Result<Arc<Self>, Error> {
-        // TODO use default/current stream format (if PCM), don't always set 48KHz S16 x2
-        log::info!("hda: create sink #{index}, tag {stream_tag}");
-        let config = SinkFormat {
-            sample_rate: SampleRate::Rate48000,
-            sample_format: SampleFormat::S16Le,
-            channels: 2,
-        };
-        let one_millisecond =
-            (config.sample_rate as usize * config.sample_format.sample_size() * config.channels)
-                / 1000;
-        let bdl_capacity = 32;
-        log::info!("buffer_count = {bdl_capacity}, buffer_size = {one_millisecond}");
-        let (stream, bdl_address) = OutputStream::new(hda.clone(), one_millisecond, bdl_capacity)?;
-
-        // TODO check if the widget actually supports such format
-        audio_widget
-            .perform_command(&*hda, Verb::SetStreamFormat, (1 << 4) | 1)
-            .await?;
-
-        {
-            let regs = hda.regs.lock();
-            regs.configure_stream(
-                index,
-                &config,
-                bdl_address,
-                one_millisecond,
-                bdl_capacity,
-                stream_tag,
-            )?;
-            regs.enable_stream_interrupts(index);
-        }
-
-        audio_widget.set_stream(&*hda, stream_tag, 0).await?;
-
-        let this = Arc::new(Self {
-            hda,
-            codec,
-            audio_widget,
-            index,
-            stream_tag,
-            stream,
-            notify: BoolEvent::new(),
-            playing: AtomicBool::new(false),
-            config: IrqSafeRwLock::new(config),
-        });
-
-        ygg_driver_sound_core::register_audio_sink(this.clone());
-
-        Ok(this)
-    }
-
-    pub async fn handle_softirq(&self, position: u32) {
-        self.stream.update_tail(position);
-        self.notify.signal_saturating();
-    }
-
-    async fn write_blocking(&self, data: &[u8]) -> Result<(), Error> {
-        let mut position = 0;
-        while position != data.len() {
-            let written = self.stream.write(&data[position..]);
-            position += written;
-            if written == 0 {
-                self.notify.wait_reset().await;
-            }
-        }
-        Ok(())
-    }
-}
-
-#[async_trait]
-impl AudioSink for HdAudioSink {
-    async fn set_format(&self, format: SinkFormat) -> Result<(), Error> {
-        todo!()
-    }
-    async fn current_format(&self) -> Result<SinkFormat, Error> {
-        todo!()
-    }
-    async fn supported_formats(&self) -> Result<SinkSupportedFormats, Error> {
-        todo!()
-    }
-
-    async fn write(&self, data: &[u8]) -> Result<usize, Error> {
-        let amount = data.len().min(48000 * 2);
-        if !self.playing.swap(true, Ordering::Acquire) {
-            self.start().await?;
-        }
-        self.write_blocking(&data[..amount]).await?;
-        Ok(data.len())
-    }
-    async fn start(&self) -> Result<(), Error> {
-        self.playing.store(true, Ordering::Release);
-        let regs = self.hda.regs.lock();
-        log::info!("hda: start stream #{}", self.index);
-        self.stream.reset();
-        regs.STREAMS[self.index].SDxCTL0.modify(SDxCTL0::RUN::SET);
-        Ok(())
-    }
-    fn stop(&self) -> Result<(), Error> {
-        self.playing.store(false, Ordering::Release);
-        let regs = self.hda.regs.lock();
-        log::info!("hda: stop stream #{}", self.index);
-        regs.STREAMS[self.index].SDxCTL0.modify(SDxCTL0::RUN::CLEAR);
-        Ok(())
-    }
-}
-
-#[async_trait]
-impl CharDevice for HdAudioSink {
-    async fn write(&self, buffer: &[u8]) -> Result<usize, Error> {
-        AudioSink::write(self, buffer).await
-    }
-}
-
-impl FileReadiness for HdAudioSink {
-    fn poll_read(&self, cx: &mut Context<'_>) -> Poll<Result<(), Error>> {
-        todo!()
-    }
-}
-
-impl Device for HdAudioSink {
-    fn display_name(&self) -> &str {
-        "HD Audio Sink"
-    }
-}

kernel/driver/sound/intel-hda/src/stream.rs

@@ -1,135 +0,0 @@
-use alloc::{sync::Arc, vec::Vec};
-use device_api::dma::DmaAllocator;
-use libk::{
-    dma::{BusAddress, DmaBuffer},
-    error::Error,
-};
-use libk_util::{
-    event::BoolEvent,
-    sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock},
-};
-use ygg_driver_sound_core::SinkFormat;
-
-use crate::HdAudio;
-
-#[derive(Debug)]
-#[repr(C)]
-pub struct BufferDescriptor {
-    address: BusAddress,
-    length: u32,
-    flags: u32,
-}
-
-pub struct BufferDescriptorList {
-    entries: DmaBuffer<[BufferDescriptor]>,
-    periods: Vec<DmaBuffer<[u8]>>,
-    wr: usize,
-    rd: usize,
-    buffer_size: usize,
-}
-
-pub enum StreamKind {
-    Input,
-    Output,
-    Bidi,
-}
-
-pub struct OutputStream {
-    pub hda: Arc<HdAudio>,
-    pub bdl: IrqSafeSpinlock<BufferDescriptorList>,
-}
-
-impl BufferDescriptorList {
-    pub fn with_capacity(
-        dma: &dyn DmaAllocator,
-        capacity: usize,
-        frame_size: usize,
-    ) -> Result<Self, Error> {
-        let periods: Vec<DmaBuffer<[u8]>> = (0..capacity)
-            .map(|_| DmaBuffer::new_slice(dma, 0, frame_size))
-            .collect::<Result<_, _>>()?;
-        let entries =
-            DmaBuffer::new_slice_with(dma, |i| BufferDescriptor::new(&periods[i]), capacity)?;
-        Ok(Self {
-            entries,
-            wr: 0,
-            rd: 0,
-            periods,
-            buffer_size: frame_size,
-        })
-    }
-
-    pub fn write(&mut self, data: &[u8]) -> usize {
-        let capacity = self.periods.len() * self.buffer_size;
-
-        let mut position = 0;
-        while (self.wr + 1) % capacity != self.rd && position != data.len() {
-            let p = self.wr / self.buffer_size;
-            let o = self.wr % self.buffer_size;
-
-            self.periods[p][o] = data[position];
-
-            position += 1;
-            self.wr = (self.wr + 1) % capacity;
-        }
-
-        position
-    }
-
-    pub fn update_tail(&mut self, tail: u32) {
-        self.rd = tail as usize;
-    }
-
-    pub fn reset(&mut self) {
-        self.wr = self.rd;
-    }
-}
-
-impl BufferDescriptor {
-    pub fn new(buffer: &DmaBuffer<[u8]>) -> Self {
-        Self {
-            address: buffer.bus_address(),
-            length: buffer.len() as _,
-            flags: 1,
-        }
-    }
-}
-
-impl OutputStream {
-    pub fn new(
-        hda: Arc<HdAudio>,
-        buffer_size: usize,
-        bdl_capacity: usize,
-    ) -> Result<(Self, BusAddress), Error> {
-        let bdl = BufferDescriptorList::with_capacity(&*hda.dma, bdl_capacity, buffer_size)?;
-        let bdl_base = bdl.entries.bus_address();
-        Ok((
-            Self {
-                hda,
-                bdl: IrqSafeSpinlock::new(bdl),
-            },
-            bdl_base,
-        ))
-    }
-
-    pub fn params(&self) -> (BusAddress, usize, usize) {
-        let bdl = self.bdl.lock();
-        (
-            bdl.entries.bus_address(),
-            bdl.buffer_size,
-            bdl.entries.len(),
-        )
-    }
-
-    pub fn reset(&self) {
-        self.bdl.lock().reset()
-    }
-
-    pub fn write(&self, bytes: &[u8]) -> usize {
-        self.bdl.lock().write(bytes)
-    }
-
-    pub fn update_tail(&self, tail: u32) {
-        self.bdl.lock().update_tail(tail)
-    }
-}

kernel/driver/usb/xhci/src/controller.rs

@@ -25,10 +25,11 @@ use tock_registers::{
 };
 use ygg_driver_pci::{device::PciDeviceInfo, PciConfigurationSpace};
 use ygg_driver_usb::{
-    bus::UsbBusManager,
-    device::{UsbBusAddress, UsbDeviceAccess, UsbSpeed},
+    address::UsbBusAddress,
+    bus::{UsbBusManager, UsbBusWrapper},
+    descriptor,
+    device::{UsbDeviceAccess, UsbSpeed},
     error::UsbError,
-    info::UsbVersion,
     pipe::control::UsbControlPipeAccess,
     UsbHostController,
 };
@@ -67,7 +68,7 @@ struct ScratchpadArray {
 }
 
 struct RootHubPort {
-    version: UsbVersion,
+    version: u16,
     slot_type: u8,
 }
 
@@ -90,6 +91,7 @@ pub struct Xhci {
     pub(crate) slots: Vec<IrqSafeRwLock<Option<Arc<XhciBusDevice>>>>,
     pub(crate) port_slot_map: Vec<AtomicU8>,
     bus_index: OneTimeInit<u16>,
+    bus: OneTimeInit<Arc<UsbBusWrapper>>,
     port_event_map: EventBitmap,
 }
 
@@ -146,9 +148,7 @@ impl Xhci {
         for cap in regs.extended_capabilities.iter() {
             match cap {
                 ExtendedCapability::ProtocolSupport(support) => {
-                    let Some(version) = support.usb_revision() else {
-                        continue;
-                    };
+                    let version = support.usb_revision();
 
                     for port in support.port_range() {
                         log::info!("* Port {port}: {version}");
@@ -195,6 +195,7 @@ impl Xhci {
 
             root_hub_ports,
             bus_index: OneTimeInit::new(),
+            bus: OneTimeInit::new(),
             endpoints: IrqSafeRwLock::new(BTreeMap::new()),
             slots,
             port_slot_map,
@@ -282,7 +283,7 @@ impl Xhci {
             .as_ref()
             .ok_or(UsbError::PortInitFailed)?;
 
-        let need_reset = !root_hub_port.version.is_version_3();
+        let need_reset = !descriptor::is_version_3(root_hub_port.version);
 
         if need_reset {
             self.reset_port(regs).await?;
@@ -341,7 +342,8 @@ impl Xhci {
             device: bus_address,
         });
 
-        let device = UsbDeviceAccess::setup(slot).await?;
+        let bus = self.bus.get();
+        let device = UsbDeviceAccess::setup(bus.clone(), slot).await?;
         UsbBusManager::register_device(device.into());
 
         Ok(())
@@ -523,8 +525,9 @@ impl Device for Xhci {
 
         op.wait_usbsts_bit(USBSTS::CNR::CLEAR, 100000000)?;
 
-        let bus = UsbBusManager::register_bus(self.clone());
-        self.bus_index.init(bus);
+        let (bus_index, bus) = UsbBusManager::register_bus(self.clone());
+        self.bus_index.init(bus_index);
+        self.bus.init(bus);
 
         runtime::spawn(self.clone().port_handler_task()).ok();
 

kernel/driver/usb/xhci/src/device.rs

@@ -6,8 +6,9 @@ use libk_util::{
 };
 use xhci_lib::context;
 use ygg_driver_usb::{
+    address::UsbBusAddress,
     communication::UsbDirection,
-    device::{UsbBusAddress, UsbDevice, UsbDeviceDetachHandler, UsbSpeed},
+    device::{UsbDevice, UsbDeviceDetachHandler, UsbSpeed},
     error::UsbError,
     info::UsbEndpointType,
     pipe::{
@@ -63,8 +64,8 @@ impl UsbDevice for XhciBusDevice {
         *self.detach_handler.lock() = Some(handler);
     }
 
-    fn controller_ref(&self) -> &dyn UsbHostController {
-        self.xhci.as_ref()
+    fn host_controller(&self) -> Arc<dyn UsbHostController> {
+        self.xhci.clone()
     }
 
     fn debug(&self) {}

kernel/driver/usb/xhci/src/lib.rs

@@ -9,7 +9,7 @@ use controller::Xhci;
 use device_api::{device::Device, dma::DmaAllocator, interrupt::InterruptAffinity};
 use regs::Regs;
 use ygg_driver_pci::{
-    capability::{power::DevicePowerState, PowerManagementCapability},
+    capability::{DevicePowerState, PowerManagementCapability},
     device::{PciDeviceInfo, PreferredInterruptMode},
     macros::pci_driver,
     PciCommandRegister, PciConfigurationSpace,

kernel/driver/usb/xhci/src/regs/extended.rs

@@ -7,7 +7,7 @@ use alloc::vec::Vec;
 use libk::error::Error;
 use libk_mm::{address::PhysicalAddress, device::DeviceMemoryIo};
 use libk_util::sync::spin_rwlock::IrqSafeRwLock;
-use ygg_driver_usb::{error::UsbError, info::UsbVersion};
+use ygg_driver_usb::error::UsbError;
 
 pub struct ProtocolSupport {
     words: [u32; 4],
@@ -69,8 +69,8 @@ impl ExtendedCapability {
 }
 
 impl ProtocolSupport {
-    pub fn usb_revision(&self) -> Option<UsbVersion> {
-        UsbVersion::from_bcd_usb((self.words[0] >> 16) as u16)
+    pub fn usb_revision(&self) -> u16 {
+        (self.words[0] >> 16) as u16
     }
 
     pub fn slot_type(&self) -> u8 {

kernel/driver/usb/xhci/src/ring/transfer.rs

@@ -94,7 +94,7 @@ impl TransferRing {
         })
     }
 
-    pub fn transaction_builder(self: &Arc<Self>) -> Result<TransactionBuilder, UsbError> {
+    pub fn transaction_builder(self: &Arc<Self>) -> Result<TransactionBuilder<'_>, UsbError> {
         if self.shutdown.load(Ordering::Acquire) {
             return Err(UsbError::DeviceDisconnected);
         }

kernel/driver/virtio/core/src/transport/pci.rs

@@ -5,7 +5,7 @@ use tock_registers::{
 };
 use ygg_driver_pci::{
     capability::{
-        virtio::VirtioCapabilityData, VirtioCommonConfigCapability, VirtioDeviceConfigCapability,
+        VirtioCapabilityData, VirtioCommonConfigCapability, VirtioDeviceConfigCapability,
         VirtioInterruptStatusCapability, VirtioNotifyConfigCapability,
     },
     PciCommandRegister, PciConfigurationSpace,
@@ -40,7 +40,7 @@ impl Transport for PciTransport {
         true
     }
 
-    fn device_cfg(&self) -> Option<&DeviceMemoryIo<[u8]>> {
+    fn device_cfg(&self) -> Option<&DeviceMemoryIo<'_, [u8]>> {
         Some(&self.device_cfg)
     }
 

kernel/driver/virtio/gpu/src/lib.rs

@@ -161,7 +161,7 @@ impl<T: Transport + 'static> VirtioGpu<T> {
         Ok(())
     }
 
-    fn begin_command(&self) -> CommandExecution<T> {
+    fn begin_command(&self) -> CommandExecution<'_, T> {
         CommandExecution {
             transport: &self.transport,
             control: &self.queues.get().control,

kernel/lib/device-tree/src/tree.rs

@@ -78,7 +78,7 @@ impl<'a> DeviceTree<'a> {
     }
 
     /// Returns the root node of this device tree
-    pub fn root(&self) -> TNode {
+    pub fn root(&self) -> TNode<'_> {
         self.index.root()
     }
 
@@ -172,12 +172,12 @@ impl<'a> DeviceTree<'a> {
     }
 
     /// Returns an iterator over the memory regions specified by this device tree
-    pub fn memory_regions(&self) -> DeviceTreeMemoryRegionIter {
+    pub fn memory_regions(&self) -> DeviceTreeMemoryRegionIter<'_> {
         DeviceTreeMemoryRegionIter::new(self)
     }
 
     /// Returns an iterator over the reserved memory regions specified by this device tree
-    pub fn reserved_regions(&self) -> DeviceTreeReservedRegionIter {
+    pub fn reserved_regions(&self) -> DeviceTreeReservedRegionIter<'_> {
         DeviceTreeReservedRegionIter::new(self)
     }
 

kernel/lib/memtables/Cargo.toml

@@ -1,13 +0,0 @@
-[package]
-name = "memtables"
-version = "0.1.0"
-edition = "2021"
-authors = ["Mark Poliakov <mark@alnyan.me>"]
-
-[dependencies]
-bitflags.workspace = true
-bytemuck.workspace = true
-
-[features]
-default = []
-all = []

kernel/lib/memtables/src/aarch64.rs

@@ -1,26 +0,0 @@
-use bytemuck::{Pod, Zeroable};
-
-use crate::RawTable;
-
-pub const KERNEL_L3_COUNT: usize = 8;
-
-#[derive(Clone, Copy, Pod, Zeroable)]
-#[repr(C)]
-pub struct FixedTables {
-    // 1GiB entries
-    pub l1: RawTable,
-
-    // 2MiB entries
-    pub l2: RawTable,
-    pub l3s: [RawTable; KERNEL_L3_COUNT],
-}
-
-impl FixedTables {
-    pub const fn zeroed() -> Self {
-        Self {
-            l1: RawTable::zeroed(),
-            l2: RawTable::zeroed(),
-            l3s: [RawTable::zeroed(); KERNEL_L3_COUNT],
-        }
-    }
-}

kernel/lib/memtables/src/any.rs

@@ -1,35 +0,0 @@
-use crate::{aarch64, riscv64, x86_64};
-
-pub enum AnyTables {
-    X86_64(x86_64::FixedTables),
-    AArch64(aarch64::FixedTables),
-    Riscv64(riscv64::FixedTables),
-}
-
-impl AnyTables {
-    pub fn as_bytes(&self) -> &[u8] {
-        match self {
-            Self::X86_64(tables) => bytemuck::bytes_of(tables),
-            Self::AArch64(tables) => bytemuck::bytes_of(tables),
-            Self::Riscv64(tables) => bytemuck::bytes_of(tables),
-        }
-    }
-}
-
-impl From<x86_64::FixedTables> for AnyTables {
-    fn from(value: x86_64::FixedTables) -> Self {
-        Self::X86_64(value)
-    }
-}
-
-impl From<aarch64::FixedTables> for AnyTables {
-    fn from(value: aarch64::FixedTables) -> Self {
-        Self::AArch64(value)
-    }
-}
-
-impl From<riscv64::FixedTables> for AnyTables {
-    fn from(value: riscv64::FixedTables) -> Self {
-        Self::Riscv64(value)
-    }
-}

kernel/lib/memtables/src/lib.rs

@@ -1,36 +0,0 @@
-#![no_std]
-
-use bytemuck::{Pod, Zeroable};
-
-// AArch64
-#[cfg(any(feature = "all", target_arch = "aarch64"))]
-pub mod aarch64;
-#[cfg(all(not(feature = "all"), target_arch = "aarch64"))]
-pub use aarch64::FixedTables;
-
-// x86-64
-#[cfg(any(feature = "all", target_arch = "x86_64"))]
-pub mod x86_64;
-#[cfg(all(not(feature = "all"), target_arch = "x86_64"))]
-pub use x86_64::FixedTables;
-
-// RISC-V 64-bit
-#[cfg(any(feature = "all", target_arch = "riscv64"))]
-pub mod riscv64;
-#[cfg(all(not(feature = "all"), target_arch = "riscv64"))]
-pub use riscv64::FixedTables;
-
-#[cfg(feature = "all")]
-pub mod any;
-
-#[derive(Clone, Copy, Pod, Zeroable)]
-#[repr(C, align(0x1000))]
-pub struct RawTable {
-    pub data: [u64; 512],
-}
-
-impl RawTable {
-    pub const fn zeroed() -> Self {
-        Self { data: [0; 512] }
-    }
-}

kernel/lib/memtables/src/riscv64.rs

@@ -1,84 +0,0 @@
-use core::fmt;
-
-use bitflags::bitflags;
-use bytemuck::{Pod, Zeroable};
-
-use crate::RawTable;
-
-pub const KERNEL_L3_COUNT: usize = 8;
-
-bitflags! {
-    #[derive(Clone, Copy, PartialEq, Eq, Debug)]
-    pub struct PageAttributes: u64 {
-        const N = 1 << 63;
-        /// Software-tracked dirty bit (RSW[0])
-        const SW_DIRTY = 1 << 9;
-        /// Dirty bit
-        const D = 1 << 7;
-        /// Access bit
-        const A = 1 << 6;
-        /// Global mapping bit, implies all lower levels are also global
-        const G = 1 << 5;
-        /// U-mode access permission
-        const U = 1 << 4;
-        /// Execute permission
-        const X = 1 << 3;
-        /// Write permission
-        const W = 1 << 2;
-        /// Read-permission
-        const R = 1 << 1;
-        /// Valid bit
-        const V = 1 << 0;
-    }
-
-    // X W R    Meaning
-    // 0 0 0    Pointer to next level of page table
-    // 0 0 1    Read-only page
-    // 0 1 0    ---
-    // 0 1 1    Read-write page
-    // 1 0 0    Execute only
-    // 1 0 1    Read-execute page
-    // 1 1 0    ---
-    // 1 1 1    Read-write-execute page
-}
-
-#[derive(Clone, Copy, Pod, Zeroable)]
-#[repr(C)]
-pub struct FixedTables {
-    pub l1: RawTable,
-    pub kernel_l2: RawTable,
-    pub kernel_l3s: [RawTable; KERNEL_L3_COUNT],
-}
-
-impl FixedTables {
-    pub const fn zeroed() -> Self {
-        Self {
-            l1: RawTable::zeroed(),
-            kernel_l2: RawTable::zeroed(),
-            kernel_l3s: [RawTable::zeroed(); KERNEL_L3_COUNT],
-        }
-    }
-}
-
-impl fmt::Display for PageAttributes {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        use fmt::Write;
-
-        macro_rules! bit {
-            ($self:ident, $field:expr, $letter:literal) => {
-                if $self.contains($field) {
-                    f.write_char($letter)
-                } else {
-                    f.write_char('-')
-                }
-            };
-        }
-
-        bit!(self, Self::R, 'r')?;
-        bit!(self, Self::W, 'w')?;
-        bit!(self, Self::X, 'x')?;
-        bit!(self, Self::U, 'u')?;
-
-        Ok(())
-    }
-}

kernel/lib/memtables/src/x86_64.rs

@@ -1,27 +0,0 @@
-use bytemuck::{Pod, Zeroable};
-
-use crate::RawTable;
-
-pub const KERNEL_L3_COUNT: usize = 16;
-
-#[derive(Clone, Copy, Pod, Zeroable)]
-#[repr(C)]
-pub struct FixedTables {
-    pub l0: RawTable,
-
-    pub kernel_l1: RawTable,
-    pub kernel_l2: RawTable,
-    pub kernel_l3s: [RawTable; KERNEL_L3_COUNT],
-}
-
-impl FixedTables {
-    pub const fn zeroed() -> Self {
-        Self {
-            l0: RawTable::zeroed(),
-
-            kernel_l1: RawTable::zeroed(),
-            kernel_l2: RawTable::zeroed(),
-            kernel_l3s: [RawTable::zeroed(); KERNEL_L3_COUNT],
-        }
-    }
-}

kernel/libk/libk-mm/interface/src/table.rs

@@ -1,6 +1,9 @@
 use core::ops::{Deref, DerefMut, Range};
 
 use bitflags::bitflags;
+use kernel_arch_interface::mem::{
+    DeviceMemoryAttributes, KernelTableManager, RawDeviceMemoryMapping,
+};
 use yggdrasil_abi::error::Error;
 
 use super::address::PhysicalAddress;
@@ -184,3 +187,124 @@ impl<T: AddressLike> EntryLevelExt for T {
         self.page_offset::<L>() == 0
     }
 }
+
+pub trait DevicePageManagerLevel {
+    type Level: EntryLevel;
+    const INDEX_RANGE: Range<usize>;
+    const VIRTUAL_BASE: usize;
+
+    fn map_page(&mut self, index: usize, physical: PhysicalAddress, attrs: &DeviceMemoryAttributes);
+    fn unmap_page(&mut self, index: usize);
+    fn is_mapped(&self, index: usize) -> bool;
+    fn flush_range(range: Range<usize>);
+
+    fn allocate_mapping(
+        &mut self,
+        base: PhysicalAddress,
+        page_count: usize,
+        attrs: &DeviceMemoryAttributes,
+    ) -> Option<usize> {
+        'l0: for i in Self::INDEX_RANGE {
+            for j in 0..page_count {
+                if self.is_mapped(i + j) {
+                    continue 'l0;
+                }
+            }
+
+            for j in 0..page_count {
+                self.map_page(i + j, base.add(j * Self::Level::SIZE), attrs);
+            }
+
+            let start = Self::VIRTUAL_BASE + i * Self::Level::SIZE;
+            Self::flush_range(i..i + page_count);
+            return Some(start);
+        }
+        None
+    }
+
+    unsafe fn remove_mapping(&mut self, base: usize, page_count: usize) {
+        let base_index = (base - Self::VIRTUAL_BASE) / Self::Level::SIZE;
+        for i in 0..page_count {
+            self.unmap_page(base_index + i);
+        }
+    }
+}
+
+pub struct DevicePageManager<N: DevicePageManagerLevel, L: DevicePageManagerLevel> {
+    pub normal: N,
+    pub large: L,
+}
+
+impl<N: DevicePageManagerLevel, L: DevicePageManagerLevel> DevicePageManager<N, L> {
+    pub const fn new(normal: N, large: L) -> Self {
+        Self { normal, large }
+    }
+
+    pub unsafe fn map_device_pages<A: KernelTableManager>(
+        &mut self,
+        base: PhysicalAddress,
+        size: usize,
+        attrs: DeviceMemoryAttributes,
+    ) -> Result<RawDeviceMemoryMapping<A>, Error> {
+        let small_aligned_base = base.page_align_down::<N::Level>();
+        let small_aligned_end = base.add(size).page_align_up::<N::Level>();
+        let small_offset = base - small_aligned_base;
+        let small_page_count = (small_aligned_end - small_aligned_base).page_count::<N::Level>();
+
+        if small_page_count > 128 {
+            // Allocate from large page pool
+            let large_aligned_base = base.page_align_down::<L::Level>();
+            let large_aligned_end = base.add(size).page_align_up::<L::Level>();
+            let large_offset = base - large_aligned_base;
+            let large_page_count =
+                (large_aligned_end - large_aligned_base).page_count::<L::Level>();
+
+            let mapped_base = self
+                .large
+                .allocate_mapping(large_aligned_base, large_page_count, &attrs)
+                .ok_or(Error::OutOfMemory)?;
+            let mapped_address = mapped_base + large_offset;
+
+            Ok(RawDeviceMemoryMapping::from_raw_parts(
+                large_aligned_base.into_u64(),
+                mapped_address,
+                mapped_base,
+                large_page_count,
+                L::Level::SIZE,
+            ))
+        } else {
+            // Allocate from small page pool
+            let mapped_base = self
+                .normal
+                .allocate_mapping(small_aligned_base, small_page_count, &attrs)
+                .ok_or(Error::OutOfMemory)?;
+            let mapped_address = mapped_base + small_offset;
+
+            Ok(RawDeviceMemoryMapping::from_raw_parts(
+                small_aligned_base.into_u64(),
+                mapped_address,
+                mapped_base,
+                small_page_count,
+                N::Level::SIZE,
+            ))
+        }
+    }
+
+    pub unsafe fn unmap_device_pages<A: KernelTableManager>(
+        &mut self,
+        mapping: &RawDeviceMemoryMapping<A>,
+    ) {
+        if mapping.page_size == N::Level::SIZE {
+            self.normal
+                .remove_mapping(mapping.base_address, mapping.page_count);
+        } else if mapping.page_size == L::Level::SIZE {
+            self.large
+                .remove_mapping(mapping.base_address, mapping.page_count);
+        } else {
+            unreachable!(
+                "Invalid device memory mapping with page size {:#x}",
+                mapping.page_size
+            )
+        }
+    }
+}

kernel/libk/libk-mm/src/phys/mod.rs

@@ -1,6 +1,8 @@
 use core::ops::Range;
 
-use kernel_arch::{mem::PhysicalMemoryAllocator, Architecture, ArchitectureImpl};
+use kernel_arch::{
+    mem::PhysicalMemoryAllocator, Architecture, ArchitectureImpl, KERNEL_VIRT_OFFSET,
+};
 use libk_mm_interface::address::PhysicalAddress;
 use libk_util::{sync::IrqSafeSpinlock, OneTimeInit};
 use yggdrasil_abi::{error::Error, system::SystemMemoryStats};
@@ -178,20 +180,14 @@ pub fn find_contiguous_region<I: Iterator<Item = PhysicalMemoryRegion>>(
 ///
 /// The caller must ensure this function has not been called before and that the regions
 /// are valid and actually available.
-pub unsafe fn init_from_iter<
-    I: Iterator<Item = PhysicalMemoryRegion> + Clone,
-    Map: FnOnce(I, PhysicalAddress, PhysicalAddress) -> Result<(), Error>,
->(
+pub unsafe fn init_from_iter<I: Iterator<Item = PhysicalMemoryRegion> + Clone>(
     it: I,
-    map_physical_memory: Map,
 ) -> Result<(), Error> {
     // Map the physical memory
     let (phys_start, phys_end) = physical_memory_range(it.clone()).unwrap();
 
     reserve_region("kernel", kernel_physical_memory_region());
 
-    map_physical_memory(it.clone(), phys_start, phys_end)?;
-
     let total_count = (phys_end - phys_start) / L3_PAGE_SIZE;
     let page_bitmap_size = (total_count + BITMAP_WORD_SIZE - 1) / (BITMAP_WORD_SIZE / 8);
     let page_bitmap_page_count = page_bitmap_size.div_ceil(L3_PAGE_SIZE);
@@ -245,18 +241,16 @@ pub unsafe fn init_from_iter<
 }
 
 fn kernel_physical_memory_region() -> PhysicalMemoryRegion {
-    use core::ptr::addr_of;
-
     extern "C" {
         static __kernel_start: u8;
         static __kernel_end: u8;
     }
 
-    let start = addr_of!(__kernel_start);
-    let end = addr_of!(__kernel_end);
+    let start = (&raw const __kernel_start).addr();
+    let end = (&raw const __kernel_end).addr();
 
-    let base = PhysicalAddress::from_usize(start.addr() - kernel_arch::KERNEL_VIRT_OFFSET);
-    let size = end.addr() - start.addr();
+    let base = PhysicalAddress::from_usize(start - KERNEL_VIRT_OFFSET);
+    let size = end - start;
 
     PhysicalMemoryRegion { base, size }
 }

kernel/libk/libk-mm/src/process.rs

@@ -524,7 +524,7 @@ impl<TA: TableAllocator> ProcessAddressSpace<TA> {
 
     /// Same as [ProcessAddressSpace::translate], except the lock on the address space is held
     /// until the resulting [TranslateGuard] is dropped.
-    pub fn translate_lock(&self, address: usize) -> Result<TranslateGuard<TA>, Error> {
+    pub fn translate_lock(&self, address: usize) -> Result<TranslateGuard<'_, TA>, Error> {
         let guard = self.inner.lock();
         let address = guard.table.translate(address).map(|e| e.0)?;
         Ok(TranslateGuard {

kernel/libk/libk-util/src/event.rs

@@ -35,11 +35,6 @@ pub struct OneTimeEvent<T> {
     notify: QueueWaker,
 }
 
-pub struct CounterEvent<N: EventNotify> {
-    counter: AtomicU64,
-    notify: N,
-}
-
 pub struct BitmapEvent<N: EventNotify> {
     value: AtomicU64,
     notify: N,
@@ -223,39 +218,3 @@ impl<N: EventNotify> BitmapEvent<N> {
         .await
     }
 }
-
-impl<N: EventNotify> CounterEvent<N> {
-    pub const fn new(notify: N) -> Self {
-        Self {
-            counter: AtomicU64::new(0),
-            notify,
-        }
-    }
-
-    pub fn signal(&self) {
-        self.counter.fetch_add(1, Ordering::Release);
-        self.notify.notify_all();
-    }
-
-    pub fn try_take(&self) -> Option<u64> {
-        let value = self.counter.swap(0, Ordering::Acquire);
-        if value > 0 {
-            Some(value)
-        } else {
-            None
-        }
-    }
-
-    pub async fn take(&self) -> u64 {
-        poll_fn(|cx| {
-            if let Some(value) = self.try_take() {
-                self.notify.unsubscribe(cx.waker());
-                Poll::Ready(value)
-            } else {
-                self.notify.subscribe(cx.waker());
-                Poll::Pending
-            }
-        })
-        .await
-    }
-}

kernel/libk/libk-util/src/lru_hash_table.rs

@@ -51,7 +51,7 @@ impl<K, V> LruCacheBucket<K, V> {
 }
 
 impl<K: Eq, V> LruCacheBucket<K, V> {
-    pub fn entry_mut(&mut self, key: &K) -> EntryMut<K, V> {
+    pub fn entry_mut(&mut self, key: &K) -> EntryMut<'_, K, V> {
         let mut cursor = self.data.cursor_front_mut();
 
         loop {
@@ -125,7 +125,7 @@ impl<K, V, H: BuildHasher> LruCache<K, V, H> {
             .find_map(|bucket| bucket.pop_front())
     }
 
-    pub fn flush(&mut self) -> FlushIter<K, V, H> {
+    pub fn flush(&mut self) -> FlushIter<'_, K, V, H> {
         FlushIter { cache: self }
     }
 }

kernel/libk/libk-util/src/ring.rs

@@ -203,7 +203,7 @@ impl<T: Copy> LossyRingQueue<T> {
     }
 
     #[inline]
-    pub fn try_read_lock(&self) -> Option<IrqSafeSpinlockGuard<RingBuffer<T>>> {
+    pub fn try_read_lock(&self) -> Option<IrqSafeSpinlockGuard<'_, RingBuffer<T>>> {
         let lock = self.ring.lock();
         lock.is_readable().then_some(lock)
     }
@@ -214,12 +214,12 @@ impl<T: Copy> LossyRingQueue<T> {
         unsafe { lock.read_single_unchecked() }
     }
 
-    pub async fn read_lock(&self) -> IrqSafeSpinlockGuard<RingBuffer<T>> {
+    pub async fn read_lock(&self) -> IrqSafeSpinlockGuard<'_, RingBuffer<T>> {
         poll_fn(|cx| self.poll_lock(cx)).await
     }
 
     #[inline]
-    pub fn poll_lock(&self, cx: &mut Context<'_>) -> Poll<IrqSafeSpinlockGuard<RingBuffer<T>>> {
+    pub fn poll_lock(&self, cx: &mut Context<'_>) -> Poll<IrqSafeSpinlockGuard<'_, RingBuffer<T>>> {
         self.read_notify.register(cx.waker());
         if let Some(lock) = self.try_read_lock() {
             self.read_notify.remove(cx.waker());
@@ -284,7 +284,7 @@ impl<T: Copy> BlockingRingQueue<T> {
         true
     }
 
-    pub async fn read_lock(&self) -> IrqSafeSpinlockGuard<RingBuffer<T>> {
+    pub async fn read_lock(&self) -> IrqSafeSpinlockGuard<'_, RingBuffer<T>> {
         poll_fn(|cx| self.poll_read_lock(cx)).await
     }
 
@@ -292,7 +292,7 @@ impl<T: Copy> BlockingRingQueue<T> {
     pub fn poll_read_lock(
         &self,
         cx: &mut Context<'_>,
-    ) -> Poll<IrqSafeSpinlockGuard<RingBuffer<T>>> {
+    ) -> Poll<IrqSafeSpinlockGuard<'_, RingBuffer<T>>> {
         if let Some(lock) = self.try_read_lock() {
             self.read_notify.remove(cx.waker());
             Poll::Ready(lock)
@@ -303,7 +303,7 @@ impl<T: Copy> BlockingRingQueue<T> {
     }
 
     #[inline]
-    pub fn try_read_lock(&self) -> Option<IrqSafeSpinlockGuard<RingBuffer<T>>> {
+    pub fn try_read_lock(&self) -> Option<IrqSafeSpinlockGuard<'_, RingBuffer<T>>> {
         let lock = self.ring.lock();
         lock.is_readable().then_some(lock)
     }

kernel/libk/libk-util/src/string.rs

@@ -28,7 +28,7 @@ impl Utf16LeStr {
         core::mem::transmute(raw)
     }
 
-    pub fn chars(&self) -> Utf16LeIter {
+    pub fn chars(&self) -> Utf16LeIter<'_> {
         Utf16LeIter {
             chars: self.raw.chunks_exact(size_of::<u16>()),
         }

kernel/libk/libk-util/src/sync/spin_rwlock.rs

@@ -131,7 +131,7 @@ impl<T> IrqSafeRwLock<T> {
         }
     }
 
-    pub fn read(&self) -> IrqSafeRwLockReadGuard<T> {
+    pub fn read(&self) -> IrqSafeRwLockReadGuard<'_, T> {
         let guard = IrqGuard::acquire();
         self.inner.acquire_read();
         IrqSafeRwLockReadGuard {
@@ -140,7 +140,7 @@ impl<T> IrqSafeRwLock<T> {
         }
     }
 
-    pub fn write(&self) -> IrqSafeRwLockWriteGuard<T> {
+    pub fn write(&self) -> IrqSafeRwLockWriteGuard<'_, T> {
         let guard = IrqGuard::acquire();
         self.inner.acquire_write();
         IrqSafeRwLockWriteGuard {