WIP usb: better driver structure, hid mouse driver

.gitignore

@@ -3,3 +3,6 @@
 /xtask.toml
 /qemu.toml
 /etc/boot/yboot.cfg
+/disk-*.img
+/tmp-*.txt
+/*.log

Cargo.lock

@@ -532,12 +532,6 @@ dependencies = [
  "hashbrown 0.14.5",
 ]
 
-[[package]]
-name = "elf"
-version = "0.7.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4445909572dbd556c457c849c4ca58623d84b27c8fff1e74b0b4227d8b90d17b"
-
 [[package]]
 name = "encode_unicode"
 version = "1.0.0"
@@ -741,18 +735,6 @@ dependencies = [
  "pin-utils",
 ]
 
-[[package]]
-name = "gentables"
-version = "0.1.0"
-dependencies = [
- "bitflags 2.8.0",
- "bytemuck",
- "clap",
- "elf 0.7.4",
- "memtables",
- "thiserror",
-]
-
 [[package]]
 name = "getrandom"
 version = "0.2.15"
@@ -1135,7 +1117,7 @@ dependencies = [
  "device-api",
  "kernel-arch-interface",
  "libk-mm-interface",
- "memtables",
+ "log",
  "static_assertions",
  "tock-registers",
  "yggdrasil-abi",
@@ -1145,6 +1127,7 @@ dependencies = [
 name = "kernel-arch-hosted"
 version = "0.1.0"
 dependencies = [
+ "device-api",
  "kernel-arch-interface",
  "libk-mm-interface",
  "yggdrasil-abi",
@@ -1183,7 +1166,6 @@ dependencies = [
  "kernel-arch-interface",
  "libk-mm-interface",
  "log",
- "memtables",
  "static_assertions",
  "tock-registers",
  "yggdrasil-abi",
@@ -1211,7 +1193,6 @@ dependencies = [
  "kernel-arch-x86",
  "libk-mm-interface",
  "log",
- "memtables",
  "static_assertions",
  "tock-registers",
  "yggdrasil-abi",
@@ -1269,7 +1250,7 @@ dependencies = [
  "cfg-if",
  "crossbeam-queue",
  "device-api",
- "elf 0.7.2",
+ "elf",
  "futures-util",
  "kernel-arch",
  "libc",
@@ -1291,6 +1272,7 @@ dependencies = [
 name = "libk-mm"
 version = "0.1.0"
 dependencies = [
+ "async-trait",
  "kernel-arch",
  "libk-mm-interface",
  "libk-util",
@@ -1356,6 +1338,10 @@ dependencies = [
  "vcpkg",
 ]
 
+[[package]]
+name = "libutil"
+version = "0.1.0"
+
 [[package]]
 name = "libyalloc"
 version = "0.1.0"
@@ -1459,14 +1445,6 @@ dependencies = [
  "autocfg",
 ]
 
-[[package]]
-name = "memtables"
-version = "0.1.0"
-dependencies = [
- "bitflags 2.8.0",
- "bytemuck",
-]
-
 [[package]]
 name = "miniz_oxide"
 version = "0.8.4"
@@ -2867,12 +2845,30 @@ dependencies = [
  "yggdrasil-abi",
 ]
 
+[[package]]
+name = "ygg_driver_virtio_blk"
+version = "0.1.0"
+dependencies = [
+ "async-trait",
+ "bytemuck",
+ "device-api",
+ "libk",
+ "libk-mm",
+ "libk-util",
+ "log",
+ "tock-registers",
+ "ygg_driver_pci",
+ "ygg_driver_virtio_core",
+ "yggdrasil-abi",
+]
+
 [[package]]
 name = "ygg_driver_virtio_core"
 version = "0.1.0"
 dependencies = [
  "bitflags 2.8.0",
  "device-api",
+ "kernel-arch-hosted",
  "libk",
  "libk-mm",
  "libk-util",
@@ -2886,6 +2882,7 @@ dependencies = [
 name = "ygg_driver_virtio_gpu"
 version = "0.1.0"
 dependencies = [
+ "async-trait",
  "bytemuck",
  "device-api",
  "libk",
@@ -2904,6 +2901,7 @@ dependencies = [
  "bitflags 2.8.0",
  "bytemuck",
  "device-api",
+ "futures-util",
  "libk",
  "libk-mm",
  "libk-util",
@@ -2948,7 +2946,7 @@ dependencies = [
  "device-api",
  "device-api-macros",
  "device-tree",
- "elf 0.7.2",
+ "elf",
  "ext2",
  "futures-util",
  "git-version",
@@ -2964,7 +2962,6 @@ dependencies = [
  "libk-util",
  "log",
  "memfs",
- "memtables",
  "prettyplease",
  "static_assertions",
  "tock-registers",
@@ -2983,6 +2980,7 @@ dependencies = [
  "ygg_driver_pci",
  "ygg_driver_usb",
  "ygg_driver_usb_xhci",
+ "ygg_driver_virtio_blk",
  "ygg_driver_virtio_gpu",
  "ygg_driver_virtio_net",
  "yggdrasil-abi",

Cargo.toml

@@ -11,13 +11,13 @@ exclude = [
 ]
 members = [
     "xtask",
-    "kernel/tools/gentables",
     "kernel",
     "lib/abi",
     "lib/libyalloc",
     "lib/runtime",
     "lib/qemu",
-    "lib/abi-serde"
+    "lib/abi-serde",
+    "lib/libutil"
 ]
 
 [workspace.dependencies]
@@ -66,7 +66,6 @@ libk-util.path = "kernel/libk/libk-util"
 libk-mm.path = "kernel/libk/libk-mm"
 libk-mm-interface.path = "kernel/libk/libk-mm/interface"
 libk-device.path = "kernel/libk/libk-device"
-memtables.path = "kernel/lib/memtables"
 vmalloc.path = "kernel/lib/vmalloc"
 device-api-macros.path = "kernel/lib/device-api/macros"
 device-tree.path = "kernel/lib/device-tree"

LICENSE.txt

@@ -0,0 +1,9 @@
+MIT License
+
+Copyright (c) 2025 Mark Poliakov <mark@alnyan.me>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the " Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice (including the next paragraph) shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

boot/yboot/src/elf.rs

@@ -1,7 +1,8 @@
 use core::mem::size_of;
 
 use bytemuck::Zeroable;
-use log::{debug, error, info};
+use log::{error, info};
+use types::{Rela, SHT_RELA};
 // TODO use 'elf' crate
 use uefi::{
     prelude::BootServices,
@@ -23,14 +24,18 @@ mod types {
     pub type Half = u16;
     pub type Word = u32;
     pub type XWord = u64;
+    pub type SXWord = i64;
 
     pub const PT_LOAD: Word = 1;
 
     pub const SHT_PROGBITS: Word = 1;
+    pub const SHT_RELA: Word = 4;
 
     pub const SHF_WRITE: XWord = 1 << 0;
     pub const SHF_ALLOC: XWord = 1 << 1;
 
+    pub const R_X86_64_RELATIVE: u32 = 8;
+
     #[derive(Clone, Copy, Zeroable, Pod)]
     #[repr(C)]
     pub struct Ehdr {
@@ -77,6 +82,20 @@ mod types {
         pub memsz: XWord,
         pub align: XWord,
     }
+
+    #[derive(Clone, Copy, Zeroable, Pod)]
+    #[repr(C)]
+    pub struct Rela {
+        pub offset: Addr,
+        pub info: XWord,
+        pub addend: SXWord,
+    }
+
+    impl Rela {
+        pub fn r_type(&self) -> u32 {
+            self.info as u32
+        }
+    }
 }
 
 // Maximum address this loader can map in the target kernel
@@ -92,6 +111,8 @@ pub struct LoadedObject {
     pub image_start: u64,
     pub image_end: u64,
 
+    pub load_address: u64,
+
     pub entry: u64,
 
     pub protocol_struct_paddr: u64,
@@ -105,6 +126,12 @@ struct LocatedProtocol {
     size: usize,
 }
 
+struct RelaSection {
+    offset: u64,
+    entry_count: usize,
+    entry_size: usize,
+}
+
 trait ReadExact {
     fn read_exact(&mut self, buf: &mut [u8]) -> Result<(), Error>;
 }
@@ -119,6 +146,23 @@ impl ReadExact for RegularFile {
     }
 }
 
+impl RelaSection {
+    pub fn from_shdr(shdr: &Shdr) -> Option<Self> {
+        if shdr.type_ != SHT_RELA {
+            return None;
+        }
+
+        let entry_size = shdr.entsize as usize;
+        let entry_count = shdr.size as usize / entry_size;
+
+        Some(Self {
+            offset: shdr.offset,
+            entry_size,
+            entry_count,
+        })
+    }
+}
+
 impl Object {
     pub fn open<D: File>(root: &mut D, path: &CStr16) -> Result<Self, Error> {
         let file = root.open(path, FileMode::Read, FileAttribute::empty())?;
@@ -140,11 +184,11 @@ impl Object {
             return Err(Error::new(Status::LOAD_ERROR, ()));
         }
 
-        // Check that the entry point is set
-        if ehdr.entry == 0 {
-            error!("Image does not have a valid entry point");
-            return Err(Error::new(Status::LOAD_ERROR, ()));
-        }
+        // // Check that the entry point is set
+        // if ehdr.entry == 0 {
+        //     error!("Image does not have a valid entry point");
+        //     return Err(Error::new(Status::LOAD_ERROR, ()));
+        // }
 
         Ok(Self { file, ehdr })
     }
@@ -170,11 +214,6 @@ impl Object {
         self.file
             .read_exact(bytemuck::bytes_of_mut(&mut proto_data))?;
 
-        info!(
-            "Kernel is virtually mapped at {:#x}",
-            proto_data.kernel_virt_offset
-        );
-
         // 2. Find the kernel's range and check that the loaded physical addresses are actually
         //    usable from UEFI
         let mut image_start = u64::MAX;
@@ -209,16 +248,22 @@ impl Object {
         assert_eq!(image_start & 0xFFF, 0);
         assert_eq!(image_end & 0xFFF, 0);
 
-        info!("Image start: {:#x}, end: {:#x}", image_start, image_end);
-        // Reserve the kernel memory
-        let reserved_addr = bs
+        // Allocate memory to load the kernel into
+        let kernel_load_address = bs
             .allocate_pages(
-                AllocateType::Address(image_start),
+                AllocateType::MaxAddress(0xFFFFFFFF),
                 MemoryType::LOADER_DATA,
                 (image_end - image_start) as usize / 0x1000,
             )
-            .expect("Could not allocate memory for kernel image");
-        assert_eq!(reserved_addr, image_start);
+            .expect("Could not allocate memory for the kernel");
+
+        // Print info
+        info!("Image start: {:#x}, end: {:#x}", image_start, image_end);
+        info!(
+            "Kernel virtual offset: {:#x}",
+            proto_data.kernel_virt_offset
+        );
+        info!("Kernel load address: {kernel_load_address:#x}");
 
         // 3. Load the segments
         for i in 0..self.ehdr.phnum {
@@ -228,53 +273,80 @@ impl Object {
                 continue;
             }
 
+            let segment_load_base = phdr.paddr + kernel_load_address;
+
             info!(
-                "Load segment {}: {:#x?}",
-                i,
-                phdr.paddr..phdr.paddr + phdr.memsz
+                "[{i}] Load {:#x?}",
+                segment_load_base..segment_load_base + phdr.memsz
             );
 
             if phdr.filesz > 0 {
-                // The section has load data
-                let dst = unsafe {
-                    core::slice::from_raw_parts_mut(phdr.paddr as *mut u8, phdr.filesz as usize)
+                let dst_slice = unsafe {
+                    core::slice::from_raw_parts_mut(
+                        segment_load_base as *mut u8,
+                        phdr.filesz as usize,
+                    )
                 };
 
-                debug!(
-                    "Load {:#x?} from ELF offset {:#x}",
-                    phdr.paddr..phdr.paddr + phdr.filesz,
-                    phdr.offset
-                );
                 self.file.set_position(phdr.offset)?;
-                self.file.read_exact(dst)?;
+                self.file.read_exact(dst_slice)?;
             }
 
             if phdr.memsz > phdr.filesz {
-                let dst = unsafe {
+                let dst_slice = unsafe {
                     core::slice::from_raw_parts_mut(
-                        (phdr.paddr + phdr.filesz) as *mut u8,
+                        (segment_load_base + phdr.filesz) as *mut u8,
                         (phdr.memsz - phdr.filesz) as usize,
                     )
                 };
 
-                debug!(
-                    "Zero data {:#x?}",
-                    phdr.paddr + phdr.filesz..phdr.paddr + phdr.memsz
-                );
+                dst_slice.fill(0);
+            }
+        }
 
-                dst.fill(0);
+        // 4. Perform kernel relocation
+        let mut rela_section = None;
+        for i in 0..self.ehdr.shnum as usize {
+            let shdr = self.read_shdr(i)?;
+            if let Some(rela) = RelaSection::from_shdr(&shdr) {
+                rela_section = Some(rela);
+                break;
+            }
+        }
+
+        if let Some(rela_section) = rela_section {
+            info!("Relocating kernel: {image_start:#x} -> {kernel_load_address:#x}");
+            info!("({} relocations)", rela_section.entry_count);
+
+            let b = (kernel_load_address + proto_data.kernel_virt_offset) as i64;
+
+            for i in 0..rela_section.entry_count {
+                let mut rela = Rela::zeroed();
+                self.file
+                    .set_position(rela_section.offset + (i * rela_section.entry_size) as u64)?;
+                self.file.read_exact(bytemuck::bytes_of_mut(&mut rela))?;
+
+                match rela.r_type() {
+                    types::R_X86_64_RELATIVE => {
+                        let qword = (rela.offset + kernel_load_address) as *mut i64;
+                        let value = rela.addend + b;
+                        unsafe { qword.write_volatile(value) };
+                    }
+                    other => todo!("Unsupported relocation type: {other}"),
+                }
             }
         }
 
         // Now that the image is in memory, protocol structure can be written in the further steps
-        let protocol_struct_paddr = (loc_proto.address as u64) - proto_data.kernel_virt_offset;
+        let protocol_struct_paddr = loc_proto.address as u64 + kernel_load_address; // (loc_proto.address as u64) - proto_data.kernel_virt_offset;
         let protocol_version = proto_data.header.version;
 
-        let entry = self.ehdr.entry;
+        let entry = self.ehdr.entry + kernel_load_address;
 
         Ok(LoadedObject {
             image_start,
             image_end,
+            load_address: kernel_load_address,
             entry,
             protocol_struct_paddr,
             protocol_version,

boot/yboot/src/main.rs

@@ -16,7 +16,7 @@ use log::{debug, error, info};
 use uefi::{
     prelude::*,
     proto::{
-        console::gop::{GraphicsOutput, PixelFormat},
+        console::gop::{self, GraphicsOutput, PixelFormat},
         device_path::DevicePath,
         loaded_image::LoadedImage,
         media::{file::Directory, fs::SimpleFileSystem},
@@ -32,23 +32,31 @@ use yboot_proto::{
     LoadProtocolV1, LOADER_MAGIC,
 };
 
-use crate::{mem::MemoryDescriptorExt, protocol_ext::GraphicsOutputExt};
+use crate::mem::MemoryDescriptorExt;
+
+fn mode_score(mode: &gop::Mode) -> usize {
+    let (w, h) = mode.info().resolution();
+    let mut size_score = w * h;
+    if w > 1920 || h > 1080 {
+        // Don't pick too large sizes
+        size_score = 0;
+    }
+    size_score
+}
 
 fn setup_framebuffer(bs: &BootServices, fb: &mut FramebufferOption) -> Result<(), Error> {
     let gop_handle = bs.get_handle_for_protocol::<GraphicsOutput>()?;
     let mut gop = bs.open_protocol_exclusive::<GraphicsOutput>(gop_handle)?;
 
-    // Find the requested mode
-    let mode = gop.match_mode(fb.req_width, fb.req_height).ok_or_else(|| {
-        error!(
-            "Requested mode is not supported: {}x{}",
-            fb.req_width, fb.req_height
-        );
+    let mode = gop.modes().max_by_key(mode_score).ok_or_else(|| {
+        error!("No mode found");
         Error::new(Status::INVALID_PARAMETER, ())
     })?;
 
     gop.set_mode(&mode)?;
 
+    let (res_width, res_height) = mode.info().resolution();
+
     let mut result = gop.frame_buffer();
 
     let format = match mode.info().pixel_format() {
@@ -57,8 +65,8 @@ fn setup_framebuffer(bs: &BootServices, fb: &mut FramebufferOption) -> Result<()
         _ => 0,
     };
 
-    fb.res_width = fb.req_width;
-    fb.res_height = fb.req_height;
+    fb.res_width = res_width as _;
+    fb.res_height = res_height as _;
     fb.res_address = result.as_mut_ptr() as _;
     fb.res_stride = mode.info().stride() as u64 * 4;
     fb.res_size = result.size() as _;
@@ -85,7 +93,7 @@ fn locate_rsdp(st: &SystemTable<Boot>) -> Option<u64> {
 fn boot_partition(
     image: Handle,
     bs: &BootServices,
-) -> Result<ScopedProtocol<SimpleFileSystem>, Error> {
+) -> Result<ScopedProtocol<'_, SimpleFileSystem>, Error> {
     let loaded_image = bs.open_protocol_exclusive::<LoadedImage>(image)?;
     let device_handle = loaded_image.device();
 
@@ -105,7 +113,7 @@ fn load_kernel<'a>(
     config: &Config,
     root: &mut Directory,
     st: &SystemTable<Boot>,
-) -> Result<(u64, u64, &'a mut LoadProtocolV1), Error> {
+) -> Result<(u64, u64, u64, &'a mut LoadProtocolV1), Error> {
     let bs = st.boot_services();
 
     let mut kernel_obj = Object::open(root, cstr16!("kernel.elf"))?;
@@ -175,13 +183,14 @@ fn load_kernel<'a>(
 
     let entry = loaded_obj.entry + proto_data.kernel_virt_offset;
 
-    Ok((entry, mmap_memory, proto_data))
+    Ok((entry, loaded_obj.load_address, mmap_memory, proto_data))
 }
 
 unsafe fn map_and_enter_kernel(
     st: SystemTable<Boot>,
     proto_data: &mut LoadProtocolV1,
     mmap_memory: u64,
+    load_base: u64,
     entry: u64,
 ) -> ! {
     let (_, mmap) = st.exit_boot_services();
@@ -208,7 +217,7 @@ unsafe fn map_and_enter_kernel(
     let cr3 = mem::map_image();
     asm!("cli; wbinvd; mov {0}, %cr3", in(reg) cr3, options(att_syntax));
 
-    asm!("jmp *{0}", in(reg) entry, in("eax") LOADER_MAGIC, options(noreturn, att_syntax));
+    asm!("jmp *{0}", in(reg) entry, in("eax") LOADER_MAGIC, in("ecx") load_base, options(noreturn, att_syntax));
 }
 
 #[entry]
@@ -235,15 +244,16 @@ fn efi_main(image_handle: Handle, mut system_table: SystemTable<Boot>) -> Status
         }
     };
 
-    let (entry, mmap_memory, proto_data) = match load_kernel(&config, &mut root, &system_table) {
-        Ok(e) => e,
-        Err(error) => {
-            error!("Failed to load the kernel/initrd: {error:?}");
-            return Status::LOAD_ERROR;
-        }
-    };
+    let (entry, load_base, mmap_memory, proto_data) =
+        match load_kernel(&config, &mut root, &system_table) {
+            Ok(e) => e,
+            Err(error) => {
+                error!("Failed to load the kernel/initrd: {error:?}");
+                return Status::LOAD_ERROR;
+            }
+        };
 
     unsafe {
-        map_and_enter_kernel(system_table, proto_data, mmap_memory, entry);
+        map_and_enter_kernel(system_table, proto_data, mmap_memory, load_base, entry);
     }
 }

boot/yboot/src/mem.rs

@@ -66,7 +66,7 @@ impl MemoryDescriptorExt for MemoryDescriptor {
     }
 }
 
-pub fn memory_map(bs: &BootServices) -> Result<MemoryMap, Error> {
+pub fn memory_map(bs: &BootServices) -> Result<MemoryMap<'_>, Error> {
     bs.memory_map(unsafe { &mut MMAP_BUFFER.data })
 }
 

etc/aarch64-unknown-none.json

@@ -3,7 +3,7 @@
   "os": "none",
   "abi": "softfloat",
   "llvm-target": "aarch64-unknown-none",
-  "data-layout": "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32",
+  "data-layout": "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32",
   "max-atomic-width": 128,
   "target-pointer-width": "64",
   "features": "+v8a,+strict-align,-neon,-fp-armv8",
@@ -13,6 +13,7 @@
   "panic-strategy": "abort",
   "dynamic-linking": true,
   "relocation-model": "pic",
+  "position-independent-executables": true,
   "eh-frame-header": false,
 
   "linker": "rust-lld",

etc/gdb/aarch64-virt.gdb

@@ -0,0 +1,4 @@
+# Lower-half
+symbol-file -o 0x40080000 target/aarch64-unknown-none/release/yggdrasil-kernel
+symbol-file -o 0xFFFFFF8040080000 target/aarch64-unknown-none/release/yggdrasil-kernel
+target remote :1234

etc/ld/arm/aarch64-unknown-none.ld

@@ -0,0 +1,67 @@
+ENTRY(__aarch64_entry);
+
+/* KERNEL_PHYS_BASE = 0x40080000; */
+KERNEL_VIRT_OFFSET = 0xFFFFFF8000000000;
+
+SECTIONS {
+    /* . = KERNEL_PHYS_BASE; */
+    . = 0x0;
+
+    PROVIDE(__kernel_start = .);
+
+    /* .text.entry : {
+        *(.text.entry)
+    }
+
+    . = ALIGN(16);
+    . = . + KERNEL_VIRT_OFFSET; */
+
+    .text : {
+        KEEP(*(.text.entry));
+        KEEP(*(.text.vectors));
+        *(.text*)
+    }
+
+    . = ALIGN(4K);
+    .rodata : {
+        *(.rodata*)
+        *(.eh_frame*)
+
+        . = ALIGN(16);
+        PROVIDE(__init_array_start = .);
+        KEEP(*(.init_array*))
+        PROVIDE(__init_array_end = .);
+    }
+
+    . = ALIGN(4K);
+    PROVIDE(__rela_start = .);
+    .rela : {
+        *(.rela*)
+    }
+    PROVIDE(__rela_end = .);
+
+    /* . = ALIGN(4K);            */
+    /* .data.tables : {          */
+    /*     KEEP(*(.data.tables)) */
+    /* }                         */
+
+    . = ALIGN(4K);
+    .data : {
+        *(.data*)
+        *(.got*)
+    }
+
+    . = ALIGN(4K);
+    PROVIDE(__bss_start = .);
+    /* PROVIDE(__bss_start_phys = . - KERNEL_VIRT_OFFSET); */
+    .bss : {
+        *(COMMON)
+        *(.bss*)
+    }
+    . = ALIGN(4K);
+    PROVIDE(__bss_end = .);
+    /* PROVIDE(__bss_end_phys = . - KERNEL_VIRT_OFFSET); */
+    /* PROVIDE(__bss_size = __bss_end_phys - __bss_start_phys); */
+
+    PROVIDE(__kernel_end = .);
+};

etc/ld/arm/aarch64-unknown-qemu.ld

@@ -1,55 +0,0 @@
-ENTRY(__aarch64_entry);
-
-KERNEL_PHYS_BASE = 0x40080000;
-KERNEL_VIRT_OFFSET = 0xFFFFFF8000000000;
-
-SECTIONS {
-    . = KERNEL_PHYS_BASE;
-    PROVIDE(__kernel_start = . + KERNEL_VIRT_OFFSET);
-
-    .text.entry : {
-        *(.text.entry)
-    }
-
-    . = ALIGN(16);
-    . = . + KERNEL_VIRT_OFFSET;
-
-    .text : AT(. - KERNEL_VIRT_OFFSET) {
-        KEEP(*(.text.vectors));
-        *(.text*)
-    }
-
-    . = ALIGN(4K);
-    .rodata : AT(. - KERNEL_VIRT_OFFSET) {
-        *(.rodata*)
-        *(.eh_frame*)
-
-        . = ALIGN(16);
-        PROVIDE(__init_array_start = .);
-        KEEP(*(.init_array*))
-        PROVIDE(__init_array_end = .);
-    }
-
-    . = ALIGN(4K);
-    .data.tables : AT (. - KERNEL_VIRT_OFFSET) {
-        KEEP(*(.data.tables))
-    }
-
-    . = ALIGN(4K);
-    .data : AT(. - KERNEL_VIRT_OFFSET) {
-        *(.data*)
-        *(.got*)
-    }
-
-    . = ALIGN(4K);
-    PROVIDE(__bss_start_phys = . - KERNEL_VIRT_OFFSET);
-    .bss : AT(. - KERNEL_VIRT_OFFSET) {
-        *(COMMON)
-        *(.bss*)
-    }
-    . = ALIGN(4K);
-    PROVIDE(__bss_end_phys = . - KERNEL_VIRT_OFFSET);
-    PROVIDE(__bss_size = __bss_end_phys - __bss_start_phys);
-
-    PROVIDE(__kernel_end = .);
-};

etc/ld/arm/aarch64-unknown-raspi4b.ld

@@ -1,55 +0,0 @@
-ENTRY(__aarch64_entry);
-
-KERNEL_PHYS_BASE = 0x80000;
-KERNEL_VIRT_OFFSET = 0xFFFFFF8000000000;
-
-SECTIONS {
-    . = KERNEL_PHYS_BASE;
-    PROVIDE(__kernel_start = . + KERNEL_VIRT_OFFSET);
-
-    .text.entry : {
-        *(.text.entry)
-    }
-
-    . = ALIGN(16);
-    . = . + KERNEL_VIRT_OFFSET;
-
-    .text : AT(. - KERNEL_VIRT_OFFSET) {
-        KEEP(*(.text.vectors));
-        *(.text*)
-    }
-
-    . = ALIGN(4K);
-    .rodata : AT(. - KERNEL_VIRT_OFFSET) {
-        *(.rodata*)
-        *(.eh_frame*)
-
-        . = ALIGN(16);
-        PROVIDE(__init_array_start = .);
-        KEEP(*(.init_array*))
-        PROVIDE(__init_array_end = .);
-    }
-
-    . = ALIGN(4K);
-    .data.tables : AT (. - KERNEL_VIRT_OFFSET) {
-        KEEP(*(.data.tables))
-    }
-
-    . = ALIGN(4K);
-    .data : AT(. - KERNEL_VIRT_OFFSET) {
-        *(.data*)
-        *(.got*)
-    }
-
-    . = ALIGN(4K);
-    PROVIDE(__bss_start_phys = . - KERNEL_VIRT_OFFSET);
-    .bss : AT(. - KERNEL_VIRT_OFFSET) {
-        *(COMMON)
-        *(.bss*)
-    }
-    . = ALIGN(4K);
-    PROVIDE(__bss_end_phys = . - KERNEL_VIRT_OFFSET);
-    PROVIDE(__bss_size = __bss_end_phys - __bss_start_phys);
-
-    PROVIDE(__kernel_end = .);
-};

etc/ld/riscv/riscv64-unknown-none.ld

@@ -1,37 +1,32 @@
 ENTRY(__rv64_entry);
 
-KERNEL_PHYS_BASE = 0x40200000;
 KERNEL_VIRT_OFFSET = 0xFFFFFFF000000000;
 
 SECTIONS {
-    . = KERNEL_PHYS_BASE;
-    PROVIDE(__kernel_start = . + KERNEL_VIRT_OFFSET);
+    . = 0;
+    PROVIDE(__kernel_start = .);
 
-    .text.entry : {
-        *(.text.entry)
-    }
-
-    . = ALIGN(16);
-    . = . + KERNEL_VIRT_OFFSET;
-
-    .text : AT(. - KERNEL_VIRT_OFFSET) {
+    .text : {
+        KEEP(*(.text.entry));
         KEEP(*(.text.vectors));
         *(.text*)
     }
 
     . = ALIGN(4K);
-    .rodata : AT(. - KERNEL_VIRT_OFFSET) {
+    .rodata : {
         *(.rodata*)
         *(.eh_frame*)
     }
 
     . = ALIGN(4K);
-    .data.tables : AT (. - KERNEL_VIRT_OFFSET) {
-        KEEP(*(.data.tables))
+    PROVIDE(__rela_start = .);
+    .rela : {
+        *(.rela*)
     }
+    PROVIDE(__rela_end = .);
 
     . = ALIGN(4K);
-    .data : AT(. - KERNEL_VIRT_OFFSET) {
+    .data : {
         *(.data*)
         . = ALIGN(8);
         /* PROVIDE(__global_pointer = . + 0x800 - KERNEL_VIRT_OFFSET); */
@@ -45,14 +40,13 @@ SECTIONS {
     }
 
     . = ALIGN(4K);
-    PROVIDE(__bss_start_phys = . - KERNEL_VIRT_OFFSET);
-    .bss : AT(. - KERNEL_VIRT_OFFSET) {
+    PROVIDE(__bss_start = .);
+    .bss : {
         *(COMMON)
         *(.bss*)
     }
     . = ALIGN(4K);
-    PROVIDE(__bss_end_phys = . - KERNEL_VIRT_OFFSET);
-    PROVIDE(__bss_size = __bss_end_phys - __bss_start_phys);
+    PROVIDE(__bss_end = .);
 
     PROVIDE(__kernel_end = .);
 };

etc/ld/riscv/riscv64-unknown-qemu.ld

@@ -1,58 +0,0 @@
-ENTRY(__rv64_entry);
-
-KERNEL_PHYS_BASE = 0x80200000;
-KERNEL_VIRT_OFFSET = 0xFFFFFFF000000000;
-
-SECTIONS {
-    . = KERNEL_PHYS_BASE;
-    PROVIDE(__kernel_start = . + KERNEL_VIRT_OFFSET);
-
-    .text.entry : {
-        *(.text.entry)
-    }
-
-    . = ALIGN(16);
-    . = . + KERNEL_VIRT_OFFSET;
-
-    .text : AT(. - KERNEL_VIRT_OFFSET) {
-        KEEP(*(.text.vectors));
-        *(.text*)
-    }
-
-    . = ALIGN(4K);
-    .rodata : AT(. - KERNEL_VIRT_OFFSET) {
-        *(.rodata*)
-        *(.eh_frame*)
-    }
-
-    . = ALIGN(4K);
-    .data.tables : AT (. - KERNEL_VIRT_OFFSET) {
-        KEEP(*(.data.tables))
-    }
-
-    . = ALIGN(4K);
-    .data : AT(. - KERNEL_VIRT_OFFSET) {
-        *(.data*)
-        . = ALIGN(8);
-        /* PROVIDE(__global_pointer = . + 0x800 - KERNEL_VIRT_OFFSET); */
-
-        . = ALIGN(16);
-        PROVIDE(__init_array_start = .);
-        KEEP(*(.init_array*))
-        PROVIDE(__init_array_end = .);
-
-        *(.got*)
-    }
-
-    . = ALIGN(4K);
-    PROVIDE(__bss_start_phys = . - KERNEL_VIRT_OFFSET);
-    .bss : AT(. - KERNEL_VIRT_OFFSET) {
-        *(COMMON)
-        *(.bss*)
-    }
-    . = ALIGN(4K);
-    PROVIDE(__bss_end_phys = . - KERNEL_VIRT_OFFSET);
-    PROVIDE(__bss_size = __bss_end_phys - __bss_start_phys);
-
-    PROVIDE(__kernel_end = .);
-};

etc/ld/x86/x86_64-unknown-none.ld

@@ -1,52 +1,52 @@
 ENTRY(__x86_64_entry);
 
-KERNEL_PHYS_BASE = 0x200000;
+/* KERNEL_PHYS_BASE = 0x200000; */
 KERNEL_VIRT_OFFSET = 0xFFFFFF8000000000;
 
 SECTIONS {
-    . = KERNEL_PHYS_BASE;
-    PROVIDE(__kernel_start = . + KERNEL_VIRT_OFFSET);
+    /* . = KERNEL_PHYS_BASE; */
+    /* PROVIDE(__kernel_start = . + KERNEL_VIRT_OFFSET); */
 
-    .text.entry : {
+    . = 0x0;
+    PROVIDE(__kernel_start = .);
+
+    .text : {
         *(.text.entry)
-    }
-
-    . = ALIGN(16);
-    . = . + KERNEL_VIRT_OFFSET;
-
-    .text : AT(. - KERNEL_VIRT_OFFSET) {
         *(.text*)
     }
 
-    .export.text : AT(. - KERNEL_VIRT_OFFSET) {
+    .export.text : {
         KEEP(*(.export.text*))
     }
 
     . = ALIGN(4K);
-    .rodata : AT(. - KERNEL_VIRT_OFFSET) {
+    .rodata : {
         *(.eh_frame*)
         *(.rodata*)
     }
 
     . = ALIGN(4K);
-    .data.tables : AT (. - KERNEL_VIRT_OFFSET) {
-        KEEP(*(.data.tables))
+    PROVIDE(__rela_start = .);
+    .rela : {
+        *(.rela*)
     }
+    PROVIDE(__rela_end = .);
 
-    .data : AT(. - KERNEL_VIRT_OFFSET) {
+    . = ALIGN(4K);
+    .data : {
         KEEP(*(.data.yboot))
         *(.data*)
         *(.got*)
     }
 
     . = ALIGN(4K);
-    PROVIDE(__bss_start_phys = . - KERNEL_VIRT_OFFSET);
-    .bss : AT(. - KERNEL_VIRT_OFFSET) {
+    PROVIDE(__bss_start = .);
+    .bss : {
         *(COMMON)
         *(.bss*)
     }
     . = ALIGN(4K);
-    PROVIDE(__bss_end_phys = . - KERNEL_VIRT_OFFSET);
+    PROVIDE(__bss_end = .);
 
     PROVIDE(__kernel_end = .);
 };

etc/riscv64-unknown-none.json

@@ -14,7 +14,8 @@
   "panic-strategy": "abort",
   "dynamic-linking": true,
   "relocation-model": "pic",
-  "code-model": "large",
+  "position-independent-executables": true,
+  "code-model": "medium",
   "eh-frame-header": false,
 
   "crt-objects-fallback": "false",

etc/x86_64-unknown-none.json

@@ -3,17 +3,19 @@
   "cpu": "x86-64",
   "os": "none",
   "abi": "softfloat",
+  "rustc-abi": "x86-softfloat",
   "llvm-target": "x86_64-unknown-linux-gnu",
   "data-layout": "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128",
   "max-atomic-width": 64,
   "target-pointer-width": "64",
-  "features": "-avx,-sse,+soft-float",
+  "features": "-avx,-sse,-avx2,+soft-float",
 
   "disable-redzone": true,
   "executables": true,
   "panic-strategy": "abort",
   "dynamic-linking": true,
   "relocation-model": "pic",
+  "position-independent-executables": true,
 
   "has-thread-local": false,
 

kernel/Cargo.toml

@@ -19,7 +19,6 @@ chrono.workspace = true
 device-api = { workspace = true, features = ["derive"] }
 device-api-macros.workspace = true
 
-memtables.workspace = true
 vmalloc.workspace = true
 kernel-arch.workspace = true
 
@@ -30,6 +29,7 @@ ygg_driver_net_core = { path = "driver/net/core" }
 ygg_driver_net_loopback = { path = "driver/net/loopback" }
 ygg_driver_virtio_net = { path = "driver/virtio/net", features = ["pci"] }
 ygg_driver_virtio_gpu = { path = "driver/virtio/gpu", features = ["pci"] }
+ygg_driver_virtio_blk = { path = "driver/virtio/blk", features = ["pci"] }
 ygg_driver_nvme = { path = "driver/block/nvme" }
 ygg_driver_ahci = { path = "driver/block/ahci" }
 ygg_driver_input = { path = "driver/input" }
@@ -98,12 +98,5 @@ ygg_driver_net_stmmac.path = "driver/net/stmmac"
 default = ["fb_console"]
 fb_console = []
 
-# TODO replace this with a better configuration mechanism
-aarch64_board_virt = ["kernel-arch-aarch64/aarch64_board_virt"]
-aarch64_board_raspi4b = ["kernel-arch-aarch64/aarch64_board_raspi4b"]
-
-riscv64_board_virt = ["kernel-arch-riscv64/riscv64_board_virt"]
-riscv64_board_jh7110 = ["kernel-arch-riscv64/riscv64_board_jh7110"]
-
 [lints]
 workspace = true

kernel/arch/aarch64/Cargo.toml

@@ -7,13 +7,13 @@ edition = "2021"
 yggdrasil-abi.workspace = true
 kernel-arch-interface.workspace = true
 libk-mm-interface.workspace = true
-memtables.workspace = true
 device-api = { workspace = true, features = ["derive"] }
 
 bitflags.workspace = true
 static_assertions.workspace = true
 aarch64-cpu.workspace = true
 tock-registers.workspace = true
+log.workspace = true
 
 [build-dependencies]
 cc = "1.0"

kernel/arch/aarch64/src/lib.rs

@@ -1,5 +1,5 @@
 #![no_std]
-#![feature(naked_functions, trait_upcasting, decl_macro)]
+#![feature(decl_macro)]
 #![allow(clippy::new_without_default)]
 
 extern crate alloc;
@@ -40,11 +40,9 @@ impl CpuData for PerCpuData {}
 static IPI_QUEUES: OneTimeInit<Vec<IpiQueue<ArchitectureImpl>>> = OneTimeInit::new();
 pub static CPU_COUNT: AtomicUsize = AtomicUsize::new(1);
 
-#[naked]
+#[unsafe(naked)]
 extern "C" fn idle_task(_: usize) -> ! {
-    unsafe {
-        core::arch::naked_asm!("1: nop; b 1b");
-    }
+    core::arch::naked_asm!("1: nop; b 1b");
 }
 
 impl ArchitectureImpl {

kernel/arch/aarch64/src/mem/device.rs

@@ -0,0 +1,95 @@
+use core::ops::Range;
+
+use kernel_arch_interface::mem::DeviceMemoryAttributes;
+use libk_mm_interface::{
+    address::PhysicalAddress,
+    table::{DevicePageManager, DevicePageManagerLevel, EntryLevel},
+};
+
+use crate::mem::table::PageEntry;
+
+use super::{
+    table::{PageTable, L2, L3},
+    tlb_flush_range_va, DEVICE_MAPPING_OFFSET, DEVICE_MEMORY_L3_COUNT,
+};
+
+#[repr(transparent)]
+pub struct L2DeviceMemory(pub PageTable<L2>);
+#[repr(transparent)]
+pub struct L3DeviceMemory(pub [PageTable<L3>; DEVICE_MEMORY_L3_COUNT]);
+
+pub(super) static mut DEVICE_MEMORY: DevicePageManager<L3DeviceMemory, L2DeviceMemory> =
+    DevicePageManager::new(
+        L3DeviceMemory([PageTable::zeroed(); DEVICE_MEMORY_L3_COUNT]),
+        L2DeviceMemory(PageTable::zeroed()),
+    );
+
+impl DevicePageManagerLevel for L2DeviceMemory {
+    type Level = L2;
+    const VIRTUAL_BASE: usize = DEVICE_MAPPING_OFFSET;
+    const INDEX_RANGE: Range<usize> = DEVICE_MEMORY_L3_COUNT..512;
+
+    fn map_page(
+        &mut self,
+        index: usize,
+        physical: PhysicalAddress,
+        attrs: &DeviceMemoryAttributes,
+    ) {
+        let _ = index;
+        let _ = physical;
+        let _ = attrs;
+        todo!()
+    }
+
+    fn unmap_page(&mut self, index: usize) {
+        let _ = index;
+        todo!()
+    }
+
+    fn is_mapped(&self, index: usize) -> bool {
+        let _ = index;
+        todo!()
+    }
+
+    fn flush_range(range: Range<usize>) {
+        let start = range.start * L2::SIZE + Self::VIRTUAL_BASE;
+        let size = (range.end - range.start) * L2::SIZE;
+        tlb_flush_range_va(start, size);
+    }
+}
+
+impl DevicePageManagerLevel for L3DeviceMemory {
+    type Level = L3;
+    const VIRTUAL_BASE: usize = DEVICE_MAPPING_OFFSET;
+    const INDEX_RANGE: Range<usize> = 0..512 * DEVICE_MEMORY_L3_COUNT;
+
+    fn map_page(
+        &mut self,
+        index: usize,
+        physical: PhysicalAddress,
+        attrs: &DeviceMemoryAttributes,
+    ) {
+        let _ = attrs;
+        let l2i = index / 512;
+        let l3i = index % 512;
+        self.0[l2i][l3i] = PageEntry::device_page(physical);
+    }
+
+    // TODO
+    fn unmap_page(&mut self, index: usize) {
+        let _ = index;
+        todo!()
+    }
+
+    fn is_mapped(&self, index: usize) -> bool {
+        let l2i = index / 512;
+        let l3i = index % 512;
+        self.0[l2i][l3i].is_present()
+    }
+
+    fn flush_range(range: Range<usize>) {
+        let start = range.start * L3::SIZE + Self::VIRTUAL_BASE;
+        let size = (range.end - range.start) * L3::SIZE;
+        tlb_flush_range_va(start, size);
+    }
+}

kernel/arch/aarch64/src/mem/intrinsics.rs

@@ -0,0 +1,93 @@
+use core::sync::atomic::{self, Ordering};
+
+use aarch64_cpu::{asm::barrier, registers::PAR_EL1};
+use libk_mm_interface::table::{EntryLevel, EntryLevelExt};
+use tock_registers::interfaces::Readable;
+
+use super::table::L3;
+
+#[inline]
+pub fn tlb_flush_asid(asid: u8) {
+    barrier::dsb(barrier::ISHST);
+    let value = (asid as u64) << 48;
+    unsafe {
+        core::arch::asm!("tlbi aside1, {value}", value = in(reg) value);
+    }
+    barrier::dsb(barrier::ISH);
+    barrier::isb(barrier::SY);
+}
+
+#[inline]
+pub fn tlb_flush_all() {
+    barrier::dsb(barrier::ISHST);
+    unsafe {
+        core::arch::asm!("tlbi vmalle1is");
+    }
+    barrier::dsb(barrier::ISH);
+    barrier::isb(barrier::SY);
+}
+
+#[inline]
+pub fn tlb_flush_vaae1(page: usize) {
+    barrier::dsb(barrier::ISHST);
+    let argument = page >> 12;
+    unsafe {
+        core::arch::asm!("tlbi vaae1, {argument}", argument = in(reg) argument);
+    }
+    barrier::dsb(barrier::ISH);
+    barrier::isb(barrier::SY);
+}
+
+pub fn tlb_flush_range_va(base: usize, size: usize) {
+    let end = (base + size).page_align_up::<L3>();
+    let base = base.page_align_down::<L3>();
+    let count = (end - base).page_count::<L3>();
+    for i in 0..count {
+        tlb_flush_vaae1(base + i * L3::SIZE);
+    }
+}
+
+pub fn at_s1e0r(input: usize) -> Option<u64> {
+    barrier::dsb(barrier::ISHST);
+    unsafe {
+        core::arch::asm!("at s1e0r, {address}", address = in(reg) input);
+    }
+    barrier::dsb(barrier::ISH);
+    barrier::isb(barrier::SY);
+    if PAR_EL1.matches_all(PAR_EL1::F::TranslationSuccessfull) {
+        Some(PAR_EL1.read(PAR_EL1::PA))
+    } else {
+        None
+    }
+}
+
+pub fn at_s1e1r(input: usize) -> Option<u64> {
+    barrier::dsb(barrier::ISHST);
+    unsafe {
+        core::arch::asm!("at s1e1r, {address}", address = in(reg) input);
+    }
+    barrier::dsb(barrier::ISH);
+    barrier::isb(barrier::SY);
+    if PAR_EL1.matches_all(PAR_EL1::F::TranslationSuccessfull) {
+        Some(PAR_EL1.read(PAR_EL1::PA))
+    } else {
+        None
+    }
+}
+
+pub fn ic_iallu() {
+    atomic::compiler_fence(Ordering::SeqCst);
+    barrier::dsb(barrier::ISH);
+    barrier::isb(barrier::SY);
+    unsafe {
+        core::arch::asm!("ic iallu");
+    }
+    barrier::isb(barrier::SY);
+}
+
+pub fn dc_cvac(input: usize) {
+    barrier::dsb(barrier::ISHST);
+    unsafe {
+        core::arch::asm!("dc cvac, {address}", address = in(reg) input);
+    }
+}

kernel/arch/aarch64/src/mem/mod.rs

@@ -1,108 +1,43 @@
-use core::{
-    alloc::Layout,
-    ops::{Deref, DerefMut},
-    sync::atomic::{self, AtomicUsize, Ordering},
-};
-
 use aarch64_cpu::{
     asm::barrier,
-    registers::{MAIR_EL1, PAR_EL1, SCTLR_EL1, TTBR0_EL1, TTBR1_EL1},
+    registers::{MAIR_EL1, SCTLR_EL1, TCR_EL1, TTBR0_EL1, TTBR1_EL1},
 };
 use kernel_arch_interface::{
     mem::{DeviceMemoryAttributes, KernelTableManager, RawDeviceMemoryMapping},
-    split_spinlock, Architecture, KERNEL_VIRT_OFFSET,
+    sync::IrqSafeSpinlock,
+    KERNEL_VIRT_OFFSET,
 };
-use libk_mm_interface::{
-    address::PhysicalAddress,
-    table::{page_index, EntryLevel, EntryLevelExt},
-};
-use memtables::aarch64::{FixedTables, KERNEL_L3_COUNT};
-use static_assertions::const_assert_eq;
-use tock_registers::interfaces::{ReadWriteable, Readable, Writeable};
+use libk_mm_interface::{address::PhysicalAddress, table::EntryLevel};
+use table::{PageAttributes, PageEntry, PageTable, L1};
+use tock_registers::interfaces::{ReadWriteable, Writeable};
+
+mod intrinsics;
+
+pub mod device;
+pub mod process;
+pub mod table;
+
+pub use intrinsics::*;
 use yggdrasil_abi::error::Error;
 
 use crate::ArchitectureImpl;
 
-use self::table::{PageAttributes, PageEntry, PageTable, L1, L2, L3};
-
-pub mod process;
-pub mod table;
-
 #[derive(Debug)]
 pub struct KernelTableManagerImpl;
 
-// TODO eliminate this requirement by using precomputed indices
-const MAPPING_OFFSET: usize = KERNEL_VIRT_OFFSET;
-
-#[cfg(any(feature = "aarch64_board_virt", rust_analyzer))]
-const KERNEL_PHYS_BASE: usize = 0x40080000;
-#[cfg(any(feature = "aarch64_board_raspi4b", rust_analyzer))]
-const KERNEL_PHYS_BASE: usize = 0x80000;
-
-// Precomputed mappings
-const KERNEL_L1_INDEX: usize = page_index::<L1>(KERNEL_VIRT_OFFSET + KERNEL_PHYS_BASE);
-const KERNEL_START_L2_INDEX: usize = page_index::<L2>(KERNEL_VIRT_OFFSET + KERNEL_PHYS_BASE);
-const KERNEL_END_L2_INDEX: usize = KERNEL_START_L2_INDEX + KERNEL_L3_COUNT;
-
-// Must not be zero, should be at 4MiB
-const_assert_eq!(KERNEL_START_L2_INDEX, 0);
-// From static mapping
-#[cfg(any(feature = "aarch64_board_raspi4b", rust_analyzer))]
-const_assert_eq!(KERNEL_L1_INDEX, 0);
-#[cfg(any(feature = "aarch64_board_virt", rust_analyzer))]
-const_assert_eq!(KERNEL_L1_INDEX, 1);
-
-// Runtime mappings
-// 2MiB max
-const EARLY_MAPPING_L2I: usize = KERNEL_END_L2_INDEX + 1;
-// 1GiB max
-const DEVICE_MAPPING_L1I: usize = KERNEL_L1_INDEX + 2;
-const DEVICE_MAPPING_L3_COUNT: usize = 4;
-// 16GiB max
-const RAM_MAPPING_START_L1I: usize = KERNEL_L1_INDEX + 3;
-pub const RAM_MAPPING_L1_COUNT: usize = 16;
-
-// 2MiB for early mappings
-const EARLY_MAPPING_OFFSET: usize =
-    MAPPING_OFFSET | (KERNEL_L1_INDEX * L1::SIZE) | (EARLY_MAPPING_L2I * L2::SIZE);
-static mut EARLY_MAPPING_L3: PageTable<L3> = PageTable::zeroed();
-// 1GiB for device MMIO mapping
-const DEVICE_MAPPING_OFFSET: usize = MAPPING_OFFSET | (DEVICE_MAPPING_L1I * L1::SIZE);
-static mut DEVICE_MAPPING_L2: PageTable<L2> = PageTable::zeroed();
-static mut DEVICE_MAPPING_L3S: [PageTable<L3>; DEVICE_MAPPING_L3_COUNT] =
-    [PageTable::zeroed(); DEVICE_MAPPING_L3_COUNT];
-// 16GiB for RAM mapping
-pub const RAM_MAPPING_OFFSET: usize = MAPPING_OFFSET | (RAM_MAPPING_START_L1I * L1::SIZE);
-pub static MEMORY_LIMIT: AtomicUsize = AtomicUsize::new(0);
-
-split_spinlock! {
-    use crate::ArchitectureImpl;
-    use crate::mem::FixedTables;
-    use libk_mm_interface::KernelImageObject;
-
-    #[link_section = ".data.tables"]
-    static KERNEL_TABLES: KernelImageObject<FixedTables> =
-        unsafe { KernelImageObject::new(FixedTables::zeroed()) };
-}
-
 impl KernelTableManager for KernelTableManagerImpl {
-    fn virtualize(address: u64) -> usize {
-        let address = address as usize;
-        if address < MEMORY_LIMIT.load(Ordering::Acquire) {
-            address + RAM_MAPPING_OFFSET
-        } else {
-            panic!("Invalid physical address: {:#x}", address);
+    fn virtualize(phys: u64) -> usize {
+        if phys as usize >= IDENTITY_SIZE {
+            unreachable!("Invalid physical address to virtualize: {phys:#x}");
         }
+        phys as usize + KERNEL_VIRT_OFFSET
     }
 
-    fn physicalize(address: usize) -> u64 {
-        if address < RAM_MAPPING_OFFSET
-            || address - RAM_MAPPING_OFFSET >= MEMORY_LIMIT.load(Ordering::Acquire)
-        {
-            panic!("Not a virtualized physical address: {:#x}", address);
+    fn physicalize(virt: usize) -> u64 {
+        if virt < KERNEL_VIRT_OFFSET || virt - KERNEL_VIRT_OFFSET >= IDENTITY_SIZE {
+            unreachable!("Invalid virtualized address: {virt:#x}");
         }
-
-        (address - RAM_MAPPING_OFFSET) as _
+        (virt - KERNEL_VIRT_OFFSET) as _
     }
 
     unsafe fn map_device_pages(
@@ -110,390 +45,71 @@ impl KernelTableManager for KernelTableManagerImpl {
         count: usize,
         attrs: DeviceMemoryAttributes,
     ) -> Result<RawDeviceMemoryMapping<Self>, Error> {
-        map_device_memory(PhysicalAddress::from_u64(base), count, attrs)
-    }
-
-    unsafe fn unmap_device_pages(mapping: &RawDeviceMemoryMapping<Self>) {
-        unmap_device_memory(mapping)
-    }
-}
-
-/// Memory mapping which may be used for performing early kernel initialization
-pub struct EarlyMapping<'a, T: ?Sized> {
-    value: &'a mut T,
-    page_count: usize,
-}
-
-impl<'a, T: Sized> EarlyMapping<'a, T> {
-    /// # Safety
-    ///
-    /// `physical` address provided must be a valid non-NULL address actually containing `T`.
-    pub unsafe fn map_slice(
-        physical: PhysicalAddress,
-        len: usize,
-    ) -> Result<EarlyMapping<'a, [T]>, Error> {
-        let layout = Layout::array::<T>(len).unwrap();
-        let aligned = physical.page_align_down::<L3>();
-        let offset = physical.page_offset::<L3>();
-        let page_count = (offset + layout.size()).div_ceil(L3::SIZE);
-
-        let virt = map_early_pages(aligned, page_count)?;
-        let value = core::slice::from_raw_parts_mut((virt + offset) as *mut T, len);
-
-        Ok(EarlyMapping { value, page_count })
-    }
-}
-
-impl<T: ?Sized> Deref for EarlyMapping<'_, T> {
-    type Target = T;
-
-    fn deref(&self) -> &Self::Target {
-        self.value
-    }
-}
-
-impl<T: ?Sized> DerefMut for EarlyMapping<'_, T> {
-    fn deref_mut(&mut self) -> &mut Self::Target {
-        self.value
-    }
-}
-
-impl<T: ?Sized> Drop for EarlyMapping<'_, T> {
-    fn drop(&mut self) {
-        let address = (self.value as *mut T).addr() & !(L3::SIZE - 1);
-
-        for i in 0..self.page_count {
-            let page = address + i * L3::SIZE;
-
-            unsafe {
-                unmap_early_page(page);
-            }
+        let _guard = DEVICE_MEMORY_LOCK.lock();
+        #[allow(static_mut_refs)]
+        unsafe {
+            device::DEVICE_MEMORY.map_device_pages(PhysicalAddress::from_u64(base), count, attrs)
         }
     }
-}
 
-fn kernel_table_flags() -> PageAttributes {
-    PageAttributes::TABLE
-        | PageAttributes::ACCESS
-        | PageAttributes::SH_INNER
-        | PageAttributes::PAGE_ATTR_NORMAL
-        | PageAttributes::PRESENT
-}
-
-fn ram_block_flags() -> PageAttributes {
-    // TODO UXN, PXN
-    PageAttributes::BLOCK
-        | PageAttributes::ACCESS
-        | PageAttributes::SH_OUTER
-        | PageAttributes::PAGE_ATTR_NORMAL
-        | PageAttributes::PRESENT
-}
-
-// Early mappings
-unsafe fn map_early_pages(physical: PhysicalAddress, count: usize) -> Result<usize, Error> {
-    for l3i in 0..512 {
-        let mut taken = false;
-        for i in 0..count {
-            if EARLY_MAPPING_L3[i + l3i].is_present() {
-                taken = true;
-                break;
-            }
-        }
-
-        if taken {
-            continue;
-        }
-
-        for i in 0..count {
-            let page = physical.add(i * L3::SIZE);
-            // TODO NX, NC
-            EARLY_MAPPING_L3[i + l3i] = PageEntry::normal_page(page, PageAttributes::empty());
-            tlb_flush_vaae1(EARLY_MAPPING_OFFSET + (l3i + i) * L3::SIZE);
-        }
-
-        return Ok(EARLY_MAPPING_OFFSET + l3i * L3::SIZE);
-    }
-
-    Err(Error::OutOfMemory)
-}
-
-unsafe fn unmap_early_page(address: usize) {
-    if !(EARLY_MAPPING_OFFSET..EARLY_MAPPING_OFFSET + L2::SIZE).contains(&address) {
-        panic!("Tried to unmap invalid early mapping: {:#x}", address);
-    }
-
-    let l3i = (address - EARLY_MAPPING_OFFSET).page_index::<L3>();
-
-    assert!(EARLY_MAPPING_L3[l3i].is_present());
-    EARLY_MAPPING_L3[l3i] = PageEntry::INVALID;
-}
-
-/// # Safety
-///
-/// Only meant to be used by the architecture initialization functions.
-pub unsafe fn map_ram_l1(index: usize) {
-    if index >= RAM_MAPPING_L1_COUNT {
-        ArchitectureImpl::halt();
-    }
-    let mut tables = KERNEL_TABLES.lock();
-    let table_index = index + RAM_MAPPING_START_L1I;
-
-    if tables.l1.data[table_index] != 0 {
-        ArchitectureImpl::halt();
-    }
-
-    tables.l1.data[table_index] = ((index * L1::SIZE) as u64) | ram_block_flags().bits();
-    tlb_flush_vaae1(RAM_MAPPING_OFFSET + index * L1::SIZE);
-}
-
-// Device mappings
-unsafe fn map_device_memory_l3(
-    base: PhysicalAddress,
-    count: usize,
-    _attrs: DeviceMemoryAttributes,
-) -> Result<usize, Error> {
-    // TODO don't map pages if already mapped
-
-    'l0: for i in 0..DEVICE_MAPPING_L3_COUNT * 512 {
-        for j in 0..count {
-            let l2i = (i + j) / 512;
-            let l3i = (i + j) % 512;
-
-            if DEVICE_MAPPING_L3S[l2i][l3i].is_present() {
-                continue 'l0;
-            }
-        }
-
-        for j in 0..count {
-            let l2i = (i + j) / 512;
-            let l3i = (i + j) % 512;
-
-            // TODO NX, NC
-            DEVICE_MAPPING_L3S[l2i][l3i] = PageEntry::device_page(base.add(j * L3::SIZE));
-            tlb_flush_vaae1(DEVICE_MAPPING_OFFSET + l2i * L2::SIZE + l3i * L3::SIZE);
-        }
-
-        return Ok(DEVICE_MAPPING_OFFSET + i * L3::SIZE);
-    }
-
-    Err(Error::OutOfMemory)
-}
-
-#[allow(unused)]
-unsafe fn map_device_memory_l2(
-    base: PhysicalAddress,
-    count: usize,
-    _attrs: DeviceMemoryAttributes,
-) -> Result<usize, Error> {
-    'l0: for i in DEVICE_MAPPING_L3_COUNT..512 {
-        for j in 0..count {
-            if DEVICE_MAPPING_L2[i + j].is_present() {
-                continue 'l0;
-            }
-        }
-
-        for j in 0..count {
-            DEVICE_MAPPING_L2[i + j] = PageEntry::<L2>::device_block(base.add(j * L2::SIZE));
-            tlb_flush_vaae1(DEVICE_MAPPING_OFFSET + (i + j) * L2::SIZE);
-        }
-
-        return Ok(DEVICE_MAPPING_OFFSET + i * L2::SIZE);
-    }
-
-    Err(Error::OutOfMemory)
-}
-
-pub(crate) unsafe fn map_device_memory(
-    base: PhysicalAddress,
-    size: usize,
-    attrs: DeviceMemoryAttributes,
-) -> Result<RawDeviceMemoryMapping<KernelTableManagerImpl>, Error> {
-    let l3_aligned = base.page_align_down::<L3>();
-    let l3_offset = base.page_offset::<L3>();
-    let page_count = (l3_offset + size).page_count::<L3>();
-
-    if page_count > 256 {
-        // Large mapping, use L2 mapping instead
-        let l2_aligned = base.page_align_down::<L2>();
-        let l2_offset = base.page_offset::<L2>();
-        let page_count = (l2_offset + size).page_count::<L2>();
-
-        let base_address = map_device_memory_l2(l2_aligned, page_count, attrs)?;
-        let address = base_address + l2_offset;
-
-        Ok(RawDeviceMemoryMapping::from_raw_parts(
-            l2_aligned.into_u64(),
-            address,
-            base_address,
-            page_count,
-            L2::SIZE,
-        ))
-    } else {
-        // Just map the pages directly
-        let base_address = map_device_memory_l3(l3_aligned, page_count, attrs)?;
-        let address = base_address + l3_offset;
-
-        Ok(RawDeviceMemoryMapping::from_raw_parts(
-            l3_aligned.into_u64(),
-            address,
-            base_address,
-            page_count,
-            L3::SIZE,
-        ))
+    unsafe fn unmap_device_pages(_mapping: &RawDeviceMemoryMapping<Self>) {
+        // TODO
     }
 }
 
-pub(crate) unsafe fn unmap_device_memory(map: &RawDeviceMemoryMapping<KernelTableManagerImpl>) {
-    match map.page_size {
-        L3::SIZE => {
-            for i in 0..map.page_count {
-                let page = map.base_address + i * L3::SIZE;
-                let l2i = page.page_index::<L2>();
-                let l3i = page.page_index::<L3>();
-                assert!(DEVICE_MAPPING_L3S[l2i][l3i].is_present());
-                DEVICE_MAPPING_L3S[l2i][l3i] = PageEntry::INVALID;
+// 64GiB of identity-mapped memory
+pub const IDENTITY_SIZE: usize = 64 * L1::SIZE;
+pub const IDENTITY_L1_START: usize = (KERNEL_VIRT_OFFSET >> L1::SHIFT) & 0x1FF;
+// 1GiB of memory-mapped devices
+pub const DEVICE_MEMORY_L1: usize = IDENTITY_L1_START + IDENTITY_SIZE / L1::SIZE;
+pub const DEVICE_MEMORY_L3_COUNT: usize = 32;
+pub const DEVICE_MAPPING_OFFSET: usize = KERNEL_VIRT_OFFSET + DEVICE_MEMORY_L1 * L1::SIZE;
+static DEVICE_MEMORY_LOCK: IrqSafeSpinlock<ArchitectureImpl, ()> = IrqSafeSpinlock::new(());
 
-                tlb_flush_vaae1(page);
-            }
-        }
-        L2::SIZE => todo!(),
-        _ => unimplemented!(),
+static mut FIXED_L1: PageTable<L1> = const {
+    if IDENTITY_L1_START != 0 {
+        panic!("Invalid KERNEL_VIRT_OFFSET");
     }
-}
-
-#[inline]
-pub fn tlb_flush_asid(asid: u8) {
-    barrier::dsb(barrier::ISHST);
-    let value = (asid as u64) << 48;
-    unsafe {
-        core::arch::asm!("tlbi aside1, {value}", value = in(reg) value);
+    if IDENTITY_L1_START + IDENTITY_SIZE / L1::SIZE >= 500 {
+        panic!("Invalid KERNEL_VIRT_OFFSET + IDENTITY_SIZE");
     }
-    barrier::dsb(barrier::ISH);
-    barrier::isb(barrier::SY);
-}
 
-#[inline]
-pub fn tlb_flush_all() {
-    barrier::dsb(barrier::ISHST);
-    unsafe {
-        core::arch::asm!("tlbi vmalle1is");
+    let mut table = PageTable::<L1>::zeroed();
+    let mut i = 0;
+
+    while i < IDENTITY_SIZE / L1::SIZE {
+        let phys = PhysicalAddress::from_usize(i * L1::SIZE);
+        table.entries[i + IDENTITY_L1_START] =
+            PageEntry::normal_block(phys, PageAttributes::empty());
+        i += 1;
     }
-    barrier::dsb(barrier::ISH);
-    barrier::isb(barrier::SY);
-}
 
-#[inline]
-pub fn tlb_flush_vaae1(page: usize) {
-    barrier::dsb(barrier::ISHST);
-    let argument = page >> 12;
-    unsafe {
-        core::arch::asm!("tlbi vaae1, {argument}", argument = in(reg) argument);
-    }
-    barrier::dsb(barrier::ISH);
-    barrier::isb(barrier::SY);
-}
+    table
+};
 
-pub fn at_s1e0r(input: usize) -> Option<u64> {
-    barrier::dsb(barrier::ISHST);
-    unsafe {
-        core::arch::asm!("at s1e0r, {address}", address = in(reg) input);
-    }
-    barrier::dsb(barrier::ISH);
-    barrier::isb(barrier::SY);
-    if PAR_EL1.matches_all(PAR_EL1::F::TranslationSuccessfull) {
-        Some(PAR_EL1.read(PAR_EL1::PA))
-    } else {
-        None
-    }
-}
-
-pub fn at_s1e1r(input: usize) -> Option<u64> {
-    barrier::dsb(barrier::ISHST);
-    unsafe {
-        core::arch::asm!("at s1e1r, {address}", address = in(reg) input);
-    }
-    barrier::dsb(barrier::ISH);
-    barrier::isb(barrier::SY);
-    if PAR_EL1.matches_all(PAR_EL1::F::TranslationSuccessfull) {
-        Some(PAR_EL1.read(PAR_EL1::PA))
-    } else {
-        None
-    }
-}
-
-pub fn ic_iallu() {
-    atomic::compiler_fence(Ordering::SeqCst);
-    barrier::dsb(barrier::ISH);
-    barrier::isb(barrier::SY);
-    unsafe {
-        core::arch::asm!("ic iallu");
-    }
-    barrier::isb(barrier::SY);
-}
-
-pub fn dc_cvac(input: usize) {
-    barrier::dsb(barrier::ISHST);
-    unsafe {
-        core::arch::asm!("dc cvac, {address}", address = in(reg) input);
-    }
-}
-
-fn auto_address<T>(value: *const T) -> usize {
-    let addr = value.addr();
-    if addr < KERNEL_VIRT_OFFSET {
-        // Called from lower half
-        addr
-    } else {
-        // Called from higher-half
-        addr - KERNEL_VIRT_OFFSET
-    }
-}
-
-/// (BSP-early init) loads precomputed kernel mapping tables for the kernel to jump to "higher-half"
-///
-/// # Safety
-///
-/// Unsafe, must only be called by BSP during its early init while still in "lower-half"
-pub unsafe fn load_fixed_tables() {
-    let ttbr0 = auto_address(&raw const KERNEL_TABLES) as u64;
-
-    TTBR0_EL1.set(ttbr0);
-    TTBR1_EL1.set(ttbr0);
-}
-
-/// Sets up additional translation tables for kernel usage
-///
-/// # Safety
-///
-/// Unsafe, must only be called by BSP during its early init, must already be in "higher-half"
-pub unsafe fn init_fixed_tables() {
-    // TODO this could be built in compile-time too?
-    let mut tables = KERNEL_TABLES.grab();
-    let early_mapping_l3_phys = auto_address(&raw const EARLY_MAPPING_L3);
-    let device_mapping_l2_phys = auto_address(&raw const DEVICE_MAPPING_L2);
-
-    for i in 0..DEVICE_MAPPING_L3_COUNT {
-        let device_mapping_l3_phys = PhysicalAddress::from_usize(
-            &DEVICE_MAPPING_L3S[i] as *const _ as usize - KERNEL_VIRT_OFFSET,
+pub unsafe fn setup_fixed_tables() {
+    let device_l2_physical = auto_lower_address(&raw const device::DEVICE_MEMORY.large);
+    FIXED_L1[DEVICE_MEMORY_L1] = PageEntry::table(
+        PhysicalAddress::from_usize(device_l2_physical),
+        PageAttributes::empty(),
+    );
+    for i in 0..DEVICE_MEMORY_L3_COUNT {
+        let device_l3_physical = auto_lower_address(&raw const device::DEVICE_MEMORY.normal.0[i]);
+        device::DEVICE_MEMORY.large.0[i] = PageEntry::table(
+            PhysicalAddress::from_usize(device_l3_physical),
+            PageAttributes::empty(),
         );
-        DEVICE_MAPPING_L2[i] = PageEntry::table(device_mapping_l3_phys, PageAttributes::empty());
     }
-
-    assert_eq!(tables.l2.data[EARLY_MAPPING_L2I], 0);
-    tables.l2.data[EARLY_MAPPING_L2I] =
-        (early_mapping_l3_phys as u64) | kernel_table_flags().bits();
-    tlb_flush_vaae1(EARLY_MAPPING_OFFSET);
-
-    assert_eq!(tables.l1.data[DEVICE_MAPPING_L1I], 0);
-    tables.l1.data[DEVICE_MAPPING_L1I] =
-        (device_mapping_l2_phys as u64) | kernel_table_flags().bits();
-
-    tlb_flush_all();
 }
 
-pub fn setup_memory_attributes() {
-    // TODO: Figure out why WriteBack_NonTransient_ReadWriteAlloc doesn't work on Pi 4B
+pub fn load_fixed_tables() {
+    let l1_physical = auto_lower_address(&raw const FIXED_L1);
+    TTBR0_EL1.set_baddr(l1_physical as _);
+    TTBR1_EL1.set_baddr(l1_physical as _);
+}
 
+pub fn configure_mmu() {
+    // TODO: Figure out why WriteBack_NonTransient_ReadWriteAlloc doesn't work on Pi 4B
     MAIR_EL1.write(
         //// Attribute 0 -- normal memory
         MAIR_EL1::Attr0_Normal_Inner::WriteBack_NonTransient +
@@ -504,6 +120,39 @@ pub fn setup_memory_attributes() {
         //// Attribute 2 -- device memory
         MAIR_EL1::Attr1_Device::nonGathering_nonReordering_EarlyWriteAck,
     );
+
+    TCR_EL1.write(
+        TCR_EL1::AS::ASID8Bits +
+             TCR_EL1::A1::TTBR0 +
+             TCR_EL1::HD::CLEAR +
+             // General
+             TCR_EL1::IPS::Bits_48 +
+             // TTBR0
+             TCR_EL1::TG0::KiB_4 + TCR_EL1::T0SZ.val(25) + TCR_EL1::SH0::Inner +
+             // TTBR1
+             TCR_EL1::TG1::KiB_4 + TCR_EL1::T1SZ.val(25) + TCR_EL1::SH1::Inner,
+    );
+    barrier::dsb(barrier::ISHST);
+    barrier::isb(barrier::SY);
+
+    SCTLR_EL1.modify(
+        SCTLR_EL1::E0E::LittleEndian
+            + SCTLR_EL1::EE::LittleEndian
+            + SCTLR_EL1::WXN::Disable
+            + SCTLR_EL1::SA0::Enable
+            + SCTLR_EL1::SA::Enable
+            + SCTLR_EL1::A::Enable
+            + SCTLR_EL1::I::NonCacheable
+            + SCTLR_EL1::C::NonCacheable,
+    );
+
+    barrier::dsb(barrier::ISH);
+    barrier::isb(barrier::SY);
+}
+
+pub fn enable_mmu() {
+    // Enable translation
+    SCTLR_EL1.modify(SCTLR_EL1::M::Enable);
 }
 
 /// Enables data cache.
@@ -549,3 +198,12 @@ pub unsafe fn disable_icache() {
     barrier::dsb(barrier::ISH);
     barrier::isb(barrier::SY);
 }
+
+fn auto_lower_address<T>(ptr: *const T) -> usize {
+    let address = ptr.addr();
+    if address < KERNEL_VIRT_OFFSET {
+        address
+    } else {
+        address - KERNEL_VIRT_OFFSET
+    }
+}

kernel/arch/aarch64/src/mem/process.rs

@@ -7,7 +7,7 @@ use core::{
 use libk_mm_interface::{
     address::{AsPhysicalAddress, PhysicalAddress},
     pointer::PhysicalRefMut,
-    process::ProcessAddressSpaceManager,
+    process::{PageAttributeUpdate, ProcessAddressSpaceManager},
     table::{
         EntryLevel, EntryLevelDrop, EntryLevelExt, MapAttributes, NextPageTable, TableAllocator,
     },
@@ -79,7 +79,15 @@ impl<TA: TableAllocator> ProcessAddressSpaceManager<TA> for ProcessAddressSpaceI
         )
     }
 
-    unsafe fn unmap_page(&mut self, address: usize) -> Result<PhysicalAddress, Error> {
+    unsafe fn update_page_attributes(
+        &mut self,
+        address: usize,
+        update: &PageAttributeUpdate,
+    ) -> Result<(), Error> {
+        self.update_l3_entry(address, |entry| entry.update(update))
+    }
+
+    unsafe fn unmap_page(&mut self, address: usize) -> Result<(PhysicalAddress, bool), Error> {
         self.pop_l3_entry(address)
     }
 
@@ -120,7 +128,32 @@ impl<TA: TableAllocator> ProcessAddressSpaceImpl<TA> {
         Ok(())
     }
 
-    fn pop_l3_entry(&mut self, virt: usize) -> Result<PhysicalAddress, Error> {
+    fn update_l3_entry<F: FnOnce(&mut PageEntry<L3>) -> Result<(), Error>>(
+        &mut self,
+        virt: usize,
+        mapper: F,
+    ) -> Result<(), Error> {
+        let l1i = virt.page_index::<L1>();
+        let l2i = virt.page_index::<L2>();
+        let l3i = virt.page_index::<L3>();
+
+        let mut l2 = self.l1.get_mut(l1i).ok_or(Error::DoesNotExist)?;
+        let mut l3 = l2.get_mut(l2i).ok_or(Error::DoesNotExist)?;
+
+        let entry = &mut l3[l3i];
+        if !entry.is_present() {
+            return Err(Error::DoesNotExist);
+        }
+
+        mapper(entry)?;
+        ic_iallu();
+        dc_cvac((&raw const l3[l3i]).addr());
+        tlb_flush_vaae1(virt);
+
+        Ok(())
+    }
+
+    fn pop_l3_entry(&mut self, virt: usize) -> Result<(PhysicalAddress, bool), Error> {
         let l1i = virt.page_index::<L1>();
         let l2i = virt.page_index::<L2>();
         let l3i = virt.page_index::<L3>();
@@ -129,14 +162,16 @@ impl<TA: TableAllocator> ProcessAddressSpaceImpl<TA> {
         let mut l2 = self.l1.get_mut(l1i).ok_or(Error::DoesNotExist)?;
         let mut l3 = l2.get_mut(l2i).ok_or(Error::DoesNotExist)?;
 
-        let page = l3[l3i].as_page().ok_or(Error::DoesNotExist)?;
+        let entry = l3[l3i];
+        let page = entry.as_page().ok_or(Error::DoesNotExist)?;
+        let dirty = entry.is_dirty();
 
         l3[l3i] = PageEntry::INVALID;
         ic_iallu();
         dc_cvac((&raw const l3[l3i]).addr());
         tlb_flush_vaae1(virt);
 
-        Ok(page)
+        Ok((page, dirty))
     }
 
     fn read_l3_entry(&self, virt: usize) -> Option<(PhysicalAddress, MapAttributes)> {

kernel/arch/aarch64/src/mem/table.rs

@@ -9,6 +9,7 @@ use kernel_arch_interface::KERNEL_VIRT_OFFSET;
 use libk_mm_interface::{
     address::{AsPhysicalAddress, PhysicalAddress},
     pointer::{PhysicalRef, PhysicalRefMut},
+    process::PageAttributeUpdate,
     table::{
         EntryLevel, EntryLevelDrop, MapAttributes, NextPageTable, NonTerminalEntryLevel,
         TableAllocator,
@@ -46,6 +47,7 @@ bitflags! {
 
         const NON_GLOBAL = 1 << 11;
 
+        const DIRTY = 1 << 51;
         const PXN = 1 << 53;
         const UXN = 1 << 54;
     }
@@ -54,7 +56,7 @@ bitflags! {
 #[derive(Clone, Copy)]
 #[repr(C, align(0x1000))]
 pub struct PageTable<L: EntryLevel> {
-    entries: [PageEntry<L>; 512],
+    pub entries: [PageEntry<L>; 512],
 }
 
 #[derive(Clone, Copy)]
@@ -136,8 +138,32 @@ impl<L: EntryLevel> PageTable<L> {
 }
 
 impl<L: EntryLevel> PageEntry<L> {
+    const ATTR_MASK: u64 = 0xFFF | (0xFFFF << 48);
     pub const INVALID: Self = Self(0, PhantomData);
 
+    pub fn update(&mut self, update: &PageAttributeUpdate) -> Result<(), Error> {
+        let mut attrs = PageAttributes::from_bits_retain(self.0);
+        if let Some(write) = update.user_write {
+            // Make writeable/non-writeable
+            if write {
+                attrs &= !PageAttributes::AP_ACCESS_MASK;
+                attrs |= PageAttributes::AP_BOTH_READWRITE;
+            } else {
+                todo!();
+            }
+        }
+        if let Some(dirty) = update.dirty {
+            if dirty {
+                attrs |= PageAttributes::DIRTY;
+            } else {
+                attrs &= !PageAttributes::DIRTY;
+            }
+        }
+        self.0 &= !Self::ATTR_MASK;
+        self.0 |= attrs.bits() & Self::ATTR_MASK;
+        Ok(())
+    }
+
     pub const fn is_present(self) -> bool {
         self.0 & PageAttributes::PRESENT.bits() != 0
     }
@@ -236,18 +262,13 @@ impl<L: NonTerminalEntryLevel> PageEntry<L> {
         )
     }
 
-    pub fn normal_block(phys: PhysicalAddress, attrs: PageAttributes) -> Self {
-        Self(
-            phys.into_u64()
-                | (PageAttributes::BLOCK
-                    | PageAttributes::PRESENT
-                    | PageAttributes::ACCESS
-                    | PageAttributes::SH_OUTER
-                    | PageAttributes::PAGE_ATTR_NORMAL
-                    | attrs)
-                    .bits(),
-            PhantomData,
-        )
+    pub const fn normal_block(phys: PhysicalAddress, attrs: PageAttributes) -> Self {
+        const ATTR: u64 = PageAttributes::BLOCK.bits()
+            | PageAttributes::PRESENT.bits()
+            | PageAttributes::ACCESS.bits()
+            | PageAttributes::SH_OUTER.bits()
+            | PageAttributes::PAGE_ATTR_NORMAL.bits();
+        Self(phys.into_u64() | attrs.bits() | ATTR, PhantomData)
     }
 
     pub fn device_block(phys: PhysicalAddress) -> Self {
@@ -271,7 +292,7 @@ impl<L: NonTerminalEntryLevel> PageEntry<L> {
         if self.0 & PageAttributes::PRESENT.bits() != 0
             && self.0 & PageAttributes::BLOCK.bits() == 0
         {
-            Some(PhysicalAddress::from_u64(self.0 & !0xFFF))
+            Some(PhysicalAddress::from_u64(self.0 & !Self::ATTR_MASK))
         } else {
             None
         }
@@ -283,7 +304,7 @@ impl<L: NonTerminalEntryLevel> PageEntry<L> {
         } else if let Some(table) = self.as_table() {
             EntryType::Table(table)
         } else {
-            EntryType::Page(PhysicalAddress::from_u64(self.0 & !0xFFF))
+            EntryType::Page(PhysicalAddress::from_u64(self.0 & !Self::ATTR_MASK))
         }
     }
 }
@@ -316,10 +337,14 @@ impl PageEntry<L3> {
         )
     }
 
+    pub fn is_dirty(&self) -> bool {
+        self.0 & PageAttributes::DIRTY.bits() != 0
+    }
+
     pub fn as_page(&self) -> Option<PhysicalAddress> {
         let mask = (PageAttributes::PRESENT | PageAttributes::PAGE).bits();
         if self.0 & mask == mask {
-            Some(PhysicalAddress::from_u64(self.0 & !0xFFF))
+            Some(PhysicalAddress::from_u64(self.0 & !Self::ATTR_MASK))
         } else {
             None
         }
@@ -355,6 +380,10 @@ impl From<MapAttributes> for PageAttributes {
             out |= PageAttributes::AP_KERNEL_READONLY;
         }
 
+        if value.contains(MapAttributes::DIRTY) {
+            out |= PageAttributes::DIRTY;
+        }
+
         if value.contains(MapAttributes::NON_GLOBAL) {
             out |= PageAttributes::NON_GLOBAL;
         }
@@ -377,6 +406,10 @@ impl From<PageAttributes> for MapAttributes {
             _ => unreachable!(),
         };
 
+        if value.contains(PageAttributes::DIRTY) {
+            out |= MapAttributes::DIRTY;
+        }
+
         if value.contains(PageAttributes::NON_GLOBAL) {
             out |= MapAttributes::NON_GLOBAL;
         }

kernel/arch/hosted/Cargo.toml

@@ -7,3 +7,4 @@ edition = "2021"
 kernel-arch-interface.workspace = true
 yggdrasil-abi.workspace = true
 libk-mm-interface.workspace = true
+device-api.workspace = true

kernel/arch/hosted/src/lib.rs

@@ -1,9 +1,11 @@
-#![feature(never_type)]
+#![feature(never_type, allocator_api, slice_ptr_get)]
 use std::{
+    alloc::{Allocator, Global, Layout},
     marker::PhantomData,
     sync::atomic::{AtomicBool, Ordering},
 };
 
+use device_api::dma::{DmaAllocation, DmaAllocator};
 use kernel_arch_interface::{
     cpu::{CpuData, IpiQueue},
     mem::{
@@ -105,6 +107,14 @@ impl Architecture for ArchitectureImpl {
     fn ipi_queue(_cpu_id: u32) -> Option<&'static IpiQueue<Self>> {
         None
     }
+
+    fn load_barrier() {}
+
+    fn store_barrier() {}
+
+    fn memory_barrier() {}
+
+    fn flush_virtual_range(_range: std::ops::Range<usize>) {}
 }
 
 impl KernelTableManager for KernelTableManagerImpl {
@@ -150,7 +160,7 @@ impl<TA: TableAllocator> ProcessAddressSpaceManager<TA> for ProcessAddressSpaceI
         unimplemented!()
     }
 
-    unsafe fn unmap_page(&mut self, _address: usize) -> Result<PhysicalAddress, Error> {
+    unsafe fn unmap_page(&mut self, _address: usize) -> Result<(PhysicalAddress, bool), Error> {
         unimplemented!()
     }
 
@@ -202,3 +212,19 @@ impl<K: KernelTableManager, PA: PhysicalMemoryAllocator> TaskContext<K, PA>
 extern "Rust" fn __signal_process_group(_group_id: ProcessGroupId, _signal: Signal) {
     unimplemented!()
 }
+
+pub struct HostedDmaAllocator;
+
+impl DmaAllocator for HostedDmaAllocator {
+    fn allocate(&self, layout: Layout) -> Result<DmaAllocation, Error> {
+        let ptr = Global.allocate(layout.align_to(0x1000).unwrap()).unwrap();
+        let base = ptr.as_non_null_ptr();
+        let addr: usize = base.addr().into();
+        Ok(DmaAllocation {
+            host_virtual: base.cast(),
+            host_physical: addr as _,
+            page_count: layout.size().div_ceil(0x1000),
+            bus_address: addr as _,
+        })
+    }
+}

kernel/arch/interface/src/sync.rs

@@ -42,7 +42,7 @@ impl<A: Architecture, T> Spinlock<A, T> {
         }
     }
 
-    pub fn lock(&self) -> SpinlockGuard<A, T> {
+    pub fn lock(&self) -> SpinlockGuard<'_, A, T> {
         // Loop until the lock can be acquired
         if LOCK_HACK.load(Ordering::Acquire) {
             return SpinlockGuard { lock: self };
@@ -103,7 +103,7 @@ impl<A: Architecture, T> IrqSafeSpinlock<A, T> {
     }
 
     /// Attempts to acquire a lock. IRQs will be disabled until the lock is released.
-    pub fn lock(&self) -> IrqSafeSpinlockGuard<A, T> {
+    pub fn lock(&self) -> IrqSafeSpinlockGuard<'_, A, T> {
         // Disable IRQs to avoid IRQ handler trying to acquire the same lock
         let irq_guard = IrqGuard::acquire();
 

kernel/arch/riscv64/Cargo.toml

@@ -7,7 +7,6 @@ edition = "2024"
 yggdrasil-abi.workspace = true
 kernel-arch-interface.workspace = true
 libk-mm-interface.workspace = true
-memtables.workspace = true
 device-api = { workspace = true, features = ["derive"] }
 
 tock-registers.workspace = true

kernel/arch/riscv64/src/context.rs

@@ -124,8 +124,7 @@ impl<K: KernelTableManager, PA: PhysicalMemoryAllocator<Address = PhysicalAddres
 
         // TODO stack is leaked
         let satp = InMemoryRegister::new(0);
-        let kernel_table_phys =
-            ((&raw const mem::KERNEL_TABLES).addr() - KERNEL_VIRT_OFFSET) as u64;
+        let kernel_table_phys = ((&raw const mem::FIXED_L1).addr() - KERNEL_VIRT_OFFSET) as u64;
         satp.write(SATP::MODE::Sv39 + SATP::ASID.val(0) + SATP::PPN.val(kernel_table_phys >> 12));
 
         Ok(Self {

kernel/arch/riscv64/src/lib.rs

@@ -1,4 +1,4 @@
-#![feature(decl_macro, naked_functions)]
+#![feature(decl_macro)]
 #![no_std]
 
 extern crate alloc;
@@ -60,11 +60,9 @@ impl CpuData for PerCpuData {
     }
 }
 
-#[naked]
+#[unsafe(naked)]
 extern "C" fn idle_task(_: usize) -> ! {
-    unsafe {
-        core::arch::naked_asm!("1: nop; j 1b");
-    }
+    core::arch::naked_asm!("1: nop; j 1b");
 }
 
 impl ArchitectureImpl {

kernel/arch/riscv64/src/mem/mod.rs

@@ -1,70 +1,38 @@
-use cfg_if::cfg_if;
-use kernel_arch_interface::{
-    mem::{DeviceMemoryAttributes, KernelTableManager, RawDeviceMemoryMapping},
-    split_spinlock,
+use kernel_arch_interface::mem::{
+    DeviceMemoryAttributes, KernelTableManager, RawDeviceMemoryMapping,
 };
 use libk_mm_interface::{
     address::PhysicalAddress,
     table::{page_index, EntryLevel, EntryLevelExt},
 };
-use memtables::riscv64::PageAttributes;
-use static_assertions::{const_assert, const_assert_eq};
-use table::{PageEntry, PageTable, L1, L2, L3};
+use table::{PageEntry, PageTable, L1, L3};
 use tock_registers::interfaces::Writeable;
 use yggdrasil_abi::error::Error;
 
-pub use memtables::riscv64::FixedTables;
-
 use crate::registers::SATP;
 
 pub mod process;
 pub mod table;
 
-split_spinlock! {
-    use crate::ArchitectureImpl;
-    use crate::mem::FixedTables;
-    use libk_mm_interface::KernelImageObject;
-
-    #[link_section = ".data.tables"]
-    #[used]
-    static KERNEL_TABLES: KernelImageObject<FixedTables> =
-        unsafe { KernelImageObject::new(FixedTables::zeroed()) };
-}
-
-cfg_if! {
-    if #[cfg(feature = "riscv64_board_virt")] {
-        pub const KERNEL_PHYS_BASE: usize = 0x80200000;
-    } else if #[cfg(feature = "riscv64_board_jh7110")] {
-        pub const KERNEL_PHYS_BASE: usize = 0x40200000;
-    } else if #[cfg(rust_analyzer)] {
-        pub const KERNEL_PHYS_BASE: usize = 0x80200000;
-    }
-}
-
 pub const KERNEL_VIRT_OFFSET: usize = kernel_arch_interface::KERNEL_VIRT_OFFSET;
 pub const SIGN_EXTEND_MASK: usize = 0xFFFFFF80_00000000;
 
-pub const KERNEL_START_L1I: usize = page_index::<L1>(KERNEL_VIRT_OFFSET + KERNEL_PHYS_BASE);
-pub const KERNEL_L2I: usize = page_index::<L2>(KERNEL_VIRT_OFFSET + KERNEL_PHYS_BASE);
-const_assert_eq!(KERNEL_L2I, 1);
+pub const IDENTITY_SIZE: usize = 64 * L1::SIZE;
+pub const IDENTITY_L1_START: usize = page_index::<L1>(KERNEL_VIRT_OFFSET);
 
-// Runtime mappings
-// 1GiB of device memory space
-const DEVICE_MAPPING_L1I: usize = KERNEL_START_L1I + 1;
-const DEVICE_MAPPING_L3_COUNT: usize = 4;
-// 32GiB of RAM space
-const RAM_MAPPING_START_L1I: usize = KERNEL_START_L1I + 2;
-const RAM_MAPPING_L1_COUNT: usize = 32;
-const_assert!(RAM_MAPPING_START_L1I + RAM_MAPPING_L1_COUNT <= 512);
-const_assert!(DEVICE_MAPPING_L1I < 512);
+pub static mut FIXED_L1: PageTable<L1> = const {
+    let mut table = PageTable::zeroed();
 
-const DEVICE_MAPPING_OFFSET: usize = (DEVICE_MAPPING_L1I << L1::SHIFT) | SIGN_EXTEND_MASK;
-const RAM_MAPPING_OFFSET: usize = (RAM_MAPPING_START_L1I << L1::SHIFT) | SIGN_EXTEND_MASK;
+    let mut i = 0;
+    while i < IDENTITY_SIZE / L1::SIZE {
+        let entry = PageEntry::identity_block(PhysicalAddress::from_usize(i * L1::SIZE));
+        table.entries[i] = entry;
+        table.entries[i + IDENTITY_L1_START] = entry;
+        i += 1;
+    }
 
-// Runtime tables
-static mut DEVICE_MAPPING_L2: PageTable<L2> = PageTable::zeroed();
-static mut DEVICE_MAPPING_L3S: [PageTable<L3>; DEVICE_MAPPING_L3_COUNT] =
-    [const { PageTable::zeroed() }; DEVICE_MAPPING_L3_COUNT];
+    table
+};
 
 /// Any VAs above this one are sign-extended
 pub const USER_BOUNDARY: usize = 0x40_00000000;
@@ -75,17 +43,17 @@ pub struct KernelTableManagerImpl;
 impl KernelTableManager for KernelTableManagerImpl {
     fn virtualize(address: u64) -> usize {
         let address = address as usize;
-        if address >= RAM_MAPPING_OFFSET {
+        if address >= IDENTITY_SIZE {
             panic!("Invalid physical address: {address:#x}");
         }
-        address + RAM_MAPPING_OFFSET
+        address + KERNEL_VIRT_OFFSET
     }
 
     fn physicalize(address: usize) -> u64 {
-        if address < RAM_MAPPING_OFFSET {
+        if address < KERNEL_VIRT_OFFSET || address - KERNEL_VIRT_OFFSET >= IDENTITY_SIZE {
             panic!("Invalid \"physicalized\" virtual address {address:#x}");
         }
-        (address - RAM_MAPPING_OFFSET) as u64
+        (address - KERNEL_VIRT_OFFSET) as u64
     }
 
     unsafe fn map_device_pages(
@@ -93,146 +61,31 @@ impl KernelTableManager for KernelTableManagerImpl {
         count: usize,
         attrs: DeviceMemoryAttributes,
     ) -> Result<RawDeviceMemoryMapping<Self>, Error> {
-        unsafe { map_device_memory(PhysicalAddress::from_u64(base), count, attrs) }
+        let _ = attrs;
+        let base = PhysicalAddress::from_u64(base);
+        let l3_aligned_base = base.page_align_down::<L3>();
+        let l3_aligned_end = base.add(count).page_align_up::<L3>();
+        let l3_offset = base - l3_aligned_base;
+        let l3_page_count = (l3_aligned_end - l3_aligned_base).page_count::<L3>();
+        let l3_aligned_virt = l3_aligned_base.add(KERNEL_VIRT_OFFSET).into_usize();
+
+        Ok(unsafe {
+            RawDeviceMemoryMapping::from_raw_parts(
+                l3_aligned_base.into_u64(),
+                l3_aligned_virt + l3_offset,
+                l3_aligned_virt,
+                l3_page_count,
+                L3::SIZE,
+            )
+        })
     }
 
     unsafe fn unmap_device_pages(mapping: &RawDeviceMemoryMapping<Self>) {
-        unsafe { unmap_device_memory(mapping) }
+        let _ = mapping;
     }
 }
 
-// Device mappings
-unsafe fn map_device_memory_l3(
-    base: PhysicalAddress,
-    count: usize,
-    _attrs: DeviceMemoryAttributes,
-) -> Result<usize, Error> {
-    // TODO don't map pages if already mapped
-
-    'l0: for i in 0..DEVICE_MAPPING_L3_COUNT * 512 {
-        for j in 0..count {
-            let l2i = (i + j) / 512;
-            let l3i = (i + j) % 512;
-
-            unsafe {
-                if DEVICE_MAPPING_L3S[l2i][l3i].is_present() {
-                    continue 'l0;
-                }
-            }
-        }
-
-        for j in 0..count {
-            let l2i = (i + j) / 512;
-            let l3i = (i + j) % 512;
-
-            unsafe {
-                DEVICE_MAPPING_L3S[l2i][l3i] =
-                    PageEntry::page(base.add(j * L3::SIZE), PageAttributes::W);
-            }
-        }
-
-        let start = DEVICE_MAPPING_OFFSET + i * L3::SIZE;
-        tlb_flush_range_va(start, count * L3::SIZE);
-        return Ok(start);
-    }
-
-    Err(Error::OutOfMemory)
-}
-
-#[allow(unused)]
-unsafe fn map_device_memory_l2(
-    base: PhysicalAddress,
-    count: usize,
-    _attrs: DeviceMemoryAttributes,
-) -> Result<usize, Error> {
-    'l0: for i in DEVICE_MAPPING_L3_COUNT..512 {
-        for j in 0..count {
-            unsafe {
-                if DEVICE_MAPPING_L2[i + j].is_present() {
-                    continue 'l0;
-                }
-            }
-        }
-
-        unsafe {
-            for j in 0..count {
-                DEVICE_MAPPING_L2[i + j] =
-                    PageEntry::<L2>::block(base.add(j * L2::SIZE), PageAttributes::W);
-            }
-        }
-
-        let start = DEVICE_MAPPING_OFFSET + i * L2::SIZE;
-        tlb_flush_range_va(start, count * L2::SIZE);
-        return Ok(start);
-    }
-
-    Err(Error::OutOfMemory)
-}
-
-pub(crate) unsafe fn map_device_memory(
-    base: PhysicalAddress,
-    size: usize,
-    attrs: DeviceMemoryAttributes,
-) -> Result<RawDeviceMemoryMapping<KernelTableManagerImpl>, Error> {
-    let l3_aligned = base.page_align_down::<L3>();
-    let l3_offset = base.page_offset::<L3>();
-    let page_count = (l3_offset + size).page_count::<L3>();
-
-    if page_count > 256 {
-        // Large mapping, use L2 mapping instead
-        let l2_aligned = base.page_align_down::<L2>();
-        let l2_offset = base.page_offset::<L2>();
-        let page_count = (l2_offset + size).page_count::<L2>();
-
-        unsafe {
-            let base_address = map_device_memory_l2(l2_aligned, page_count, attrs)?;
-            let address = base_address + l2_offset;
-
-            Ok(RawDeviceMemoryMapping::from_raw_parts(
-                l2_aligned.into_u64(),
-                address,
-                base_address,
-                page_count,
-                L2::SIZE,
-            ))
-        }
-    } else {
-        // Just map the pages directly
-        unsafe {
-            let base_address = map_device_memory_l3(l3_aligned, page_count, attrs)?;
-            let address = base_address + l3_offset;
-
-            Ok(RawDeviceMemoryMapping::from_raw_parts(
-                l3_aligned.into_u64(),
-                address,
-                base_address,
-                page_count,
-                L3::SIZE,
-            ))
-        }
-    }
-}
-
-pub(crate) unsafe fn unmap_device_memory(map: &RawDeviceMemoryMapping<KernelTableManagerImpl>) {
-    match map.page_size {
-        L3::SIZE => {
-            for i in 0..map.page_count {
-                let page = map.base_address + i * L3::SIZE;
-                let l2i = page.page_index::<L2>();
-                let l3i = page.page_index::<L3>();
-                unsafe {
-                    assert!(DEVICE_MAPPING_L3S[l2i][l3i].is_present());
-                    DEVICE_MAPPING_L3S[l2i][l3i] = PageEntry::INVALID;
-                }
-            }
-            tlb_flush_range_va(map.base_address, map.page_count * L3::SIZE);
-        }
-        L2::SIZE => todo!(),
-        _ => unimplemented!(),
-    }
-}
-
-pub fn auto_address<T>(x: *const T) -> usize {
+pub fn auto_lower_address<T>(x: *const T) -> usize {
     let x = x.addr();
     if x >= KERNEL_VIRT_OFFSET {
         x - KERNEL_VIRT_OFFSET
@@ -247,7 +100,7 @@ pub fn auto_address<T>(x: *const T) -> usize {
 ///
 /// Only meant to be called once per each HART during their early init.
 pub unsafe fn enable_mmu() {
-    let l1_phys = auto_address(&raw const KERNEL_TABLES) as u64;
+    let l1_phys = auto_lower_address(&raw const FIXED_L1) as u64;
     tlb_flush_full();
     SATP.write(SATP::PPN.val(l1_phys >> 12) + SATP::MODE::Sv39);
 }
@@ -258,49 +111,10 @@ pub unsafe fn enable_mmu() {
 ///
 /// Needs to be called once after secondary HARTs are initialized.
 pub unsafe fn unmap_lower_half() {
-    let mut tables = KERNEL_TABLES.lock();
-    let kernel_l1i_lower = page_index::<L1>(KERNEL_PHYS_BASE);
-    tables.l1.data[kernel_l1i_lower] = 0;
-    tlb_flush_range_va(0x0, L1::SIZE);
-}
-
-/// Sets up run-time kernel translation tables.
-///
-/// # Safety
-///
-/// The caller must ensure MMU is already enabled.
-pub unsafe fn setup_fixed_tables() {
-    let mut tables = KERNEL_TABLES.lock();
-
-    let device_mapping_l2_phys = auto_address(&raw const DEVICE_MAPPING_L2);
-
-    // Set up static runtime mappings
-    for i in 0..DEVICE_MAPPING_L3_COUNT {
-        unsafe {
-            let device_mapping_l3_phys = PhysicalAddress::from_usize(
-                (&raw const DEVICE_MAPPING_L3S[i]).addr() - KERNEL_VIRT_OFFSET,
-            );
-            DEVICE_MAPPING_L2[i] =
-                PageEntry::table(device_mapping_l3_phys, PageAttributes::empty());
-        }
-    }
-
-    assert_eq!(tables.l1.data[DEVICE_MAPPING_L1I], 0);
-    tables.l1.data[DEVICE_MAPPING_L1I] =
-        ((device_mapping_l2_phys as u64) >> 2) | PageAttributes::V.bits();
-
-    for l1i in 0..RAM_MAPPING_L1_COUNT {
-        let physical = (l1i as u64) << L1::SHIFT;
-        tables.l1.data[l1i + RAM_MAPPING_START_L1I] = (physical >> 2)
-            | (PageAttributes::R
-                | PageAttributes::W
-                | PageAttributes::A
-                | PageAttributes::D
-                | PageAttributes::V)
-                .bits();
-    }
-
-    tlb_flush_full();
+    // for i in 0..(IDENTITY_SIZE / L1::SIZE) {
+    //     unsafe { FIXED_L1[i] = PageEntry::INVALID };
+    // }
+    // tlb_flush_full();
 }
 
 pub fn tlb_flush_global_full() {
@@ -352,8 +166,9 @@ pub fn tlb_flush_va_asid(va: usize, asid: usize) {
 }
 
 pub fn clone_kernel_tables(dst: &mut PageTable<L1>) {
-    let tables = KERNEL_TABLES.lock();
+    // let tables = KERNEL_TABLES.lock();
     for l1i in page_index::<L1>(USER_BOUNDARY)..512 {
-        dst[l1i] = unsafe { PageEntry::from_raw(tables.l1.data[l1i]) };
+        // dst[l1i] = unsafe { PageEntry::from_raw(tables.l1.data[l1i]) };
+        dst[l1i] = unsafe { FIXED_L1[l1i] };
     }
 }

kernel/arch/riscv64/src/mem/process.rs

@@ -6,18 +6,17 @@ use core::{
 use libk_mm_interface::{
     address::{AsPhysicalAddress, PhysicalAddress},
     pointer::PhysicalRefMut,
-    process::ProcessAddressSpaceManager,
+    process::{PageAttributeUpdate, ProcessAddressSpaceManager},
     table::{
         EntryLevel, EntryLevelDrop, EntryLevelExt, MapAttributes, NextPageTable, TableAllocator,
     },
 };
-use memtables::riscv64::PageAttributes;
 use yggdrasil_abi::error::Error;
 
 use crate::mem::{clone_kernel_tables, table::PageEntry};
 
 use super::{
-    table::{DroppableRange, PageTable, L1, L2, L3},
+    table::{DroppableRange, PageAttributes, PageTable, L1, L2, L3},
     KernelTableManagerImpl, USER_BOUNDARY,
 };
 
@@ -70,7 +69,15 @@ impl<TA: TableAllocator> ProcessAddressSpaceManager<TA> for ProcessAddressSpaceI
         Ok(())
     }
 
-    unsafe fn unmap_page(&mut self, address: usize) -> Result<PhysicalAddress, Error> {
+    unsafe fn update_page_attributes(
+        &mut self,
+        address: usize,
+        update: &PageAttributeUpdate,
+    ) -> Result<(), Error> {
+        self.update_l3_entry(address, |entry| entry.update(update))
+    }
+
+    unsafe fn unmap_page(&mut self, address: usize) -> Result<(PhysicalAddress, bool), Error> {
         self.pop_l3_entry(address)
     }
 
@@ -118,7 +125,11 @@ impl<TA: TableAllocator> ProcessAddressSpaceImpl<TA> {
         Ok(())
     }
 
-    fn pop_l3_entry(&mut self, virt: usize) -> Result<PhysicalAddress, Error> {
+    fn update_l3_entry<F: FnOnce(&mut PageEntry<L3>) -> Result<(), Error>>(
+        &mut self,
+        virt: usize,
+        mapper: F,
+    ) -> Result<(), Error> {
         let l1i = virt.page_index::<L1>();
         let l2i = virt.page_index::<L2>();
         let l3i = virt.page_index::<L3>();
@@ -127,12 +138,33 @@ impl<TA: TableAllocator> ProcessAddressSpaceImpl<TA> {
         let mut l2 = self.l1.get_mut(l1i).ok_or(Error::DoesNotExist)?;
         let mut l3 = l2.get_mut(l2i).ok_or(Error::DoesNotExist)?;
 
-        let page = l3[l3i].as_page().ok_or(Error::DoesNotExist)?;
+        let entry = &mut l3[l3i];
+        if !entry.is_present() {
+            return Err(Error::DoesNotExist);
+        }
+        mapper(entry)?;
+        super::tlb_flush_va_asid(virt, self.asid as usize);
+
+        Ok(())
+    }
+
+    fn pop_l3_entry(&mut self, virt: usize) -> Result<(PhysicalAddress, bool), Error> {
+        let l1i = virt.page_index::<L1>();
+        let l2i = virt.page_index::<L2>();
+        let l3i = virt.page_index::<L3>();
+
+        // TODO somehow drop tables if they're known to be empty?
+        let mut l2 = self.l1.get_mut(l1i).ok_or(Error::DoesNotExist)?;
+        let mut l3 = l2.get_mut(l2i).ok_or(Error::DoesNotExist)?;
+
+        let entry = l3[l3i];
+        let page = entry.as_page().ok_or(Error::DoesNotExist)?;
+        let dirty = entry.is_dirty();
 
         l3[l3i] = PageEntry::INVALID;
         super::tlb_flush_va_asid(virt, self.asid as usize);
 
-        Ok(page)
+        Ok((page, dirty))
     }
 
     fn read_l3_entry(&self, virt: usize) -> Option<(PhysicalAddress, MapAttributes)> {
@@ -178,6 +210,9 @@ fn to_page_attributes(src: MapAttributes) -> PageAttributes {
     if src.intersects(MapAttributes::USER_READ | MapAttributes::USER_WRITE) {
         result |= PageAttributes::U;
     }
+    if src.contains(MapAttributes::DIRTY) {
+        result |= PageAttributes::SW_DIRTY;
+    }
     result
 }
 
@@ -192,5 +227,9 @@ fn to_map_attributes(src: PageAttributes) -> MapAttributes {
         }
     }
 
+    if src.contains(PageAttributes::SW_DIRTY) {
+        result |= MapAttributes::DIRTY;
+    }
+
     result
 }

kernel/arch/riscv64/src/mem/table.rs

@@ -3,9 +3,11 @@ use core::{
     ops::{Index, IndexMut, Range},
 };
 
+use bitflags::bitflags;
 use libk_mm_interface::{
     address::{AsPhysicalAddress, PhysicalAddress},
     pointer::{PhysicalRef, PhysicalRefMut},
+    process::PageAttributeUpdate,
     table::{
         page_index, EntryLevel, EntryLevelDrop, NextPageTable, NonTerminalEntryLevel,
         TableAllocator,
@@ -15,7 +17,41 @@ use yggdrasil_abi::error::Error;
 
 use super::{KernelTableManagerImpl, USER_BOUNDARY};
 
-pub use memtables::riscv64::PageAttributes;
+// pub use memtables::riscv64::PageAttributes;
+bitflags! {
+    #[derive(Clone, Copy, PartialEq, Eq, Debug)]
+    pub struct PageAttributes: u64 {
+        const N = 1 << 63;
+        /// Software-tracked dirty bit (RSW[0])
+        const SW_DIRTY = 1 << 9;
+        /// Dirty bit
+        const D = 1 << 7;
+        /// Access bit
+        const A = 1 << 6;
+        /// Global mapping bit, implies all lower levels are also global
+        const G = 1 << 5;
+        /// U-mode access permission
+        const U = 1 << 4;
+        /// Execute permission
+        const X = 1 << 3;
+        /// Write permission
+        const W = 1 << 2;
+        /// Read-permission
+        const R = 1 << 1;
+        /// Valid bit
+        const V = 1 << 0;
+    }
+
+    // X W R    Meaning
+    // 0 0 0    Pointer to next level of page table
+    // 0 0 1    Read-only page
+    // 0 1 0    ---
+    // 0 1 1    Read-write page
+    // 1 0 0    Execute only
+    // 1 0 1    Read-execute page
+    // 1 1 0    ---
+    // 1 1 1    Read-write-execute page
+}
 
 /// L3 - entry is 4KiB
 #[derive(Debug, Clone, Copy)]
@@ -39,9 +75,10 @@ impl EntryLevel for L1 {
     const SHIFT: usize = 30;
 }
 
+#[derive(Clone, Copy)]
 #[repr(C, align(0x1000))]
 pub struct PageTable<L: EntryLevel> {
-    entries: [PageEntry<L>; 512],
+    pub entries: [PageEntry<L>; 512],
 }
 
 #[derive(Clone, Copy, Debug, PartialEq)]
@@ -88,6 +125,8 @@ impl<L: EntryLevel> PageTable<L> {
 }
 
 impl<L: EntryLevel> PageEntry<L> {
+    // Upper + lower 10 bits
+    const ATTR_MASK: u64 = 0xFFC00000000003FF;
     pub const INVALID: Self = Self(0, PhantomData);
 
     /// Constructs a [PageEntry] from its raw representation.
@@ -103,6 +142,23 @@ impl<L: EntryLevel> PageEntry<L> {
         self.0 & PageAttributes::V.bits() != 0
     }
 
+    pub fn update(&mut self, update: &PageAttributeUpdate) -> Result<(), Error> {
+        let mut attrs = self.attributes();
+        if let Some(write) = update.user_write {
+            attrs.set(PageAttributes::W, write);
+        }
+        if let Some(dirty) = update.dirty {
+            attrs.set(PageAttributes::SW_DIRTY, dirty);
+        }
+        self.0 &= !Self::ATTR_MASK;
+        self.0 |= attrs.bits() & Self::ATTR_MASK;
+        Ok(())
+    }
+
+    pub const fn is_dirty(&self) -> bool {
+        self.0 & PageAttributes::SW_DIRTY.bits() != 0
+    }
+
     pub fn attributes(self) -> PageAttributes {
         PageAttributes::from_bits_retain(self.0)
     }
@@ -184,6 +240,17 @@ impl<L: NonTerminalEntryLevel + 'static> NextPageTable for PageTable<L> {
 }
 
 impl<L: NonTerminalEntryLevel> PageEntry<L> {
+    pub const fn identity_block(address: PhysicalAddress) -> Self {
+        const ATTR: u64 = PageAttributes::R.bits()
+            | PageAttributes::W.bits()
+            | PageAttributes::X.bits()
+            | PageAttributes::A.bits()
+            | PageAttributes::D.bits()
+            | PageAttributes::V.bits();
+
+        Self((address.into_u64() >> 2) | ATTR, PhantomData)
+    }
+
     pub fn block(address: PhysicalAddress, attrs: PageAttributes) -> Self {
         // TODO validate address alignment
         Self(
@@ -211,7 +278,7 @@ impl<L: NonTerminalEntryLevel> PageEntry<L> {
             & (PageAttributes::R | PageAttributes::W | PageAttributes::X | PageAttributes::V)
                 .bits()
             == PageAttributes::V.bits())
-        .then_some((self.0 << 2) & !0xFFF)
+        .then_some((self.0 & !Self::ATTR_MASK) << 2)
         .map(PhysicalAddress::from_u64)
     }
 }
@@ -232,7 +299,7 @@ impl PageEntry<L3> {
 
     pub fn as_page(&self) -> Option<PhysicalAddress> {
         (self.0 & PageAttributes::V.bits() != 0)
-            .then_some((self.0 << 2) & !0xFFF)
+            .then_some((self.0 & !Self::ATTR_MASK) << 2)
             .map(PhysicalAddress::from_u64)
     }
 }

kernel/arch/x86_64/Cargo.toml

@@ -7,7 +7,6 @@ edition = "2021"
 yggdrasil-abi.workspace = true
 kernel-arch-interface.workspace = true
 libk-mm-interface.workspace = true
-memtables.workspace = true
 device-api = { workspace = true, features = ["derive"] }
 kernel-arch-x86.workspace = true
 

kernel/arch/x86_64/src/context.rs

@@ -5,11 +5,14 @@ use kernel_arch_interface::{
     task::{ForkFrame, StackBuilder, TaskContext, TaskFrame, UserContextInfo},
 };
 use kernel_arch_x86::registers::{FpuContext, CR3, MSR_IA32_FS_BASE};
-use libk_mm_interface::address::{AsPhysicalAddress, PhysicalAddress};
+use libk_mm_interface::address::PhysicalAddress;
 use tock_registers::interfaces::Writeable;
 use yggdrasil_abi::{arch::SavedFrame, error::Error};
 
-use crate::{mem::KERNEL_TABLES, ArchitectureImpl};
+use crate::{
+    mem::{auto_lower_address, FIXED_PML4},
+    ArchitectureImpl,
+};
 
 /// Frame saved onto the stack when taking an IRQ
 #[derive(Debug)]
@@ -431,7 +434,7 @@ impl<K: KernelTableManager, PA: PhysicalMemoryAllocator<Address = PhysicalAddres
     fn kernel(entry: extern "C" fn(usize) -> !, arg: usize) -> Result<Self, Error> {
         const KERNEL_TASK_PAGES: usize = 32;
 
-        let cr3: usize = unsafe { KERNEL_TABLES.lock().as_physical_address() }.into();
+        let cr3 = auto_lower_address(&raw const FIXED_PML4);
 
         let stack_base_phys = PA::allocate_contiguous_pages(KERNEL_TASK_PAGES)?;
         let stack_base = stack_base_phys.raw_virtualize::<K>();

kernel/arch/x86_64/src/lib.rs

@@ -1,6 +1,5 @@
 #![no_std]
 #![allow(clippy::new_without_default)]
-#![feature(naked_functions, trait_upcasting)]
 
 extern crate alloc;
 
@@ -69,18 +68,16 @@ impl PerCpuData {
 static IPI_QUEUES: OneTimeInit<Vec<IpiQueue<ArchitectureImpl>>> = OneTimeInit::new();
 pub static CPU_COUNT: AtomicUsize = AtomicUsize::new(1);
 
-#[naked]
+#[unsafe(naked)]
 extern "C" fn idle_task(_: usize) -> ! {
-    unsafe {
-        core::arch::naked_asm!(
-            r#"
+    core::arch::naked_asm!(
+        r#"
             1:
                 nop
                 jmp 1b
             "#,
-            options(att_syntax)
-        );
-    }
+        options(att_syntax)
+    );
 }
 
 impl ArchitectureImpl {

kernel/arch/x86_64/src/mem/device.rs

@@ -0,0 +1,87 @@
+use core::ops::Range;
+
+use kernel_arch_interface::mem::DeviceMemoryAttributes;
+use libk_mm_interface::{
+    address::PhysicalAddress,
+    table::{DevicePageManager, DevicePageManagerLevel},
+};
+
+use crate::mem::table::PageEntry;
+
+use super::{
+    table::{PageAttributes, PageTable, L2, L3},
+    DEVICE_MAPPING_OFFSET, DEVICE_MEMORY_L3_COUNT,
+};
+
+#[repr(transparent)]
+pub struct L2DeviceMemory(pub PageTable<L2>);
+#[repr(transparent)]
+pub struct L3DeviceMemory(pub [PageTable<L3>; DEVICE_MEMORY_L3_COUNT]);
+
+pub(super) static mut DEVICE_MEMORY: DevicePageManager<L3DeviceMemory, L2DeviceMemory> =
+    DevicePageManager::new(
+        L3DeviceMemory([PageTable::zeroed(); DEVICE_MEMORY_L3_COUNT]),
+        L2DeviceMemory(PageTable::zeroed()),
+    );
+
+impl DevicePageManagerLevel for L2DeviceMemory {
+    type Level = L2;
+    const VIRTUAL_BASE: usize = DEVICE_MAPPING_OFFSET;
+    const INDEX_RANGE: Range<usize> = DEVICE_MEMORY_L3_COUNT..512;
+
+    fn map_page(
+        &mut self,
+        index: usize,
+        physical: PhysicalAddress,
+        attrs: &DeviceMemoryAttributes,
+    ) {
+        let _ = attrs;
+        self.0[index] = PageEntry::<L2>::block(physical, PageAttributes::WRITABLE);
+    }
+
+    fn unmap_page(&mut self, index: usize) {
+        self.0[index] = PageEntry::INVALID;
+    }
+
+    fn is_mapped(&self, index: usize) -> bool {
+        self.0[index].is_present()
+    }
+
+    fn flush_range(range: Range<usize>) {
+        let _ = range;
+    }
+}
+
+impl DevicePageManagerLevel for L3DeviceMemory {
+    type Level = L3;
+    const VIRTUAL_BASE: usize = DEVICE_MAPPING_OFFSET;
+    const INDEX_RANGE: Range<usize> = 0..512 * DEVICE_MEMORY_L3_COUNT;
+
+    fn map_page(
+        &mut self,
+        index: usize,
+        physical: PhysicalAddress,
+        attrs: &DeviceMemoryAttributes,
+    ) {
+        let _ = attrs;
+        let l2i = index / 512;
+        let l3i = index % 512;
+        self.0[l2i][l3i] = PageEntry::page(physical, PageAttributes::WRITABLE);
+    }
+
+    fn unmap_page(&mut self, index: usize) {
+        let l2i = index / 512;
+        let l3i = index % 512;
+        self.0[l2i][l3i] = PageEntry::INVALID;
+    }
+
+    fn is_mapped(&self, index: usize) -> bool {
+        let l2i = index / 512;
+        let l3i = index % 512;
+        self.0[l2i][l3i].is_present()
+    }
+
+    fn flush_range(range: Range<usize>) {
+        let _ = range;
+    }
+}

kernel/arch/x86_64/src/mem/mod.rs

@@ -1,98 +1,39 @@
-use core::{
-    alloc::Layout,
-    ops::{Deref, DerefMut},
-    sync::atomic::{AtomicUsize, Ordering},
-};
-
 use kernel_arch_interface::{
     mem::{DeviceMemoryAttributes, KernelTableManager, RawDeviceMemoryMapping},
-    split_spinlock,
+    sync::IrqSafeSpinlock,
 };
 use kernel_arch_x86::registers::CR3;
 use libk_mm_interface::{
     address::PhysicalAddress,
-    table::{page_index, EntryLevel, EntryLevelExt},
+    table::{EntryLevel, EntryLevelExt},
 };
-use static_assertions::{const_assert_eq, const_assert_ne};
 use yggdrasil_abi::error::Error;
 
-use crate::KERNEL_VIRT_OFFSET;
+use crate::{ArchitectureImpl, KERNEL_VIRT_OFFSET};
 
-use self::table::{PageAttributes, PageEntry, PageTable, L0, L1, L2, L3};
+use self::table::{PageAttributes, PageEntry, PageTable, L0, L1, L2};
 
+pub mod device;
 pub mod process;
 pub mod table;
 
 #[derive(Debug)]
 pub struct KernelTableManagerImpl;
 
-const CANONICAL_ADDRESS_MASK: usize = 0xFFFF000000000000;
-const KERNEL_PHYS_BASE: usize = 0x200000;
-
-// Mapped at compile time
-const KERNEL_MAPPING_BASE: usize = KERNEL_VIRT_OFFSET + KERNEL_PHYS_BASE;
-const KERNEL_L0_INDEX: usize = page_index::<L0>(KERNEL_MAPPING_BASE);
-const KERNEL_L1_INDEX: usize = page_index::<L1>(KERNEL_MAPPING_BASE);
-const KERNEL_START_L2_INDEX: usize = page_index::<L2>(KERNEL_MAPPING_BASE);
-
-// Must not be zero, should be at 4MiB
-const_assert_ne!(KERNEL_START_L2_INDEX, 0);
-// From static mapping
-const_assert_eq!(KERNEL_L0_INDEX, 511);
-const_assert_eq!(KERNEL_L1_INDEX, 0);
-
-// Mapped at boot
-const EARLY_MAPPING_L2I: usize = KERNEL_START_L2_INDEX - 1;
-const DEVICE_MAPPING_L1I: usize = KERNEL_L1_INDEX + 2;
-const RAM_MAPPING_L0I: usize = KERNEL_L0_INDEX - 1;
-
-const DEVICE_MAPPING_L3_COUNT: usize = 4;
-
-split_spinlock! {
-    use libk_mm_interface::KernelImageObject;
-    use memtables::x86_64::FixedTables;
-    use crate::ArchitectureImpl;
-
-    #[link_section = ".data.tables"]
-    static KERNEL_TABLES: KernelImageObject<FixedTables> =
-        unsafe { KernelImageObject::new(FixedTables::zeroed()) };
-}
-
-// 2MiB for early mappings
-const EARLY_MAPPING_OFFSET: usize = CANONICAL_ADDRESS_MASK
-    | (KERNEL_L0_INDEX * L0::SIZE)
-    | (KERNEL_L1_INDEX * L1::SIZE)
-    | (EARLY_MAPPING_L2I * L2::SIZE);
-static mut EARLY_MAPPING_L3: PageTable<L3> = PageTable::zeroed();
-// 1GiB for device MMIO mapping
-const DEVICE_MAPPING_OFFSET: usize =
-    CANONICAL_ADDRESS_MASK | (KERNEL_L0_INDEX * L0::SIZE) | (DEVICE_MAPPING_L1I * L1::SIZE);
-static mut DEVICE_MAPPING_L2: PageTable<L2> = PageTable::zeroed();
-static mut DEVICE_MAPPING_L3S: [PageTable<L3>; DEVICE_MAPPING_L3_COUNT] =
-    [PageTable::zeroed(); DEVICE_MAPPING_L3_COUNT];
-// 512GiB for whole RAM mapping
-pub const RAM_MAPPING_OFFSET: usize = CANONICAL_ADDRESS_MASK | (RAM_MAPPING_L0I * L0::SIZE);
-pub static MEMORY_LIMIT: AtomicUsize = AtomicUsize::new(0);
-pub static mut RAM_MAPPING_L1: PageTable<L1> = PageTable::zeroed();
-
 impl KernelTableManager for KernelTableManagerImpl {
     fn virtualize(address: u64) -> usize {
         let address = address as usize;
-        if address < MEMORY_LIMIT.load(Ordering::Acquire) {
-            address + RAM_MAPPING_OFFSET
-        } else {
-            panic!("Invalid physical address: {:#x}", address);
+        if address >= IDENTITY_SIZE {
+            panic!("Invalid physical address to virtualize: {address:#x}");
         }
+        address + KERNEL_VIRT_OFFSET
     }
 
     fn physicalize(address: usize) -> u64 {
-        if address < RAM_MAPPING_OFFSET
-            || address - RAM_MAPPING_OFFSET >= MEMORY_LIMIT.load(Ordering::Acquire)
-        {
-            panic!("Not a virtualized physical address: {:#x}", address);
+        if address < KERNEL_VIRT_OFFSET || address - KERNEL_VIRT_OFFSET >= IDENTITY_SIZE {
+            panic!("Invalid virtualized address: {address:#x}");
         }
-
-        (address - RAM_MAPPING_OFFSET) as _
+        (address - KERNEL_VIRT_OFFSET) as _
     }
 
     unsafe fn map_device_pages(
@@ -100,245 +41,42 @@ impl KernelTableManager for KernelTableManagerImpl {
         count: usize,
         attrs: DeviceMemoryAttributes,
     ) -> Result<RawDeviceMemoryMapping<Self>, Error> {
-        map_device_memory(PhysicalAddress::from_u64(base), count, attrs)
+        let _guard = DEVICE_MEMORY_LOCK.lock();
+        #[allow(static_mut_refs)]
+        {
+            device::DEVICE_MEMORY.map_device_pages(PhysicalAddress::from_u64(base), count, attrs)
+        }
     }
 
     unsafe fn unmap_device_pages(mapping: &RawDeviceMemoryMapping<Self>) {
-        unmap_device_memory(mapping)
-    }
-}
-
-// Early mappings
-unsafe fn map_early_pages(physical: PhysicalAddress, count: usize) -> Result<usize, Error> {
-    for l3i in 0..512 {
-        let mut taken = false;
-        for i in 0..count {
-            if EARLY_MAPPING_L3[i + l3i].is_present() {
-                taken = true;
-                break;
-            }
-        }
-
-        if taken {
-            continue;
-        }
-
-        for i in 0..count {
-            // TODO NX, NC
-            EARLY_MAPPING_L3[i + l3i] =
-                PageEntry::page(physical.add(i * L3::SIZE), PageAttributes::WRITABLE);
-            flush_tlb_entry(EARLY_MAPPING_OFFSET + (i + l3i) * L3::SIZE);
-        }
-
-        return Ok(EARLY_MAPPING_OFFSET + l3i * L3::SIZE);
-    }
-
-    Err(Error::OutOfMemory)
-}
-
-unsafe fn unmap_early_page(address: usize) {
-    if !(EARLY_MAPPING_OFFSET..EARLY_MAPPING_OFFSET + L2::SIZE).contains(&address) {
-        panic!("Tried to unmap invalid early mapping: {:#x}", address);
-    }
-
-    let l3i = (address - EARLY_MAPPING_OFFSET).page_index::<L3>();
-
-    assert!(EARLY_MAPPING_L3[l3i].is_present());
-    EARLY_MAPPING_L3[l3i] = PageEntry::INVALID;
-}
-
-// Device mappings
-unsafe fn map_device_memory_l3(
-    base: PhysicalAddress,
-    count: usize,
-    _attrs: DeviceMemoryAttributes,
-) -> Result<usize, Error> {
-    // TODO don't map pages if already mapped
-
-    'l0: for i in 0..DEVICE_MAPPING_L3_COUNT * 512 {
-        for j in 0..count {
-            let l2i = (i + j) / 512;
-            let l3i = (i + j) % 512;
-
-            if DEVICE_MAPPING_L3S[l2i][l3i].is_present() {
-                continue 'l0;
-            }
-        }
-
-        for j in 0..count {
-            let l2i = (i + j) / 512;
-            let l3i = (i + j) % 512;
-
-            // TODO NX, NC
-            DEVICE_MAPPING_L3S[l2i][l3i] =
-                PageEntry::page(base.add(j * L3::SIZE), PageAttributes::WRITABLE);
-        }
-
-        return Ok(DEVICE_MAPPING_OFFSET + i * L3::SIZE);
-    }
-
-    Err(Error::OutOfMemory)
-}
-
-unsafe fn map_device_memory_l2(
-    base: PhysicalAddress,
-    count: usize,
-    _attrs: DeviceMemoryAttributes,
-) -> Result<usize, Error> {
-    'l0: for i in DEVICE_MAPPING_L3_COUNT..512 {
-        for j in 0..count {
-            if DEVICE_MAPPING_L2[i + j].is_present() {
-                continue 'l0;
-            }
-        }
-
-        for j in 0..count {
-            DEVICE_MAPPING_L2[i + j] =
-                PageEntry::<L2>::block(base.add(j * L2::SIZE), PageAttributes::WRITABLE);
-        }
-
-        return Ok(DEVICE_MAPPING_OFFSET + i * L2::SIZE);
-    }
-
-    Err(Error::OutOfMemory)
-}
-
-unsafe fn map_device_memory(
-    base: PhysicalAddress,
-    size: usize,
-    attrs: DeviceMemoryAttributes,
-) -> Result<RawDeviceMemoryMapping<KernelTableManagerImpl>, Error> {
-    let l3_aligned = base.page_align_down::<L3>();
-    let l3_offset = base.page_offset::<L3>();
-    let page_count = (l3_offset + size).page_count::<L3>();
-
-    if page_count > 256 {
-        // Large mapping, use L2 mapping instead
-        let l2_aligned = base.page_align_down::<L2>();
-        let l2_offset = base.page_offset::<L2>();
-        let page_count = (l2_offset + size).page_count::<L2>();
-
-        let base_address = map_device_memory_l2(l2_aligned, page_count, attrs)?;
-        let address = base_address + l2_offset;
-
-        Ok(RawDeviceMemoryMapping::from_raw_parts(
-            l2_aligned.into_u64(),
-            address,
-            base_address,
-            page_count,
-            L2::SIZE,
-        ))
-    } else {
-        // Just map the pages directly
-        let base_address = map_device_memory_l3(l3_aligned, page_count, attrs)?;
-        let address = base_address + l3_offset;
-
-        Ok(RawDeviceMemoryMapping::from_raw_parts(
-            l3_aligned.into_u64(),
-            address,
-            base_address,
-            page_count,
-            L3::SIZE,
-        ))
-    }
-}
-
-unsafe fn unmap_device_memory(map: &RawDeviceMemoryMapping<KernelTableManagerImpl>) {
-    match map.page_size {
-        L3::SIZE => {
-            for i in 0..map.page_count {
-                let page = map.base_address + i * L3::SIZE;
-                let l2i = page.page_index::<L2>();
-                let l3i = page.page_index::<L3>();
-                assert!(DEVICE_MAPPING_L3S[l2i][l3i].is_present());
-                DEVICE_MAPPING_L3S[l2i][l3i] = PageEntry::INVALID;
-                flush_tlb_entry(page);
-            }
-        }
-        L2::SIZE => todo!(),
-        _ => unimplemented!(),
-    }
-}
-
-/// Memory mapping which may be used for performing early kernel initialization
-pub struct EarlyMapping<'a, T: ?Sized> {
-    value: &'a mut T,
-    page_count: usize,
-}
-
-impl<'a, T: Sized> EarlyMapping<'a, T> {
-    /// # Safety
-    ///
-    /// `physical` address provided must be a valid non-NULL address actually containing `T`.
-    pub unsafe fn map(physical: PhysicalAddress) -> Result<EarlyMapping<'a, T>, Error> {
-        let layout = Layout::new::<T>();
-        let aligned = physical.page_align_down::<L3>();
-        let offset = physical.page_offset::<L3>();
-        let page_count = (offset + layout.size()).div_ceil(L3::SIZE);
-
-        let virt = map_early_pages(aligned, page_count)?;
-        let value = &mut *((virt + offset) as *mut T);
-
-        Ok(EarlyMapping { value, page_count })
-    }
-
-    /// # Safety
-    ///
-    /// `physical` address provided must be a valid non-NULL address actually containing a `T`
-    /// slice of given `len`.
-    pub unsafe fn map_slice(
-        physical: PhysicalAddress,
-        len: usize,
-    ) -> Result<EarlyMapping<'a, [T]>, Error> {
-        let layout = Layout::array::<T>(len).unwrap();
-        let aligned = physical.page_align_down::<L3>();
-        let offset = physical.page_offset::<L3>();
-        let page_count = (offset + layout.size()).div_ceil(L3::SIZE);
-
-        let virt = map_early_pages(aligned, page_count)?;
-        let value = core::slice::from_raw_parts_mut((virt + offset) as *mut T, len);
-
-        Ok(EarlyMapping { value, page_count })
-    }
-}
-
-impl<T: ?Sized> Deref for EarlyMapping<'_, T> {
-    type Target = T;
-
-    fn deref(&self) -> &Self::Target {
-        self.value
-    }
-}
-
-impl<T: ?Sized> DerefMut for EarlyMapping<'_, T> {
-    fn deref_mut(&mut self) -> &mut Self::Target {
-        self.value
-    }
-}
-
-impl<T: ?Sized> Drop for EarlyMapping<'_, T> {
-    fn drop(&mut self) {
-        let address = (self.value as *mut T).addr() & !(L3::SIZE - 1);
-
-        for i in 0..self.page_count {
-            let page = address + i * L3::SIZE;
-
-            unsafe {
-                unmap_early_page(page);
-            }
+        let _guard = DEVICE_MEMORY_LOCK.lock();
+        #[allow(static_mut_refs)]
+        {
+            device::DEVICE_MEMORY.unmap_device_pages(mapping);
         }
     }
 }
 
 pub fn clone_kernel_tables(dst: &mut PageTable<L0>) {
-    let tables = KERNEL_TABLES.lock();
     unsafe {
-        dst[KERNEL_L0_INDEX] = PageEntry::from_raw(tables.l0.data[KERNEL_L0_INDEX]);
-        dst[RAM_MAPPING_L0I] = PageEntry::from_raw(tables.l0.data[RAM_MAPPING_L0I]);
+        let kernel_l0i = KERNEL_VIRT_OFFSET.page_index::<L0>();
+        dst[kernel_l0i] = FIXED_PML4[kernel_l0i];
     }
 }
 
-pub fn auto_address<T>(pointer: *const T) -> usize {
+const FIXED_PD_COUNT: usize = 32;
+const IDENTITY_SIZE: usize = FIXED_PD_COUNT * L1::SIZE;
+
+const DEVICE_MAPPING_L1I: usize = FIXED_PD_COUNT;
+const DEVICE_MEMORY_L3_COUNT: usize = 16;
+const DEVICE_MAPPING_OFFSET: usize = KERNEL_VIRT_OFFSET + IDENTITY_SIZE;
+
+static DEVICE_MEMORY_LOCK: IrqSafeSpinlock<ArchitectureImpl, ()> = IrqSafeSpinlock::new(());
+static mut FIXED_PDS: [PageTable<L2>; FIXED_PD_COUNT] = [PageTable::zeroed(); FIXED_PD_COUNT];
+static mut FIXED_PDPT: PageTable<L1> = PageTable::zeroed();
+pub static mut FIXED_PML4: PageTable<L0> = PageTable::zeroed();
+
+pub fn auto_lower_address<T>(pointer: *const T) -> usize {
     let address = pointer.addr();
     if address < KERNEL_VIRT_OFFSET {
         address
@@ -347,53 +85,80 @@ pub fn auto_address<T>(pointer: *const T) -> usize {
     }
 }
 
-/// Sets up the following memory map:
-/// ...: KERNEL_TABLES.l0:
-///     * 0xFFFFFF0000000000 .. 0xFFFFFFFF8000000000        : RAM_MAPPING_L1
-///     * 0xFFFFFF8000000000 .. ...                         : KERNEL_TABLES.kernel_l1:
-///         * 0xFFFFFF8000000000 .. 0xFFFFFF8040000000      : KERNEL_TABLES.kernel_l2
-///             * 0xFFFFFF8000000000 .. 0xFFFFFF8000200000  : ---
-///             * 0xFFFFFF8000200000 .. 0xFFFFFF8000400000  : EARLY_MAPPING_L3
-///             * 0xFFFFFF8000400000 .. ...                 : KERNEL_TABLES.kernel_l3s
-///         * 0xFFFFFF8040000000 .. 0xFFFFFF8080000000      : ---
-///         * 0xFFFFFF8080000000 .. 0xFFFFFF8100000000      : DEVICE_MAPPING_L2
-///             * 0xFFFFFF8080000000 .. 0xFFFFFF8080800000  : DEVICE_MAPPING_L3S
-///             * 0xFFFFFF8080800000 .. 0xFFFFFF8100000000  : ...
-///
-/// # Safety
-///
-/// Unsafe, must only be called by BSP during its early init, must already be in "higher-half"
-pub unsafe fn init_fixed_tables() {
-    let mut tables = KERNEL_TABLES.lock();
-
-    // TODO this could be built in compile-time too?
-    let early_mapping_l3_phys = auto_address(&raw const EARLY_MAPPING_L3);
-    let device_mapping_l2_phys = auto_address(&raw const DEVICE_MAPPING_L2);
-    let ram_mapping_l1_phys = auto_address(&raw const RAM_MAPPING_L1);
-
-    for i in 0..DEVICE_MAPPING_L3_COUNT {
-        let device_mapping_l3_phys =
-            PhysicalAddress::from_usize(auto_address(&raw const DEVICE_MAPPING_L3S[i]));
-        DEVICE_MAPPING_L2[i] = PageEntry::table(device_mapping_l3_phys, PageAttributes::WRITABLE);
+pub unsafe fn init_fixed_tables(have_1gib_pages: bool) {
+    if have_1gib_pages {
+        for i in 0..IDENTITY_SIZE / L1::SIZE {
+            FIXED_PDPT[i] = PageEntry::<L1>::block(
+                PhysicalAddress::from_usize(i * L1::SIZE),
+                PageAttributes::WRITABLE,
+            );
+        }
+    } else {
+        for i in 0..IDENTITY_SIZE / L1::SIZE {
+            for j in 0..512 {
+                FIXED_PDS[i][j] = PageEntry::<L2>::block(
+                    PhysicalAddress::from_usize(i * L1::SIZE + j * L2::SIZE),
+                    PageAttributes::WRITABLE,
+                );
+            }
+            let pd_physical =
+                PhysicalAddress::from_usize(auto_lower_address(&raw const FIXED_PDS[i]));
+            FIXED_PDPT[i] = PageEntry::table(pd_physical, PageAttributes::WRITABLE);
+        }
     }
 
-    assert_eq!(tables.kernel_l2.data[EARLY_MAPPING_L2I], 0);
-    tables.kernel_l2.data[EARLY_MAPPING_L2I] = (early_mapping_l3_phys as u64)
-        | (PageAttributes::WRITABLE | PageAttributes::PRESENT).bits();
+    // Device memory
+    let device_pd_physical =
+        PhysicalAddress::from_usize(auto_lower_address(&raw const device::DEVICE_MEMORY.large));
 
-    assert_eq!(tables.kernel_l1.data[DEVICE_MAPPING_L1I], 0);
-    tables.kernel_l1.data[DEVICE_MAPPING_L1I] = (device_mapping_l2_phys as u64)
-        | (PageAttributes::WRITABLE | PageAttributes::PRESENT).bits();
+    for i in 0..DEVICE_MEMORY_L3_COUNT {
+        let device_pt_physical = PhysicalAddress::from_usize(auto_lower_address(
+            &raw const device::DEVICE_MEMORY.normal.0[i],
+        ));
+        device::DEVICE_MEMORY.large.0[i] =
+            PageEntry::table(device_pt_physical, PageAttributes::WRITABLE);
+    }
 
-    assert_eq!(tables.l0.data[RAM_MAPPING_L0I], 0);
-    tables.l0.data[RAM_MAPPING_L0I] =
-        (ram_mapping_l1_phys as u64) | (PageAttributes::WRITABLE | PageAttributes::PRESENT).bits();
+    FIXED_PDPT[DEVICE_MAPPING_L1I] =
+        PageEntry::<L1>::table(device_pd_physical, PageAttributes::WRITABLE);
 
-    // TODO ENABLE EFER.NXE
-    let cr3 = auto_address(&raw const tables.l0);
-    CR3.set_address(cr3);
+    let pdpt_physical = PhysicalAddress::from_usize(auto_lower_address(&raw const FIXED_PDPT));
+
+    FIXED_PML4[KERNEL_VIRT_OFFSET.page_index::<L0>()] =
+        PageEntry::table(pdpt_physical, PageAttributes::WRITABLE);
+
+    let pml4_physical = auto_lower_address(&raw const FIXED_PML4);
+    CR3.set_address(pml4_physical);
 }
-
+//     let mut tables = KERNEL_TABLES.lock();
+//
+//     // TODO this could be built in compile-time too?
+//     let early_mapping_l3_phys = auto_address(&raw const EARLY_MAPPING_L3);
+//     let device_mapping_l2_phys = auto_address(&raw const DEVICE_MAPPING_L2);
+//     let ram_mapping_l1_phys = auto_address(&raw const RAM_MAPPING_L1);
+//
+//     for i in 0..DEVICE_MAPPING_L3_COUNT {
+//         let device_mapping_l3_phys =
+//             PhysicalAddress::from_usize(auto_address(&raw const DEVICE_MAPPING_L3S[i]));
+//         DEVICE_MAPPING_L2[i] = PageEntry::table(device_mapping_l3_phys, PageAttributes::WRITABLE);
+//     }
+//
+//     assert_eq!(tables.kernel_l2.data[EARLY_MAPPING_L2I], 0);
+//     tables.kernel_l2.data[EARLY_MAPPING_L2I] = (early_mapping_l3_phys as u64)
+//         | (PageAttributes::WRITABLE | PageAttributes::PRESENT).bits();
+//
+//     assert_eq!(tables.kernel_l1.data[DEVICE_MAPPING_L1I], 0);
+//     tables.kernel_l1.data[DEVICE_MAPPING_L1I] = (device_mapping_l2_phys as u64)
+//         | (PageAttributes::WRITABLE | PageAttributes::PRESENT).bits();
+//
+//     assert_eq!(tables.l0.data[RAM_MAPPING_L0I], 0);
+//     tables.l0.data[RAM_MAPPING_L0I] =
+//         (ram_mapping_l1_phys as u64) | (PageAttributes::WRITABLE | PageAttributes::PRESENT).bits();
+//
+//     // TODO ENABLE EFER.NXE
+//     let cr3 = auto_address(&raw const tables.l0);
+// }
+//
 /// # Safety
 ///
 /// `address` must be page-aligned.

kernel/arch/x86_64/src/mem/process.rs

@@ -4,7 +4,7 @@ use core::marker::PhantomData;
 use libk_mm_interface::{
     address::{AsPhysicalAddress, PhysicalAddress},
     pointer::PhysicalRefMut,
-    process::ProcessAddressSpaceManager,
+    process::{PageAttributeUpdate, ProcessAddressSpaceManager},
     table::{
         EntryLevel, EntryLevelDrop, EntryLevelExt, MapAttributes, NextPageTable, TableAllocator,
     },
@@ -61,7 +61,15 @@ impl<TA: TableAllocator> ProcessAddressSpaceManager<TA> for ProcessAddressSpaceI
         self.write_l3_entry(address, PageEntry::page(physical, flags.into()), false)
     }
 
-    unsafe fn unmap_page(&mut self, address: usize) -> Result<PhysicalAddress, Error> {
+    unsafe fn update_page_attributes(
+        &mut self,
+        address: usize,
+        update: &PageAttributeUpdate,
+    ) -> Result<(), Error> {
+        self.update_l3_entry(address, |entry| entry.update(update))
+    }
+
+    unsafe fn unmap_page(&mut self, address: usize) -> Result<(PhysicalAddress, bool), Error> {
         self.pop_l3_entry(address)
     }
 
@@ -111,7 +119,33 @@ impl<TA: TableAllocator> ProcessAddressSpaceImpl<TA> {
         Ok(())
     }
 
-    fn pop_l3_entry(&mut self, virt: usize) -> Result<PhysicalAddress, Error> {
+    fn update_l3_entry<F: FnOnce(&mut PageEntry<L3>) -> Result<(), Error>>(
+        &mut self,
+        virt: usize,
+        mapper: F,
+    ) -> Result<(), Error> {
+        let l0i = virt.page_index::<L0>();
+        let l1i = virt.page_index::<L1>();
+        let l2i = virt.page_index::<L2>();
+        let l3i = virt.page_index::<L3>();
+
+        let mut l1 = self.l0.get_mut(l0i).ok_or(Error::DoesNotExist)?;
+        let mut l2 = l1.get_mut(l1i).ok_or(Error::DoesNotExist)?;
+        let mut l3 = l2.get_mut(l2i).ok_or(Error::DoesNotExist)?;
+
+        let entry = &mut l3[l3i];
+        if !entry.is_present() {
+            return Err(Error::DoesNotExist);
+        }
+        mapper(entry)?;
+        unsafe {
+            flush_tlb_entry(virt);
+        }
+
+        Ok(())
+    }
+
+    fn pop_l3_entry(&mut self, virt: usize) -> Result<(PhysicalAddress, bool), Error> {
         let l0i = virt.page_index::<L0>();
         let l1i = virt.page_index::<L1>();
         let l2i = virt.page_index::<L2>();
@@ -123,13 +157,14 @@ impl<TA: TableAllocator> ProcessAddressSpaceImpl<TA> {
         let mut l3 = l2.get_mut(l2i).ok_or(Error::DoesNotExist)?;
 
         let page = l3[l3i].as_page().ok_or(Error::DoesNotExist)?;
+        let dirty = l3[l3i].is_dirty();
 
         l3[l3i] = PageEntry::INVALID;
         unsafe {
             flush_tlb_entry(virt);
         }
 
-        Ok(page)
+        Ok((page, dirty))
     }
 
     fn read_l3_entry(&self, virt: usize) -> Option<(PhysicalAddress, MapAttributes)> {

kernel/arch/x86_64/src/mem/table.rs

@@ -8,6 +8,7 @@ use bitflags::bitflags;
 use libk_mm_interface::{
     address::{AsPhysicalAddress, PhysicalAddress},
     pointer::{PhysicalRef, PhysicalRefMut},
+    process::PageAttributeUpdate,
     table::{
         EntryLevel, EntryLevelDrop, MapAttributes, NextPageTable, NonTerminalEntryLevel,
         TableAllocator,
@@ -31,6 +32,8 @@ bitflags! {
         /// For tables, allows user access to further translation levels, for pages/blocks, allows
         /// user access to the region covered by the entry
         const USER = 1 << 2;
+        /// If set, the page has been written to
+        const DIRTY = 1 << 6;
     }
 }
 
@@ -98,11 +101,15 @@ impl PageEntry<L3> {
     /// not
     pub fn as_page(self) -> Option<PhysicalAddress> {
         if self.0 & PageAttributes::PRESENT.bits() != 0 {
-            Some(PhysicalAddress::from_u64(self.0 & !0xFFF))
+            Some(PhysicalAddress::from_u64(self.0 & !Self::ATTR_MASK))
         } else {
             None
         }
     }
+
+    pub fn is_dirty(&self) -> bool {
+        self.0 & PageAttributes::DIRTY.bits() != 0
+    }
 }
 
 impl PageEntry<L2> {
@@ -145,7 +152,7 @@ impl<L: NonTerminalEntryLevel> PageEntry<L> {
         if self.0 & PageAttributes::PRESENT.bits() != 0
             && self.0 & PageAttributes::BLOCK.bits() == 0
         {
-            Some(PhysicalAddress::from_u64(self.0 & !0xFFF))
+            Some(PhysicalAddress::from_u64(self.0 & !Self::ATTR_MASK))
         } else {
             None
         }
@@ -158,6 +165,8 @@ impl<L: NonTerminalEntryLevel> PageEntry<L> {
 }
 
 impl<L: EntryLevel> PageEntry<L> {
+    const ATTR_MASK: u64 = 0xFFF | (1 << 63);
+
     /// An entry that is not mapped
     pub const INVALID: Self = Self(0, PhantomData);
 
@@ -179,6 +188,21 @@ impl<L: EntryLevel> PageEntry<L> {
     pub fn is_present(&self) -> bool {
         self.0 & PageAttributes::PRESENT.bits() != 0
     }
+
+    pub fn update(&mut self, update: &PageAttributeUpdate) -> Result<(), Error> {
+        let mut attrs = PageAttributes::from_bits_retain(self.0);
+        if let Some(write) = update.user_write {
+            if write {
+                attrs |= PageAttributes::WRITABLE;
+            } else {
+                attrs &= !PageAttributes::WRITABLE;
+            }
+        }
+        // Dirty is ignored, it's hardware-managed
+        self.0 &= !Self::ATTR_MASK;
+        self.0 |= attrs.bits() & Self::ATTR_MASK;
+        Ok(())
+    }
 }
 
 impl<L: EntryLevel> PageTable<L> {

kernel/driver/block/ahci/src/port.rs

@@ -16,7 +16,8 @@ use libk::{
     error::Error,
 };
 use libk_mm::{
-    address::PhysicalAddress, device::DeviceMemoryIo, table::MapAttributes, PageProvider,
+    address::PhysicalAddress, device::DeviceMemoryIo, table::MapAttributes, OnDemandPage,
+    PageProvider, VirtualPage,
 };
 use libk_util::{sync::IrqSafeSpinlock, waker::QueueWaker, OneTimeInit};
 use tock_registers::interfaces::{Readable, Writeable};
@@ -234,7 +235,7 @@ impl AhciPort {
         .await
     }
 
-    async fn submit<C: AtaCommand>(&self, command: &C) -> Result<SubmittedCommand, AhciError> {
+    async fn submit<C: AtaCommand>(&self, command: &C) -> Result<SubmittedCommand<'_>, AhciError> {
         if command.prd_count() > 2 {
             log::warn!("TODO: AHCI doesn't like 3+ PRD transfers");
             return Err(AhciError::RegionTooLarge);
@@ -365,8 +366,21 @@ impl Device for AhciPort {
 }
 
 impl PageProvider for AhciPort {
-    fn get_page(&self, _offset: u64) -> Result<PhysicalAddress, Error> {
-        todo!()
+    fn ondemand_fetch(&self, _opaque: u64) -> Result<OnDemandPage, Error> {
+        unimplemented!()
+    }
+
+    fn get_page(&self, _offset: u64) -> Result<VirtualPage, Error> {
+        unimplemented!()
+    }
+
+    fn release_page(
+        &self,
+        _offset: u64,
+        _phys: PhysicalAddress,
+        _dirty: bool,
+    ) -> Result<(), Error> {
+        unimplemented!()
     }
 
     fn clone_page(
@@ -375,10 +389,6 @@ impl PageProvider for AhciPort {
         _src_phys: PhysicalAddress,
         _src_attrs: MapAttributes,
     ) -> Result<PhysicalAddress, Error> {
-        todo!()
-    }
-
-    fn release_page(&self, _offset: u64, _phys: PhysicalAddress) -> Result<(), Error> {
-        todo!()
+        unimplemented!()
     }
 }

kernel/driver/block/nvme/src/drive.rs

@@ -11,7 +11,7 @@ use libk::{
 use libk_mm::{
     address::{AsPhysicalAddress, PhysicalAddress},
     table::MapAttributes,
-    PageProvider, PageSlice,
+    OnDemandPage, PageProvider, PageSlice, VirtualPage,
 };
 
 use crate::{command::IdentifyNamespaceRequest, register_nvme_namespace, IoDirection};
@@ -151,8 +151,21 @@ impl BlockDevice for NvmeNamespace {
 }
 
 impl PageProvider for NvmeNamespace {
-    fn get_page(&self, _offset: u64) -> Result<PhysicalAddress, Error> {
-        todo!()
+    fn ondemand_fetch(&self, _opaque: u64) -> Result<OnDemandPage, Error> {
+        unimplemented!()
+    }
+
+    fn get_page(&self, _offset: u64) -> Result<VirtualPage, Error> {
+        unimplemented!()
+    }
+
+    fn release_page(
+        &self,
+        _offset: u64,
+        _phys: PhysicalAddress,
+        _dirty: bool,
+    ) -> Result<(), Error> {
+        unimplemented!()
     }
 
     fn clone_page(
@@ -161,10 +174,6 @@ impl PageProvider for NvmeNamespace {
         _src_phys: PhysicalAddress,
         _src_attrs: MapAttributes,
     ) -> Result<PhysicalAddress, Error> {
-        todo!()
-    }
-
-    fn release_page(&self, _offset: u64, _phys: PhysicalAddress) -> Result<(), Error> {
-        todo!()
+        unimplemented!()
     }
 }

kernel/driver/block/scsi/src/lib.rs

@@ -22,7 +22,9 @@ use libk::{
     fs::devfs,
     task::{runtime, sync::AsyncMutex},
 };
-use libk_mm::{address::PhysicalAddress, table::MapAttributes, PageProvider};
+use libk_mm::{
+    address::PhysicalAddress, table::MapAttributes, OnDemandPage, PageProvider, VirtualPage,
+};
 use libk_util::{
     sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock},
     OneTimeInit,
@@ -278,12 +280,21 @@ impl BlockDevice for ScsiUnit {
 }
 
 impl PageProvider for ScsiUnit {
-    fn get_page(&self, _offset: u64) -> Result<PhysicalAddress, Error> {
-        Err(Error::NotImplemented)
+    fn ondemand_fetch(&self, _opaque: u64) -> Result<OnDemandPage, Error> {
+        unimplemented!()
     }
 
-    fn release_page(&self, _offset: u64, _phys: PhysicalAddress) -> Result<(), Error> {
-        Err(Error::NotImplemented)
+    fn get_page(&self, _offset: u64) -> Result<VirtualPage, Error> {
+        unimplemented!()
+    }
+
+    fn release_page(
+        &self,
+        _offset: u64,
+        _phys: PhysicalAddress,
+        _dirty: bool,
+    ) -> Result<(), Error> {
+        unimplemented!()
     }
 
     fn clone_page(
@@ -292,7 +303,7 @@ impl PageProvider for ScsiUnit {
         _src_phys: PhysicalAddress,
         _src_attrs: MapAttributes,
     ) -> Result<PhysicalAddress, Error> {
-        Err(Error::NotImplemented)
+        unimplemented!()
     }
 }
 

kernel/driver/block/scsi/src/transport.rs

@@ -84,8 +84,7 @@ impl ScsiTransportWrapper {
                 response_buffer.slice_mut(0..R::RESPONSE_LEN),
             )
             .await?;
-        let response_bytes =
-            unsafe { MaybeUninit::slice_assume_init_ref(&response_buffer[..response_len]) };
+        let response_bytes = unsafe { response_buffer[..response_len].assume_init_ref() };
 
         R::parse_response(response_bytes)
     }

kernel/driver/bus/pci/src/space/mod.rs

@@ -349,7 +349,7 @@ pub trait PciConfigurationSpace {
     }
 
     /// Returns an iterator over the PCI capabilities
-    fn capability_iter(&self) -> CapabilityIterator<Self> {
+    fn capability_iter(&self) -> CapabilityIterator<'_, Self> {
         let status = PciStatusRegister::from_bits_retain(self.status());
 
         let current = if status.contains(PciStatusRegister::CAPABILITIES_LIST) {

kernel/driver/bus/usb/src/address.rs

@@ -0,0 +1,58 @@
+use core::fmt;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
+pub struct UsbRoute {
+    bus: u16,
+    ports: [u8; 8],
+    len: u8,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
+pub struct UsbBusAddress {
+    pub bus: u16,
+    pub device: u8,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
+pub struct UsbInterfaceAddress {
+    pub device: UsbBusAddress,
+    pub interface: u8,
+}
+
+impl UsbBusAddress {
+    pub fn with_interface(self, interface: u8) -> UsbInterfaceAddress {
+        UsbInterfaceAddress {
+            device: self,
+            interface,
+        }
+    }
+}
+
+impl fmt::Display for UsbBusAddress {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "<Bus {} Device {}>", self.bus, self.device)
+    }
+}
+
+impl fmt::Display for UsbInterfaceAddress {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(
+            f,
+            "<Bus {} Device {} Interface {}>",
+            self.device.bus, self.device.device, self.interface
+        )
+    }
+}
+
+impl fmt::Display for UsbRoute {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}-", self.bus)?;
+        for (i, &port) in self.ports[..self.len as usize].iter().enumerate() {
+            if i != 0 {
+                write!(f, ".")?;
+            }
+            write!(f, "{port}")?;
+        }
+        Ok(())
+    }
+}

kernel/driver/bus/usb/src/bus.rs

@@ -1,34 +1,56 @@
 use core::sync::atomic::{AtomicU16, Ordering};
 
-use alloc::{collections::BTreeMap, sync::Arc};
-use libk_util::{queue::UnboundedMpmcQueue, sync::spin_rwlock::IrqSafeRwLock};
+use alloc::{collections::BTreeMap, format, sync::Arc};
+use libk_util::{queue::UnboundedMpmcQueue, sync::spin_rwlock::IrqSafeRwLock, OneTimeInit};
 
 use crate::{
+    address::UsbBusAddress,
     class_driver,
-    device::{UsbBusAddress, UsbDeviceAccess},
+    device::{UsbDevice, UsbDeviceAccess},
+    sysfs::{self, UsbBusKObject},
     UsbHostController,
 };
 
+pub struct UsbBusWrapper {
+    pub(crate) hc: Arc<dyn UsbHostController>,
+    pub(crate) index: u16,
+    kobject: OneTimeInit<UsbBusKObject>,
+}
+
 pub struct UsbBusManager {
-    busses: IrqSafeRwLock<BTreeMap<u16, Arc<dyn UsbHostController>>>,
+    busses: IrqSafeRwLock<BTreeMap<u16, Arc<UsbBusWrapper>>>,
     devices: IrqSafeRwLock<BTreeMap<UsbBusAddress, Arc<UsbDeviceAccess>>>,
 
     last_bus_address: AtomicU16,
 }
 
+impl UsbBusWrapper {
+    pub fn kobject(&self) -> &UsbBusKObject {
+        self.kobject.get()
+    }
+}
+
 impl UsbBusManager {
-    pub fn register_bus(hc: Arc<dyn UsbHostController>) -> u16 {
+    pub fn register_bus(hc: Arc<dyn UsbHostController>) -> (u16, Arc<UsbBusWrapper>) {
         let i = BUS_MANAGER.last_bus_address.fetch_add(1, Ordering::AcqRel);
-        BUS_MANAGER.busses.write().insert(i, hc);
-        i
+        let wrapper = Arc::new(UsbBusWrapper {
+            hc,
+            index: i,
+            kobject: OneTimeInit::new(),
+        });
+        BUS_MANAGER.busses.write().insert(i, wrapper.clone());
+        wrapper.kobject.init(sysfs::register_bus_kobject(&wrapper));
+        (i, wrapper)
     }
 
     pub fn register_device(device: Arc<UsbDeviceAccess>) {
         log::info!("usb: register device {}", device.bus_address());
+
         BUS_MANAGER
             .devices
             .write()
             .insert(device.bus_address(), device.clone());
+        device.kobject.init(sysfs::register_device_kobject(&device));
 
         QUEUE.push_back(device);
     }
@@ -51,7 +73,11 @@ pub async fn bus_handler() {
             new_device.bus_address()
         );
 
-        class_driver::spawn_driver(new_device).await.ok();
+        let address = new_device.bus_address();
+        if let Err(error) = class_driver::setup_device(new_device).await {
+            log::warn!("USB device {address} setup error: {error:?}",);
+        }
+        // class_driver::spawn_driver(new_device).await.ok();
     }
 }
 

kernel/driver/bus/usb/src/class_driver/hid_keyboard.rs

@@ -4,9 +4,12 @@ use alloc::{boxed::Box, sync::Arc};
 use async_trait::async_trait;
 use yggdrasil_abi::io::{KeyboardKey, KeyboardKeyEvent};
 
-use crate::{device::UsbDeviceAccess, error::UsbError, info::UsbDeviceClass};
-
-use super::{UsbClassInfo, UsbDriver};
+use crate::{
+    class_driver::{UsbInterfaceClass, UsbInterfaceDriver},
+    device::UsbDeviceAccess,
+    error::UsbError,
+    info::UsbInterfaceInfo,
+};
 
 pub struct UsbHidKeyboardDriver;
 
@@ -125,10 +128,14 @@ impl KeyboardState {
 }
 
 #[async_trait]
-impl UsbDriver for UsbHidKeyboardDriver {
-    async fn run(self: Arc<Self>, device: Arc<UsbDeviceAccess>) -> Result<(), UsbError> {
+impl UsbInterfaceDriver for UsbHidKeyboardDriver {
+    async fn run(
+        self: Arc<Self>,
+        device: Arc<UsbDeviceAccess>,
+        interface: UsbInterfaceInfo,
+    ) -> Result<(), UsbError> {
         // TODO not sure whether to use boot protocol (easy) or GetReport
-        let config = device.select_configuration(|_| true).await?.unwrap();
+        let config = device.current_configuration().unwrap();
 
         log::info!("Setup HID keyboard");
         let pipe = device
@@ -152,11 +159,11 @@ impl UsbDriver for UsbHidKeyboardDriver {
             event_count += state.retain(&buffer[2..], &mut events[event_count..]);
             event_count += state.press(&buffer[2..], &mut events[event_count..]);
 
-            let events = unsafe { MaybeUninit::slice_assume_init_ref(&events[..event_count]) };
+            let events = unsafe { events[..event_count].assume_init_ref() };
 
             for &event in events {
                 log::trace!("Generic Keyboard: {:?}", event);
-                ygg_driver_input::send_event(event);
+                ygg_driver_input::send_keyboard_event(event);
             }
         }
     }
@@ -165,12 +172,18 @@ impl UsbDriver for UsbHidKeyboardDriver {
         "USB HID Keyboard"
     }
 
-    fn probe(&self, class: &UsbClassInfo, _device: &UsbDeviceAccess) -> bool {
-        log::info!(
-            "class = {:?}, subclass = {:02x}",
-            class.class,
-            class.subclass
-        );
-        class.class == UsbDeviceClass::Hid && (class.subclass == 0x00 || class.subclass == 0x01)
+    fn probe(
+        &self,
+        device: &UsbDeviceAccess,
+        interface: &UsbInterfaceInfo,
+        class: UsbInterfaceClass,
+    ) -> bool {
+        let _ = (device, interface);
+        class
+            == UsbInterfaceClass {
+                class: 3,
+                subclass: 1,
+                protocol: 1,
+            }
     }
 }

kernel/driver/bus/usb/src/class_driver/hid_mouse.rs

@@ -0,0 +1,64 @@
+use alloc::{boxed::Box, sync::Arc};
+use async_trait::async_trait;
+use yggdrasil_abi::io::{ButtonMask, MouseEvent};
+
+use crate::{
+    class_driver::{UsbInterfaceClass, UsbInterfaceDriver},
+    device::UsbDeviceAccess,
+    error::UsbError,
+    info::UsbInterfaceInfo,
+};
+
+pub struct UsbHidMouseDriver;
+
+#[async_trait]
+impl UsbInterfaceDriver for UsbHidMouseDriver {
+    async fn run(
+        self: Arc<Self>,
+        device: Arc<UsbDeviceAccess>,
+        interface: UsbInterfaceInfo,
+    ) -> Result<(), UsbError> {
+        let config = device.current_configuration().unwrap();
+
+        log::info!("Setup HID mouse");
+        let pipe = device
+            .open_interrupt_in_pipe(1, config.endpoints[0].max_packet_size as u16)
+            .await?;
+
+        let mut buffer = [0; 16];
+        let mut button_state = 0;
+
+        loop {
+            let len = pipe.read(&mut buffer).await?;
+            if len < 4 {
+                continue;
+            }
+
+            let event = MouseEvent {
+                buttons: ButtonMask(buffer[0]),
+                dx: (buffer[1] as i8) as i32,
+                dy: (buffer[2] as i8) as i32,
+            };
+
+            ygg_driver_input::send_mouse_event(event);
+        }
+    }
+
+    fn name(&self) -> &'static str {
+        "USB HID Mouse"
+    }
+
+    fn probe(
+        &self,
+        device: &UsbDeviceAccess,
+        interface: &UsbInterfaceInfo,
+        class: UsbInterfaceClass,
+    ) -> bool {
+        class
+            == UsbInterfaceClass {
+                class: 3,
+                subclass: 1,
+                protocol: 2,
+            }
+    }
+}

kernel/driver/bus/usb/src/class_driver/mass_storage.rs

@@ -1,273 +1,273 @@
-use core::mem::MaybeUninit;
-
-use alloc::{boxed::Box, sync::Arc};
-use async_trait::async_trait;
-use bytemuck::{Pod, Zeroable};
-use libk::{
-    dma::{DmaBuffer, DmaSliceMut},
-    error::Error,
-};
-use ygg_driver_scsi::{transport::ScsiTransport, ScsiEnclosure};
-
-use crate::{
-    communication::UsbDirection,
-    device::{UsbDeviceAccess, UsbDeviceDetachHandler},
-    error::UsbError,
-    info::{UsbDeviceClass, UsbEndpointType},
-    pipe::{
-        control::{ControlTransferSetup, UsbClassSpecificRequest},
-        normal::{UsbBulkInPipeAccess, UsbBulkOutPipeAccess},
-    },
-};
-
-use super::{UsbClassInfo, UsbDriver};
-
-pub struct UsbMassStorageDriverBulkOnly;
-
-#[derive(Debug, Clone, Copy, Zeroable, Pod)]
-#[repr(C)]
-struct Cbw {
-    signature: u32,       // 0x00
-    tag: u32,             // 0x04
-    transfer_length: u32, // 0x08
-    flags: u8,            // 0x0C
-    lun: u8,              // 0x0D
-    cb_length: u8,        // 0x0E
-    cb_data: [u8; 16],    // 0x0F
-    // Not sent
-    _0: u8,
-}
-
-#[derive(Debug, Clone, Copy, Zeroable, Pod)]
-#[repr(C)]
-struct Csw {
-    signature: u32,
-    tag: u32,
-    data_residue: u32,
-    status: u8,
-    _0: [u8; 3],
-}
-
-struct Bbb {
-    #[allow(unused)]
-    device: Arc<UsbDeviceAccess>,
-    in_pipe: UsbBulkInPipeAccess,
-    out_pipe: UsbBulkOutPipeAccess,
-    last_tag: u32,
-}
-
-struct DetachHandler(Arc<ScsiEnclosure>);
-
-impl Bbb {
-    pub fn new(
-        device: Arc<UsbDeviceAccess>,
-        in_pipe: UsbBulkInPipeAccess,
-        out_pipe: UsbBulkOutPipeAccess,
-    ) -> Result<Self, UsbError> {
-        Ok(Self {
-            device,
-            in_pipe,
-            out_pipe,
-            last_tag: 0,
-        })
-    }
-}
-
-impl Bbb {
-    async fn send_cbw(
-        &mut self,
-        lun: u8,
-        host_to_dev: bool,
-        command: &[u8],
-        response_len: usize,
-    ) -> Result<u32, Error> {
-        self.last_tag = self.last_tag.wrapping_add(1);
-
-        let flags = if !host_to_dev { 1 << 7 } else { 0 };
-        let tag = self.last_tag;
-        let mut cbw_bytes = [0; 32];
-        let cbw = bytemuck::from_bytes_mut::<Cbw>(&mut cbw_bytes);
-
-        cbw.signature = 0x43425355;
-        cbw.transfer_length = response_len as u32;
-        cbw.flags = flags;
-        cbw.tag = tag;
-        cbw.lun = lun;
-        cbw.cb_length = command.len() as u8;
-        cbw.cb_data[..command.len()].copy_from_slice(command);
-
-        self.out_pipe
-            .write(&cbw_bytes[..31])
-            .await
-            .inspect_err(|error| log::error!("msc: CBW send error: {error:?}"))?;
-
-        Ok(tag)
-    }
-
-    async fn read_csw(&mut self, tag: u32) -> Result<(), Error> {
-        let mut csw_bytes = [0; 16];
-        self.in_pipe
-            .read_exact(&mut csw_bytes[..13])
-            .await
-            .inspect_err(|error| log::error!("msc: CSW receive error: {error:?}"))?;
-        let csw = bytemuck::from_bytes::<Csw>(&csw_bytes);
-
-        if csw.signature != 0x53425355 {
-            log::warn!("msc: invalid csw signature");
-            return Err(Error::InvalidArgument);
-        }
-        if csw.tag != tag {
-            let csw_tag = csw.tag;
-            log::warn!("msc: invalid csw tag (got {}, expected {tag})", csw_tag);
-            return Err(Error::InvalidArgument);
-        }
-        if csw.status != 0x00 {
-            return Err(Error::InvalidArgument);
-        }
-        Ok(())
-    }
-
-    async fn read_response_data(
-        &mut self,
-        buffer: DmaSliceMut<'_, MaybeUninit<u8>>,
-    ) -> Result<usize, Error> {
-        if buffer.len() == 0 {
-            return Ok(0);
-        }
-        let len = self
-            .in_pipe
-            .read_dma(buffer)
-            .await
-            .inspect_err(|error| log::error!("msc: DMA read error: {error:?}"))?;
-        Ok(len)
-    }
-}
-
-#[async_trait]
-impl ScsiTransport for Bbb {
-    fn allocate_buffer(&self, size: usize) -> Result<DmaBuffer<[MaybeUninit<u8>]>, Error> {
-        Ok(self.in_pipe.allocate_dma_buffer(size)?)
-    }
-
-    async fn perform_request_raw(
-        &mut self,
-        lun: u8,
-        request_data: &[u8],
-        response_buffer: DmaSliceMut<'_, MaybeUninit<u8>>,
-    ) -> Result<usize, Error> {
-        if request_data.len() > 16 || response_buffer.len() > self.max_bytes_per_request() {
-            return Err(Error::InvalidArgument);
-        }
-
-        let tag = self
-            .send_cbw(lun, false, request_data, response_buffer.len())
-            .await?;
-        let response_len = self.read_response_data(response_buffer).await?;
-        self.read_csw(tag).await?;
-        Ok(response_len)
-    }
-
-    fn max_bytes_per_request(&self) -> usize {
-        32768
-    }
-}
-
-impl UsbDeviceDetachHandler for DetachHandler {
-    fn handle_device_detach(&self) {
-        log::info!("Mass storage detached");
-        self.0.detach();
-    }
-}
-
-#[derive(Debug, Pod, Zeroable, Clone, Copy)]
-#[repr(C)]
-pub struct BulkOnlyMassStorageReset;
-
-#[derive(Debug, Pod, Zeroable, Clone, Copy)]
-#[repr(C)]
-pub struct GetMaxLun;
-
-impl UsbClassSpecificRequest for BulkOnlyMassStorageReset {
-    const BM_REQUEST_TYPE: u8 = 0b00100001;
-    const B_REQUEST: u8 = 0b11111111;
-}
-
-impl UsbClassSpecificRequest for GetMaxLun {
-    const BM_REQUEST_TYPE: u8 = 0b10100001;
-    const B_REQUEST: u8 = 0b11111110;
-}
-
-#[async_trait]
-impl UsbDriver for UsbMassStorageDriverBulkOnly {
-    async fn run(self: Arc<Self>, device: Arc<UsbDeviceAccess>) -> Result<(), UsbError> {
-        // TODO filter to only accept BBB config
-        let config = device.select_configuration(|_| true).await?.unwrap();
-        // Bulk-in, bulk-out
-        assert_eq!(config.endpoints.len(), 2);
-        let control_pipe = device.control_pipe();
-        let (in_index, in_info) = config
-            .find_endpoint(|ep| ep.is(UsbEndpointType::Bulk, UsbDirection::In))
-            .ok_or(UsbError::InvalidConfiguration)?;
-        let (out_index, out_info) = config
-            .find_endpoint(|ep| ep.is(UsbEndpointType::Bulk, UsbDirection::Out))
-            .ok_or(UsbError::InvalidConfiguration)?;
-        let in_pipe = device
-            .open_bulk_in_pipe(in_index, in_info.max_packet_size as u16)
-            .await?;
-        let out_pipe = device
-            .open_bulk_out_pipe(out_index, out_info.max_packet_size as u16)
-            .await?;
-
-        // Perform a Bulk-Only Mass Storage Reset
-        // TODO interface id?
-        control_pipe
-            .control_transfer(ControlTransferSetup {
-                bm_request_type: BulkOnlyMassStorageReset::BM_REQUEST_TYPE,
-                b_request: BulkOnlyMassStorageReset::B_REQUEST,
-                w_value: 0,
-                w_index: 0,
-                w_length: 0,
-            })
-            .await?;
-
-        // Get max LUN
-        // TODO on devices which do not support multiple LUNs, this command may STALL
-        let mut buffer = [MaybeUninit::uninit()];
-        let len = control_pipe
-            .control_transfer_in(
-                ControlTransferSetup {
-                    bm_request_type: GetMaxLun::BM_REQUEST_TYPE,
-                    b_request: GetMaxLun::B_REQUEST,
-                    w_value: 0,
-                    w_index: 0,
-                    w_length: 1,
-                },
-                &mut buffer,
-            )
-            .await?;
-        let max_lun = if len < 1 {
-            0
-        } else {
-            unsafe { buffer[0].assume_init() }
-        };
-
-        let bbb = Bbb::new(device.clone(), in_pipe, out_pipe)?;
-        let scsi = ScsiEnclosure::setup(Box::new(bbb), max_lun as usize + 1)
-            .await
-            .inspect_err(|error| log::error!("msc: scsi error {error:?}"))
-            .map_err(|_| UsbError::DriverError)?;
-        let detach = DetachHandler(scsi.clone());
-        device.set_detach_handler(Arc::new(detach));
-
-        Ok(())
-    }
-
-    fn name(&self) -> &'static str {
-        "USB Mass Storage"
-    }
-
-    fn probe(&self, class: &UsbClassInfo, _device: &UsbDeviceAccess) -> bool {
-        // TODO support other protocols
-        class.class == UsbDeviceClass::MassStorage && class.interface_protocol_number == 0x50
-    }
-}
+// use core::mem::MaybeUninit;
+//
+// use alloc::{boxed::Box, sync::Arc};
+// use async_trait::async_trait;
+// use bytemuck::{Pod, Zeroable};
+// use libk::{
+//     dma::{DmaBuffer, DmaSliceMut},
+//     error::Error,
+// };
+// use ygg_driver_scsi::{transport::ScsiTransport, ScsiEnclosure};
+//
+// use crate::{
+//     communication::UsbDirection,
+//     device::{UsbDeviceAccess, UsbDeviceDetachHandler},
+//     error::UsbError,
+//     info::{UsbDeviceClass, UsbEndpointType},
+//     pipe::{
+//         control::{ControlTransferSetup, UsbClassSpecificRequest},
+//         normal::{UsbBulkInPipeAccess, UsbBulkOutPipeAccess},
+//     },
+// };
+//
+// use super::{UsbClassInfo, UsbDriver};
+//
+// pub struct UsbMassStorageDriverBulkOnly;
+//
+// #[derive(Debug, Clone, Copy, Zeroable, Pod)]
+// #[repr(C)]
+// struct Cbw {
+//     signature: u32,       // 0x00
+//     tag: u32,             // 0x04
+//     transfer_length: u32, // 0x08
+//     flags: u8,            // 0x0C
+//     lun: u8,              // 0x0D
+//     cb_length: u8,        // 0x0E
+//     cb_data: [u8; 16],    // 0x0F
+//     // Not sent
+//     _0: u8,
+// }
+//
+// #[derive(Debug, Clone, Copy, Zeroable, Pod)]
+// #[repr(C)]
+// struct Csw {
+//     signature: u32,
+//     tag: u32,
+//     data_residue: u32,
+//     status: u8,
+//     _0: [u8; 3],
+// }
+//
+// struct Bbb {
+//     #[allow(unused)]
+//     device: Arc<UsbDeviceAccess>,
+//     in_pipe: UsbBulkInPipeAccess,
+//     out_pipe: UsbBulkOutPipeAccess,
+//     last_tag: u32,
+// }
+//
+// struct DetachHandler(Arc<ScsiEnclosure>);
+//
+// impl Bbb {
+//     pub fn new(
+//         device: Arc<UsbDeviceAccess>,
+//         in_pipe: UsbBulkInPipeAccess,
+//         out_pipe: UsbBulkOutPipeAccess,
+//     ) -> Result<Self, UsbError> {
+//         Ok(Self {
+//             device,
+//             in_pipe,
+//             out_pipe,
+//             last_tag: 0,
+//         })
+//     }
+// }
+//
+// impl Bbb {
+//     async fn send_cbw(
+//         &mut self,
+//         lun: u8,
+//         host_to_dev: bool,
+//         command: &[u8],
+//         response_len: usize,
+//     ) -> Result<u32, Error> {
+//         self.last_tag = self.last_tag.wrapping_add(1);
+//
+//         let flags = if !host_to_dev { 1 << 7 } else { 0 };
+//         let tag = self.last_tag;
+//         let mut cbw_bytes = [0; 32];
+//         let cbw = bytemuck::from_bytes_mut::<Cbw>(&mut cbw_bytes);
+//
+//         cbw.signature = 0x43425355;
+//         cbw.transfer_length = response_len as u32;
+//         cbw.flags = flags;
+//         cbw.tag = tag;
+//         cbw.lun = lun;
+//         cbw.cb_length = command.len() as u8;
+//         cbw.cb_data[..command.len()].copy_from_slice(command);
+//
+//         self.out_pipe
+//             .write(&cbw_bytes[..31])
+//             .await
+//             .inspect_err(|error| log::error!("msc: CBW send error: {error:?}"))?;
+//
+//         Ok(tag)
+//     }
+//
+//     async fn read_csw(&mut self, tag: u32) -> Result<(), Error> {
+//         let mut csw_bytes = [0; 16];
+//         self.in_pipe
+//             .read_exact(&mut csw_bytes[..13])
+//             .await
+//             .inspect_err(|error| log::error!("msc: CSW receive error: {error:?}"))?;
+//         let csw = bytemuck::from_bytes::<Csw>(&csw_bytes);
+//
+//         if csw.signature != 0x53425355 {
+//             log::warn!("msc: invalid csw signature");
+//             return Err(Error::InvalidArgument);
+//         }
+//         if csw.tag != tag {
+//             let csw_tag = csw.tag;
+//             log::warn!("msc: invalid csw tag (got {}, expected {tag})", csw_tag);
+//             return Err(Error::InvalidArgument);
+//         }
+//         if csw.status != 0x00 {
+//             return Err(Error::InvalidArgument);
+//         }
+//         Ok(())
+//     }
+//
+//     async fn read_response_data(
+//         &mut self,
+//         buffer: DmaSliceMut<'_, MaybeUninit<u8>>,
+//     ) -> Result<usize, Error> {
+//         if buffer.len() == 0 {
+//             return Ok(0);
+//         }
+//         let len = self
+//             .in_pipe
+//             .read_dma(buffer)
+//             .await
+//             .inspect_err(|error| log::error!("msc: DMA read error: {error:?}"))?;
+//         Ok(len)
+//     }
+// }
+//
+// #[async_trait]
+// impl ScsiTransport for Bbb {
+//     fn allocate_buffer(&self, size: usize) -> Result<DmaBuffer<[MaybeUninit<u8>]>, Error> {
+//         Ok(self.in_pipe.allocate_dma_buffer(size)?)
+//     }
+//
+//     async fn perform_request_raw(
+//         &mut self,
+//         lun: u8,
+//         request_data: &[u8],
+//         response_buffer: DmaSliceMut<'_, MaybeUninit<u8>>,
+//     ) -> Result<usize, Error> {
+//         if request_data.len() > 16 || response_buffer.len() > self.max_bytes_per_request() {
+//             return Err(Error::InvalidArgument);
+//         }
+//
+//         let tag = self
+//             .send_cbw(lun, false, request_data, response_buffer.len())
+//             .await?;
+//         let response_len = self.read_response_data(response_buffer).await?;
+//         self.read_csw(tag).await?;
+//         Ok(response_len)
+//     }
+//
+//     fn max_bytes_per_request(&self) -> usize {
+//         32768
+//     }
+// }
+//
+// impl UsbDeviceDetachHandler for DetachHandler {
+//     fn handle_device_detach(&self) {
+//         log::info!("Mass storage detached");
+//         self.0.detach();
+//     }
+// }
+//
+// #[derive(Debug, Pod, Zeroable, Clone, Copy)]
+// #[repr(C)]
+// pub struct BulkOnlyMassStorageReset;
+//
+// #[derive(Debug, Pod, Zeroable, Clone, Copy)]
+// #[repr(C)]
+// pub struct GetMaxLun;
+//
+// impl UsbClassSpecificRequest for BulkOnlyMassStorageReset {
+//     const BM_REQUEST_TYPE: u8 = 0b00100001;
+//     const B_REQUEST: u8 = 0b11111111;
+// }
+//
+// impl UsbClassSpecificRequest for GetMaxLun {
+//     const BM_REQUEST_TYPE: u8 = 0b10100001;
+//     const B_REQUEST: u8 = 0b11111110;
+// }
+//
+// #[async_trait]
+// impl UsbDriver for UsbMassStorageDriverBulkOnly {
+//     async fn run(self: Arc<Self>, device: Arc<UsbDeviceAccess>) -> Result<(), UsbError> {
+//         // TODO filter to only accept BBB config
+//         let config = device.select_configuration(|_| true).await?.unwrap();
+//         // Bulk-in, bulk-out
+//         assert_eq!(config.endpoints.len(), 2);
+//         let control_pipe = device.control_pipe();
+//         let (in_index, in_info) = config
+//             .find_endpoint(|ep| ep.is(UsbEndpointType::Bulk, UsbDirection::In))
+//             .ok_or(UsbError::InvalidConfiguration)?;
+//         let (out_index, out_info) = config
+//             .find_endpoint(|ep| ep.is(UsbEndpointType::Bulk, UsbDirection::Out))
+//             .ok_or(UsbError::InvalidConfiguration)?;
+//         let in_pipe = device
+//             .open_bulk_in_pipe(in_index, in_info.max_packet_size as u16)
+//             .await?;
+//         let out_pipe = device
+//             .open_bulk_out_pipe(out_index, out_info.max_packet_size as u16)
+//             .await?;
+//
+//         // Perform a Bulk-Only Mass Storage Reset
+//         // TODO interface id?
+//         control_pipe
+//             .control_transfer(ControlTransferSetup {
+//                 bm_request_type: BulkOnlyMassStorageReset::BM_REQUEST_TYPE,
+//                 b_request: BulkOnlyMassStorageReset::B_REQUEST,
+//                 w_value: 0,
+//                 w_index: 0,
+//                 w_length: 0,
+//             })
+//             .await?;
+//
+//         // Get max LUN
+//         // TODO on devices which do not support multiple LUNs, this command may STALL
+//         let mut buffer = [MaybeUninit::uninit()];
+//         let len = control_pipe
+//             .control_transfer_in(
+//                 ControlTransferSetup {
+//                     bm_request_type: GetMaxLun::BM_REQUEST_TYPE,
+//                     b_request: GetMaxLun::B_REQUEST,
+//                     w_value: 0,
+//                     w_index: 0,
+//                     w_length: 1,
+//                 },
+//                 &mut buffer,
+//             )
+//             .await?;
+//         let max_lun = if len < 1 {
+//             0
+//         } else {
+//             unsafe { buffer[0].assume_init() }
+//         };
+//
+//         let bbb = Bbb::new(device.clone(), in_pipe, out_pipe)?;
+//         let scsi = ScsiEnclosure::setup(Box::new(bbb), max_lun as usize + 1)
+//             .await
+//             .inspect_err(|error| log::error!("msc: scsi error {error:?}"))
+//             .map_err(|_| UsbError::DriverError)?;
+//         let detach = DetachHandler(scsi.clone());
+//         device.set_detach_handler(Arc::new(detach));
+//
+//         Ok(())
+//     }
+//
+//     fn name(&self) -> &'static str {
+//         "USB Mass Storage"
+//     }
+//
+//     fn probe(&self, class: &UsbClassInfo, _device: &UsbDeviceAccess) -> bool {
+//         // TODO support other protocols
+//         class.class == UsbDeviceClass::MassStorage && class.interface_protocol_number == 0x50
+//     }
+// }

kernel/driver/bus/usb/src/class_driver/mod.rs

@@ -1,117 +1,208 @@
+use core::mem::MaybeUninit;
+
 use alloc::{boxed::Box, sync::Arc, vec::Vec};
 use async_trait::async_trait;
 use libk::task::runtime;
 use libk_util::sync::spin_rwlock::IrqSafeRwLock;
 
 use crate::{
+    address::UsbInterfaceAddress,
     device::UsbDeviceAccess,
     error::UsbError,
-    info::{UsbDeviceClass, UsbDeviceProtocol},
+    info::{UsbInterfaceInfo, CLASS_FROM_INTERFACE},
+    pipe::control::{ControlTransferSetup, UsbClassSpecificRequest},
 };
 
+// use alloc::{boxed::Box, sync::Arc, vec::Vec};
+// use async_trait::async_trait;
+// use libk::task::runtime;
+// use libk_util::sync::spin_rwlock::IrqSafeRwLock;
+//
+// use crate::{
+//     device::UsbDeviceAccess,
+//     error::UsbError,
+//     info::{UsbDeviceClass, UsbDeviceProtocol},
+// };
+//
 pub mod hid_keyboard;
-pub mod mass_storage;
-
-#[derive(Debug)]
-pub struct UsbClassInfo {
-    pub class: UsbDeviceClass,
+pub mod hid_mouse;
+// pub mod mass_storage;
+//
+// #[derive(Debug)]
+// pub struct UsbClassInfo {
+//     pub class: UsbDeviceClass,
+//     pub subclass: u8,
+//     pub protocol: UsbDeviceProtocol,
+//     pub device_protocol_number: u8,
+//     pub interface_protocol_number: u8,
+// }
+//
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct UsbInterfaceClass {
+    pub class: u8,
     pub subclass: u8,
-    pub protocol: UsbDeviceProtocol,
-    pub device_protocol_number: u8,
-    pub interface_protocol_number: u8,
+    pub protocol: u8,
 }
 
 #[async_trait]
-pub trait UsbDriver: Send + Sync {
-    async fn run(self: Arc<Self>, device: Arc<UsbDeviceAccess>) -> Result<(), UsbError>;
+pub trait UsbInterfaceDriver: Send + Sync {
+    async fn run(
+        self: Arc<Self>,
+        device: Arc<UsbDeviceAccess>,
+        interface: UsbInterfaceInfo,
+    ) -> Result<(), UsbError>;
 
     fn name(&self) -> &'static str;
-    fn probe(&self, class: &UsbClassInfo, device: &UsbDeviceAccess) -> bool;
+    fn probe(
+        &self,
+        device: &UsbDeviceAccess,
+        interface: &UsbInterfaceInfo,
+        class: UsbInterfaceClass,
+    ) -> bool;
 }
 
-async fn extract_class_info(device: &UsbDeviceAccess) -> Result<Option<UsbClassInfo>, UsbError> {
-    if device.info.num_configurations != 1 {
-        return Ok(None);
-    }
-    let device_info = &device.info;
-    let config_info = device.query_configuration_info(0).await?;
+// async fn extract_class_info(device: &UsbDeviceAccess) -> Result<Option<UsbClassInfo>, UsbError> {
+//     if device.info.num_configurations != 1 {
+//         return Ok(None);
+//     }
+//     let device_info = &device.info;
+//     let config_info = device.query_configuration_info(0).await?;
+//
+//     if config_info.interfaces.len() >= 1 {
+//         let if_info = &config_info.interfaces[0];
+//
+//         let class = if device_info.device_class == UsbDeviceClass::FromInterface {
+//             if_info.interface_class
+//         } else {
+//             device_info.device_class
+//         };
+//         let subclass = if device_info.device_subclass == 0 {
+//             if_info.interface_subclass
+//         } else {
+//             device_info.device_subclass
+//         };
+//         let protocol = if device_info.device_protocol == UsbDeviceProtocol::FromInterface {
+//             if_info.interface_protocol
+//         } else {
+//             device_info.device_protocol
+//         };
+//
+//         Ok(Some(UsbClassInfo {
+//             class,
+//             subclass,
+//             protocol,
+//             interface_protocol_number: if_info.interface_protocol_number,
+//             device_protocol_number: device_info.device_protocol_number,
+//         }))
+//     } else {
+//         Ok(None)
+//     }
+// }
+//
+// async fn pick_driver(
+//     device: &UsbDeviceAccess,
+// ) -> Result<Option<Arc<dyn UsbDriver + 'static>>, UsbError> {
+//     let Some(class) = extract_class_info(device).await? else {
+//         return Ok(None);
+//     };
+//
+//     for driver in USB_DEVICE_DRIVERS.read().iter() {
+//         if driver.probe(&class, device) {
+//             return Ok(Some(driver.clone()));
+//         }
+//     }
+//     Ok(None)
+// }
+//
+// pub async fn spawn_driver(device: Arc<UsbDeviceAccess>) -> Result<(), UsbError> {
+//     // if let Some(driver) = pick_driver(&device).await? {
+//     //     runtime::spawn(async move {
+//     //         let name = driver.name();
+//     //         match driver.run(device).await {
+//     //             e @ Err(UsbError::DeviceDisconnected) => {
+//     //                 log::warn!(
+//     //                     "Driver {:?} did not exit cleanly: device disconnected",
+//     //                     name,
+//     //                 );
+//
+//     //                 e
+//     //             }
+//     //             e => e,
+//     //         }
+//     //     })
+//     //     .map_err(UsbError::SystemError)?;
+//     // }
+//     Ok(())
+// }
 
-    if config_info.interfaces.len() >= 1 {
-        let if_info = &config_info.interfaces[0];
-
-        let class = if device_info.device_class == UsbDeviceClass::FromInterface {
-            if_info.interface_class
-        } else {
-            device_info.device_class
-        };
-        let subclass = if device_info.device_subclass == 0 {
-            if_info.interface_subclass
-        } else {
-            device_info.device_subclass
-        };
-        let protocol = if device_info.device_protocol == UsbDeviceProtocol::FromInterface {
-            if_info.interface_protocol
-        } else {
-            device_info.device_protocol
-        };
-
-        Ok(Some(UsbClassInfo {
-            class,
-            subclass,
-            protocol,
-            interface_protocol_number: if_info.interface_protocol_number,
-            device_protocol_number: device_info.device_protocol_number,
-        }))
-    } else {
-        Ok(None)
-    }
-}
-
-async fn pick_driver(
-    device: &UsbDeviceAccess,
-) -> Result<Option<Arc<dyn UsbDriver + 'static>>, UsbError> {
-    let Some(class) = extract_class_info(device).await? else {
-        return Ok(None);
+async fn setup_interface(
+    device: &Arc<UsbDeviceAccess>,
+    address: UsbInterfaceAddress,
+    interface: &UsbInterfaceInfo,
+) -> Result<(), UsbError> {
+    let class = UsbInterfaceClass {
+        class: interface.interface_class,
+        subclass: interface.interface_subclass,
+        protocol: interface.interface_protocol,
     };
-
-    for driver in USB_DEVICE_DRIVERS.read().iter() {
-        if driver.probe(&class, device) {
-            return Ok(Some(driver.clone()));
-        }
-    }
-    Ok(None)
-}
-
-pub async fn spawn_driver(device: Arc<UsbDeviceAccess>) -> Result<(), UsbError> {
-    if let Some(driver) = pick_driver(&device).await? {
-        runtime::spawn(async move {
-            let name = driver.name();
-            match driver.run(device).await {
-                e @ Err(UsbError::DeviceDisconnected) => {
-                    log::warn!(
-                        "Driver {:?} did not exit cleanly: device disconnected",
-                        name,
-                    );
-
-                    e
+    let drivers = USB_INTERFACE_DRIVERS.read();
+    for driver in drivers.iter() {
+        if driver.probe(device, interface, class) {
+            let driver = driver.clone();
+            let device = device.clone();
+            let interface = interface.clone();
+            runtime::spawn(async move {
+                let name = driver.name();
+                match driver.run(device, interface).await {
+                    e @ Err(UsbError::DeviceDisconnected) => {
+                        log::warn!("{address} did not exit cleanly: device disconnected ({name})");
+                        e
+                    }
+                    e => e,
                 }
-                e => e,
-            }
-        })
-        .map_err(UsbError::SystemError)?;
+            })
+            .map_err(UsbError::SystemError)?;
+            break;
+        }
     }
     Ok(())
 }
 
-pub fn register_driver(driver: Arc<dyn UsbDriver + 'static>) {
+pub async fn setup_device(device: Arc<UsbDeviceAccess>) -> Result<(), UsbError> {
+    // If device has only one configuration available, use it
+    // TODO support devices with multiple configurations
+    let address = device.bus_address();
+    log::info!("Setup USB device @ {address}");
+
+    let Some(config_info) = device.use_default_configuration().await? else {
+        log::warn!("{address} has multiple configurations, not supported yet",);
+        return Ok(());
+    };
+
+    // Setup drivers for interfaces
+    log::info!("{address}: {config_info:#?}");
+
+    // TODO device-level drivers
+    for interface in config_info.interfaces.iter() {
+        let address = address.with_interface(interface.number);
+        if let Err(error) = setup_interface(&device, address, interface).await {
+            log::error!("{}: {:?}", address, error);
+        }
+    }
+
+    Ok(())
+}
+
+pub fn register_driver(driver: Arc<dyn UsbInterfaceDriver + 'static>) {
     // TODO check for duplicates
-    USB_DEVICE_DRIVERS.write().push(driver);
+    USB_INTERFACE_DRIVERS.write().push(driver);
 }
 
 pub fn register_default_class_drivers() {
     register_driver(Arc::new(hid_keyboard::UsbHidKeyboardDriver));
-    register_driver(Arc::new(mass_storage::UsbMassStorageDriverBulkOnly));
+    register_driver(Arc::new(hid_mouse::UsbHidMouseDriver));
+    // register_driver(Arc::new(mass_storage::UsbMassStorageDriverBulkOnly));
 }
 
-static USB_DEVICE_DRIVERS: IrqSafeRwLock<Vec<Arc<dyn UsbDriver + 'static>>> =
+static USB_INTERFACE_DRIVERS: IrqSafeRwLock<Vec<Arc<dyn UsbInterfaceDriver + 'static>>> =
     IrqSafeRwLock::new(Vec::new());

kernel/driver/bus/usb/src/descriptor.rs

@@ -2,9 +2,9 @@ use bytemuck::{Pod, Zeroable};
 
 use crate::{
     communication::UsbDirection,
-    device::UsbSpeed,
+    device::{self, UsbSpeed},
     error::UsbError,
-    info::{UsbDeviceClass, UsbDeviceProtocol, UsbEndpointType, UsbVersion},
+    info::UsbEndpointType,
 };
 
 #[derive(Clone, Copy, Debug, Default, Pod, Zeroable)]
@@ -91,15 +91,15 @@ pub struct UsbOtherSpeedConfiguration {
     pub max_power: u8,
 }
 
-impl UsbInterfaceDescriptor {
-    pub fn class(&self) -> UsbDeviceClass {
-        UsbDeviceClass::try_from(self.interface_class).unwrap_or(UsbDeviceClass::Unknown)
-    }
-
-    pub fn protocol(&self) -> UsbDeviceProtocol {
-        UsbDeviceProtocol::try_from(self.interface_protocol).unwrap_or(UsbDeviceProtocol::Unknown)
-    }
-}
+// impl UsbInterfaceDescriptor {
+//     pub fn class(&self) -> UsbDeviceClass {
+//         UsbDeviceClass::try_from(self.interface_class).unwrap_or(UsbDeviceClass::Unknown)
+//     }
+//
+//     pub fn protocol(&self) -> UsbDeviceProtocol {
+//         UsbDeviceProtocol::try_from(self.interface_protocol).unwrap_or(UsbDeviceProtocol::Unknown)
+//     }
+// }
 
 impl UsbEndpointDescriptor {
     pub fn direction(&self) -> UsbDirection {
@@ -127,16 +127,16 @@ impl UsbEndpointDescriptor {
 }
 
 impl UsbDeviceDescriptor {
-    pub fn class(&self) -> UsbDeviceClass {
-        UsbDeviceClass::try_from(self.device_class).unwrap_or(UsbDeviceClass::Unknown)
-    }
+    // pub fn class(&self) -> UsbDeviceClass {
+    //     UsbDeviceClass::try_from(self.device_class).unwrap_or(UsbDeviceClass::Unknown)
+    // }
 
-    pub fn protocol(&self) -> UsbDeviceProtocol {
-        UsbDeviceProtocol::try_from(self.device_protocol).unwrap_or(UsbDeviceProtocol::Unknown)
-    }
+    // pub fn protocol(&self) -> UsbDeviceProtocol {
+    //     UsbDeviceProtocol::try_from(self.device_protocol).unwrap_or(UsbDeviceProtocol::Unknown)
+    // }
 
-    pub fn max_packet_size(&self, version: UsbVersion, speed: UsbSpeed) -> Result<usize, UsbError> {
-        match (version.is_version_3(), speed, self.max_packet_size_0) {
+    pub fn max_packet_size(&self, version: u16, speed: UsbSpeed) -> Result<usize, UsbError> {
+        match (is_version_3(version), speed, self.max_packet_size_0) {
             (true, UsbSpeed::Super, 9) => Ok(1 << 9),
             (true, _, _) => todo!("Non-GenX speed USB3+ maxpacketsize0"),
             (false, _, 8) => Ok(8),
@@ -147,3 +147,7 @@ impl UsbDeviceDescriptor {
         }
     }
 }
+
+pub fn is_version_3(version: u16) -> bool {
+    version & 0xFF00 == 0x300
+}

kernel/driver/bus/usb/src/device.rs

@@ -1,14 +1,28 @@
-use core::{fmt, ops::Deref};
+use core::{
+    fmt,
+    ops::{Deref, Sub},
+};
 
-use alloc::{boxed::Box, sync::Arc, vec::Vec};
+use alloc::{
+    boxed::Box,
+    format,
+    string::{String, ToString},
+    sync::Arc,
+    vec::Vec,
+};
 use async_trait::async_trait;
-use libk_util::sync::spin_rwlock::{IrqSafeRwLock, IrqSafeRwLockReadGuard};
+use libk::error::Error;
+use libk_util::{
+    sync::spin_rwlock::{IrqSafeRwLock, IrqSafeRwLockReadGuard},
+    OneTimeInit,
+};
 
 use crate::{
+    address::UsbBusAddress,
+    bus::UsbBusWrapper,
     error::UsbError,
     info::{
         UsbConfigurationInfo, UsbDeviceInfo, UsbEndpointInfo, UsbEndpointType, UsbInterfaceInfo,
-        UsbVersion,
     },
     pipe::{
         control::{ConfigurationDescriptorEntry, UsbControlPipeAccess},
@@ -17,21 +31,26 @@ use crate::{
             UsbNormalPipeOut,
         },
     },
+    sysfs::UsbDeviceKObject,
     UsbHostController,
 };
 
 // High-level structures for info provided through descriptors
 
-#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)]
-pub struct UsbBusAddress {
-    pub bus: u16,
-    pub device: u8,
-}
+// #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)]
+// pub struct UsbBusAddress {
+//     pub bus: u16,
+//     pub device: u8,
+// }
 
 pub struct UsbDeviceAccess {
     pub device: Arc<dyn UsbDevice>,
+    pub bus: Arc<UsbBusWrapper>,
     pub info: UsbDeviceInfo,
-    pub current_configuration: IrqSafeRwLock<Option<UsbConfigurationInfo>>,
+    pub configurations: Vec<UsbConfigurationInfo>,
+    current_configuration: IrqSafeRwLock<Option<usize>>,
+
+    pub(crate) kobject: OneTimeInit<UsbDeviceKObject>,
 }
 
 #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)]
@@ -69,7 +88,7 @@ pub trait UsbDevice: Send + Sync {
     fn port_number(&self) -> u8;
     fn bus_address(&self) -> UsbBusAddress;
     fn speed(&self) -> UsbSpeed;
-    fn controller_ref(&self) -> &dyn UsbHostController;
+    fn host_controller(&self) -> Arc<dyn UsbHostController>;
 
     fn set_detach_handler(&self, handler: Arc<dyn UsbDeviceDetachHandler>);
     fn handle_detach(&self);
@@ -84,47 +103,43 @@ impl UsbDeviceAccess {
     /// * Device is not yet configured
     /// * Control pipe for the device has been properly set up
     /// * Device has been assigned a bus address
-    pub async fn setup(raw: Arc<dyn UsbDevice>) -> Result<Self, UsbError> {
+    pub async fn setup(bus: Arc<UsbBusWrapper>, raw: Arc<dyn UsbDevice>) -> Result<Self, UsbError> {
         let control = raw.control_pipe();
 
         let device_desc = control.query_device_descriptor().await?;
 
-        let bcd_usb = device_desc.bcd_usb;
-        let usb_version = UsbVersion::from_bcd_usb(device_desc.bcd_usb)
-            .ok_or(UsbError::InvalidDescriptorField)
-            .inspect_err(|_| {
-                log::error!(
-                    "{}: unsupported/invalid USB version: {:#x}",
-                    raw.bus_address(),
-                    bcd_usb
-                )
-            })?;
-
         let manufacturer = control.query_string(device_desc.manufacturer_str).await?;
         let product = control.query_string(device_desc.product_str).await?;
 
+        // Query device
         let info = UsbDeviceInfo {
             manufacturer,
             product,
-            usb_version,
+            usb_version: device_desc.bcd_usb,
 
             id_vendor: device_desc.id_vendor,
             id_product: device_desc.id_product,
 
-            device_class: device_desc.class(),
+            device_class: device_desc.device_class,
             device_subclass: device_desc.device_subclass,
-            device_protocol: device_desc.protocol(),
-            device_protocol_number: device_desc.device_protocol,
+            device_protocol: device_desc.device_protocol,
 
             num_configurations: device_desc.num_configurations,
 
-            max_packet_size: device_desc.max_packet_size(usb_version, raw.speed())?,
+            max_packet_size: device_desc.max_packet_size(device_desc.bcd_usb, raw.speed())?,
         };
 
+        let configurations =
+            Self::query_configurations(control, device_desc.num_configurations).await?;
+
         Ok(Self {
             device: raw,
+            bus,
             info,
             current_configuration: IrqSafeRwLock::new(None),
+            configurations,
+
+            kobject: OneTimeInit::new(),
         })
     }
 
@@ -164,45 +179,54 @@ impl UsbDeviceAccess {
         Ok(UsbBulkOutPipeAccess(pipe))
     }
 
-    pub fn read_current_configuration(
-        &self,
-    ) -> IrqSafeRwLockReadGuard<Option<UsbConfigurationInfo>> {
-        self.current_configuration.read()
+    pub fn current_configuration(&self) -> Option<&UsbConfigurationInfo> {
+        let index = (*self.current_configuration.read())?;
+        Some(&self.configurations[index])
     }
 
-    pub async fn select_configuration<F: Fn(&UsbConfigurationInfo) -> bool>(
+    pub async fn use_default_configuration(
         &self,
-        predicate: F,
     ) -> Result<Option<UsbConfigurationInfo>, UsbError> {
-        let mut current_config = self.current_configuration.write();
-        let control_pipe = self.control_pipe();
-
-        for i in 0..self.info.num_configurations {
-            let info = self.query_configuration_info(i).await?;
-
-            if predicate(&info) {
-                log::debug!("Selected configuration: {:#?}", info);
-                let config = current_config.insert(info);
-
-                control_pipe
-                    .set_configuration(config.config_value as _)
-                    .await?;
-
-                return Ok(Some(config.clone()));
-            }
+        if self.configurations.len() != 1 {
+            return Ok(None);
         }
 
-        Ok(None)
+        self.set_configuration(0).await.map(Some)
     }
 
-    pub async fn query_configuration_info(
-        &self,
-        index: u8,
-    ) -> Result<UsbConfigurationInfo, UsbError> {
-        if index >= self.info.num_configurations {
+    pub async fn set_configuration(&self, index: usize) -> Result<UsbConfigurationInfo, UsbError> {
+        if index >= self.configurations.len() {
             return Err(UsbError::InvalidConfiguration);
         }
+
+        let mut current = self.current_configuration.write();
         let control_pipe = self.control_pipe();
+        let info = self.configurations[index].clone();
+
+        control_pipe
+            .set_configuration(info.config_value as _)
+            .await?;
+        *current = Some(index);
+
+        Ok(info)
+    }
+
+    async fn query_configurations(
+        control_pipe: &UsbControlPipeAccess,
+        num_configurations: u8,
+    ) -> Result<Vec<UsbConfigurationInfo>, UsbError> {
+        let mut configurations = Vec::new();
+        for i in 0..num_configurations {
+            let configuration = Self::query_configuration(control_pipe, i).await?;
+            configurations.push(configuration);
+        }
+        Ok(configurations)
+    }
+
+    async fn query_configuration(
+        control_pipe: &UsbControlPipeAccess,
+        index: u8,
+    ) -> Result<UsbConfigurationInfo, UsbError> {
         let query = control_pipe.query_configuration_descriptor(index).await?;
 
         let configuration_name = control_pipe
@@ -228,10 +252,9 @@ impl UsbDeviceAccess {
                         name,
                         number: iface.interface_number,
 
-                        interface_class: iface.class(),
+                        interface_class: iface.interface_class,
                         interface_subclass: iface.interface_subclass,
-                        interface_protocol: iface.protocol(),
-                        interface_protocol_number: iface.interface_protocol,
+                        interface_protocol: iface.interface_protocol,
                     });
                 }
                 _ => (),
@@ -248,6 +271,13 @@ impl UsbDeviceAccess {
         Ok(info)
     }
 
+    // pub async fn query_configuration_info(
+    //     &self,
+    //     index: u8,
+    // ) -> Result<UsbConfigurationInfo, UsbError> {
+    //     let control_pipe = self.control_pipe();
+    // }
+
     pub fn set_detach_handler(&self, handler: Arc<dyn UsbDeviceDetachHandler>) {
         self.device.set_detach_handler(handler);
     }
@@ -260,9 +290,3 @@ impl Deref for UsbDeviceAccess {
         &*self.device
     }
 }
-
-impl fmt::Display for UsbBusAddress {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "{}:{}", self.bus, self.device)
-    }
-}

kernel/driver/bus/usb/src/info.rs

@@ -29,41 +29,51 @@ pub enum UsbUsageType {
     Reserved,
 }
 
-#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)]
-pub enum UsbVersion {
-    Usb11,
-    Usb20,
-    Usb21,
-    Usb30,
-    Usb31,
-    Usb32,
-}
+// #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)]
+// pub enum UsbVersion {
+//     Usb11,
+//     Usb20,
+//     Usb21,
+//     Usb30,
+//     Usb31,
+//     Usb32,
+// }
 
-primitive_enum! {
-    pub enum UsbDeviceClass: u8 {
-        FromInterface = 0x00,
-        Hid = 0x03,
-        MassStorage = 0x08,
-        Unknown = 0xFF,
-    }
-}
+pub const CLASS_FROM_INTERFACE: u8 = 0x00;
+pub const CLASS_HID: u8 = 0x03;
+pub const CLASS_MASS_STORAGE: u8 = 0x08;
 
-primitive_enum! {
-    pub enum UsbDeviceProtocol: u8 {
-        FromInterface = 0x00,
-        Unknown = 0xFF,
-    }
-}
+// primitive_enum! {
+//     pub enum UsbDeviceClass: u8 {
+//         FromInterface = 0x00,
+//         Hid = 0x03,
+//         MassStorage = 0x08,
+//         Unknown = 0xFF,
+//     }
+// }
+//
+// primitive_enum! {
+//     pub enum UsbDeviceProtocol: u8 {
+//         FromInterface = 0x00,
+//         Unknown = 0xFF,
+//     }
+// }
 
 #[derive(Debug, Clone)]
 pub struct UsbInterfaceInfo {
     pub name: String,
     pub number: u8,
 
-    pub interface_class: UsbDeviceClass,
+    pub interface_class: u8,
     pub interface_subclass: u8,
-    pub interface_protocol: UsbDeviceProtocol,
-    pub interface_protocol_number: u8,
+    pub interface_protocol: u8,
+    // pub name: String,
+    // pub number: u8,
+
+    // pub interface_class: UsbDeviceClass,
+    // pub interface_subclass: u8,
+    // pub interface_protocol: UsbDeviceProtocol,
+    // pub interface_protocol_number: u8,
 }
 
 #[derive(Debug, Clone)]
@@ -87,15 +97,14 @@ pub struct UsbDeviceInfo {
     pub manufacturer: String,
     pub product: String,
 
-    pub usb_version: UsbVersion,
+    pub usb_version: u16,
 
     pub id_vendor: u16,
     pub id_product: u16,
 
-    pub device_class: UsbDeviceClass,
+    pub device_class: u8,
     pub device_subclass: u8,
-    pub device_protocol: UsbDeviceProtocol,
-    pub device_protocol_number: u8,
+    pub device_protocol: u8,
 
     /// Max packet size for endpoint zero
     pub max_packet_size: usize,
@@ -103,37 +112,37 @@ pub struct UsbDeviceInfo {
     pub num_configurations: u8,
 }
 
-impl UsbVersion {
-    pub fn is_version_3(&self) -> bool {
-        matches!(self, Self::Usb30 | Self::Usb31 | Self::Usb32)
-    }
-
-    pub fn from_bcd_usb(value: u16) -> Option<Self> {
-        match value {
-            0x110 => Some(UsbVersion::Usb11),
-            0x200..=0x20F => Some(UsbVersion::Usb20),
-            0x210..=0x21F => Some(UsbVersion::Usb21),
-            0x300 => Some(UsbVersion::Usb30),
-            0x310 => Some(UsbVersion::Usb31),
-            0x320 => Some(UsbVersion::Usb32),
-            _ => None,
-        }
-    }
-}
-
-impl fmt::Display for UsbVersion {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        let string = match self {
-            Self::Usb11 => "USB1.1",
-            Self::Usb20 => "USB2.0",
-            Self::Usb21 => "USB2.1",
-            Self::Usb30 => "USB3.0",
-            Self::Usb31 => "USB3.1",
-            Self::Usb32 => "USB3.2",
-        };
-        f.write_str(string)
-    }
-}
+// impl UsbVersion {
+//     pub fn is_version_3(&self) -> bool {
+//         matches!(self, Self::Usb30 | Self::Usb31 | Self::Usb32)
+//     }
+//
+//     pub fn from_bcd_usb(value: u16) -> Option<Self> {
+//         match value {
+//             0x110 => Some(UsbVersion::Usb11),
+//             0x200..=0x20F => Some(UsbVersion::Usb20),
+//             0x210..=0x21F => Some(UsbVersion::Usb21),
+//             0x300 => Some(UsbVersion::Usb30),
+//             0x310 => Some(UsbVersion::Usb31),
+//             0x320 => Some(UsbVersion::Usb32),
+//             _ => None,
+//         }
+//     }
+// }
+//
+// impl fmt::Display for UsbVersion {
+//     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+//         let string = match self {
+//             Self::Usb11 => "USB1.1",
+//             Self::Usb20 => "USB2.0",
+//             Self::Usb21 => "USB2.1",
+//             Self::Usb30 => "USB3.0",
+//             Self::Usb31 => "USB3.1",
+//             Self::Usb32 => "USB3.2",
+//         };
+//         f.write_str(string)
+//     }
+// }
 
 impl UsbEndpointInfo {
     pub fn is(&self, ty: UsbEndpointType, dir: UsbDirection) -> bool {

kernel/driver/bus/usb/src/interface.rs

@@ -0,0 +1 @@
+pub struct UsbInterface {}

kernel/driver/bus/usb/src/lib.rs

@@ -4,20 +4,24 @@
     iter_array_chunks,
     maybe_uninit_slice,
     maybe_uninit_as_bytes,
-    maybe_uninit_uninit_array,
     maybe_uninit_write_slice,
     maybe_uninit_fill
 )]
 
+use crate::sysfs::UsbBusKObject;
+
 extern crate alloc;
 
+pub mod address;
 pub mod bus;
 pub mod communication;
 pub mod descriptor;
 pub mod device;
 pub mod error;
 pub mod info;
+pub mod interface;
 pub mod pipe;
+pub mod sysfs;
 pub mod util;
 
 pub mod class_driver;
@@ -26,4 +30,8 @@ pub mod class_driver;
 
 pub trait UsbEndpoint: Sync {}
 
-pub trait UsbHostController: Sync + Send {}
+pub trait UsbHostController: Sync + Send {
+    fn register_sysfs_properties(&self, kobject: &UsbBusKObject) {
+        let _ = kobject;
+    }
+}

kernel/driver/bus/usb/src/pipe/control.rs

@@ -37,6 +37,7 @@ pub trait UsbDeviceRequest: Sized + Pod {
 pub trait UsbClassSpecificRequest: Sized + Pod {
     const BM_REQUEST_TYPE: u8;
     const B_REQUEST: u8;
+    const W_VALUE: u16 = 0;
 }
 
 pub trait UsbDescriptorRequest: UsbDeviceRequest {
@@ -215,7 +216,7 @@ impl UsbControlPipeAccess {
     }
 
     pub async fn query_string(&self, index: u8) -> Result<String, UsbError> {
-        let mut buffer = MaybeUninit::uninit_array::<256>();
+        let mut buffer = [MaybeUninit::uninit(); 256];
 
         let len = self
             .control_transfer_in(
@@ -229,7 +230,7 @@ impl UsbControlPipeAccess {
                 &mut buffer[..],
             )
             .await?;
-        let data = unsafe { MaybeUninit::slice_assume_init_ref(&buffer[..len]) };
+        let data = unsafe { buffer[..len].assume_init_ref() };
         let len = data[0] as usize;
 
         decode_usb_string(&data[2..len])

kernel/driver/bus/usb/src/pipe/normal.rs

@@ -15,7 +15,7 @@ pub trait UsbNormalPipeIn: UsbGenericPipe {
     async fn read(&self, buffer: &mut [u8]) -> Result<usize, UsbError> {
         let mut dma_buffer = self.allocate_dma_buffer(buffer.len())?;
         let len = self.read_dma(dma_buffer.slice_mut(0..buffer.len())).await?;
-        let dma_slice = unsafe { MaybeUninit::slice_assume_init_ref(&dma_buffer[..len]) };
+        let dma_slice = unsafe { dma_buffer[..len].assume_init_ref() };
         buffer[..len].copy_from_slice(dma_slice);
         Ok(len)
     }
@@ -35,7 +35,7 @@ pub trait UsbNormalPipeOut: UsbGenericPipe {
 
     async fn write(&self, buffer: &[u8]) -> Result<usize, UsbError> {
         let mut dma_buffer = self.allocate_dma_buffer(buffer.len())?;
-        MaybeUninit::copy_from_slice(&mut dma_buffer, buffer);
+        dma_buffer[..].write_copy_of_slice(buffer);
         let dma_buffer = unsafe { DmaBuffer::assume_init_slice(dma_buffer) };
 
         self.write_dma(dma_buffer.slice(0..buffer.len())).await

kernel/driver/bus/usb/src/sysfs.rs

@@ -0,0 +1,129 @@
+use alloc::{format, sync::Arc};
+use libk::{
+    error::Error,
+    fs::sysfs::{
+        self,
+        attribute::{IntegerAttribute, IntegerAttributeFormat, IntegerAttributeOps},
+        object::KObject,
+    },
+};
+use libk_util::OneTimeInit;
+
+use crate::{bus::UsbBusWrapper, device::UsbDeviceAccess};
+
+pub type UsbBusKObject = Arc<KObject<Arc<UsbBusWrapper>>>;
+pub type UsbDeviceKObject = Arc<KObject<Arc<UsbDeviceAccess>>>;
+
+pub(crate) fn register_bus_kobject(bus: &Arc<UsbBusWrapper>) -> UsbBusKObject {
+    let root = sysfs_usb_root();
+    let bus_kobject = KObject::new(bus.clone());
+    bus.hc.register_sysfs_properties(&bus_kobject);
+    root.add_object(format!("{}", bus.index), bus_kobject.clone())
+        .ok();
+    bus_kobject
+}
+
+pub(crate) fn register_device_kobject(device: &Arc<UsbDeviceAccess>) -> UsbDeviceKObject {
+    struct Class;
+    struct Subclass;
+    struct Protocol;
+    struct Version;
+    struct IdVendor;
+    struct IdProduct;
+
+    impl IntegerAttributeOps<u8> for Class {
+        type Data = Arc<UsbDeviceAccess>;
+        const NAME: &'static str = "class";
+        const FORMAT: IntegerAttributeFormat = IntegerAttributeFormat::Hex;
+
+        fn read(state: &Self::Data) -> Result<u8, Error> {
+            Ok(state.info.device_class)
+        }
+    }
+    impl IntegerAttributeOps<u8> for Subclass {
+        type Data = Arc<UsbDeviceAccess>;
+        const NAME: &'static str = "subclass";
+        const FORMAT: IntegerAttributeFormat = IntegerAttributeFormat::Hex;
+
+        fn read(state: &Self::Data) -> Result<u8, Error> {
+            Ok(state.info.device_subclass)
+        }
+    }
+    impl IntegerAttributeOps<u8> for Protocol {
+        type Data = Arc<UsbDeviceAccess>;
+        const NAME: &'static str = "protocol";
+        const FORMAT: IntegerAttributeFormat = IntegerAttributeFormat::Hex;
+
+        fn read(state: &Self::Data) -> Result<u8, Error> {
+            Ok(state.info.device_protocol)
+        }
+    }
+    impl IntegerAttributeOps<u16> for Version {
+        type Data = Arc<UsbDeviceAccess>;
+        const NAME: &'static str = "version";
+        const FORMAT: IntegerAttributeFormat = IntegerAttributeFormat::Hex;
+
+        fn read(state: &Self::Data) -> Result<u16, Error> {
+            Ok(state.info.usb_version)
+        }
+    }
+    impl IntegerAttributeOps<u16> for IdVendor {
+        type Data = Arc<UsbDeviceAccess>;
+        const NAME: &'static str = "vendor";
+        const FORMAT: IntegerAttributeFormat = IntegerAttributeFormat::Hex;
+
+        fn read(state: &Self::Data) -> Result<u16, Error> {
+            Ok(state.info.id_vendor)
+        }
+    }
+    impl IntegerAttributeOps<u16> for IdProduct {
+        type Data = Arc<UsbDeviceAccess>;
+        const NAME: &'static str = "product";
+        const FORMAT: IntegerAttributeFormat = IntegerAttributeFormat::Hex;
+
+        fn read(state: &Self::Data) -> Result<u16, Error> {
+            Ok(state.info.id_product)
+        }
+    }
+
+    let bus_kobject = device.bus.kobject();
+    let device_kobject = KObject::new(device.clone());
+
+    device_kobject
+        .add_attribute(IntegerAttribute::from(Class))
+        .ok();
+    device_kobject
+        .add_attribute(IntegerAttribute::from(Subclass))
+        .ok();
+    device_kobject
+        .add_attribute(IntegerAttribute::from(Protocol))
+        .ok();
+    device_kobject
+        .add_attribute(IntegerAttribute::from(Version))
+        .ok();
+    device_kobject
+        .add_attribute(IntegerAttribute::from(IdVendor))
+        .ok();
+    device_kobject
+        .add_attribute(IntegerAttribute::from(IdProduct))
+        .ok();
+
+    let address = device.bus_address();
+
+    bus_kobject
+        .add_object(format!("{}", address.device), device_kobject.clone())
+        .ok();
+
+    device_kobject
+}
+
+fn sysfs_usb_root() -> &'static Arc<KObject<()>> {
+    static USB_ROOT: OneTimeInit<Arc<KObject<()>>> = OneTimeInit::new();
+
+    USB_ROOT.or_init_with(|| {
+        let bus_object = sysfs::bus().expect("bus object");
+        let usb_object = KObject::new(());
+        bus_object.add_object("usb", usb_object.clone()).ok();
+        usb_object
+    })
+}

kernel/driver/fs/ext2/src/access.rs

@@ -2,6 +2,22 @@ use libk::error::Error;
 
 use crate::{BlockGroupDescriptor, Ext2Fs, ExtendedSuperblock, Inode};
 
+pub enum InodeBlock<'a> {
+    Hole,
+    Data(&'a [u8]),
+}
+
+impl InodeBlock<'_> {
+    pub fn copy_to(&self, offset: usize, destination: &mut [u8]) {
+        match self {
+            Self::Hole => destination.fill(0),
+            Self::Data(data) => {
+                destination.copy_from_slice(&data[offset..offset + destination.len()])
+            }
+        }
+    }
+}
+
 impl Ext2Fs {
     pub async fn with_block<T, F: FnOnce(&[u8]) -> Result<T, Error>>(
         &self,
@@ -47,14 +63,19 @@ impl Ext2Fs {
             .await
     }
 
-    pub async fn with_inode_block<T, F: FnOnce(&[u8]) -> Result<T, Error>>(
+    pub async fn with_inode_block<T, F: FnOnce(InodeBlock) -> Result<T, Error>>(
         &self,
         inode: &Inode,
         block: u32,
         mapper: F,
     ) -> Result<T, Error> {
         let block_index = self.inode_block_index(inode, block).await?;
-        self.with_block(block_index, mapper).await
+        if block_index == 0 {
+            mapper(InodeBlock::Hole)
+        } else {
+            self.with_block(block_index, |block| mapper(InodeBlock::Data(block)))
+                .await
+        }
     }
 
     pub async fn with_inode_block_mut<T, F: FnOnce(&mut [u8]) -> Result<T, Error>>(

kernel/driver/fs/ext2/src/data.rs

@@ -5,6 +5,7 @@ use libk::vfs::Metadata;
 use yggdrasil_abi::{
     bitflags,
     io::{FileMode, FileType, GroupId, UserId},
+    time::SystemTime,
 };
 
 use crate::Ext2Fs;
@@ -253,8 +254,9 @@ impl Inode {
             inode: Some(ino),
             block_count: self.blocks(fs) as _,
             block_size: fs.block_size as _,
-            ctime: self.ctime as _,
-            mtime: self.mtime as _,
+            ctime: SystemTime::from_seconds(self.ctime as u64),
+            mtime: SystemTime::from_seconds(self.mtime as u64),
+            atime: SystemTime::from_seconds(self.atime as u64),
         }
     }
 }

kernel/driver/fs/ext2/src/dir/walk.rs

@@ -9,7 +9,7 @@ use yggdrasil_abi::{
     util::FixedString,
 };
 
-use crate::{data::FsRequiredFeatures, Dirent, Ext2Fs};
+use crate::{access::InodeBlock, data::FsRequiredFeatures, Dirent, Ext2Fs};
 
 use super::DirentName;
 
@@ -110,15 +110,24 @@ impl<'a, D: DerefMut<Target = [u8]> + 'a> Record<'a, D> {
 }
 
 impl<'a> DirentIter<'a> {
-    pub fn new(fs: &'a Ext2Fs, block: &'a [u8], offset: usize) -> Self {
-        Self { fs, block, offset }
+    pub fn new(fs: &'a Ext2Fs, block: InodeBlock<'a>, offset: usize) -> Self {
+        let data = if let InodeBlock::Data(data) = block {
+            data
+        } else {
+            &[]
+        };
+        Self {
+            fs,
+            block: data,
+            offset,
+        }
     }
 
     pub fn offset(&self) -> usize {
         self.offset
     }
 
-    pub fn next_record(&mut self) -> Option<Record<&[u8]>> {
+    pub fn next_record(&mut self) -> Option<Record<'_, &[u8]>> {
         if self.offset + size_of::<Dirent>() > self.block.len() {
             return None;
         }
@@ -126,8 +135,9 @@ impl<'a> DirentIter<'a> {
         let dirent_struct_end = self.offset + size_of::<Dirent>();
         let dirent: &Dirent = bytemuck::from_bytes(&self.block[self.offset..dirent_struct_end]);
         let record_size = (dirent.ent_size as usize).max(size_of::<Dirent>());
+        let end = (self.offset + record_size).min(self.block.len());
         let record = Record {
-            data: &self.block[self.offset..self.offset + record_size],
+            data: &self.block[self.offset..end],
             fs: self.fs,
             offset: self.offset,
         };
@@ -173,7 +183,7 @@ impl<'a> DirentIterMut<'a> {
         Self { fs, block, offset }
     }
 
-    pub fn next_record(&mut self) -> Option<Record<&mut [u8]>> {
+    pub fn next_record(&mut self) -> Option<Record<'_, &mut [u8]>> {
         if self.offset + size_of::<Dirent>() > self.block.len() {
             return None;
         }

kernel/driver/fs/ext2/src/inode/cache.rs

@@ -15,7 +15,8 @@ use yggdrasil_abi::{
 };
 
 use crate::{
-    data::InodeMode, dir::DirectoryNode, file::RegularNode, symlink::SymlinkNode, Ext2Fs, Inode,
+    access::InodeBlock, data::InodeMode, dir::DirectoryNode, file::RegularNode,
+    symlink::SymlinkNode, Ext2Fs, Inode,
 };
 
 pub struct InodeHolder {
@@ -103,7 +104,7 @@ impl InodeAccess {
         result
     }
 
-    pub async fn map_blocks<T, F: Fn(&Inode, usize, &[u8]) -> Result<Option<T>, Error>>(
+    pub async fn map_blocks<T, F: Fn(&Inode, usize, InodeBlock) -> Result<Option<T>, Error>>(
         &self,
         mapper: F,
     ) -> Result<Option<T>, Error> {
@@ -175,9 +176,9 @@ impl InodeAccess {
             .map_err(|_| Error::InvalidArgument)?;
 
         self.map_mut(|inode| {
-            inode.mtime = metadata.mtime as _;
-            inode.atime = metadata.mtime as _;
-            inode.ctime = metadata.ctime as _;
+            inode.mtime = metadata.mtime.seconds() as u32;
+            inode.atime = metadata.mtime.seconds() as u32;
+            inode.ctime = metadata.ctime.seconds() as u32;
 
             inode.mode.update_permissions(metadata.mode);
             inode.uid = uid;

kernel/driver/fs/ext2/src/lib.rs

@@ -1,6 +1,6 @@
 #![feature(if_let_guard, async_drop, impl_trait_in_assoc_type)]
 #![cfg_attr(not(test), no_std)]
-#![allow(clippy::new_ret_no_self)]
+#![allow(clippy::new_ret_no_self, incomplete_features)]
 
 extern crate alloc;
 
@@ -390,12 +390,17 @@ impl Ext2Fs {
             let block_offset = (pos % self.block_size as u64) as usize;
             let amount = remaining.min(self.block_size - block_offset);
 
-            self.with_inode_block(inode, block_index as u32, |block| {
-                buffer[offset..offset + amount]
-                    .copy_from_slice(&block[block_offset..block_offset + amount]);
-                Ok(())
-            })
-            .await?;
+            let block = self.inode_block_index(inode, block_index as u32).await?;
+            if block == 0 {
+                buffer[offset..offset + amount].fill(0);
+            } else {
+                self.with_block(block, |block| {
+                    buffer[offset..offset + amount]
+                        .copy_from_slice(&block[block_offset..block_offset + amount]);
+                    Ok(())
+                })
+                .await?;
+            }
 
             pos += amount as u64;
             offset += amount;

kernel/driver/fs/ext2/src/symlink.rs

@@ -54,8 +54,9 @@ impl SymlinkNode {
                 } else {
                     self.fs
                         .with_inode_block(inode, 0, |block| {
-                            write.extend_from_slice(&block[..len]);
-                            buf[..len].copy_from_slice(&block[..len]);
+                            write.resize(len, 0);
+                            block.copy_to(0, &mut write[..]);
+                            block.copy_to(0, &mut buf[..len]);
                             Ok(())
                         })
                         .await?;

kernel/driver/fs/fat32/src/directory.rs

@@ -15,6 +15,7 @@ use libk_util::{
 };
 use yggdrasil_abi::{
     io::{DirectoryEntry, FileMode, GroupId, UserId},
+    time::SystemTime,
     util::FixedString,
 };
 
@@ -254,8 +255,10 @@ impl DirectoryNode {
                 FileMode::default_file()
             },
             inode: Some(cluster.0),
-            ctime: 0,
-            mtime: 0,
+            // TODO
+            ctime: SystemTime::ZERO,
+            mtime: SystemTime::ZERO,
+            atime: SystemTime::ZERO,
             block_count: (size.div_ceil(self.fs.layout.bytes_per_sector as u32)) as u64,
             block_size: self.fs.layout.bytes_per_sector as u64,
         };

kernel/driver/fs/fat32/src/lib.rs

@@ -10,7 +10,10 @@ use libk::{
     vfs::{Filesystem, FilesystemMountOption, Metadata, Node, NodeFlags, NodeRef},
 };
 use libk_util::get_le_u32;
-use yggdrasil_abi::io::{FileMode, GroupId, UserId};
+use yggdrasil_abi::{
+    io::{FileMode, GroupId, UserId},
+    time::SystemTime,
+};
 
 extern crate alloc;
 
@@ -64,8 +67,10 @@ impl Fat32Fs {
             uid: UserId::root(),
             gid: GroupId::root(),
             mode: FileMode::default_dir(),
-            ctime: 0,
-            mtime: 0,
+            // TODO
+            ctime: SystemTime::ZERO,
+            mtime: SystemTime::ZERO,
+            atime: SystemTime::ZERO,
             inode: Some(fs.layout.root_directory_cluster.0),
             block_size: fs.layout.bytes_per_sector as u64,
             block_count: 0,

kernel/driver/fs/memfs/src/bvec.rs

@@ -28,8 +28,8 @@ impl<'a, A: BlockAllocator> BVec<'a, A> {
     ///
     /// The function is guaranteed to make no allocations before the vector is actually written to.
     pub fn new() -> Self {
-        let mut l0 = MaybeUninit::uninit_array();
-        let mut l1 = MaybeUninit::uninit_array();
+        let mut l0 = [const { MaybeUninit::uninit() }; L0_BLOCKS];
+        let mut l1 = [const { MaybeUninit::uninit() }; L1_BLOCKS];
 
         for it in l0.iter_mut() {
             it.write(BlockData::null());
@@ -183,6 +183,7 @@ impl<'a, A: BlockAllocator> BVec<'a, A> {
                     let block = unsafe { self.index_unchecked_mut(index) };
                     assert!(block.is_null());
                     *block = BlockData::new()?;
+                    block.fill(0);
                 }
             }
 

kernel/driver/fs/memfs/src/dir.rs

@@ -1,3 +1,5 @@
+use core::sync::atomic::Ordering;
+
 use alloc::sync::Arc;
 
 use libk::vfs::{
@@ -6,7 +8,7 @@ use libk::vfs::{
 };
 use yggdrasil_abi::error::Error;
 
-use crate::{block::BlockAllocator, file::FileNode, MemoryFilesystem};
+use crate::{block::BlockAllocator, file::FileNode, MemoryFilesystem, INO_COUNTER};
 
 pub(crate) struct DirectoryNode<A: BlockAllocator> {
     fs: Arc<MemoryFilesystem<A>>,
@@ -31,7 +33,8 @@ impl<A: BlockAllocator> DirectoryImpl for DirectoryNode<A> {
     }
 
     fn create_node(&self, _parent: &NodeRef, info: &CreateInfo) -> Result<NodeRef, Error> {
-        let metadata = Metadata::now(info.uid, info.gid, info.mode);
+        let ino = INO_COUNTER.fetch_add(1, Ordering::Relaxed);
+        let metadata = Metadata::now(info.uid, info.gid, info.mode, ino);
         match info.ty {
             CreateFileType::File => Ok(FileNode::<A>::new(self.fs.clone(), metadata)),
             CreateFileType::Directory => Ok(DirectoryNode::<A>::new(self.fs.clone(), metadata)),

kernel/driver/fs/memfs/src/file.rs

@@ -2,13 +2,14 @@ use alloc::sync::Arc;
 use core::any::Any;
 
 use libk::vfs::{CommonImpl, InstanceData, Metadata, Node, NodeFlags, NodeRef, RegularImpl};
-use libk_util::sync::IrqSafeSpinlock;
+use libk_util::sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock};
 use yggdrasil_abi::{error::Error, io::OpenOptions};
 
 use crate::{block::BlockAllocator, bvec::BVec, MemoryFilesystem};
 
 pub(crate) struct FileNode<A: BlockAllocator> {
     pub(crate) data: IrqSafeSpinlock<BVec<'static, A>>,
+    pub(crate) metadata: IrqSafeRwLock<Metadata>,
 }
 
 impl<A: BlockAllocator> FileNode<A> {
@@ -16,9 +17,10 @@ impl<A: BlockAllocator> FileNode<A> {
         Node::regular(
             Self {
                 data: IrqSafeSpinlock::new(BVec::new()),
+                metadata: IrqSafeRwLock::new(metadata),
             },
-            NodeFlags::IN_MEMORY_PROPS,
-            Some(metadata),
+            NodeFlags::empty(),
+            None,
             Some(fs),
         )
     }
@@ -32,6 +34,15 @@ impl<A: BlockAllocator> CommonImpl for FileNode<A> {
     fn size(&self, _node: &NodeRef) -> Result<u64, Error> {
         Ok(self.data.lock().size() as u64)
     }
+
+    fn metadata(&self, _node: &NodeRef) -> Result<Metadata, Error> {
+        Ok(*self.metadata.read())
+    }
+
+    fn set_metadata(&self, _node: &NodeRef, metadata: &Metadata) -> Result<(), Error> {
+        *self.metadata.write() = *metadata;
+        Ok(())
+    }
 }
 
 impl<A: BlockAllocator> RegularImpl for FileNode<A> {
@@ -55,6 +66,7 @@ impl<A: BlockAllocator> RegularImpl for FileNode<A> {
         pos: u64,
         buf: &mut [u8],
     ) -> Result<usize, Error> {
+        self.metadata.write().set_atime_now();
         self.data.lock().read(pos, buf)
     }
 
@@ -65,10 +77,12 @@ impl<A: BlockAllocator> RegularImpl for FileNode<A> {
         pos: u64,
         buf: &[u8],
     ) -> Result<usize, Error> {
+        self.metadata.write().set_mtime_now();
         self.data.lock().write(pos, buf)
     }
 
     fn truncate(&self, _node: &NodeRef, new_size: u64) -> Result<(), Error> {
+        self.metadata.write().set_mtime_now();
         self.data.lock().truncate(new_size)
     }
 

kernel/driver/fs/memfs/src/lib.rs

@@ -2,9 +2,12 @@
 #![no_std]
 #![deny(missing_docs)]
 #![allow(clippy::new_without_default, clippy::new_ret_no_self)]
-#![feature(maybe_uninit_uninit_array, maybe_uninit_array_assume_init)]
+#![feature(maybe_uninit_array_assume_init)]
 
-use core::marker::PhantomData;
+use core::{
+    marker::PhantomData,
+    sync::atomic::{AtomicU32, Ordering},
+};
 
 use alloc::sync::Arc;
 use block::BlockAllocator;
@@ -57,6 +60,8 @@ mod dir;
 mod file;
 mod tar;
 
+static INO_COUNTER: AtomicU32 = AtomicU32::new(1);
+
 /// In-memory read/write filesystem
 pub struct MemoryFilesystem<A: BlockAllocator> {
     root: IrqSafeSpinlock<Option<NodeRef>>,
@@ -91,7 +96,8 @@ impl<A: BlockAllocator> MemoryFilesystem<A> {
                     return Err(Error::DoesNotExist);
                 }
 
-                let node = DirectoryNode::<A>::new(self.clone(), Metadata::now_root(mode));
+                let ino = INO_COUNTER.fetch_add(1, Ordering::Relaxed);
+                let node = DirectoryNode::<A>::new(self.clone(), Metadata::now_root(mode, ino));
                 at.add_child(filename, node.clone())?;
 
                 node
@@ -114,11 +120,15 @@ impl<A: BlockAllocator> MemoryFilesystem<A> {
         let kind = hdr.node_kind();
         let mode = usize::from(&hdr.mode);
         let mode = FileMode::new(0o777 & (mode as u32));
+        let ino = INO_COUNTER.fetch_add(1, Ordering::Relaxed);
         match kind {
-            FileType::File => Ok(FileNode::<A>::new(self.clone(), Metadata::now_root(mode))),
+            FileType::File => Ok(FileNode::<A>::new(
+                self.clone(),
+                Metadata::now_root(mode, ino),
+            )),
             FileType::Directory => Ok(DirectoryNode::<A>::new(
                 self.clone(),
-                Metadata::now_root(mode),
+                Metadata::now_root(mode, ino),
             )),
             FileType::Symlink => {
                 let target = hdr.symlink_target()?;
@@ -129,7 +139,8 @@ impl<A: BlockAllocator> MemoryFilesystem<A> {
     }
 
     fn from_slice_internal(self: &Arc<Self>, tar_data: &'static [u8]) -> Result<NodeRef, Error> {
-        let root = DirectoryNode::<A>::new(self.clone(), Metadata::now_root(FileMode::new(0o755)));
+        let root =
+            DirectoryNode::<A>::new(self.clone(), Metadata::now_root(FileMode::new(0o755), 0));
 
         // 1. Create paths in tar
         for item in TarIterator::new(tar_data) {

kernel/driver/input/src/lib.rs

@@ -9,14 +9,20 @@ use async_trait::async_trait;
 use device_api::device::Device;
 use libk::{device::char::CharDevice, vfs::FileReadiness};
 use libk_util::{ring::LossyRingQueue, OneTimeInit};
-use yggdrasil_abi::{error::Error, io::KeyboardKeyEvent};
+use yggdrasil_abi::{
+    abi_serde::wire,
+    error::Error,
+    io::{KeyboardKeyEvent, MouseEvent},
+};
 
 #[derive(Clone, Copy)]
 pub struct KeyboardDevice;
+#[derive(Clone, Copy)]
+pub struct MouseDevice;
 
 impl FileReadiness for KeyboardDevice {
     fn poll_read(&self, cx: &mut Context<'_>) -> Poll<Result<(), Error>> {
-        INPUT_QUEUE.poll_readable(cx).map(Ok)
+        KEYBOARD_INPUT_QUEUE.poll_readable(cx).map(Ok)
     }
 }
 
@@ -33,7 +39,7 @@ impl CharDevice for KeyboardDevice {
             return Ok(0);
         }
 
-        let ev = INPUT_QUEUE.read().await;
+        let ev = KEYBOARD_INPUT_QUEUE.read().await;
 
         buf[..4].copy_from_slice(&ev.as_bytes());
 
@@ -45,7 +51,7 @@ impl CharDevice for KeyboardDevice {
             return Ok(0);
         }
 
-        let ev = INPUT_QUEUE.try_read().ok_or(Error::WouldBlock)?;
+        let ev = KEYBOARD_INPUT_QUEUE.try_read().ok_or(Error::WouldBlock)?;
 
         buf[..4].copy_from_slice(&ev.as_bytes());
 
@@ -68,15 +74,68 @@ impl CharDevice for KeyboardDevice {
     }
 }
 
-static INPUT_QUEUE: LossyRingQueue<KeyboardKeyEvent> = LossyRingQueue::with_capacity(32);
+impl FileReadiness for MouseDevice {
+    fn poll_read(&self, cx: &mut Context<'_>) -> Poll<Result<(), Error>> {
+        MOUSE_INPUT_QUEUE.poll_readable(cx).map(Ok)
+    }
+}
+
+impl Device for MouseDevice {
+    fn display_name(&self) -> &str {
+        "Mouse input pseudo-device"
+    }
+}
+
+#[async_trait]
+impl CharDevice for MouseDevice {
+    async fn read(&self, buf: &mut [u8]) -> Result<usize, Error> {
+        let ev = MOUSE_INPUT_QUEUE.read().await;
+        let len = wire::to_slice(&ev, buf)?;
+        Ok(len)
+    }
+
+    fn read_nonblocking(&self, buf: &mut [u8]) -> Result<usize, Error> {
+        let ev = MOUSE_INPUT_QUEUE.try_read().ok_or(Error::WouldBlock)?;
+        let len = wire::to_slice(&ev, buf)?;
+        Ok(len)
+    }
+
+    fn is_writeable(&self) -> bool {
+        false
+    }
+
+    fn device_request(&self, option: u32, buffer: &mut [u8], len: usize) -> Result<usize, Error> {
+        let _ = option;
+        let _ = buffer;
+        let _ = len;
+        Err(Error::InvalidOperation)
+    }
+
+    fn is_terminal(&self) -> bool {
+        false
+    }
+}
+
+static KEYBOARD_INPUT_QUEUE: LossyRingQueue<KeyboardKeyEvent> = LossyRingQueue::with_capacity(32);
 static KEYBOARD_DEVICE: OneTimeInit<Arc<KeyboardDevice>> = OneTimeInit::new();
 
-pub fn setup() -> Arc<KeyboardDevice> {
+static MOUSE_INPUT_QUEUE: LossyRingQueue<MouseEvent> = LossyRingQueue::with_capacity(32);
+static MOUSE_DEVICE: OneTimeInit<Arc<MouseDevice>> = OneTimeInit::new();
+
+pub fn setup_keyboard() -> Arc<KeyboardDevice> {
     KEYBOARD_DEVICE
         .or_init_with(|| Arc::new(KeyboardDevice))
         .clone()
 }
 
-pub fn send_event(ev: KeyboardKeyEvent) {
-    INPUT_QUEUE.write(ev);
+pub fn setup_mouse() -> Arc<MouseDevice> {
+    MOUSE_DEVICE.or_init_with(|| Arc::new(MouseDevice)).clone()
+}
+
+pub fn send_keyboard_event(ev: KeyboardKeyEvent) {
+    KEYBOARD_INPUT_QUEUE.write(ev);
+}
+
+pub fn send_mouse_event(ev: MouseEvent) {
+    MOUSE_INPUT_QUEUE.write(ev);
 }

kernel/driver/net/core/src/ephy.rs

@@ -125,24 +125,31 @@ impl<'a, M: MdioBus> PhyAccess<'a, M> {
         })
     }
 
-    pub fn setup_link(&self, have_pause: bool, force_gbesr: GBESR) -> Result<(), Error> {
+    pub fn setup_link(&self, have_pause: bool, force_gbesr: Option<GBESR>) -> Result<(), Error> {
         let bmsr = BMSR::from(self.read_reg(REG_BMSR)?);
-        let mut gbesr = if bmsr.contains(BMSR::EXT_STATUS_1000BASET) {
-            GBESR::from(self.read_reg(REG_GBESR)?)
+        let gbesr = if let Some(force_gbesr) = force_gbesr {
+            let mut gbesr = if bmsr.contains(BMSR::EXT_STATUS_1000BASET) {
+                GBESR::from(self.read_reg(REG_GBESR)?)
+            } else {
+                GBESR::empty()
+            };
+            gbesr |= force_gbesr;
+            Some(gbesr)
         } else {
-            GBESR::empty()
+            None
         };
-        gbesr |= force_gbesr;
         let mut anar = ANAR::from_capabilities(bmsr);
         if have_pause {
             anar |= ANAR::HAVE_PAUSE | ANAR::ASM_DIR;
         }
         let mut gbcr = GBCR::empty();
-        if gbesr.contains(GBESR::HAVE_1000BASET_HALF) {
-            gbcr |= GBCR::HAVE_1000BASET_HALF;
-        }
-        if gbesr.contains(GBESR::HAVE_1000BASET_FULL) {
-            gbcr |= GBCR::HAVE_1000BASET_FULL;
+        if let Some(gbesr) = gbesr {
+            if gbesr.contains(GBESR::HAVE_1000BASET_HALF) {
+                gbcr |= GBCR::HAVE_1000BASET_HALF;
+            }
+            if gbesr.contains(GBESR::HAVE_1000BASET_FULL) {
+                gbcr |= GBCR::HAVE_1000BASET_FULL;
+            }
         }
 
         self.write_reg(REG_ANAR, anar.bits())?;

kernel/driver/net/core/src/lib.rs

@@ -1,4 +1,4 @@
-#![feature(map_try_insert, let_chains, result_flattening)]
+#![feature(map_try_insert, let_chains)]
 #![allow(clippy::type_complexity, clippy::new_without_default)]
 #![no_std]
 

kernel/driver/net/core/src/socket/local/mod.rs

@@ -15,7 +15,7 @@ enum OwnedAddress {
     Anonymous(u64),
 }
 
-pub fn load_address(bytes: &[u8]) -> Result<LocalSocketAddress, Error> {
+pub fn load_address(bytes: &[u8]) -> Result<LocalSocketAddress<'_>, Error> {
     Ok(wire::from_slice(bytes)?)
 }
 
@@ -76,7 +76,7 @@ pub fn write_ancillary(
 }
 
 impl OwnedAddress {
-    fn to_borrowed(&self) -> LocalSocketAddress {
+    fn to_borrowed(&self) -> LocalSocketAddress<'_> {
         match self {
             Self::Path(path) => LocalSocketAddress::Path(path.as_ref()),
             Self::Anonymous(anon) => LocalSocketAddress::Anonymous(*anon),

kernel/driver/net/core/src/socket/tcp/listener.rs

@@ -70,7 +70,7 @@ impl TcpListener {
     pub(super) fn poll_accept(
         &self,
         cx: &mut Context<'_>,
-    ) -> Poll<IrqSafeSpinlockGuard<Vec<Arc<TcpStream>>>> {
+    ) -> Poll<IrqSafeSpinlockGuard<'_, Vec<Arc<TcpStream>>>> {
         let lock = self.pending_accept.lock();
         self.accept_notify.register(cx.waker());
         if !lock.is_empty() {

kernel/driver/net/core/src/socket/tcp/stream.rs

@@ -214,7 +214,7 @@ impl TcpStream {
     pub(super) fn poll_receive(
         &self,
         cx: &mut Context<'_>,
-    ) -> Poll<Result<IrqSafeRwLockWriteGuard<TcpConnection>, Error>> {
+    ) -> Poll<Result<IrqSafeRwLockWriteGuard<'_, TcpConnection>, Error>> {
         let lock = self.connection.write();
         match lock.poll_receive(cx) {
             Poll::Ready(Ok(())) => Poll::Ready(Ok(lock)),

kernel/driver/net/igbe/src/lib.rs

@@ -26,12 +26,15 @@ use ygg_driver_pci::{
 };
 use yggdrasil_abi::net::{link::LinkState, MacAddress};
 
+use crate::regs::Revision;
+
 extern crate alloc;
 
 mod regs;
 mod ring;
 
 struct Igbe {
+    chip: Revision,
     regs: IrqSafeSpinlock<Regs>,
     dma: Arc<dyn DmaAllocator>,
     pci: PciDeviceInfo,
@@ -43,8 +46,9 @@ struct Igbe {
 }
 
 impl Igbe {
-    pub fn new(dma: Arc<dyn DmaAllocator>, regs: Regs, pci: PciDeviceInfo) -> Self {
+    pub fn new(dma: Arc<dyn DmaAllocator>, regs: Regs, chip: Revision, pci: PciDeviceInfo) -> Self {
         Self {
+            chip,
             dma,
             pci,
             mac: OneTimeInit::new(),
@@ -74,7 +78,7 @@ impl Device for Igbe {
         regs.reset(Duration::from_millis(200))?;
         // Intel 8257x manuals say an additional interrupt disable is needed after a global reset
         regs.disable_interrupts();
-        regs.set_link_up()?;
+        regs.set_link_up(self.chip)?;
 
         // Initialize Rx
         regs.initialize_receiver(&rx_ring);
@@ -175,6 +179,10 @@ impl NetworkDevice for Igbe {
 
 pci_driver! {
     matches: [
+        device (0x8086:0x100E), // 82540EM (E1000)
+        device (0x8086:0x100C), // 82544GC (E1000)
+        device (0x8086:0x100F), // 82545EM (E1000)
+        device (0x8086:0x10D3), // 82574L (E1000E) [[BROKEN]]
         device (0x8086:0x10C9), // 82576 GbE
         device (0x8086:0x1502), // 82579LM GbE (Lewisville)
     ],
@@ -197,11 +205,22 @@ pci_driver! {
                 }
             };
 
+            let chip = match info.device_id {
+                0x100E | 0x100C | 0x100F => Revision::I8254x,
+                0x10D3 => Revision::I82574L,
+                0x10C9 => Revision::I82576,
+                0x1502 => Revision::I82579LM,
+                id => {
+                    log::error!("Invalid igbe chip variant: {id:#04x}");
+                    return Err(Error::InvalidOperation)
+                },
+            };
+
             info.init_interrupts(PreferredInterruptMode::Msi(true))?;
             info.set_command(true, use_mmio, !use_mmio, true);
 
             let regs = unsafe { Regs::map(base) }?;
-            let device = Igbe::new(dma.clone(), regs, info.clone());
+            let device = Igbe::new(dma.clone(), regs, chip, info.clone());
             Ok(Arc::new(device))
         }
 

kernel/driver/net/igbe/src/regs.rs

@@ -42,6 +42,14 @@ pub trait Reg {
     const OFFSET: u16;
 }
 
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub enum Revision {
+    I8254x,
+    I82574L,
+    I82576,
+    I82579LM,
+}
+
 register_bitfields! {
     u32,
     pub CTRL [
@@ -329,6 +337,7 @@ impl MdioBus for Regs {
 
         let mdic = self.inner.extract();
         if mdic.matches_all(MDIC::E::SET) {
+            log::warn!("MDIO read error: phyaddr={phyaddr:#x}, regaddr={regaddr:#x}");
             return Err(Error::InvalidOperation);
         }
 
@@ -350,6 +359,9 @@ impl MdioBus for Regs {
         )?;
 
         if self.inner.matches_all(MDIC::E::SET) {
+            log::warn!(
+                "MDIO write error: phyaddr={phyaddr:#x}, regaddr={regaddr:#x}, value={value:#x}"
+            );
             return Err(Error::InvalidOperation);
         }
 
@@ -403,7 +415,7 @@ impl Regs {
         })
     }
 
-    pub fn set_link_up(&mut self) -> Result<(), Error> {
+    pub fn set_link_up(&mut self, chip: Revision) -> Result<(), Error> {
         self.inner
             .modify(CTRL::SLU::SET + CTRL::RFCE::SET + CTRL::TFCE::SET);
 
@@ -412,8 +424,14 @@ impl Regs {
         let (id0, id1) = phy.id()?;
         log::info!("PHY {:04x}:{:04x}", id0, id1);
 
-        phy.reset(Duration::from_millis(200))?;
-        phy.setup_link(true, GBESR::empty())?;
+        phy.reset(Duration::from_millis(200))
+            .inspect_err(|e| log::error!("PHY reset error {e:?}"))?;
+        let force_gbesr = match chip {
+            Revision::I82576 | Revision::I82579LM => Some(GBESR::empty()),
+            _ => None,
+        };
+        phy.setup_link(true, force_gbesr)
+            .inspect_err(|e| log::error!("PHY setup error: {e:?}"))?;
 
         Ok(())
     }

kernel/driver/net/rtl81xx/src/rtl8168.rs

@@ -498,7 +498,7 @@ impl Regs {
         phy.write_reg(0x0E, 0x00)?;
 
         phy.reset(timeout)?;
-        phy.setup_link(true, GBESR::empty())?;
+        phy.setup_link(true, Some(GBESR::empty()))?;
 
         psleep(Duration::from_millis(100));
 

kernel/driver/net/stmmac/src/lib.rs

@@ -1,4 +1,5 @@
 #![no_std]
+#![recursion_limit = "256"]
 
 use core::{mem::MaybeUninit, time::Duration};
 
@@ -336,7 +337,7 @@ impl Device for Stmmac {
         let (id0, id1) = phy.id()?;
         log::info!("stmmac: PHY {id0:04x}:{id1:04x}");
         phy.reset(Duration::from_millis(100))?;
-        phy.setup_link(true, GBESR::empty())?;
+        phy.setup_link(true, Some(GBESR::empty()))?;
 
         self.inner.init(Inner {
             regs: IrqSafeSpinlock::new(regs),

kernel/driver/usb/xhci/src/controller.rs

@@ -25,10 +25,11 @@ use tock_registers::{
 };
 use ygg_driver_pci::{device::PciDeviceInfo, PciConfigurationSpace};
 use ygg_driver_usb::{
-    bus::UsbBusManager,
-    device::{UsbBusAddress, UsbDeviceAccess, UsbSpeed},
+    address::UsbBusAddress,
+    bus::{UsbBusManager, UsbBusWrapper},
+    descriptor,
+    device::{UsbDeviceAccess, UsbSpeed},
     error::UsbError,
-    info::UsbVersion,
     pipe::control::UsbControlPipeAccess,
     UsbHostController,
 };
@@ -67,7 +68,7 @@ struct ScratchpadArray {
 }
 
 struct RootHubPort {
-    version: UsbVersion,
+    version: u16,
     slot_type: u8,
 }
 
@@ -90,6 +91,7 @@ pub struct Xhci {
     pub(crate) slots: Vec<IrqSafeRwLock<Option<Arc<XhciBusDevice>>>>,
     pub(crate) port_slot_map: Vec<AtomicU8>,
     bus_index: OneTimeInit<u16>,
+    bus: OneTimeInit<Arc<UsbBusWrapper>>,
     port_event_map: EventBitmap,
 }
 
@@ -146,9 +148,7 @@ impl Xhci {
         for cap in regs.extended_capabilities.iter() {
             match cap {
                 ExtendedCapability::ProtocolSupport(support) => {
-                    let Some(version) = support.usb_revision() else {
-                        continue;
-                    };
+                    let version = support.usb_revision();
 
                     for port in support.port_range() {
                         log::info!("* Port {port}: {version}");
@@ -195,6 +195,7 @@ impl Xhci {
 
             root_hub_ports,
             bus_index: OneTimeInit::new(),
+            bus: OneTimeInit::new(),
             endpoints: IrqSafeRwLock::new(BTreeMap::new()),
             slots,
             port_slot_map,
@@ -282,7 +283,7 @@ impl Xhci {
             .as_ref()
             .ok_or(UsbError::PortInitFailed)?;
 
-        let need_reset = !root_hub_port.version.is_version_3();
+        let need_reset = !descriptor::is_version_3(root_hub_port.version);
 
         if need_reset {
             self.reset_port(regs).await?;
@@ -341,7 +342,8 @@ impl Xhci {
             device: bus_address,
         });
 
-        let device = UsbDeviceAccess::setup(slot).await?;
+        let bus = self.bus.get();
+        let device = UsbDeviceAccess::setup(bus.clone(), slot).await?;
         UsbBusManager::register_device(device.into());
 
         Ok(())
@@ -523,8 +525,9 @@ impl Device for Xhci {
 
         op.wait_usbsts_bit(USBSTS::CNR::CLEAR, 100000000)?;
 
-        let bus = UsbBusManager::register_bus(self.clone());
-        self.bus_index.init(bus);
+        let (bus_index, bus) = UsbBusManager::register_bus(self.clone());
+        self.bus_index.init(bus_index);
+        self.bus.init(bus);
 
         runtime::spawn(self.clone().port_handler_task()).ok();
 

kernel/driver/usb/xhci/src/device.rs

@@ -6,8 +6,9 @@ use libk_util::{
 };
 use xhci_lib::context;
 use ygg_driver_usb::{
+    address::UsbBusAddress,
     communication::UsbDirection,
-    device::{UsbBusAddress, UsbDevice, UsbDeviceDetachHandler, UsbSpeed},
+    device::{UsbDevice, UsbDeviceDetachHandler, UsbSpeed},
     error::UsbError,
     info::UsbEndpointType,
     pipe::{
@@ -63,8 +64,8 @@ impl UsbDevice for XhciBusDevice {
         *self.detach_handler.lock() = Some(handler);
     }
 
-    fn controller_ref(&self) -> &dyn UsbHostController {
-        self.xhci.as_ref()
+    fn host_controller(&self) -> Arc<dyn UsbHostController> {
+        self.xhci.clone()
     }
 
     fn debug(&self) {}

kernel/driver/usb/xhci/src/regs/extended.rs

@@ -7,7 +7,7 @@ use alloc::vec::Vec;
 use libk::error::Error;
 use libk_mm::{address::PhysicalAddress, device::DeviceMemoryIo};
 use libk_util::sync::spin_rwlock::IrqSafeRwLock;
-use ygg_driver_usb::{error::UsbError, info::UsbVersion};
+use ygg_driver_usb::error::UsbError;
 
 pub struct ProtocolSupport {
     words: [u32; 4],
@@ -69,8 +69,8 @@ impl ExtendedCapability {
 }
 
 impl ProtocolSupport {
-    pub fn usb_revision(&self) -> Option<UsbVersion> {
-        UsbVersion::from_bcd_usb((self.words[0] >> 16) as u16)
+    pub fn usb_revision(&self) -> u16 {
+        (self.words[0] >> 16) as u16
     }
 
     pub fn slot_type(&self) -> u8 {

kernel/driver/usb/xhci/src/ring/transfer.rs

@@ -94,7 +94,7 @@ impl TransferRing {
         })
     }
 
-    pub fn transaction_builder(self: &Arc<Self>) -> Result<TransactionBuilder, UsbError> {
+    pub fn transaction_builder(self: &Arc<Self>) -> Result<TransactionBuilder<'_>, UsbError> {
         if self.shutdown.load(Ordering::Acquire) {
             return Err(UsbError::DeviceDisconnected);
         }

kernel/driver/virtio/blk/Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+name = "ygg_driver_virtio_blk"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+yggdrasil-abi.workspace = true
+libk-util.workspace = true
+libk-mm.workspace = true
+libk.workspace = true
+device-api = { workspace = true, features = ["derive"] }
+
+ygg_driver_virtio_core = { path = "../core" }
+ygg_driver_pci = { path = "../../bus/pci", optional = true }
+
+log.workspace = true
+bytemuck.workspace = true
+tock-registers.workspace = true
+async-trait.workspace = true
+
+[features]
+default = []
+pci = ["ygg_driver_pci", "ygg_driver_virtio_core/pci"]

kernel/driver/virtio/blk/src/lib.rs

@@ -0,0 +1,414 @@
+#![no_std]
+
+use core::mem::MaybeUninit;
+
+use alloc::{boxed::Box, format, sync::Arc, vec::Vec};
+use async_trait::async_trait;
+use bytemuck::{Pod, Zeroable};
+use device_api::{
+    device::{Device, DeviceInitContext},
+    dma::DmaAllocator,
+    interrupt::{InterruptAffinity, InterruptHandler, IrqVector},
+};
+use libk::{
+    device::{block::BlockDevice, manager::probe_partitions},
+    dma::{DmaBuffer, DmaSlice, DmaSliceMut},
+    error::Error,
+    fs::devfs,
+    task::runtime,
+};
+use libk_mm::{
+    address::PhysicalAddress, table::MapAttributes, OnDemandPage, PageProvider, VirtualPage,
+};
+use libk_util::sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock};
+use ygg_driver_pci::{
+    device::{PciDeviceInfo, PreferredInterruptMode},
+    macros::pci_driver,
+};
+use ygg_driver_virtio_core::{
+    queue::VirtQueue,
+    transport::{pci::PciTransport, Transport},
+    DeviceStatus,
+};
+use yggdrasil_abi::{bitflags, io::FileMode};
+
+extern crate alloc;
+
+bitflags! {
+    pub struct Features: u64 {
+        const F_SIZE_MAX: bit 1;
+        const F_SEG_MAX: bit 2;
+        const F_RO: bit 5;
+    }
+}
+
+#[derive(Clone, Copy, Pod, Zeroable)]
+#[repr(C)]
+struct DeviceConfig {
+    capacity: u64,
+    size_max: u32,
+    seg_max: u32,
+    // virtio_blk_geometry {
+    cylinders: u16,
+    heads: u8,
+    sectors: u8,
+    // }
+    blk_size: u32,
+    // virtio_blk_topology {
+    physical_block_exp: u8,
+    alignment_offset: u8,
+    min_io_size: u16,
+    opt_io_size: u32,
+    // }
+    writeback: u8,
+    _0: u8,
+    num_queues: u16,
+    max_discard_sectors: u32,
+    max_discard_seg: u32,
+    discard_sector_alignment: u32,
+    max_write_zeroes_sectors: u32,
+    max_write_zeroes_seg: u32,
+    write_zeroes_may_unmap: u8,
+    _1: [u8; 3],
+    max_secure_erase_sectors: u32,
+    max_secure_erase_seg: u32,
+    secure_erase_sector_alignment: u32,
+}
+
+#[derive(Clone, Copy, Pod, Zeroable)]
+#[repr(C)]
+struct CommandHeader {
+    ty: u32,
+    _0: u32,
+    sector: u64,
+}
+
+pub struct VirtioBlk<T: Transport + 'static> {
+    transport: IrqSafeSpinlock<T>,
+    pci_device_info: PciDeviceInfo,
+    dma: Arc<dyn DmaAllocator>,
+
+    segment_size: usize,
+    read_only: bool,
+    capacity: u64,
+    request_queue: VirtQueue,
+}
+
+impl CommandHeader {
+    const TYPE_READ: u32 = 0;
+    const TYPE_WRITE: u32 = 1;
+
+    pub fn for_read(lba: u64) -> Self {
+        Self {
+            ty: Self::TYPE_READ,
+            _0: 0,
+            sector: lba,
+        }
+    }
+
+    pub fn for_write(lba: u64) -> Self {
+        Self {
+            ty: Self::TYPE_WRITE,
+            _0: 0,
+            sector: lba,
+        }
+    }
+}
+
+impl<T: Transport + 'static> VirtioBlk<T> {
+    // Only one VQ
+    const VQ_REQUEST_0: u16 = 0;
+
+    fn new(
+        dma: Arc<dyn DmaAllocator>,
+        mut transport: T,
+        pci_device_info: PciDeviceInfo,
+    ) -> Result<Self, Error> {
+        let features = Features::from(transport.read_device_features());
+        let device_cfg = transport
+            .device_cfg()
+            .ok_or(Error::InvalidArgument)
+            .inspect_err(|_| log::error!("virtio-blk does not expose device configuration"))?;
+        let device_cfg: &DeviceConfig =
+            bytemuck::from_bytes(&device_cfg[..size_of::<DeviceConfig>()]);
+
+        let read_only = features.contains(Features::F_RO);
+        let segment_size = if features.contains(Features::F_SIZE_MAX) {
+            device_cfg.size_max as usize
+        } else {
+            // I guess no limit then?
+            262144
+        };
+        let segment_limit = if features.contains(Features::F_SEG_MAX) {
+            device_cfg.seg_max as usize
+        } else {
+            // I guess no limit then?
+            8
+        };
+        let capacity = device_cfg.capacity;
+
+        if segment_limit < 3 {
+            // Won't be able to send header + data + status
+            log::error!("virtio-blk: allowed segment count too small");
+            return Err(Error::InvalidArgument);
+        }
+
+        let request_queue = VirtQueue::with_capacity(&*dma, Self::VQ_REQUEST_0, 256)?;
+
+        Ok(Self {
+            transport: IrqSafeSpinlock::new(transport),
+            dma,
+            pci_device_info,
+
+            segment_size,
+            read_only,
+            capacity,
+            request_queue,
+        })
+    }
+
+    fn begin_init(&self) -> Result<DeviceStatus, Error> {
+        let mut transport = self.transport.lock();
+        let mut status = DeviceStatus::RESET_VALUE;
+
+        log::debug!("Reset device");
+        transport.write_device_status(status);
+        status |= DeviceStatus::ACKNOWLEDGE;
+        transport.write_device_status(status);
+        status |= DeviceStatus::DRIVER;
+        transport.write_device_status(status);
+
+        let _device_features = transport.read_device_features();
+
+        // TODO blah blah blah
+
+        transport.write_driver_features(0);
+
+        status |= DeviceStatus::FEATURES_OK;
+        transport.write_device_status(status);
+
+        if !transport
+            .read_device_status()
+            .contains(DeviceStatus::FEATURES_OK)
+        {
+            return Err(Error::InvalidOperation);
+        }
+
+        Ok(status)
+    }
+
+    fn finish_init(&self, status: DeviceStatus) {
+        let mut transport = self.transport.lock();
+
+        transport.write_device_status(status | DeviceStatus::DRIVER_OK);
+    }
+
+    fn setup_queues(self: &Arc<Self>) -> Result<(), Error> {
+        self.pci_device_info
+            .init_interrupts(PreferredInterruptMode::Msi(true))?;
+        let msi_info = self
+            .pci_device_info
+            .map_interrupt(InterruptAffinity::Any, self.clone())?;
+        let vector = msi_info.map(|msi| msi.vector as u16);
+
+        let mut transport = self.transport.lock();
+
+        transport.set_queue(Self::VQ_REQUEST_0, &self.request_queue, vector);
+
+        Ok(())
+    }
+}
+
+impl<T: Transport + 'static> InterruptHandler for VirtioBlk<T> {
+    fn handle_irq(self: Arc<Self>, _vector: IrqVector) -> bool {
+        // Only one queue
+        self.request_queue.handle_notify();
+        true
+    }
+}
+
+impl<T: Transport + 'static> Device for VirtioBlk<T> {
+    unsafe fn init(self: Arc<Self>, _cx: DeviceInitContext) -> Result<(), Error> {
+        let status = self.begin_init()?;
+        self.setup_queues()?;
+        self.finish_init(status);
+        register_virtio_block_device(self.clone());
+        Ok(())
+    }
+
+    fn display_name(&self) -> &str {
+        "VirtIO Block Device"
+    }
+}
+
+#[async_trait]
+impl<T: Transport + 'static> BlockDevice for VirtioBlk<T> {
+    fn allocate_buffer(&self, size: usize) -> Result<DmaBuffer<[MaybeUninit<u8>]>, Error> {
+        DmaBuffer::new_uninit_slice(&*self.dma, size)
+    }
+
+    async fn read_aligned(
+        &self,
+        position: u64,
+        buffer: DmaSliceMut<'_, MaybeUninit<u8>>,
+    ) -> Result<(), Error> {
+        if position % 512 != 0 || buffer.len() % 512 != 0 {
+            return Err(Error::InvalidArgument);
+        }
+        let lba = position / 512;
+        let lba_count = buffer.len() / 512;
+        if lba + lba_count as u64 >= self.capacity {
+            return Err(Error::InvalidArgument);
+        }
+
+        let mut header = DmaBuffer::new_slice(&*self.dma, 0, size_of::<CommandHeader>())?;
+        *bytemuck::from_bytes_mut(&mut header[..]) = CommandHeader::for_read(lba);
+        let mut status = DmaBuffer::new_uninit_slice(&*self.dma, 1)?;
+
+        self.request_queue
+            .enqueue_wait(
+                &[header.slice(0..size_of::<CommandHeader>())],
+                &[buffer, status.slice_mut(0..1)],
+                || {
+                    self.transport.lock().notify(Self::VQ_REQUEST_0);
+                },
+            )
+            .await?;
+
+        let status = unsafe { DmaBuffer::assume_init_slice(status) }[0];
+        if status == 0 {
+            Ok(())
+        } else {
+            Err(Error::InvalidOperation)
+        }
+    }
+    async fn write_aligned(&self, position: u64, buffer: DmaSlice<'_, u8>) -> Result<(), Error> {
+        if self.read_only {
+            return Err(Error::ReadOnly);
+        }
+
+        if position % 512 != 0 || buffer.len() % 512 != 0 {
+            return Err(Error::InvalidArgument);
+        }
+        let lba = position / 512;
+        let lba_count = buffer.len() / 512;
+        if lba + lba_count as u64 >= self.capacity {
+            return Err(Error::InvalidArgument);
+        }
+
+        let mut header = DmaBuffer::new_slice(&*self.dma, 0, size_of::<CommandHeader>())?;
+        *bytemuck::from_bytes_mut(&mut header[..]) = CommandHeader::for_write(lba);
+        let mut status = DmaBuffer::new_uninit_slice(&*self.dma, 1)?;
+
+        self.request_queue
+            .enqueue_wait(
+                &[header.slice(0..size_of::<CommandHeader>()), buffer],
+                &[status.slice_mut(0..1)],
+                || {
+                    self.transport.lock().notify(Self::VQ_REQUEST_0);
+                },
+            )
+            .await?;
+
+        let status = unsafe { DmaBuffer::assume_init_slice(status) }[0];
+        if status == 0 {
+            Ok(())
+        } else {
+            Err(Error::InvalidOperation)
+        }
+    }
+
+    fn block_size(&self) -> usize {
+        512
+    }
+    fn block_count(&self) -> u64 {
+        self.capacity
+    }
+    fn max_blocks_per_request(&self) -> usize {
+        // TODO this limit can be bumped or scatter-gather operations
+        self.segment_size / 512
+    }
+}
+
+impl<T: Transport + 'static> PageProvider for VirtioBlk<T> {
+    fn ondemand_fetch(&self, _opaque: u64) -> Result<OnDemandPage, Error> {
+        unimplemented!()
+    }
+
+    fn get_page(&self, _offset: u64) -> Result<VirtualPage, Error> {
+        unimplemented!()
+    }
+
+    fn release_page(
+        &self,
+        _offset: u64,
+        _phys: PhysicalAddress,
+        _dirty: bool,
+    ) -> Result<(), Error> {
+        unimplemented!()
+    }
+
+    fn clone_page(
+        &self,
+        _offset: u64,
+        _src_phys: PhysicalAddress,
+        _src_attrs: MapAttributes,
+    ) -> Result<PhysicalAddress, Error> {
+        unimplemented!()
+    }
+}
+
+static DEVICES: IrqSafeRwLock<Vec<Arc<dyn BlockDevice>>> = IrqSafeRwLock::new(Vec::new());
+
+fn register_virtio_block_device(device: Arc<dyn BlockDevice>) {
+    let index = {
+        let mut devices = DEVICES.write();
+        let index = devices.len();
+        devices.push(device.clone());
+        index
+    };
+
+    let name = format!("vb{index}");
+    devfs::add_named_block_device(device.clone(), name.clone(), FileMode::new(0o600)).ok();
+
+    runtime::spawn(async move {
+        let name = name;
+        log::info!("Probing partitions for {name}");
+        probe_partitions(device, |index, partition| {
+            let partition_name = format!("{name}p{}", index + 1);
+            devfs::add_named_block_device(
+                Arc::new(partition),
+                partition_name,
+                FileMode::new(0o600),
+            )
+            .ok();
+        })
+        .await
+        .ok();
+    })
+    .ok();
+}
+
+pci_driver! {
+    matches: [device (0x1AF4:0x1001)],
+    driver: {
+        fn probe(
+            &self,
+            info: &PciDeviceInfo,
+            dma: &Arc<dyn DmaAllocator>,
+        ) -> Result<Arc<dyn Device>, Error> {
+            let space = &info.config_space;
+
+            let transport = PciTransport::from_config_space(space).unwrap();
+            let device = VirtioBlk::new(dma.clone(), transport, info.clone())?;
+
+            let device = Arc::new(device);
+
+            Ok(device)
+        }
+
+        fn driver_name(&self) -> &str {
+            "virtio-blk"
+        }
+    }
+}

kernel/driver/virtio/core/Cargo.toml

@@ -16,6 +16,12 @@ log.workspace = true
 bitflags.workspace = true
 tock-registers.workspace = true
 
+[dev-dependencies]
+kernel-arch-hosted.path = "../../../arch/hosted"
+
 [features]
 default = []
 pci = ["ygg_driver_pci"]
+
+[lints]
+workspace = true

kernel/driver/virtio/core/src/queue.rs

@@ -1,361 +1,535 @@
 //! VirtIO queue implementation.
-//!
-//! # Note
-//!
-//! The code is poorly borrowed from `virtio-drivers` crate. I want to rewrite it properly myself.
-use core::{
-    mem::MaybeUninit,
-    sync::atomic::{fence, Ordering},
+
+use core::{future::poll_fn, mem::MaybeUninit, task::Poll};
+
+use alloc::{boxed::Box, sync::Arc};
+use device_api::dma::DmaAllocator;
+use libk::{
+    dma::{BusAddress, DmaBuffer, DmaSlice, DmaSliceMut},
+    error::Error,
+};
+use libk_util::{
+    event::OneTimeEvent, hash_table::DefaultHashTable, sync::IrqSafeSpinlock, waker::QueueWaker,
 };
 
-use device_api::dma::DmaAllocator;
-use libk::dma::{BusAddress, DmaBuffer};
-
-use crate::{error::Error, transport::Transport};
-
-#[derive(Debug)]
+#[derive(Clone, Copy, Debug)]
 #[repr(C)]
 struct Descriptor {
     address: BusAddress,
-    len: u32,
+    length: u32,
     flags: u16,
     next: u16,
 }
 
-// Layout:
-//  {
-//      flags: u16,
-//      idx: u16,
-//      ring: [u16; QUEUE_SIZE],
-//      used_event: u16
-//  }
 struct AvailableRing {
-    data: DmaBuffer<[MaybeUninit<u16>]>,
+    mapping: DmaBuffer<[u16]>,
+    capacity: usize,
 }
 
-// Layout:
-//  {
-//      flags: u16,
-//      idx: u16,
-//      ring: [UsedElem; QUEUE_SIZE],
-//      avail_event: u16,
-//      _pad: u16
-//  }
 struct UsedRing {
-    data: DmaBuffer<[MaybeUninit<u32>]>,
-    used_count: usize,
+    mapping: DmaBuffer<[u32]>,
+    last_seen_used: u16,
+    capacity: usize,
 }
 
-pub struct VirtQueue {
-    descriptor_table: DmaBuffer<[MaybeUninit<Descriptor>]>,
+pub struct DescriptorTable {
+    descriptors: DmaBuffer<[Descriptor]>,
+    free_count: usize,
+    first_free: Option<u16>,
+    last_free: Option<u16>,
+}
+
+struct VqInner {
+    descriptors: DescriptorTable,
     available: AvailableRing,
     used: UsedRing,
+}
 
+pub struct VirtQueue<N: VqNotificationMechanism = VqAsyncNotification> {
+    inner: IrqSafeSpinlock<VqInner>,
+    free_descriptor_notify: QueueWaker,
+
+    #[allow(unused)]
+    index: u16,
     capacity: usize,
 
-    queue_index: u16,
-    free_head: u16,
+    used_notify: N,
+}
 
-    avail_idx: u16,
-    last_used_idx: u16,
+pub trait VqNotificationMechanism {
+    type Token;
 
-    msix_vector: u16,
+    fn notify_used(&self, head: u16, length: u32);
+    fn create_token(&self, head: u16) -> Self::Token;
+}
+
+pub struct VqAsyncNotification {
+    completions: IrqSafeSpinlock<DefaultHashTable<u16, Arc<OneTimeEvent<u32>>>>,
+}
+
+pub struct VqManualNotification;
+
+pub struct VqCallbackNotification(Box<dyn Fn(u16, u32) + Sync + Send>);
+
+impl VqNotificationMechanism for VqAsyncNotification {
+    type Token = Arc<OneTimeEvent<u32>>;
+
+    fn notify_used(&self, head: u16, length: u32) {
+        let mut completions = self.completions.lock();
+        if let Some(completion) = completions.remove(&head) {
+            log::trace!("vq: completion #{head}");
+            completion.signal(length);
+        }
+    }
+
+    fn create_token(&self, head: u16) -> Self::Token {
+        let mut completions = self.completions.lock();
+        let token = Arc::new(OneTimeEvent::new());
+        completions.insert(head, token.clone());
+        token
+    }
+}
+
+impl VqNotificationMechanism for VqCallbackNotification {
+    type Token = u16;
+
+    fn create_token(&self, head: u16) -> Self::Token {
+        head
+    }
+
+    fn notify_used(&self, head: u16, length: u32) {
+        (self.0)(head, length);
+    }
+}
+
+impl VqNotificationMechanism for VqManualNotification {
+    type Token = u16;
+
+    fn create_token(&self, head: u16) -> Self::Token {
+        head
+    }
+
+    fn notify_used(&self, _head: u16, _length: u32) {
+        unreachable!()
+    }
 }
 
 impl AvailableRing {
+    const FLAGS: usize = 0;
+    const IDX: usize = 1;
+    const RING: usize = 2;
+
     pub fn with_capacity(
         dma: &dyn DmaAllocator,
-        no_irq: bool,
         capacity: usize,
+        no_interrupt: bool,
     ) -> Result<Self, Error> {
-        let mut data = DmaBuffer::new_zeroed_slice(dma, capacity + 3)?;
-
-        if no_irq {
-            data[0].write(1);
+        // flags + idx + [ring] + used_event
+        let mut mapping = DmaBuffer::new_slice(dma, 0u16, (capacity + 6) & !3)?;
+        if no_interrupt {
+            mapping[Self::FLAGS] |= 1 << 0;
         }
-
-        data[1].write(0);
-
-        Ok(Self { data })
+        Ok(Self { mapping, capacity })
     }
 
-    pub fn set_head(&mut self, slot: u16, head: u16) {
-        self.data[slot as usize + 2].write(head);
-    }
-
-    pub fn set_index(&mut self, index: u16) {
-        self.data[1].write(index);
+    pub fn push(&mut self, head: u16) -> u16 {
+        log::trace!("enqueue #{head}");
+        let idx = self.mapping[Self::IDX];
+        let index = idx as usize % self.capacity;
+        self.mapping[Self::RING + index] = head;
+        self.mapping.cache_flush_element(Self::RING + index, true);
+        let idx = idx.wrapping_add(1);
+        self.mapping[Self::IDX] = idx;
+        self.mapping.cache_flush_element(Self::IDX, true);
+        idx
     }
 }
 
 impl UsedRing {
-    pub fn with_capacity(dma: &dyn DmaAllocator, capacity: usize) -> Result<Self, Error> {
-        let mut data = DmaBuffer::new_zeroed_slice(dma, capacity * 2 + 2)?;
-
-        data[0].write(0);
+    const FLAGS_IDX: usize = 0;
+    const RING: usize = 1;
 
+    pub fn with_capacity(
+        dma: &dyn DmaAllocator,
+        capacity: usize,
+        no_notify: bool,
+    ) -> Result<Self, Error> {
+        // 2x u16 (flags + idx) + [ring x 2 x u32] + avail_event
+        let mut mapping = DmaBuffer::new_slice(dma, 0, capacity * 2 + 1)?;
+        if no_notify {
+            mapping[Self::FLAGS_IDX] |= 1 << 0;
+        }
+        mapping.cache_flush_element(Self::FLAGS_IDX, true);
         Ok(Self {
-            data,
-            used_count: 0,
+            mapping,
+            capacity,
+            last_seen_used: 0,
         })
     }
 
-    pub fn read_slot(&self, index: u16) -> (u32, u32) {
-        let index = unsafe { self.data[1 + index as usize * 2].assume_init() };
-        let len = unsafe { self.data[2 + index as usize * 2].assume_init() };
-        (index, len)
-    }
-
-    pub fn index(&self) -> u16 {
-        unsafe { (self.data[0].assume_init() >> 16) as u16 }
+    pub fn consume<F: FnMut(u16, u32)>(&mut self, mut handler: F) -> usize {
+        self.mapping.cache_flush_element(Self::FLAGS_IDX, false);
+        let idx = (self.mapping[Self::FLAGS_IDX] >> 16) as u16;
+        let mut count = 0;
+        while self.last_seen_used != idx {
+            let index = self.last_seen_used as usize % self.capacity;
+            self.mapping
+                .cache_flush_range(Self::RING + index..Self::RING + index + 1, false);
+            let head = self.mapping[Self::RING + index * 2] as u16;
+            let len = self.mapping[Self::RING + index * 2 + 1];
+            handler(head, len);
+            count += 1;
+            self.last_seen_used = self.last_seen_used.wrapping_add(1);
+        }
+        count
     }
 }
 
-impl VirtQueue {
-    pub fn with_capacity<T: Transport>(
-        transport: &mut T,
-        dma: &dyn DmaAllocator,
-        index: u16,
-        capacity: usize,
-        msix_vector: Option<u16>,
-        no_avail_irq: bool,
-    ) -> Result<Self, Error> {
-        // TODO check if queue is already set up
+impl Descriptor {
+    pub const EMPTY: Self = Self {
+        address: BusAddress::ZERO,
+        length: 0,
+        flags: 0,
+        next: 0,
+    };
 
-        let max_capacity = transport.max_queue_size(index);
+    pub const F_NEXT: u16 = 1 << 0;
+    pub const F_WRITE: u16 = 1 << 1;
+}
 
-        if !capacity.is_power_of_two() || capacity > u16::MAX.into() {
-            return Err(Error::InvalidQueueSize);
+impl DescriptorTable {
+    const FREE_CHAIN_END: u16 = u16::MAX;
+
+    pub fn with_capacity(dma: &dyn DmaAllocator, capacity: usize) -> Result<Self, Error> {
+        if capacity >= Self::FREE_CHAIN_END as usize - 1 {
+            return Err(Error::InvalidArgument);
         }
-
-        if capacity > max_capacity as usize {
-            return Err(Error::QueueTooLarge);
+        let mut descriptors = DmaBuffer::new_slice(dma, Descriptor::EMPTY, capacity)?;
+        for i in 0..capacity {
+            if i == capacity - 1 {
+                // Last descriptor of the free chain
+                descriptors[i].next = Self::FREE_CHAIN_END;
+            } else {
+                descriptors[i].next = (i + 1) as u16;
+            }
         }
 
-        let descriptor_table = DmaBuffer::new_zeroed_slice(dma, capacity)?;
-        let available = AvailableRing::with_capacity(dma, no_avail_irq, capacity)?;
-        let used = UsedRing::with_capacity(dma, capacity)?;
-
-        transport.set_queue(
-            index,
-            capacity as u16,
-            descriptor_table.bus_address(),
-            available.data.bus_address(),
-            used.data.bus_address(),
-            msix_vector,
-        );
-
         Ok(Self {
-            descriptor_table,
-            available,
-            used,
-
-            capacity,
-
-            queue_index: index,
-            free_head: 0,
-
-            avail_idx: 0,
-            last_used_idx: 0,
-
-            msix_vector: msix_vector.unwrap_or(0xFFFF),
+            descriptors,
+            free_count: capacity,
+            first_free: Some(0),
+            last_free: Some(capacity as u16 - 1),
         })
     }
 
-    pub fn capacity(&self) -> usize {
-        self.capacity
-    }
-
-    pub fn with_max_capacity<T: Transport>(
-        transport: &mut T,
-        dma: &dyn DmaAllocator,
-        index: u16,
-        capacity: usize,
-        msix_vector: Option<u16>,
-        no_avail_irq: bool,
-    ) -> Result<Self, Error> {
-        let max_capacity = transport.max_queue_size(index);
-        let capacity = capacity.min(max_capacity as usize);
-
-        Self::with_capacity(transport, dma, index, capacity, msix_vector, no_avail_irq)
+    // Allocate a chain of descriptors
+    fn alloc_descriptors(
+        &mut self,
+        h2d: &[DmaSlice<u8>],
+        d2h: &[DmaSliceMut<MaybeUninit<u8>>],
+    ) -> Result<u16, Error> {
+        if d2h.len() + h2d.len() == 0 {
+            // Empty transfer
+            return Err(Error::InvalidArgument);
+        }
+        if self.free_count < d2h.len() + h2d.len() {
+            // Not enough descriptor "slots" to place the buffers
+            return Err(Error::WouldBlock);
+        }
+        // Implied by free_count
+        debug_assert!(self.first_free.is_some());
+        debug_assert!(self.last_free.is_some());
+        let head = unsafe {
+            self.follow_descriptor_chain(h2d.len() + d2h.len(), |i, desc| {
+                if i < h2d.len() {
+                    desc.address = h2d[i].bus_address();
+                    desc.length = h2d[i].len() as u32;
+                    desc.flags = 0;
+                } else {
+                    let i = i - h2d.len();
+                    desc.address = d2h[i].bus_address();
+                    desc.length = d2h[i].len() as u32;
+                    desc.flags = Descriptor::F_WRITE;
+                }
+            })
+        };
+        Ok(head)
     }
 
     /// # Safety
     ///
-    /// Invariants: DmaBuffer remains valid and allocated until it is properly dequeued.
-    pub unsafe fn add<'a, 'b>(
+    /// The following invariants must hold:
+    ///
+    /// * **There must actually be `count` free descriptors**.
+    /// * **`count` should not be zero**.
+    /// * The VQ must be in a consistent state: first_free/last_free/free_count must be consistent
+    ///   in terms of tracking the current state of the descriptor allocation.
+    /// * Free descriptors in the table must be chained.
+    /// * Last free descriptor must have [Self::FREE_CHAIN_END] as its `next` field.
+    unsafe fn follow_descriptor_chain<F: Fn(usize, &mut Descriptor)>(
         &mut self,
-        input: &'a [&'b mut DmaBuffer<[MaybeUninit<u8>]>],
-        output: &'a [&'b DmaBuffer<[u8]>],
-    ) -> Result<u16, Error> {
-        if input.is_empty() && output.is_empty() {
-            return Err(Error::EmptyTransaction);
-        }
-        let n_desc = input.len() + output.len();
-
-        if self.used.used_count + 1 > self.capacity || self.used.used_count + n_desc > self.capacity
-        {
-            return Err(Error::QueueFull);
-        }
-
-        let head = self.add_direct(input, output);
-        let avail_slot = self.avail_idx % self.capacity as u16;
-
-        self.available.set_head(avail_slot, head);
-
-        fence(Ordering::SeqCst);
-
-        self.avail_idx = self.avail_idx.wrapping_add(1);
-        self.available.set_index(self.avail_idx);
-
-        fence(Ordering::SeqCst);
-
-        Ok(head)
-    }
-
-    unsafe fn add_direct<'a, 'b>(
-        &mut self,
-        input: &'a [&'b mut DmaBuffer<[MaybeUninit<u8>]>],
-        output: &'a [&'b DmaBuffer<[u8]>],
+        count: usize,
+        visitor: F,
     ) -> u16 {
-        let head = self.free_head;
-        let mut last = self.free_head;
-
-        for item in output {
-            assert_ne!(item.len(), 0);
-            let desc = &mut self.descriptor_table[usize::from(self.free_head)];
-            let next = (self.free_head + 1) % self.capacity as u16;
-
-            desc.write(Descriptor {
-                address: item.bus_address(),
-                len: item.len().try_into().unwrap(),
-                // TODO
-                flags: (1 << 0),
-                next,
-            });
-
-            last = self.free_head;
-            self.free_head = next;
+        debug_assert_ne!(count, 0);
+        let mut current = self.first_free.unwrap_unchecked();
+        let head = current;
+        for i in 0..count {
+            let descriptor = &mut self.descriptors[current as usize];
+            log::trace!("vq: alloc desc #{current}");
+            visitor(i, descriptor);
+            if i == count - 1 {
+                debug_assert_eq!(descriptor.flags & Descriptor::F_NEXT, 0);
+                current = descriptor.next;
+                descriptor.next = 0;
+            } else {
+                current = descriptor.next;
+                descriptor.flags |= Descriptor::F_NEXT;
+                debug_assert_ne!(current, Self::FREE_CHAIN_END);
+            };
         }
-
-        for item in input {
-            assert_ne!(item.len(), 0);
-            let desc = &mut self.descriptor_table[usize::from(self.free_head)];
-            let next = (self.free_head + 1) % self.capacity as u16;
-
-            desc.write(Descriptor {
-                address: item.bus_address(),
-                len: item.len().try_into().unwrap(),
-                // TODO MAGIC
-                flags: (1 << 0) | (1 << 1),
-                next,
-            });
-
-            last = self.free_head;
-            self.free_head = next;
+        if current == Self::FREE_CHAIN_END {
+            // No free descriptors left
+            debug_assert_eq!(self.free_count, count);
+            self.first_free = None;
+            self.last_free = None;
+        } else {
+            self.first_free = Some(current);
         }
-
-        {
-            let last_desc = self.descriptor_table[last as usize].assume_init_mut();
-
-            // TODO
-            last_desc.flags &= !(1 << 0);
-        }
-
-        self.used.used_count += input.len() + output.len();
-
-        fence(Ordering::SeqCst);
+        self.free_count -= count;
 
         head
     }
 
-    pub fn add_notify_wait_pop<'a, 'b, T: Transport>(
-        &mut self,
-        input: &'a [&'b mut DmaBuffer<[MaybeUninit<u8>]>],
-        output: &'a [&'b DmaBuffer<[u8]>],
-        transport: &mut T,
-    ) -> Result<u32, Error> {
-        let token = unsafe { self.add(input, output) }?;
+    fn add_free_descriptor(&mut self, idx: u16) {
+        log::trace!("vq: free descriptor #{idx}");
+        self.descriptors[idx as usize] = Descriptor {
+            next: Self::FREE_CHAIN_END,
+            ..Descriptor::EMPTY
+        };
 
-        transport.notify(self.queue_index);
-
-        while self.is_used_empty() {
-            core::hint::spin_loop();
-        }
-
-        fence(Ordering::SeqCst);
-
-        unsafe { self.pop_used(token) }
-    }
-
-    pub fn is_used_empty(&self) -> bool {
-        fence(Ordering::SeqCst);
-
-        self.last_used_idx == self.used.index()
-    }
-
-    pub fn pop_last_used(&mut self) -> Option<(u16, u32)> {
-        let token = self.peek_used()?;
-        let len = unsafe { self.pop_used(token) }.unwrap();
-
-        Some((token, len))
-    }
-
-    fn peek_used(&mut self) -> Option<u16> {
-        if !self.is_used_empty() {
-            let last_used = self.last_used_idx % self.capacity as u16;
-            Some(self.used.read_slot(last_used).0 as u16)
+        if let Some(last) = self.last_free {
+            // Implies first free is Some as well
+            self.descriptors[last as usize].next = idx;
+            // last -> idx
+            self.last_free = Some(idx);
         } else {
-            None
+            // Implies first free is None as well
+            self.first_free = Some(idx);
+            self.last_free = Some(idx);
         }
+
+        self.free_count += 1;
     }
 
-    unsafe fn pop_used(&mut self, token: u16) -> Result<u32, Error> {
-        if self.is_used_empty() {
-            return Err(Error::QueueEmpty);
-        }
-
-        let last_used_slot = self.last_used_idx % self.capacity as u16;
-        let (index, len) = self.used.read_slot(last_used_slot);
-
-        if index != token as u32 {
-            return Err(Error::WrongToken);
-        }
-
-        self.free_descriptor_chain(token);
-
-        fence(Ordering::SeqCst);
-
-        self.last_used_idx = self.last_used_idx.wrapping_add(1);
-
-        Ok(len)
-    }
-
-    unsafe fn free_descriptor_chain(&mut self, head: u16) -> usize {
-        let mut current_node = Some(self.descriptor_table[usize::from(head)].assume_init_mut());
+    fn free_descriptor_chain(&mut self, first: u16) -> usize {
+        let mut current = first;
         let mut count = 0;
+        loop {
+            let descriptor = &self.descriptors[current as usize];
+            let next = if descriptor.flags & Descriptor::F_NEXT != 0 {
+                Some(descriptor.next)
+            } else {
+                None
+            };
 
-        while let Some(current) = current_node {
-            assert_ne!(current.len, 0);
-            let next_head = (current.flags & (1 << 0) != 0).then_some(current.next);
-
-            current.address = BusAddress::ZERO;
-            current.flags = 0;
-            current.next = 0;
-            current.len = 0;
-
-            self.used.used_count -= 1;
             count += 1;
+            self.add_free_descriptor(current);
 
-            current_node =
-                next_head.map(|head| self.descriptor_table[usize::from(head)].assume_init_mut());
+            if let Some(next) = next {
+                current = next;
+            } else {
+                break;
+            }
         }
-
-        self.free_head = head;
         count
     }
+}
 
-    pub fn msix_vector(&self) -> u16 {
-        self.msix_vector
+impl VqInner {
+    fn with_capacity(dma: &dyn DmaAllocator, capacity: usize) -> Result<Self, Error> {
+        let descriptors = DescriptorTable::with_capacity(dma, capacity)?;
+        let available = AvailableRing::with_capacity(dma, capacity, false)?;
+        let used = UsedRing::with_capacity(dma, capacity, false)?;
+
+        Ok(Self {
+            descriptors,
+            available,
+            used,
+        })
+    }
+
+    fn try_enqueue(
+        &mut self,
+        h2d: &[DmaSlice<u8>],
+        d2h: &[DmaSliceMut<MaybeUninit<u8>>],
+    ) -> Result<u16, Error> {
+        let head = self.descriptors.alloc_descriptors(h2d, d2h)?;
+        self.available.push(head);
+        Ok(head)
+    }
+
+    fn consume<F: FnMut(u16, u32)>(&mut self, free_notify: &QueueWaker, mut handler: F) -> usize {
+        self.used.consume(|head, len| {
+            log::trace!("vq: used #{head}, len={len}");
+
+            self.descriptors.free_descriptor_chain(head);
+            free_notify.wake_all();
+
+            handler(head, len);
+        })
+    }
+}
+
+impl VirtQueue<VqAsyncNotification> {
+    pub fn with_capacity(
+        dma: &dyn DmaAllocator,
+        index: u16,
+        capacity: usize,
+    ) -> Result<Self, Error> {
+        let used_notify = VqAsyncNotification::new();
+        Self::with_capacity_and_notify(dma, index, capacity, used_notify)
+    }
+
+    pub async fn enqueue_wait<F: FnOnce()>(
+        &self,
+        h2d: &[DmaSlice<'_, u8>],
+        d2h: &[DmaSliceMut<'_, MaybeUninit<u8>>],
+        notify_queue: F,
+    ) -> Result<u32, Error> {
+        let completion = self.enqueue(h2d, d2h).await?;
+        notify_queue();
+        let result = completion.wait_copy().await;
+        Ok(result)
+    }
+}
+
+impl VirtQueue<VqCallbackNotification> {
+    pub fn with_capacity_and_callback(
+        dma: &dyn DmaAllocator,
+        index: u16,
+        capacity: usize,
+        used_callback: Box<dyn Fn(u16, u32) + Sync + Send>,
+    ) -> Result<Self, Error> {
+        let used_notify = VqCallbackNotification(used_callback);
+        Self::with_capacity_and_notify(dma, index, capacity, used_notify)
+    }
+}
+
+impl VirtQueue<VqManualNotification> {
+    pub fn with_capacity_manual(
+        dma: &dyn DmaAllocator,
+        index: u16,
+        capacity: usize,
+    ) -> Result<Self, Error> {
+        let used_notify = VqManualNotification;
+        Self::with_capacity_and_notify(dma, index, capacity, used_notify)
+    }
+
+    pub fn handle_notify_manual<F: FnMut(u16, u32)>(&self, handler: F) -> usize {
+        self.inner
+            .lock()
+            .consume(&self.free_descriptor_notify, handler)
+    }
+
+    // Used when queue does not support device-side used notification
+    pub fn enqueue_blocking<F: FnOnce()>(
+        &self,
+        h2d: &[DmaSlice<u8>],
+        d2h: &[DmaSliceMut<MaybeUninit<u8>>],
+        notify_queue: F,
+    ) -> Result<u32, Error> {
+        let token = self.try_enqueue(h2d, d2h)?;
+        let mut length = 0;
+        notify_queue();
+        loop {
+            self.handle_notify_manual(|head, len| {
+                assert_eq!(head, token);
+                length = len;
+            });
+            if length != 0 {
+                break;
+            }
+            core::hint::spin_loop();
+        }
+        Ok(length)
+    }
+}
+
+impl<N: VqNotificationMechanism> VirtQueue<N> {
+    pub fn with_capacity_and_notify(
+        dma: &dyn DmaAllocator,
+        index: u16,
+        capacity: usize,
+        used_notify: N,
+    ) -> Result<Self, Error> {
+        let inner = VqInner::with_capacity(dma, capacity)?;
+        Ok(Self {
+            inner: IrqSafeSpinlock::new(inner),
+
+            index,
+            capacity,
+            used_notify,
+            free_descriptor_notify: QueueWaker::new(),
+        })
+    }
+
+    pub fn descriptor_table_base(&self) -> BusAddress {
+        self.inner.lock().descriptors.descriptors.bus_address()
+    }
+
+    pub fn available_ring_base(&self) -> BusAddress {
+        self.inner.lock().available.mapping.bus_address()
+    }
+
+    pub fn used_ring_base(&self) -> BusAddress {
+        self.inner.lock().used.mapping.bus_address()
+    }
+
+    pub fn capacity(&self) -> u16 {
+        self.capacity as u16
+    }
+
+    pub fn handle_notify(&self) -> usize {
+        self.inner
+            .lock()
+            .consume(&self.free_descriptor_notify, |head, len| {
+                self.used_notify.notify_used(head, len);
+            })
+    }
+
+    pub fn try_enqueue(
+        &self,
+        h2d: &[DmaSlice<'_, u8>],
+        d2h: &[DmaSliceMut<'_, MaybeUninit<u8>>],
+    ) -> Result<N::Token, Error> {
+        let head = self.inner.lock().try_enqueue(h2d, d2h)?;
+        Ok(self.used_notify.create_token(head))
+    }
+
+    pub async fn enqueue(
+        &self,
+        h2d: &[DmaSlice<'_, u8>],
+        d2h: &[DmaSliceMut<'_, MaybeUninit<u8>>],
+    ) -> Result<N::Token, Error> {
+        poll_fn(|cx| match self.try_enqueue(h2d, d2h) {
+            Err(Error::WouldBlock) => {
+                self.free_descriptor_notify.register(cx.waker());
+                Poll::Pending
+            }
+            result => {
+                self.free_descriptor_notify.remove(cx.waker());
+                Poll::Ready(result)
+            }
+        })
+        .await
+    }
+}
+
+impl VqAsyncNotification {
+    pub fn new() -> Self {
+        Self {
+            completions: IrqSafeSpinlock::new(DefaultHashTable::new()),
+        }
     }
 }

kernel/driver/virtio/core/src/transport/mod.rs

@@ -7,8 +7,12 @@ use tock_registers::{
     registers::WriteOnly,
 };
 
-use crate::{CommonConfiguration, DeviceStatus};
+use crate::{
+    queue::{VirtQueue, VqNotificationMechanism},
+    CommonConfiguration, DeviceStatus,
+};
 
+#[cfg(any(feature = "pci", rust_analyzer))]
 pub mod pci;
 
 pub trait Transport: Send {
@@ -53,7 +57,23 @@ pub trait Transport: Send {
         cfg.queue_size.get().into()
     }
 
-    fn set_queue(
+    fn set_queue<N: VqNotificationMechanism>(
+        &mut self,
+        index: u16,
+        queue: &VirtQueue<N>,
+        msix_vector: Option<u16>,
+    ) {
+        self.set_queue_raw(
+            index,
+            queue.capacity(),
+            queue.descriptor_table_base(),
+            queue.available_ring_base(),
+            queue.used_ring_base(),
+            msix_vector,
+        );
+    }
+
+    fn set_queue_raw(
         &mut self,
         queue: u16,
         capacity: u16,

kernel/driver/virtio/core/src/transport/pci.rs

@@ -40,7 +40,7 @@ impl Transport for PciTransport {
         true
     }
 
-    fn device_cfg(&self) -> Option<&DeviceMemoryIo<[u8]>> {
+    fn device_cfg(&self) -> Option<&DeviceMemoryIo<'_, [u8]>> {
         Some(&self.device_cfg)
     }
 

kernel/driver/virtio/gpu/Cargo.toml

@@ -15,6 +15,7 @@ ygg_driver_pci = { path = "../../bus/pci", optional = true }
 
 log.workspace = true
 bytemuck.workspace = true
+async-trait.workspace = true
 
 [features]
 default = []

kernel/driver/virtio/gpu/src/command.rs

@@ -7,8 +7,11 @@ use libk::{
     dma::{BusAddress, DmaBuffer},
     error::Error,
 };
-use libk_util::sync::IrqSafeSpinlockGuard;
-use ygg_driver_virtio_core::{queue::VirtQueue, transport::Transport};
+use libk_util::sync::IrqSafeSpinlock;
+use ygg_driver_virtio_core::{
+    queue::{VirtQueue, VqManualNotification},
+    transport::Transport,
+};
 
 #[derive(Clone, Copy, Pod, Zeroable)]
 #[repr(C)]
@@ -93,19 +96,12 @@ pub struct TransferToHost2d {
     pub _0: u32,
 }
 
-pub struct ControlLock<'a, T: Transport> {
-    control: IrqSafeSpinlockGuard<'a, VirtQueue>,
-    transport: IrqSafeSpinlockGuard<'a, T>,
+pub struct CommandExecution<'a, T: Transport> {
+    pub(super) transport: &'a IrqSafeSpinlock<T>,
+    pub(super) control: &'a VirtQueue<VqManualNotification>,
 }
 
-impl<'a, T: Transport> ControlLock<'a, T> {
-    pub const fn new(
-        control: IrqSafeSpinlockGuard<'a, VirtQueue>,
-        transport: IrqSafeSpinlockGuard<'a, T>,
-    ) -> Self {
-        Self { control, transport }
-    }
-
+impl<'a, T: Transport> CommandExecution<'a, T> {
     fn send_recv<'r, Req: Pod>(
         &mut self,
         dma: &dyn DmaAllocator,
@@ -116,20 +112,18 @@ impl<'a, T: Transport> ControlLock<'a, T> {
         let mut request = unsafe { DmaBuffer::assume_init_slice(request) };
         request.copy_from_slice(bytemuck::bytes_of(req));
 
-        let len = self
-            .control
-            .add_notify_wait_pop(&[buffer], &[&request], &mut *self.transport)
-            .inspect_err(|error| {
-                log::warn!("virtio queue: {error:?}");
-            })
-            .map_err(|_| Error::InvalidArgument)? as usize;
+        let len = self.control.enqueue_blocking(
+            &[request.slice(0..size_of::<Req>())],
+            &[buffer.slice_mut(0..buffer.len())],
+            || self.transport.lock().notify(0),
+        )? as usize;
 
         if len < size_of::<ControlHeader>() {
             log::warn!("virtio-gpu: invalid device response length: {len}");
             return Err(Error::InvalidArgument);
         }
 
-        let payload = unsafe { MaybeUninit::slice_assume_init_ref(&buffer[..len]) };
+        let payload = unsafe { buffer[..len].assume_init_ref() };
         let header = bytemuck::from_bytes(&payload[..size_of::<ControlHeader>()]);
         let data = &payload[size_of::<ControlHeader>()..len];
 

kernel/driver/virtio/gpu/src/lib.rs

@@ -6,7 +6,7 @@ extern crate alloc;
 use core::mem::MaybeUninit;
 
 use alloc::{sync::Arc, vec::Vec};
-use command::{ControlLock, ScanoutInfo};
+use command::{CommandExecution, ScanoutInfo};
 use device_api::{
     device::{Device, DeviceInitContext},
     dma::DmaAllocator,
@@ -23,7 +23,7 @@ use libk::{
 use libk_mm::{
     address::{AsPhysicalAddress, PhysicalAddress},
     table::MapAttributes,
-    PageProvider, L3_PAGE_SIZE,
+    OnDemandPage, PageProvider, VirtualPage, L3_PAGE_SIZE,
 };
 use libk_util::{
     sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock},
@@ -31,7 +31,7 @@ use libk_util::{
 };
 use ygg_driver_pci::{device::PciDeviceInfo, macros::pci_driver};
 use ygg_driver_virtio_core::{
-    queue::VirtQueue,
+    queue::{VirtQueue, VqManualNotification},
     transport::{pci::PciTransport, Transport},
     DeviceStatus,
 };
@@ -40,7 +40,7 @@ use yggdrasil_abi::error::Error;
 mod command;
 
 struct Queues {
-    control: IrqSafeSpinlock<VirtQueue>,
+    control: VirtQueue<VqManualNotification>,
 }
 
 struct Framebuffer {
@@ -64,7 +64,7 @@ struct Config {
 pub struct VirtioGpu<T: Transport> {
     transport: IrqSafeSpinlock<T>,
     #[allow(unused)]
-    pci_device_info: Option<PciDeviceInfo>,
+    pci_device_info: PciDeviceInfo,
 
     queues: OneTimeInit<Queues>,
     config: IrqSafeRwLock<Config>,
@@ -78,7 +78,7 @@ impl<T: Transport + 'static> VirtioGpu<T> {
     pub fn new(
         dma: Arc<dyn DmaAllocator>,
         transport: T,
-        info: Option<PciDeviceInfo>,
+        info: PciDeviceInfo,
     ) -> Result<Self, Error> {
         // Read num-scanouts from device config
         let Some(device_cfg) = transport.device_cfg() else {
@@ -149,34 +149,31 @@ impl<T: Transport + 'static> VirtioGpu<T> {
         transport.write_device_status(status | DeviceStatus::DRIVER_OK);
     }
 
-    fn setup_queues(&self) -> Result<(), Error> {
+    fn setup_queues(self: &Arc<Self>) -> Result<(), Error> {
         // TODO cursorq
         let mut transport = self.transport.lock();
 
-        let control = VirtQueue::with_max_capacity(&mut *transport, &*self.dma, 0, 128, None, true)
-            .map_err(|_| Error::InvalidArgument)?;
+        let control = VirtQueue::with_capacity_manual(&*self.dma, 0, 128)?;
+        transport.set_queue(0, &control, None);
 
-        self.queues.init(Queues {
-            control: IrqSafeSpinlock::new(control),
-        });
+        self.queues.init(Queues { control });
 
         Ok(())
     }
 
-    fn control(&self) -> ControlLock<T> {
-        let queues = self.queues.get();
-        let control = queues.control.lock();
-        let transport = self.transport.lock();
-
-        ControlLock::new(control, transport)
+    fn begin_command(&self) -> CommandExecution<'_, T> {
+        CommandExecution {
+            transport: &self.transport,
+            control: &self.queues.get().control,
+        }
     }
 
     fn setup_display(&self) -> Result<(), Error> {
-        let mut control = self.control();
         let mut config = self.config.write();
+        let mut command = self.begin_command();
 
         let scanouts =
-            control.query_scanouts(&*self.dma, self.num_scanouts, &mut config.response)?;
+            command.query_scanouts(&*self.dma, self.num_scanouts, &mut config.response)?;
         for (i, scanout) in scanouts.iter().enumerate() {
             log::info!(
                 "virtio-gpu: [{i}] {}x{} + {},{}",
@@ -214,23 +211,23 @@ impl<T: Transport + 'static> VirtioGpu<T> {
 
         let dma_buffer = DmaBuffer::new_uninit_slice(&*self.dma, size)?;
 
-        let mut control = self.control();
+        let mut command = self.begin_command();
 
-        let resource_id = control.create_resource_2d(
+        let resource_id = command.create_resource_2d(
             &*self.dma,
             &mut config.response,
             w,
             h,
             PixelFormat::R8G8B8A8,
         )?;
-        control.attach_backing(
+        command.attach_backing(
             &*self.dma,
             &mut config.response,
             resource_id,
             dma_buffer.bus_address(),
             size.try_into().unwrap(),
         )?;
-        control.set_scanout(
+        command.set_scanout(
             &*self.dma,
             &mut config.response,
             index as u32,
@@ -259,7 +256,7 @@ impl<T: Transport + 'static> VirtioGpu<T> {
         let framebuffer = config.framebuffer.as_ref().ok_or(Error::DoesNotExist)?;
         let r = config.scanouts[framebuffer.scanout_index].r;
 
-        let mut control = self.control();
+        let mut command = self.begin_command();
 
         if framebuffer.double {
             // Flip the buffer
@@ -267,8 +264,8 @@ impl<T: Transport + 'static> VirtioGpu<T> {
         } else {
             let resource_id = framebuffer.resource_id;
 
-            control.transfer_to_host_2d(&*self.dma, &mut config.response, resource_id, r)?;
-            control.resource_flush(&*self.dma, &mut config.response, resource_id, r)?;
+            command.transfer_to_host_2d(&*self.dma, &mut config.response, resource_id, r)?;
+            command.resource_flush(&*self.dma, &mut config.response, resource_id, r)?;
 
             Ok(())
         }
@@ -283,7 +280,6 @@ impl<T: Transport + 'static> Device for VirtioGpu<T> {
 
         // Set up some initial mode
         self.setup_display()?;
-
         self.setup_mode(0)?;
 
         DEVICE_REGISTRY.display.register(self.clone(), false)?;
@@ -297,7 +293,11 @@ impl<T: Transport + 'static> Device for VirtioGpu<T> {
 }
 
 impl<T: Transport + 'static> PageProvider for VirtioGpu<T> {
-    fn get_page(&self, offset: u64) -> Result<PhysicalAddress, Error> {
+    fn ondemand_fetch(&self, _opaque: u64) -> Result<OnDemandPage, Error> {
+        unreachable!()
+    }
+
+    fn get_page(&self, offset: u64) -> Result<VirtualPage, Error> {
         // TODO check that the page is mapped by framebuffer owner
         let config = self.config.read();
         let framebuffer = config.framebuffer.as_ref().ok_or(Error::DoesNotExist)?;
@@ -311,7 +311,16 @@ impl<T: Transport + 'static> PageProvider for VirtioGpu<T> {
         }
         let phys = unsafe { framebuffer.dma_buffer.as_physical_address() }.add(offset as usize);
 
-        Ok(phys)
+        Ok(VirtualPage::Immediate(phys))
+    }
+
+    fn release_page(
+        &self,
+        _offset: u64,
+        _phys: PhysicalAddress,
+        _dirty: bool,
+    ) -> Result<(), Error> {
+        Ok(())
     }
 
     fn clone_page(
@@ -320,11 +329,7 @@ impl<T: Transport + 'static> PageProvider for VirtioGpu<T> {
         _src_phys: PhysicalAddress,
         _src_attrs: MapAttributes,
     ) -> Result<PhysicalAddress, Error> {
-        todo!()
-    }
-
-    fn release_page(&self, _offset: u64, _phys: PhysicalAddress) -> Result<(), Error> {
-        Ok(())
+        unimplemented!()
     }
 }
 
@@ -427,7 +432,7 @@ pci_driver! {
                     log::error!("Couldn't set up PCI virtio transport: {error:?}");
                 })
                 .map_err(|_| Error::InvalidArgument)?;
-            let device = VirtioGpu::new(dma.clone(), transport, Some(info.clone()))?;
+            let device = VirtioGpu::new(dma.clone(), transport, info.clone())?;
             let device = Arc::new(device);
 
             Ok(device)

kernel/driver/virtio/net/Cargo.toml

@@ -18,6 +18,7 @@ log.workspace = true
 bitflags.workspace = true
 tock-registers.workspace = true
 bytemuck.workspace = true
+futures-util.workspace = true
 
 [features]
 default = []

kernel/driver/virtio/net/src/lib.rs

@@ -5,16 +5,19 @@ extern crate alloc;
 
 use core::mem::{size_of, MaybeUninit};
 
-use alloc::{collections::BTreeMap, sync::Arc};
+use alloc::{boxed::Box, sync::Arc};
 use bytemuck::{Pod, Zeroable};
 use device_api::{
     device::{Device, DeviceInitContext},
     dma::DmaAllocator,
     interrupt::{InterruptAffinity, InterruptHandler, IrqVector},
 };
-use libk::dma::DmaBuffer;
+use futures_util::task::AtomicWaker;
+use libk::{dma::DmaBuffer, task::runtime};
 use libk_util::{
-    sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock, IrqSafeSpinlockGuard},
+    event::BitmapEvent,
+    hash_table::DefaultHashTable,
+    sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock},
     OneTimeInit,
 };
 use ygg_driver_net_core::{
@@ -26,28 +29,25 @@ use ygg_driver_pci::{
     macros::pci_driver,
 };
 use ygg_driver_virtio_core::{
-    queue::VirtQueue,
+    queue::{VirtQueue, VqCallbackNotification, VqManualNotification},
     transport::{pci::PciTransport, Transport},
     DeviceStatus,
 };
 use yggdrasil_abi::{error::Error, net::MacAddress};
 
-struct Queues {
-    receive: IrqSafeSpinlock<VirtQueue>,
-    transmit: IrqSafeSpinlock<VirtQueue>,
-}
-
 pub struct VirtioNet<T: Transport> {
     transport: IrqSafeSpinlock<T>,
-    queues: OneTimeInit<Queues>,
-    interface_id: OneTimeInit<u32>,
-
-    mac: IrqSafeRwLock<MacAddress>,
-
-    pending_packets: IrqSafeRwLock<BTreeMap<u16, DmaBuffer<[MaybeUninit<u8>]>>>,
+    pci_device_info: PciDeviceInfo,
     dma: Arc<dyn DmaAllocator>,
 
-    pci_device_info: Option<PciDeviceInfo>,
+    interface_id: OneTimeInit<u32>,
+    mac: IrqSafeRwLock<MacAddress>,
+
+    rx_queue: OneTimeInit<VirtQueue<VqManualNotification>>,
+    tx_queue: OneTimeInit<VirtQueue<VqCallbackNotification>>,
+    tx_in_flight: IrqSafeSpinlock<DefaultHashTable<u16, DmaBuffer<[u8]>>>,
+
+    softirq: BitmapEvent<AtomicWaker>,
 }
 
 #[derive(Clone, Copy, Debug, Pod, Zeroable)]
@@ -61,23 +61,15 @@ struct VirtioPacketHeader {
     csum_offset: u16,
 }
 
-impl Queues {
-    pub fn try_receive(&self, _index: usize) -> Option<(u16, IrqSafeSpinlockGuard<VirtQueue>)> {
-        let mut queue = self.receive.lock();
-        // TODO use len for packet size hint
-        let (token, _len) = queue.pop_last_used()?;
-        Some((token, queue))
-    }
-}
-
 impl<T: Transport + 'static> VirtioNet<T> {
-    const PACKET_SIZE: usize = 4096;
+    const VQ_RX: u16 = 0;
+    const VQ_TX: u16 = 1;
 
     pub fn new(
         dma: Arc<dyn DmaAllocator>,
         transport: T,
-        pci_device_info: Option<PciDeviceInfo>,
-    ) -> Self {
+        pci_device_info: PciDeviceInfo,
+    ) -> Result<Self, Error> {
         // Read MAC from device config
         let device_cfg = transport
             .device_cfg()
@@ -86,60 +78,21 @@ impl<T: Transport + 'static> VirtioNet<T> {
         mac_bytes.copy_from_slice(&device_cfg[..6]);
         let mac = MacAddress::from(mac_bytes);
 
-        Self {
+        pci_device_info.init_interrupts(PreferredInterruptMode::Msi(true))?;
+
+        Ok(Self {
             transport: IrqSafeSpinlock::new(transport),
-            queues: OneTimeInit::new(),
-            interface_id: OneTimeInit::new(),
-
-            mac: IrqSafeRwLock::new(mac),
-
-            pending_packets: IrqSafeRwLock::new(BTreeMap::new()),
-
             pci_device_info,
             dma,
-        }
-    }
 
-    pub fn listen(&self, buffers: usize) {
-        let queues = self.queues.get();
-        let mut queue = queues.receive.lock();
-        let mut packets = self.pending_packets.write();
+            interface_id: OneTimeInit::new(),
+            mac: IrqSafeRwLock::new(mac),
 
-        for _ in 0..buffers {
-            let mut packet = DmaBuffer::new_uninit_slice(&*self.dma, Self::PACKET_SIZE).unwrap();
-            let token = unsafe { queue.add(&[&mut packet], &[]).unwrap() };
-            packets.insert(token, packet);
-        }
-
-        let mut transport = self.transport.lock();
-        transport.notify(0);
-    }
-
-    fn handle_receive_interrupt(&self, queue: usize) -> bool {
-        let queues = self.queues.get();
-        let interface_id = *self.interface_id.get();
-        let mut count = 0;
-
-        while let Some((token, mut queue)) = queues.try_receive(queue) {
-            let mut pending_packets = self.pending_packets.write();
-            let packet = pending_packets.remove(&token).unwrap();
-
-            let mut buffer = DmaBuffer::new_uninit_slice(&*self.dma, Self::PACKET_SIZE).unwrap();
-
-            let token = unsafe { queue.add(&[&mut buffer], &[]).unwrap() };
-            pending_packets.insert(token, buffer);
-
-            let packet = unsafe { DmaBuffer::assume_init_slice(packet) };
-            let packet = RxPacket::new(packet, size_of::<VirtioPacketHeader>(), interface_id);
-            ygg_driver_net_core::receive_packet(packet).unwrap();
-            count += 1
-        }
-
-        if count != 0 {
-            self.transport.lock().notify(0);
-        }
-
-        count != 0
+            rx_queue: OneTimeInit::new(),
+            tx_queue: OneTimeInit::new(),
+            tx_in_flight: IrqSafeSpinlock::new(DefaultHashTable::new()),
+            softirq: BitmapEvent::new(AtomicWaker::new()),
+        })
     }
 
     fn begin_init(&self) -> Result<DeviceStatus, Error> {
@@ -178,46 +131,87 @@ impl<T: Transport + 'static> VirtioNet<T> {
         transport.write_device_status(status | DeviceStatus::DRIVER_OK);
     }
 
-    unsafe fn setup_queues(
-        self: Arc<Self>,
-        receive_count: usize,
-        transmit_count: usize,
-    ) -> Result<(), Error> {
-        let receive_vector = if let Some(pci) = self.pci_device_info.as_ref() {
-            pci.init_interrupts(PreferredInterruptMode::Msi(true))?;
-            let info = pci.map_interrupt(InterruptAffinity::Any, self.clone())?;
-
-            info.map(|info| info.vector as u16)
+    unsafe fn setup_queues(self: &Arc<Self>) -> Result<(), Error> {
+        let (rx_vector, tx_vector) = if let Ok(msis) =
+            self.pci_device_info
+                .map_interrupt_multiple(0..2, InterruptAffinity::Any, self.clone())
+        {
+            // Bound a MSI(-x) range, use per-queue vectors
+            (Some(msis[0].vector as u16), Some(msis[1].vector as u16))
         } else {
-            None
+            // TODO support non-MSI-x/non-multivec setups
+            self.pci_device_info
+                .map_interrupt(InterruptAffinity::Any, self.clone())?;
+            (None, None)
         };
 
-        // TODO multiqueue capability
-        assert_eq!(receive_count, 1);
-        assert_eq!(transmit_count, 1);
+        // Setup a callback to remove pending buffers and pass them to the network stack
+        let rx_queue = VirtQueue::with_capacity_manual(&*self.dma, Self::VQ_RX, 64)?;
+
+        let p = self.clone();
+        // Setup a callback to remove buffers from "in flight owned buffers" list
+        let tx_queue = VirtQueue::with_capacity_and_callback(
+            &*self.dma,
+            Self::VQ_TX,
+            64,
+            Box::new(move |head, _| {
+                p.tx_in_flight.lock().remove(&head);
+            }),
+        )?;
+
+        let rx_queue = self.rx_queue.init(rx_queue);
+        let tx_queue = self.tx_queue.init(tx_queue);
 
         let mut transport = self.transport.lock();
 
-        // Setup the virtqs
-        let rx = VirtQueue::with_max_capacity(
-            &mut *transport,
-            &*self.dma,
-            0,
-            128,
-            receive_vector,
-            false,
-        )
-        .map_err(cvt_error)?;
-        let tx = VirtQueue::with_max_capacity(&mut *transport, &*self.dma, 1, 128, None, true)
-            .map_err(cvt_error)?;
-
-        self.queues.init(Queues {
-            receive: IrqSafeSpinlock::new(rx),
-            transmit: IrqSafeSpinlock::new(tx),
-        });
+        transport.set_queue(Self::VQ_RX, rx_queue, rx_vector);
+        transport.set_queue(Self::VQ_TX, tx_queue, tx_vector);
 
         Ok(())
     }
+
+    async fn softirq(&self) -> Result<(), Error> {
+        const RX_SIZE: usize = 4096;
+        const RX_IN_FLIGHT: usize = 32;
+
+        let rx_queue = self.rx_queue.get();
+        let tx_queue = self.tx_queue.get();
+        let nic = *self.interface_id.get();
+
+        let mut rx_in_flight = DefaultHashTable::<u16, DmaBuffer<[MaybeUninit<u8>]>>::new();
+
+        // Setup initial Rx set
+        for _ in 0..RX_IN_FLIGHT {
+            let mut buffer = DmaBuffer::new_uninit_slice(&*self.dma, RX_SIZE)?;
+            let token = rx_queue.try_enqueue(&[], &[buffer.slice_mut(0..RX_SIZE)])?;
+            rx_in_flight.insert(token, buffer);
+        }
+
+        loop {
+            let events = self.softirq.wait().await;
+
+            if events & (1 << Self::VQ_RX) != 0 {
+                let refill_rx = rx_queue.handle_notify_manual(|head, _| {
+                    if let Some(packet) = rx_in_flight.remove(&head) {
+                        let packet = unsafe { DmaBuffer::assume_init_slice(packet) };
+                        let packet = RxPacket::new(packet, size_of::<VirtioPacketHeader>(), nic);
+                        ygg_driver_net_core::receive_packet(packet).ok();
+                    }
+                });
+
+                // Refill Rx buffers
+                for _ in 0..refill_rx {
+                    let mut buffer = DmaBuffer::new_uninit_slice(&*self.dma, RX_SIZE)?;
+                    let token = rx_queue.try_enqueue(&[], &[buffer.slice_mut(0..RX_SIZE)])?;
+                    rx_in_flight.insert(token, buffer);
+                }
+            }
+
+            if events & (1 << Self::VQ_TX) != 0 {
+                tx_queue.handle_notify();
+            }
+        }
+    }
 }
 
 impl<T: Transport + 'static> NetworkDevice for VirtioNet<T> {
@@ -226,13 +220,13 @@ impl<T: Transport + 'static> NetworkDevice for VirtioNet<T> {
     }
 
     fn transmit_buffer(&self, mut packet: DmaBuffer<[u8]>) -> Result<(), Error> {
-        let queues = self.queues.get();
-        let mut tx = queues.transmit.lock();
-        let mut transport = self.transport.lock();
+        let tx_queue = self.tx_queue.get();
         packet[..size_of::<VirtioPacketHeader>()].fill(0);
-        let _len = tx
-            .add_notify_wait_pop(&[], &[&packet], &mut *transport)
-            .unwrap();
+        let token = tx_queue.try_enqueue(&[packet.slice(0..packet.len())], &[])?;
+        // Add the packet to "in flight" list to make sure it doesn't get dropped and invalidated
+        // immediately after returning from this function
+        self.tx_in_flight.lock().insert(token, packet);
+        self.transport.lock().notify(Self::VQ_TX);
         Ok(())
     }
 
@@ -248,19 +242,14 @@ impl<T: Transport + 'static> NetworkDevice for VirtioNet<T> {
 impl<T: Transport + 'static> InterruptHandler for VirtioNet<T> {
     fn handle_irq(self: Arc<Self>, vector: IrqVector) -> bool {
         match vector {
-            IrqVector::Msi(_) => {
-                // MSI/MSI-X
-                self.handle_receive_interrupt(0)
+            IrqVector::Msi(vector) => {
+                self.softirq.signal(1 << vector);
+                true
             }
+            // TODO non-multivec/legacy IRQ setup
             IrqVector::Irq(_) => {
-                // Legacy IRQ
-                let (queue_irq, config_irq) = self.transport.lock().read_interrupt_status();
-
-                if queue_irq {
-                    self.handle_receive_interrupt(0);
-                }
-
-                queue_irq || config_irq
+                self.softirq.signal((1 << Self::VQ_RX) | (1 << Self::VQ_TX));
+                true
             }
         }
     }
@@ -274,15 +263,16 @@ impl<T: Transport + 'static> Device for VirtioNet<T> {
     unsafe fn init(self: Arc<Self>, _cx: DeviceInitContext) -> Result<(), Error> {
         let status = self.begin_init()?;
 
-        // TODO multiqueue
-        self.clone().setup_queues(1, 1)?;
+        self.setup_queues()?;
 
         self.finish_init(status);
 
         let iface =
             ygg_driver_net_core::register_interface(NetworkInterfaceType::Ethernet, self.clone());
         self.interface_id.init(iface.id());
-        self.listen(64);
+
+        let p = self.clone();
+        runtime::spawn(async move { p.softirq().await })?;
 
         Ok(())
     }
@@ -292,14 +282,6 @@ impl<T: Transport + 'static> Device for VirtioNet<T> {
     }
 }
 
-fn cvt_error(error: ygg_driver_virtio_core::error::Error) -> Error {
-    use ygg_driver_virtio_core::error::Error as VirtioError;
-    match error {
-        VirtioError::OsError(err) => err,
-        _ => Error::InvalidOperation,
-    }
-}
-
 pci_driver! {
     matches: [device (0x1AF4:0x1000)],
     driver: {
@@ -311,7 +293,7 @@ pci_driver! {
             let space = &info.config_space;
 
             let transport = PciTransport::from_config_space(space).unwrap();
-            let device = VirtioNet::new(dma.clone(), transport, Some(info.clone()));
+            let device = VirtioNet::new(dma.clone(), transport, info.clone())?;
 
             let device = Arc::new(device);
 

kernel/lib/device-tree/src/tree.rs

@@ -78,7 +78,7 @@ impl<'a> DeviceTree<'a> {
     }
 
     /// Returns the root node of this device tree
-    pub fn root(&self) -> TNode {
+    pub fn root(&self) -> TNode<'_> {
         self.index.root()
     }
 
@@ -172,12 +172,12 @@ impl<'a> DeviceTree<'a> {
     }
 
     /// Returns an iterator over the memory regions specified by this device tree
-    pub fn memory_regions(&self) -> DeviceTreeMemoryRegionIter {
+    pub fn memory_regions(&self) -> DeviceTreeMemoryRegionIter<'_> {
         DeviceTreeMemoryRegionIter::new(self)
     }
 
     /// Returns an iterator over the reserved memory regions specified by this device tree
-    pub fn reserved_regions(&self) -> DeviceTreeReservedRegionIter {
+    pub fn reserved_regions(&self) -> DeviceTreeReservedRegionIter<'_> {
         DeviceTreeReservedRegionIter::new(self)
     }
 

kernel/lib/memtables/Cargo.toml

@@ -1,13 +0,0 @@
-[package]
-name = "memtables"
-version = "0.1.0"
-edition = "2021"
-authors = ["Mark Poliakov <mark@alnyan.me>"]
-
-[dependencies]
-bitflags.workspace = true
-bytemuck.workspace = true
-
-[features]
-default = []
-all = []

kernel/lib/memtables/src/aarch64.rs

@@ -1,26 +0,0 @@
-use bytemuck::{Pod, Zeroable};
-
-use crate::RawTable;
-
-pub const KERNEL_L3_COUNT: usize = 8;
-
-#[derive(Clone, Copy, Pod, Zeroable)]
-#[repr(C)]
-pub struct FixedTables {
-    // 1GiB entries
-    pub l1: RawTable,
-
-    // 2MiB entries
-    pub l2: RawTable,
-    pub l3s: [RawTable; KERNEL_L3_COUNT],
-}
-
-impl FixedTables {
-    pub const fn zeroed() -> Self {
-        Self {
-            l1: RawTable::zeroed(),
-            l2: RawTable::zeroed(),
-            l3s: [RawTable::zeroed(); KERNEL_L3_COUNT],
-        }
-    }
-}