WIP usb: better driver structure, hid mouse driver

Cargo.lock

@@ -532,12 +532,6 @@ dependencies = [
  "hashbrown 0.14.5",
 ]
 
-[[package]]
-name = "elf"
-version = "0.7.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4445909572dbd556c457c849c4ca58623d84b27c8fff1e74b0b4227d8b90d17b"
-
 [[package]]
 name = "encode_unicode"
 version = "1.0.0"
@@ -741,18 +735,6 @@ dependencies = [
  "pin-utils",
 ]
 
-[[package]]
-name = "gentables"
-version = "0.1.0"
-dependencies = [
- "bitflags 2.8.0",
- "bytemuck",
- "clap",
- "elf 0.7.4",
- "memtables",
- "thiserror",
-]
-
 [[package]]
 name = "getrandom"
 version = "0.2.15"
@@ -1135,7 +1117,7 @@ dependencies = [
  "device-api",
  "kernel-arch-interface",
  "libk-mm-interface",
- "memtables",
+ "log",
  "static_assertions",
  "tock-registers",
  "yggdrasil-abi",
@@ -1184,7 +1166,6 @@ dependencies = [
  "kernel-arch-interface",
  "libk-mm-interface",
  "log",
- "memtables",
  "static_assertions",
  "tock-registers",
  "yggdrasil-abi",
@@ -1212,7 +1193,6 @@ dependencies = [
  "kernel-arch-x86",
  "libk-mm-interface",
  "log",
- "memtables",
  "static_assertions",
  "tock-registers",
  "yggdrasil-abi",
@@ -1270,7 +1250,7 @@ dependencies = [
  "cfg-if",
  "crossbeam-queue",
  "device-api",
- "elf 0.7.2",
+ "elf",
  "futures-util",
  "kernel-arch",
  "libc",
@@ -1465,14 +1445,6 @@ dependencies = [
  "autocfg",
 ]
 
-[[package]]
-name = "memtables"
-version = "0.1.0"
-dependencies = [
- "bitflags 2.8.0",
- "bytemuck",
-]
-
 [[package]]
 name = "miniz_oxide"
 version = "0.8.4"
@@ -2974,7 +2946,7 @@ dependencies = [
  "device-api",
  "device-api-macros",
  "device-tree",
- "elf 0.7.2",
+ "elf",
  "ext2",
  "futures-util",
  "git-version",
@@ -2990,7 +2962,6 @@ dependencies = [
  "libk-util",
  "log",
  "memfs",
- "memtables",
  "prettyplease",
  "static_assertions",
  "tock-registers",

Cargo.toml

@@ -11,7 +11,6 @@ exclude = [
 ]
 members = [
     "xtask",
-    "kernel/tools/gentables",
     "kernel",
     "lib/abi",
     "lib/libyalloc",
@@ -67,7 +66,6 @@ libk-util.path = "kernel/libk/libk-util"
 libk-mm.path = "kernel/libk/libk-mm"
 libk-mm-interface.path = "kernel/libk/libk-mm/interface"
 libk-device.path = "kernel/libk/libk-device"
-memtables.path = "kernel/lib/memtables"
 vmalloc.path = "kernel/lib/vmalloc"
 device-api-macros.path = "kernel/lib/device-api/macros"
 device-tree.path = "kernel/lib/device-tree"

boot/yboot/src/elf.rs

@@ -1,7 +1,8 @@
 use core::mem::size_of;
 
 use bytemuck::Zeroable;
-use log::{debug, error, info};
+use log::{error, info};
+use types::{Rela, SHT_RELA};
 // TODO use 'elf' crate
 use uefi::{
     prelude::BootServices,
@@ -23,14 +24,18 @@ mod types {
     pub type Half = u16;
     pub type Word = u32;
     pub type XWord = u64;
+    pub type SXWord = i64;
 
     pub const PT_LOAD: Word = 1;
 
     pub const SHT_PROGBITS: Word = 1;
+    pub const SHT_RELA: Word = 4;
 
     pub const SHF_WRITE: XWord = 1 << 0;
     pub const SHF_ALLOC: XWord = 1 << 1;
 
+    pub const R_X86_64_RELATIVE: u32 = 8;
+
     #[derive(Clone, Copy, Zeroable, Pod)]
     #[repr(C)]
     pub struct Ehdr {
@@ -77,6 +82,20 @@ mod types {
         pub memsz: XWord,
         pub align: XWord,
     }
+
+    #[derive(Clone, Copy, Zeroable, Pod)]
+    #[repr(C)]
+    pub struct Rela {
+        pub offset: Addr,
+        pub info: XWord,
+        pub addend: SXWord,
+    }
+
+    impl Rela {
+        pub fn r_type(&self) -> u32 {
+            self.info as u32
+        }
+    }
 }
 
 // Maximum address this loader can map in the target kernel
@@ -92,6 +111,8 @@ pub struct LoadedObject {
     pub image_start: u64,
     pub image_end: u64,
 
+    pub load_address: u64,
+
     pub entry: u64,
 
     pub protocol_struct_paddr: u64,
@@ -105,6 +126,12 @@ struct LocatedProtocol {
     size: usize,
 }
 
+struct RelaSection {
+    offset: u64,
+    entry_count: usize,
+    entry_size: usize,
+}
+
 trait ReadExact {
     fn read_exact(&mut self, buf: &mut [u8]) -> Result<(), Error>;
 }
@@ -119,6 +146,23 @@ impl ReadExact for RegularFile {
     }
 }
 
+impl RelaSection {
+    pub fn from_shdr(shdr: &Shdr) -> Option<Self> {
+        if shdr.type_ != SHT_RELA {
+            return None;
+        }
+
+        let entry_size = shdr.entsize as usize;
+        let entry_count = shdr.size as usize / entry_size;
+
+        Some(Self {
+            offset: shdr.offset,
+            entry_size,
+            entry_count,
+        })
+    }
+}
+
 impl Object {
     pub fn open<D: File>(root: &mut D, path: &CStr16) -> Result<Self, Error> {
         let file = root.open(path, FileMode::Read, FileAttribute::empty())?;
@@ -140,11 +184,11 @@ impl Object {
             return Err(Error::new(Status::LOAD_ERROR, ()));
         }
 
-        // Check that the entry point is set
+        // // Check that the entry point is set
-        if ehdr.entry == 0 {
+        // if ehdr.entry == 0 {
-            error!("Image does not have a valid entry point");
+        //     error!("Image does not have a valid entry point");
-            return Err(Error::new(Status::LOAD_ERROR, ()));
+        //     return Err(Error::new(Status::LOAD_ERROR, ()));
-        }
+        // }
 
         Ok(Self { file, ehdr })
     }
@@ -170,11 +214,6 @@ impl Object {
         self.file
             .read_exact(bytemuck::bytes_of_mut(&mut proto_data))?;
 
-        info!(
-            "Kernel is virtually mapped at {:#x}",
-            proto_data.kernel_virt_offset
-        );
-
         // 2. Find the kernel's range and check that the loaded physical addresses are actually
         //    usable from UEFI
         let mut image_start = u64::MAX;
@@ -209,16 +248,22 @@ impl Object {
         assert_eq!(image_start & 0xFFF, 0);
         assert_eq!(image_end & 0xFFF, 0);
 
-        info!("Image start: {:#x}, end: {:#x}", image_start, image_end);
+        // Allocate memory to load the kernel into
-        // Reserve the kernel memory
+        let kernel_load_address = bs
-        let reserved_addr = bs
             .allocate_pages(
-                AllocateType::Address(image_start),
+                AllocateType::MaxAddress(0xFFFFFFFF),
                 MemoryType::LOADER_DATA,
                 (image_end - image_start) as usize / 0x1000,
             )
-            .expect("Could not allocate memory for kernel image");
+            .expect("Could not allocate memory for the kernel");
-        assert_eq!(reserved_addr, image_start);
+
+        // Print info
+        info!("Image start: {:#x}, end: {:#x}", image_start, image_end);
+        info!(
+            "Kernel virtual offset: {:#x}",
+            proto_data.kernel_virt_offset
+        );
+        info!("Kernel load address: {kernel_load_address:#x}");
 
         // 3. Load the segments
         for i in 0..self.ehdr.phnum {
@@ -228,53 +273,80 @@ impl Object {
                 continue;
             }
 
+            let segment_load_base = phdr.paddr + kernel_load_address;
+
             info!(
-                "Load segment {}: {:#x?}",
+                "[{i}] Load {:#x?}",
-                i,
+                segment_load_base..segment_load_base + phdr.memsz
-                phdr.paddr..phdr.paddr + phdr.memsz
             );
 
             if phdr.filesz > 0 {
-                // The section has load data
+                let dst_slice = unsafe {
-                let dst = unsafe {
+                    core::slice::from_raw_parts_mut(
-                    core::slice::from_raw_parts_mut(phdr.paddr as *mut u8, phdr.filesz as usize)
+                        segment_load_base as *mut u8,
+                        phdr.filesz as usize,
+                    )
                 };
 
-                debug!(
-                    "Load {:#x?} from ELF offset {:#x}",
-                    phdr.paddr..phdr.paddr + phdr.filesz,
-                    phdr.offset
-                );
                 self.file.set_position(phdr.offset)?;
-                self.file.read_exact(dst)?;
+                self.file.read_exact(dst_slice)?;
             }
 
             if phdr.memsz > phdr.filesz {
-                let dst = unsafe {
+                let dst_slice = unsafe {
                     core::slice::from_raw_parts_mut(
-                        (phdr.paddr + phdr.filesz) as *mut u8,
+                        (segment_load_base + phdr.filesz) as *mut u8,
                         (phdr.memsz - phdr.filesz) as usize,
                     )
                 };
 
-                debug!(
+                dst_slice.fill(0);
-                    "Zero data {:#x?}",
+            }
-                    phdr.paddr + phdr.filesz..phdr.paddr + phdr.memsz
+        }
-                );
 
-                dst.fill(0);
+        // 4. Perform kernel relocation
+        let mut rela_section = None;
+        for i in 0..self.ehdr.shnum as usize {
+            let shdr = self.read_shdr(i)?;
+            if let Some(rela) = RelaSection::from_shdr(&shdr) {
+                rela_section = Some(rela);
+                break;
+            }
+        }
+
+        if let Some(rela_section) = rela_section {
+            info!("Relocating kernel: {image_start:#x} -> {kernel_load_address:#x}");
+            info!("({} relocations)", rela_section.entry_count);
+
+            let b = (kernel_load_address + proto_data.kernel_virt_offset) as i64;
+
+            for i in 0..rela_section.entry_count {
+                let mut rela = Rela::zeroed();
+                self.file
+                    .set_position(rela_section.offset + (i * rela_section.entry_size) as u64)?;
+                self.file.read_exact(bytemuck::bytes_of_mut(&mut rela))?;
+
+                match rela.r_type() {
+                    types::R_X86_64_RELATIVE => {
+                        let qword = (rela.offset + kernel_load_address) as *mut i64;
+                        let value = rela.addend + b;
+                        unsafe { qword.write_volatile(value) };
+                    }
+                    other => todo!("Unsupported relocation type: {other}"),
+                }
             }
         }
 
         // Now that the image is in memory, protocol structure can be written in the further steps
-        let protocol_struct_paddr = (loc_proto.address as u64) - proto_data.kernel_virt_offset;
+        let protocol_struct_paddr = loc_proto.address as u64 + kernel_load_address; // (loc_proto.address as u64) - proto_data.kernel_virt_offset;
         let protocol_version = proto_data.header.version;
 
-        let entry = self.ehdr.entry;
+        let entry = self.ehdr.entry + kernel_load_address;
 
         Ok(LoadedObject {
             image_start,
             image_end,
+            load_address: kernel_load_address,
             entry,
             protocol_struct_paddr,
             protocol_version,

boot/yboot/src/main.rs

@@ -93,7 +93,7 @@ fn locate_rsdp(st: &SystemTable<Boot>) -> Option<u64> {
 fn boot_partition(
     image: Handle,
     bs: &BootServices,
-) -> Result<ScopedProtocol<SimpleFileSystem>, Error> {
+) -> Result<ScopedProtocol<'_, SimpleFileSystem>, Error> {
     let loaded_image = bs.open_protocol_exclusive::<LoadedImage>(image)?;
     let device_handle = loaded_image.device();
 
@@ -113,7 +113,7 @@ fn load_kernel<'a>(
     config: &Config,
     root: &mut Directory,
     st: &SystemTable<Boot>,
-) -> Result<(u64, u64, &'a mut LoadProtocolV1), Error> {
+) -> Result<(u64, u64, u64, &'a mut LoadProtocolV1), Error> {
     let bs = st.boot_services();
 
     let mut kernel_obj = Object::open(root, cstr16!("kernel.elf"))?;
@@ -183,13 +183,14 @@ fn load_kernel<'a>(
 
     let entry = loaded_obj.entry + proto_data.kernel_virt_offset;
 
-    Ok((entry, mmap_memory, proto_data))
+    Ok((entry, loaded_obj.load_address, mmap_memory, proto_data))
 }
 
 unsafe fn map_and_enter_kernel(
     st: SystemTable<Boot>,
     proto_data: &mut LoadProtocolV1,
     mmap_memory: u64,
+    load_base: u64,
     entry: u64,
 ) -> ! {
     let (_, mmap) = st.exit_boot_services();
@@ -216,7 +217,7 @@ unsafe fn map_and_enter_kernel(
     let cr3 = mem::map_image();
     asm!("cli; wbinvd; mov {0}, %cr3", in(reg) cr3, options(att_syntax));
 
-    asm!("jmp *{0}", in(reg) entry, in("eax") LOADER_MAGIC, options(noreturn, att_syntax));
+    asm!("jmp *{0}", in(reg) entry, in("eax") LOADER_MAGIC, in("ecx") load_base, options(noreturn, att_syntax));
 }
 
 #[entry]
@@ -243,15 +244,16 @@ fn efi_main(image_handle: Handle, mut system_table: SystemTable<Boot>) -> Status
         }
     };
 
-    let (entry, mmap_memory, proto_data) = match load_kernel(&config, &mut root, &system_table) {
+    let (entry, load_base, mmap_memory, proto_data) =
-        Ok(e) => e,
+        match load_kernel(&config, &mut root, &system_table) {
-        Err(error) => {
+            Ok(e) => e,
-            error!("Failed to load the kernel/initrd: {error:?}");
+            Err(error) => {
-            return Status::LOAD_ERROR;
+                error!("Failed to load the kernel/initrd: {error:?}");
-        }
+                return Status::LOAD_ERROR;
-    };
+            }
+        };
 
     unsafe {
-        map_and_enter_kernel(system_table, proto_data, mmap_memory, entry);
+        map_and_enter_kernel(system_table, proto_data, mmap_memory, load_base, entry);
     }
 }

boot/yboot/src/mem.rs

@@ -66,7 +66,7 @@ impl MemoryDescriptorExt for MemoryDescriptor {
     }
 }
 
-pub fn memory_map(bs: &BootServices) -> Result<MemoryMap, Error> {
+pub fn memory_map(bs: &BootServices) -> Result<MemoryMap<'_>, Error> {
     bs.memory_map(unsafe { &mut MMAP_BUFFER.data })
 }
 

etc/aarch64-unknown-none.json

@@ -13,6 +13,7 @@
   "panic-strategy": "abort",
   "dynamic-linking": true,
   "relocation-model": "pic",
+  "position-independent-executables": true,
   "eh-frame-header": false,
 
   "linker": "rust-lld",

etc/gdb/aarch64-virt.gdb

@@ -0,0 +1,4 @@
+# Lower-half
+symbol-file -o 0x40080000 target/aarch64-unknown-none/release/yggdrasil-kernel
+symbol-file -o 0xFFFFFF8040080000 target/aarch64-unknown-none/release/yggdrasil-kernel
+target remote :1234

etc/ld/arm/aarch64-unknown-none.ld

@@ -0,0 +1,67 @@
+ENTRY(__aarch64_entry);
+
+/* KERNEL_PHYS_BASE = 0x40080000; */
+KERNEL_VIRT_OFFSET = 0xFFFFFF8000000000;
+
+SECTIONS {
+    /* . = KERNEL_PHYS_BASE; */
+    . = 0x0;
+
+    PROVIDE(__kernel_start = .);
+
+    /* .text.entry : {
+        *(.text.entry)
+    }
+
+    . = ALIGN(16);
+    . = . + KERNEL_VIRT_OFFSET; */
+
+    .text : {
+        KEEP(*(.text.entry));
+        KEEP(*(.text.vectors));
+        *(.text*)
+    }
+
+    . = ALIGN(4K);
+    .rodata : {
+        *(.rodata*)
+        *(.eh_frame*)
+
+        . = ALIGN(16);
+        PROVIDE(__init_array_start = .);
+        KEEP(*(.init_array*))
+        PROVIDE(__init_array_end = .);
+    }
+
+    . = ALIGN(4K);
+    PROVIDE(__rela_start = .);
+    .rela : {
+        *(.rela*)
+    }
+    PROVIDE(__rela_end = .);
+
+    /* . = ALIGN(4K);            */
+    /* .data.tables : {          */
+    /*     KEEP(*(.data.tables)) */
+    /* }                         */
+
+    . = ALIGN(4K);
+    .data : {
+        *(.data*)
+        *(.got*)
+    }
+
+    . = ALIGN(4K);
+    PROVIDE(__bss_start = .);
+    /* PROVIDE(__bss_start_phys = . - KERNEL_VIRT_OFFSET); */
+    .bss : {
+        *(COMMON)
+        *(.bss*)
+    }
+    . = ALIGN(4K);
+    PROVIDE(__bss_end = .);
+    /* PROVIDE(__bss_end_phys = . - KERNEL_VIRT_OFFSET); */
+    /* PROVIDE(__bss_size = __bss_end_phys - __bss_start_phys); */
+
+    PROVIDE(__kernel_end = .);
+};

etc/ld/arm/aarch64-unknown-qemu.ld

@@ -1,55 +0,0 @@
-ENTRY(__aarch64_entry);
-
-KERNEL_PHYS_BASE = 0x40080000;
-KERNEL_VIRT_OFFSET = 0xFFFFFF8000000000;
-
-SECTIONS {
-    . = KERNEL_PHYS_BASE;
-    PROVIDE(__kernel_start = . + KERNEL_VIRT_OFFSET);
-
-    .text.entry : {
-        *(.text.entry)
-    }
-
-    . = ALIGN(16);
-    . = . + KERNEL_VIRT_OFFSET;
-
-    .text : AT(. - KERNEL_VIRT_OFFSET) {
-        KEEP(*(.text.vectors));
-        *(.text*)
-    }
-
-    . = ALIGN(4K);
-    .rodata : AT(. - KERNEL_VIRT_OFFSET) {
-        *(.rodata*)
-        *(.eh_frame*)
-
-        . = ALIGN(16);
-        PROVIDE(__init_array_start = .);
-        KEEP(*(.init_array*))
-        PROVIDE(__init_array_end = .);
-    }
-
-    . = ALIGN(4K);
-    .data.tables : AT (. - KERNEL_VIRT_OFFSET) {
-        KEEP(*(.data.tables))
-    }
-
-    . = ALIGN(4K);
-    .data : AT(. - KERNEL_VIRT_OFFSET) {
-        *(.data*)
-        *(.got*)
-    }
-
-    . = ALIGN(4K);
-    PROVIDE(__bss_start_phys = . - KERNEL_VIRT_OFFSET);
-    .bss : AT(. - KERNEL_VIRT_OFFSET) {
-        *(COMMON)
-        *(.bss*)
-    }
-    . = ALIGN(4K);
-    PROVIDE(__bss_end_phys = . - KERNEL_VIRT_OFFSET);
-    PROVIDE(__bss_size = __bss_end_phys - __bss_start_phys);
-
-    PROVIDE(__kernel_end = .);
-};

etc/ld/arm/aarch64-unknown-raspi4b.ld

@@ -1,55 +0,0 @@
-ENTRY(__aarch64_entry);
-
-KERNEL_PHYS_BASE = 0x80000;
-KERNEL_VIRT_OFFSET = 0xFFFFFF8000000000;
-
-SECTIONS {
-    . = KERNEL_PHYS_BASE;
-    PROVIDE(__kernel_start = . + KERNEL_VIRT_OFFSET);
-
-    .text.entry : {
-        *(.text.entry)
-    }
-
-    . = ALIGN(16);
-    . = . + KERNEL_VIRT_OFFSET;
-
-    .text : AT(. - KERNEL_VIRT_OFFSET) {
-        KEEP(*(.text.vectors));
-        *(.text*)
-    }
-
-    . = ALIGN(4K);
-    .rodata : AT(. - KERNEL_VIRT_OFFSET) {
-        *(.rodata*)
-        *(.eh_frame*)
-
-        . = ALIGN(16);
-        PROVIDE(__init_array_start = .);
-        KEEP(*(.init_array*))
-        PROVIDE(__init_array_end = .);
-    }
-
-    . = ALIGN(4K);
-    .data.tables : AT (. - KERNEL_VIRT_OFFSET) {
-        KEEP(*(.data.tables))
-    }
-
-    . = ALIGN(4K);
-    .data : AT(. - KERNEL_VIRT_OFFSET) {
-        *(.data*)
-        *(.got*)
-    }
-
-    . = ALIGN(4K);
-    PROVIDE(__bss_start_phys = . - KERNEL_VIRT_OFFSET);
-    .bss : AT(. - KERNEL_VIRT_OFFSET) {
-        *(COMMON)
-        *(.bss*)
-    }
-    . = ALIGN(4K);
-    PROVIDE(__bss_end_phys = . - KERNEL_VIRT_OFFSET);
-    PROVIDE(__bss_size = __bss_end_phys - __bss_start_phys);
-
-    PROVIDE(__kernel_end = .);
-};

etc/ld/riscv/riscv64-unknown-none.ld

@@ -1,37 +1,32 @@
 ENTRY(__rv64_entry);
 
-KERNEL_PHYS_BASE = 0x40200000;
 KERNEL_VIRT_OFFSET = 0xFFFFFFF000000000;
 
 SECTIONS {
-    . = KERNEL_PHYS_BASE;
+    . = 0;
-    PROVIDE(__kernel_start = . + KERNEL_VIRT_OFFSET);
+    PROVIDE(__kernel_start = .);
 
-    .text.entry : {
+    .text : {
-        *(.text.entry)
+        KEEP(*(.text.entry));
-    }
-
-    . = ALIGN(16);
-    . = . + KERNEL_VIRT_OFFSET;
-
-    .text : AT(. - KERNEL_VIRT_OFFSET) {
         KEEP(*(.text.vectors));
         *(.text*)
     }
 
     . = ALIGN(4K);
-    .rodata : AT(. - KERNEL_VIRT_OFFSET) {
+    .rodata : {
         *(.rodata*)
         *(.eh_frame*)
     }
 
     . = ALIGN(4K);
-    .data.tables : AT (. - KERNEL_VIRT_OFFSET) {
+    PROVIDE(__rela_start = .);
-        KEEP(*(.data.tables))
+    .rela : {
+        *(.rela*)
     }
+    PROVIDE(__rela_end = .);
 
     . = ALIGN(4K);
-    .data : AT(. - KERNEL_VIRT_OFFSET) {
+    .data : {
         *(.data*)
         . = ALIGN(8);
         /* PROVIDE(__global_pointer = . + 0x800 - KERNEL_VIRT_OFFSET); */
@@ -45,14 +40,13 @@ SECTIONS {
     }
 
     . = ALIGN(4K);
-    PROVIDE(__bss_start_phys = . - KERNEL_VIRT_OFFSET);
+    PROVIDE(__bss_start = .);
-    .bss : AT(. - KERNEL_VIRT_OFFSET) {
+    .bss : {
         *(COMMON)
         *(.bss*)
     }
     . = ALIGN(4K);
-    PROVIDE(__bss_end_phys = . - KERNEL_VIRT_OFFSET);
+    PROVIDE(__bss_end = .);
-    PROVIDE(__bss_size = __bss_end_phys - __bss_start_phys);
 
     PROVIDE(__kernel_end = .);
 };

etc/ld/riscv/riscv64-unknown-qemu.ld

@@ -1,58 +0,0 @@
-ENTRY(__rv64_entry);
-
-KERNEL_PHYS_BASE = 0x80200000;
-KERNEL_VIRT_OFFSET = 0xFFFFFFF000000000;
-
-SECTIONS {
-    . = KERNEL_PHYS_BASE;
-    PROVIDE(__kernel_start = . + KERNEL_VIRT_OFFSET);
-
-    .text.entry : {
-        *(.text.entry)
-    }
-
-    . = ALIGN(16);
-    . = . + KERNEL_VIRT_OFFSET;
-
-    .text : AT(. - KERNEL_VIRT_OFFSET) {
-        KEEP(*(.text.vectors));
-        *(.text*)
-    }
-
-    . = ALIGN(4K);
-    .rodata : AT(. - KERNEL_VIRT_OFFSET) {
-        *(.rodata*)
-        *(.eh_frame*)
-    }
-
-    . = ALIGN(4K);
-    .data.tables : AT (. - KERNEL_VIRT_OFFSET) {
-        KEEP(*(.data.tables))
-    }
-
-    . = ALIGN(4K);
-    .data : AT(. - KERNEL_VIRT_OFFSET) {
-        *(.data*)
-        . = ALIGN(8);
-        /* PROVIDE(__global_pointer = . + 0x800 - KERNEL_VIRT_OFFSET); */
-
-        . = ALIGN(16);
-        PROVIDE(__init_array_start = .);
-        KEEP(*(.init_array*))
-        PROVIDE(__init_array_end = .);
-
-        *(.got*)
-    }
-
-    . = ALIGN(4K);
-    PROVIDE(__bss_start_phys = . - KERNEL_VIRT_OFFSET);
-    .bss : AT(. - KERNEL_VIRT_OFFSET) {
-        *(COMMON)
-        *(.bss*)
-    }
-    . = ALIGN(4K);
-    PROVIDE(__bss_end_phys = . - KERNEL_VIRT_OFFSET);
-    PROVIDE(__bss_size = __bss_end_phys - __bss_start_phys);
-
-    PROVIDE(__kernel_end = .);
-};

etc/ld/x86/x86_64-unknown-none.ld

@@ -1,52 +1,52 @@
 ENTRY(__x86_64_entry);
 
-KERNEL_PHYS_BASE = 0x200000;
+/* KERNEL_PHYS_BASE = 0x200000; */
 KERNEL_VIRT_OFFSET = 0xFFFFFF8000000000;
 
 SECTIONS {
-    . = KERNEL_PHYS_BASE;
+    /* . = KERNEL_PHYS_BASE; */
-    PROVIDE(__kernel_start = . + KERNEL_VIRT_OFFSET);
+    /* PROVIDE(__kernel_start = . + KERNEL_VIRT_OFFSET); */
 
-    .text.entry : {
+    . = 0x0;
+    PROVIDE(__kernel_start = .);
+
+    .text : {
         *(.text.entry)
-    }
-
-    . = ALIGN(16);
-    . = . + KERNEL_VIRT_OFFSET;
-
-    .text : AT(. - KERNEL_VIRT_OFFSET) {
         *(.text*)
     }
 
-    .export.text : AT(. - KERNEL_VIRT_OFFSET) {
+    .export.text : {
         KEEP(*(.export.text*))
     }
 
     . = ALIGN(4K);
-    .rodata : AT(. - KERNEL_VIRT_OFFSET) {
+    .rodata : {
         *(.eh_frame*)
         *(.rodata*)
     }
 
     . = ALIGN(4K);
-    .data.tables : AT (. - KERNEL_VIRT_OFFSET) {
+    PROVIDE(__rela_start = .);
-        KEEP(*(.data.tables))
+    .rela : {
+        *(.rela*)
     }
+    PROVIDE(__rela_end = .);
 
-    .data : AT(. - KERNEL_VIRT_OFFSET) {
+    . = ALIGN(4K);
+    .data : {
         KEEP(*(.data.yboot))
         *(.data*)
         *(.got*)
     }
 
     . = ALIGN(4K);
-    PROVIDE(__bss_start_phys = . - KERNEL_VIRT_OFFSET);
+    PROVIDE(__bss_start = .);
-    .bss : AT(. - KERNEL_VIRT_OFFSET) {
+    .bss : {
         *(COMMON)
         *(.bss*)
     }
     . = ALIGN(4K);
-    PROVIDE(__bss_end_phys = . - KERNEL_VIRT_OFFSET);
+    PROVIDE(__bss_end = .);
 
     PROVIDE(__kernel_end = .);
 };

etc/riscv64-unknown-none.json

@@ -14,6 +14,7 @@
   "panic-strategy": "abort",
   "dynamic-linking": true,
   "relocation-model": "pic",
+  "position-independent-executables": true,
   "code-model": "medium",
   "eh-frame-header": false,
 

etc/x86_64-unknown-none.json

@@ -15,6 +15,7 @@
   "panic-strategy": "abort",
   "dynamic-linking": true,
   "relocation-model": "pic",
+  "position-independent-executables": true,
 
   "has-thread-local": false,
 

kernel/Cargo.toml

@@ -19,7 +19,6 @@ chrono.workspace = true
 device-api = { workspace = true, features = ["derive"] }
 device-api-macros.workspace = true
 
-memtables.workspace = true
 vmalloc.workspace = true
 kernel-arch.workspace = true
 
@@ -99,12 +98,5 @@ ygg_driver_net_stmmac.path = "driver/net/stmmac"
 default = ["fb_console"]
 fb_console = []
 
-# TODO replace this with a better configuration mechanism
-aarch64_board_virt = ["kernel-arch-aarch64/aarch64_board_virt"]
-aarch64_board_raspi4b = ["kernel-arch-aarch64/aarch64_board_raspi4b"]
-
-riscv64_board_virt = ["kernel-arch-riscv64/riscv64_board_virt"]
-riscv64_board_jh7110 = ["kernel-arch-riscv64/riscv64_board_jh7110"]
-
 [lints]
 workspace = true

kernel/arch/aarch64/Cargo.toml

@@ -7,13 +7,13 @@ edition = "2021"
 yggdrasil-abi.workspace = true
 kernel-arch-interface.workspace = true
 libk-mm-interface.workspace = true
-memtables.workspace = true
 device-api = { workspace = true, features = ["derive"] }
 
 bitflags.workspace = true
 static_assertions.workspace = true
 aarch64-cpu.workspace = true
 tock-registers.workspace = true
+log.workspace = true
 
 [build-dependencies]
 cc = "1.0"

kernel/arch/aarch64/src/mem/device.rs

@@ -0,0 +1,95 @@
+use core::ops::Range;
+
+use kernel_arch_interface::mem::DeviceMemoryAttributes;
+use libk_mm_interface::{
+    address::PhysicalAddress,
+    table::{DevicePageManager, DevicePageManagerLevel, EntryLevel},
+};
+
+use crate::mem::table::PageEntry;
+
+use super::{
+    table::{PageTable, L2, L3},
+    tlb_flush_range_va, DEVICE_MAPPING_OFFSET, DEVICE_MEMORY_L3_COUNT,
+};
+
+#[repr(transparent)]
+pub struct L2DeviceMemory(pub PageTable<L2>);
+#[repr(transparent)]
+pub struct L3DeviceMemory(pub [PageTable<L3>; DEVICE_MEMORY_L3_COUNT]);
+
+pub(super) static mut DEVICE_MEMORY: DevicePageManager<L3DeviceMemory, L2DeviceMemory> =
+    DevicePageManager::new(
+        L3DeviceMemory([PageTable::zeroed(); DEVICE_MEMORY_L3_COUNT]),
+        L2DeviceMemory(PageTable::zeroed()),
+    );
+
+impl DevicePageManagerLevel for L2DeviceMemory {
+    type Level = L2;
+    const VIRTUAL_BASE: usize = DEVICE_MAPPING_OFFSET;
+    const INDEX_RANGE: Range<usize> = DEVICE_MEMORY_L3_COUNT..512;
+
+    fn map_page(
+        &mut self,
+        index: usize,
+        physical: PhysicalAddress,
+        attrs: &DeviceMemoryAttributes,
+    ) {
+        let _ = index;
+        let _ = physical;
+        let _ = attrs;
+        todo!()
+    }
+
+    fn unmap_page(&mut self, index: usize) {
+        let _ = index;
+        todo!()
+    }
+
+    fn is_mapped(&self, index: usize) -> bool {
+        let _ = index;
+        todo!()
+    }
+
+    fn flush_range(range: Range<usize>) {
+        let start = range.start * L2::SIZE + Self::VIRTUAL_BASE;
+        let size = (range.end - range.start) * L2::SIZE;
+        tlb_flush_range_va(start, size);
+    }
+}
+
+impl DevicePageManagerLevel for L3DeviceMemory {
+    type Level = L3;
+    const VIRTUAL_BASE: usize = DEVICE_MAPPING_OFFSET;
+    const INDEX_RANGE: Range<usize> = 0..512 * DEVICE_MEMORY_L3_COUNT;
+
+    fn map_page(
+        &mut self,
+        index: usize,
+        physical: PhysicalAddress,
+        attrs: &DeviceMemoryAttributes,
+    ) {
+        let _ = attrs;
+        let l2i = index / 512;
+        let l3i = index % 512;
+        self.0[l2i][l3i] = PageEntry::device_page(physical);
+    }
+
+    // TODO
+    fn unmap_page(&mut self, index: usize) {
+        let _ = index;
+        todo!()
+    }
+
+    fn is_mapped(&self, index: usize) -> bool {
+        let l2i = index / 512;
+        let l3i = index % 512;
+        self.0[l2i][l3i].is_present()
+    }
+
+    fn flush_range(range: Range<usize>) {
+        let start = range.start * L3::SIZE + Self::VIRTUAL_BASE;
+        let size = (range.end - range.start) * L3::SIZE;
+        tlb_flush_range_va(start, size);
+    }
+}

kernel/arch/aarch64/src/mem/intrinsics.rs

@@ -0,0 +1,93 @@
+use core::sync::atomic::{self, Ordering};
+
+use aarch64_cpu::{asm::barrier, registers::PAR_EL1};
+use libk_mm_interface::table::{EntryLevel, EntryLevelExt};
+use tock_registers::interfaces::Readable;
+
+use super::table::L3;
+
+#[inline]
+pub fn tlb_flush_asid(asid: u8) {
+    barrier::dsb(barrier::ISHST);
+    let value = (asid as u64) << 48;
+    unsafe {
+        core::arch::asm!("tlbi aside1, {value}", value = in(reg) value);
+    }
+    barrier::dsb(barrier::ISH);
+    barrier::isb(barrier::SY);
+}
+
+#[inline]
+pub fn tlb_flush_all() {
+    barrier::dsb(barrier::ISHST);
+    unsafe {
+        core::arch::asm!("tlbi vmalle1is");
+    }
+    barrier::dsb(barrier::ISH);
+    barrier::isb(barrier::SY);
+}
+
+#[inline]
+pub fn tlb_flush_vaae1(page: usize) {
+    barrier::dsb(barrier::ISHST);
+    let argument = page >> 12;
+    unsafe {
+        core::arch::asm!("tlbi vaae1, {argument}", argument = in(reg) argument);
+    }
+    barrier::dsb(barrier::ISH);
+    barrier::isb(barrier::SY);
+}
+
+pub fn tlb_flush_range_va(base: usize, size: usize) {
+    let end = (base + size).page_align_up::<L3>();
+    let base = base.page_align_down::<L3>();
+    let count = (end - base).page_count::<L3>();
+    for i in 0..count {
+        tlb_flush_vaae1(base + i * L3::SIZE);
+    }
+}
+
+pub fn at_s1e0r(input: usize) -> Option<u64> {
+    barrier::dsb(barrier::ISHST);
+    unsafe {
+        core::arch::asm!("at s1e0r, {address}", address = in(reg) input);
+    }
+    barrier::dsb(barrier::ISH);
+    barrier::isb(barrier::SY);
+    if PAR_EL1.matches_all(PAR_EL1::F::TranslationSuccessfull) {
+        Some(PAR_EL1.read(PAR_EL1::PA))
+    } else {
+        None
+    }
+}
+
+pub fn at_s1e1r(input: usize) -> Option<u64> {
+    barrier::dsb(barrier::ISHST);
+    unsafe {
+        core::arch::asm!("at s1e1r, {address}", address = in(reg) input);
+    }
+    barrier::dsb(barrier::ISH);
+    barrier::isb(barrier::SY);
+    if PAR_EL1.matches_all(PAR_EL1::F::TranslationSuccessfull) {
+        Some(PAR_EL1.read(PAR_EL1::PA))
+    } else {
+        None
+    }
+}
+
+pub fn ic_iallu() {
+    atomic::compiler_fence(Ordering::SeqCst);
+    barrier::dsb(barrier::ISH);
+    barrier::isb(barrier::SY);
+    unsafe {
+        core::arch::asm!("ic iallu");
+    }
+    barrier::isb(barrier::SY);
+}
+
+pub fn dc_cvac(input: usize) {
+    barrier::dsb(barrier::ISHST);
+    unsafe {
+        core::arch::asm!("dc cvac, {address}", address = in(reg) input);
+    }
+}

kernel/arch/aarch64/src/mem/mod.rs

@@ -1,108 +1,43 @@
-use core::{
-    alloc::Layout,
-    ops::{Deref, DerefMut},
-    sync::atomic::{self, AtomicUsize, Ordering},
-};
-
 use aarch64_cpu::{
     asm::barrier,
-    registers::{MAIR_EL1, PAR_EL1, SCTLR_EL1, TTBR0_EL1, TTBR1_EL1},
+    registers::{MAIR_EL1, SCTLR_EL1, TCR_EL1, TTBR0_EL1, TTBR1_EL1},
 };
 use kernel_arch_interface::{
     mem::{DeviceMemoryAttributes, KernelTableManager, RawDeviceMemoryMapping},
-    split_spinlock, Architecture, KERNEL_VIRT_OFFSET,
+    sync::IrqSafeSpinlock,
+    KERNEL_VIRT_OFFSET,
 };
-use libk_mm_interface::{
+use libk_mm_interface::{address::PhysicalAddress, table::EntryLevel};
-    address::PhysicalAddress,
+use table::{PageAttributes, PageEntry, PageTable, L1};
-    table::{page_index, EntryLevel, EntryLevelExt},
+use tock_registers::interfaces::{ReadWriteable, Writeable};
-};
+
-use memtables::aarch64::{FixedTables, KERNEL_L3_COUNT};
+mod intrinsics;
-use static_assertions::const_assert_eq;
+
-use tock_registers::interfaces::{ReadWriteable, Readable, Writeable};
+pub mod device;
+pub mod process;
+pub mod table;
+
+pub use intrinsics::*;
 use yggdrasil_abi::error::Error;
 
 use crate::ArchitectureImpl;
 
-use self::table::{PageAttributes, PageEntry, PageTable, L1, L2, L3};
-
-pub mod process;
-pub mod table;
-
 #[derive(Debug)]
 pub struct KernelTableManagerImpl;
 
-// TODO eliminate this requirement by using precomputed indices
-const MAPPING_OFFSET: usize = KERNEL_VIRT_OFFSET;
-
-#[cfg(any(feature = "aarch64_board_virt", rust_analyzer))]
-const KERNEL_PHYS_BASE: usize = 0x40080000;
-#[cfg(any(feature = "aarch64_board_raspi4b", rust_analyzer))]
-const KERNEL_PHYS_BASE: usize = 0x80000;
-
-// Precomputed mappings
-const KERNEL_L1_INDEX: usize = page_index::<L1>(KERNEL_VIRT_OFFSET + KERNEL_PHYS_BASE);
-const KERNEL_START_L2_INDEX: usize = page_index::<L2>(KERNEL_VIRT_OFFSET + KERNEL_PHYS_BASE);
-const KERNEL_END_L2_INDEX: usize = KERNEL_START_L2_INDEX + KERNEL_L3_COUNT;
-
-// Must not be zero, should be at 4MiB
-const_assert_eq!(KERNEL_START_L2_INDEX, 0);
-// From static mapping
-#[cfg(any(feature = "aarch64_board_raspi4b", rust_analyzer))]
-const_assert_eq!(KERNEL_L1_INDEX, 0);
-#[cfg(any(feature = "aarch64_board_virt", rust_analyzer))]
-const_assert_eq!(KERNEL_L1_INDEX, 1);
-
-// Runtime mappings
-// 2MiB max
-const EARLY_MAPPING_L2I: usize = KERNEL_END_L2_INDEX + 1;
-// 1GiB max
-const DEVICE_MAPPING_L1I: usize = KERNEL_L1_INDEX + 2;
-const DEVICE_MAPPING_L3_COUNT: usize = 4;
-// 16GiB max
-const RAM_MAPPING_START_L1I: usize = KERNEL_L1_INDEX + 3;
-pub const RAM_MAPPING_L1_COUNT: usize = 16;
-
-// 2MiB for early mappings
-const EARLY_MAPPING_OFFSET: usize =
-    MAPPING_OFFSET | (KERNEL_L1_INDEX * L1::SIZE) | (EARLY_MAPPING_L2I * L2::SIZE);
-static mut EARLY_MAPPING_L3: PageTable<L3> = PageTable::zeroed();
-// 1GiB for device MMIO mapping
-const DEVICE_MAPPING_OFFSET: usize = MAPPING_OFFSET | (DEVICE_MAPPING_L1I * L1::SIZE);
-static mut DEVICE_MAPPING_L2: PageTable<L2> = PageTable::zeroed();
-static mut DEVICE_MAPPING_L3S: [PageTable<L3>; DEVICE_MAPPING_L3_COUNT] =
-    [PageTable::zeroed(); DEVICE_MAPPING_L3_COUNT];
-// 16GiB for RAM mapping
-pub const RAM_MAPPING_OFFSET: usize = MAPPING_OFFSET | (RAM_MAPPING_START_L1I * L1::SIZE);
-pub static MEMORY_LIMIT: AtomicUsize = AtomicUsize::new(0);
-
-split_spinlock! {
-    use crate::ArchitectureImpl;
-    use crate::mem::FixedTables;
-    use libk_mm_interface::KernelImageObject;
-
-    #[link_section = ".data.tables"]
-    static KERNEL_TABLES: KernelImageObject<FixedTables> =
-        unsafe { KernelImageObject::new(FixedTables::zeroed()) };
-}
-
 impl KernelTableManager for KernelTableManagerImpl {
-    fn virtualize(address: u64) -> usize {
+    fn virtualize(phys: u64) -> usize {
-        let address = address as usize;
+        if phys as usize >= IDENTITY_SIZE {
-        if address < MEMORY_LIMIT.load(Ordering::Acquire) {
+            unreachable!("Invalid physical address to virtualize: {phys:#x}");
-            address + RAM_MAPPING_OFFSET
-        } else {
-            panic!("Invalid physical address: {:#x}", address);
         }
+        phys as usize + KERNEL_VIRT_OFFSET
     }
 
-    fn physicalize(address: usize) -> u64 {
+    fn physicalize(virt: usize) -> u64 {
-        if address < RAM_MAPPING_OFFSET
+        if virt < KERNEL_VIRT_OFFSET || virt - KERNEL_VIRT_OFFSET >= IDENTITY_SIZE {
-            || address - RAM_MAPPING_OFFSET >= MEMORY_LIMIT.load(Ordering::Acquire)
+            unreachable!("Invalid virtualized address: {virt:#x}");
-        {
-            panic!("Not a virtualized physical address: {:#x}", address);
         }
-
+        (virt - KERNEL_VIRT_OFFSET) as _
-        (address - RAM_MAPPING_OFFSET) as _
     }
 
     unsafe fn map_device_pages(
@@ -110,390 +45,71 @@ impl KernelTableManager for KernelTableManagerImpl {
         count: usize,
         attrs: DeviceMemoryAttributes,
     ) -> Result<RawDeviceMemoryMapping<Self>, Error> {
-        map_device_memory(PhysicalAddress::from_u64(base), count, attrs)
+        let _guard = DEVICE_MEMORY_LOCK.lock();
-    }
+        #[allow(static_mut_refs)]
-
+        unsafe {
-    unsafe fn unmap_device_pages(mapping: &RawDeviceMemoryMapping<Self>) {
+            device::DEVICE_MEMORY.map_device_pages(PhysicalAddress::from_u64(base), count, attrs)
-        unmap_device_memory(mapping)
-    }
-}
-
-/// Memory mapping which may be used for performing early kernel initialization
-pub struct EarlyMapping<'a, T: ?Sized> {
-    value: &'a mut T,
-    page_count: usize,
-}
-
-impl<'a, T: Sized> EarlyMapping<'a, T> {
-    /// # Safety
-    ///
-    /// `physical` address provided must be a valid non-NULL address actually containing `T`.
-    pub unsafe fn map_slice(
-        physical: PhysicalAddress,
-        len: usize,
-    ) -> Result<EarlyMapping<'a, [T]>, Error> {
-        let layout = Layout::array::<T>(len).unwrap();
-        let aligned = physical.page_align_down::<L3>();
-        let offset = physical.page_offset::<L3>();
-        let page_count = (offset + layout.size()).div_ceil(L3::SIZE);
-
-        let virt = map_early_pages(aligned, page_count)?;
-        let value = core::slice::from_raw_parts_mut((virt + offset) as *mut T, len);
-
-        Ok(EarlyMapping { value, page_count })
-    }
-}
-
-impl<T: ?Sized> Deref for EarlyMapping<'_, T> {
-    type Target = T;
-
-    fn deref(&self) -> &Self::Target {
-        self.value
-    }
-}
-
-impl<T: ?Sized> DerefMut for EarlyMapping<'_, T> {
-    fn deref_mut(&mut self) -> &mut Self::Target {
-        self.value
-    }
-}
-
-impl<T: ?Sized> Drop for EarlyMapping<'_, T> {
-    fn drop(&mut self) {
-        let address = (self.value as *mut T).addr() & !(L3::SIZE - 1);
-
-        for i in 0..self.page_count {
-            let page = address + i * L3::SIZE;
-
-            unsafe {
-                unmap_early_page(page);
-            }
         }
     }
-}
 
-fn kernel_table_flags() -> PageAttributes {
+    unsafe fn unmap_device_pages(_mapping: &RawDeviceMemoryMapping<Self>) {
-    PageAttributes::TABLE
+        // TODO
-        | PageAttributes::ACCESS
-        | PageAttributes::SH_INNER
-        | PageAttributes::PAGE_ATTR_NORMAL
-        | PageAttributes::PRESENT
-}
-
-fn ram_block_flags() -> PageAttributes {
-    // TODO UXN, PXN
-    PageAttributes::BLOCK
-        | PageAttributes::ACCESS
-        | PageAttributes::SH_OUTER
-        | PageAttributes::PAGE_ATTR_NORMAL
-        | PageAttributes::PRESENT
-}
-
-// Early mappings
-unsafe fn map_early_pages(physical: PhysicalAddress, count: usize) -> Result<usize, Error> {
-    for l3i in 0..512 {
-        let mut taken = false;
-        for i in 0..count {
-            if EARLY_MAPPING_L3[i + l3i].is_present() {
-                taken = true;
-                break;
-            }
-        }
-
-        if taken {
-            continue;
-        }
-
-        for i in 0..count {
-            let page = physical.add(i * L3::SIZE);
-            // TODO NX, NC
-            EARLY_MAPPING_L3[i + l3i] = PageEntry::normal_page(page, PageAttributes::empty());
-            tlb_flush_vaae1(EARLY_MAPPING_OFFSET + (l3i + i) * L3::SIZE);
-        }
-
-        return Ok(EARLY_MAPPING_OFFSET + l3i * L3::SIZE);
-    }
-
-    Err(Error::OutOfMemory)
-}
-
-unsafe fn unmap_early_page(address: usize) {
-    if !(EARLY_MAPPING_OFFSET..EARLY_MAPPING_OFFSET + L2::SIZE).contains(&address) {
-        panic!("Tried to unmap invalid early mapping: {:#x}", address);
-    }
-
-    let l3i = (address - EARLY_MAPPING_OFFSET).page_index::<L3>();
-
-    assert!(EARLY_MAPPING_L3[l3i].is_present());
-    EARLY_MAPPING_L3[l3i] = PageEntry::INVALID;
-}
-
-/// # Safety
-///
-/// Only meant to be used by the architecture initialization functions.
-pub unsafe fn map_ram_l1(index: usize) {
-    if index >= RAM_MAPPING_L1_COUNT {
-        ArchitectureImpl::halt();
-    }
-    let mut tables = KERNEL_TABLES.lock();
-    let table_index = index + RAM_MAPPING_START_L1I;
-
-    if tables.l1.data[table_index] != 0 {
-        ArchitectureImpl::halt();
-    }
-
-    tables.l1.data[table_index] = ((index * L1::SIZE) as u64) | ram_block_flags().bits();
-    tlb_flush_vaae1(RAM_MAPPING_OFFSET + index * L1::SIZE);
-}
-
-// Device mappings
-unsafe fn map_device_memory_l3(
-    base: PhysicalAddress,
-    count: usize,
-    _attrs: DeviceMemoryAttributes,
-) -> Result<usize, Error> {
-    // TODO don't map pages if already mapped
-
-    'l0: for i in 0..DEVICE_MAPPING_L3_COUNT * 512 {
-        for j in 0..count {
-            let l2i = (i + j) / 512;
-            let l3i = (i + j) % 512;
-
-            if DEVICE_MAPPING_L3S[l2i][l3i].is_present() {
-                continue 'l0;
-            }
-        }
-
-        for j in 0..count {
-            let l2i = (i + j) / 512;
-            let l3i = (i + j) % 512;
-
-            // TODO NX, NC
-            DEVICE_MAPPING_L3S[l2i][l3i] = PageEntry::device_page(base.add(j * L3::SIZE));
-            tlb_flush_vaae1(DEVICE_MAPPING_OFFSET + l2i * L2::SIZE + l3i * L3::SIZE);
-        }
-
-        return Ok(DEVICE_MAPPING_OFFSET + i * L3::SIZE);
-    }
-
-    Err(Error::OutOfMemory)
-}
-
-#[allow(unused)]
-unsafe fn map_device_memory_l2(
-    base: PhysicalAddress,
-    count: usize,
-    _attrs: DeviceMemoryAttributes,
-) -> Result<usize, Error> {
-    'l0: for i in DEVICE_MAPPING_L3_COUNT..512 {
-        for j in 0..count {
-            if DEVICE_MAPPING_L2[i + j].is_present() {
-                continue 'l0;
-            }
-        }
-
-        for j in 0..count {
-            DEVICE_MAPPING_L2[i + j] = PageEntry::<L2>::device_block(base.add(j * L2::SIZE));
-            tlb_flush_vaae1(DEVICE_MAPPING_OFFSET + (i + j) * L2::SIZE);
-        }
-
-        return Ok(DEVICE_MAPPING_OFFSET + i * L2::SIZE);
-    }
-
-    Err(Error::OutOfMemory)
-}
-
-pub(crate) unsafe fn map_device_memory(
-    base: PhysicalAddress,
-    size: usize,
-    attrs: DeviceMemoryAttributes,
-) -> Result<RawDeviceMemoryMapping<KernelTableManagerImpl>, Error> {
-    let l3_aligned = base.page_align_down::<L3>();
-    let l3_offset = base.page_offset::<L3>();
-    let page_count = (l3_offset + size).page_count::<L3>();
-
-    if page_count > 256 {
-        // Large mapping, use L2 mapping instead
-        let l2_aligned = base.page_align_down::<L2>();
-        let l2_offset = base.page_offset::<L2>();
-        let page_count = (l2_offset + size).page_count::<L2>();
-
-        let base_address = map_device_memory_l2(l2_aligned, page_count, attrs)?;
-        let address = base_address + l2_offset;
-
-        Ok(RawDeviceMemoryMapping::from_raw_parts(
-            l2_aligned.into_u64(),
-            address,
-            base_address,
-            page_count,
-            L2::SIZE,
-        ))
-    } else {
-        // Just map the pages directly
-        let base_address = map_device_memory_l3(l3_aligned, page_count, attrs)?;
-        let address = base_address + l3_offset;
-
-        Ok(RawDeviceMemoryMapping::from_raw_parts(
-            l3_aligned.into_u64(),
-            address,
-            base_address,
-            page_count,
-            L3::SIZE,
-        ))
     }
 }
 
-pub(crate) unsafe fn unmap_device_memory(map: &RawDeviceMemoryMapping<KernelTableManagerImpl>) {
+// 64GiB of identity-mapped memory
-    match map.page_size {
+pub const IDENTITY_SIZE: usize = 64 * L1::SIZE;
-        L3::SIZE => {
+pub const IDENTITY_L1_START: usize = (KERNEL_VIRT_OFFSET >> L1::SHIFT) & 0x1FF;
-            for i in 0..map.page_count {
+// 1GiB of memory-mapped devices
-                let page = map.base_address + i * L3::SIZE;
+pub const DEVICE_MEMORY_L1: usize = IDENTITY_L1_START + IDENTITY_SIZE / L1::SIZE;
-                let l2i = page.page_index::<L2>();
+pub const DEVICE_MEMORY_L3_COUNT: usize = 32;
-                let l3i = page.page_index::<L3>();
+pub const DEVICE_MAPPING_OFFSET: usize = KERNEL_VIRT_OFFSET + DEVICE_MEMORY_L1 * L1::SIZE;
-                assert!(DEVICE_MAPPING_L3S[l2i][l3i].is_present());
+static DEVICE_MEMORY_LOCK: IrqSafeSpinlock<ArchitectureImpl, ()> = IrqSafeSpinlock::new(());
-                DEVICE_MAPPING_L3S[l2i][l3i] = PageEntry::INVALID;
 
-                tlb_flush_vaae1(page);
+static mut FIXED_L1: PageTable<L1> = const {
-            }
+    if IDENTITY_L1_START != 0 {
-        }
+        panic!("Invalid KERNEL_VIRT_OFFSET");
-        L2::SIZE => todo!(),
-        _ => unimplemented!(),
     }
-}
+    if IDENTITY_L1_START + IDENTITY_SIZE / L1::SIZE >= 500 {
-
+        panic!("Invalid KERNEL_VIRT_OFFSET + IDENTITY_SIZE");
-#[inline]
-pub fn tlb_flush_asid(asid: u8) {
-    barrier::dsb(barrier::ISHST);
-    let value = (asid as u64) << 48;
-    unsafe {
-        core::arch::asm!("tlbi aside1, {value}", value = in(reg) value);
     }
-    barrier::dsb(barrier::ISH);
-    barrier::isb(barrier::SY);
-}
 
-#[inline]
+    let mut table = PageTable::<L1>::zeroed();
-pub fn tlb_flush_all() {
+    let mut i = 0;
-    barrier::dsb(barrier::ISHST);
+
-    unsafe {
+    while i < IDENTITY_SIZE / L1::SIZE {
-        core::arch::asm!("tlbi vmalle1is");
+        let phys = PhysicalAddress::from_usize(i * L1::SIZE);
+        table.entries[i + IDENTITY_L1_START] =
+            PageEntry::normal_block(phys, PageAttributes::empty());
+        i += 1;
     }
-    barrier::dsb(barrier::ISH);
-    barrier::isb(barrier::SY);
-}
 
-#[inline]
+    table
-pub fn tlb_flush_vaae1(page: usize) {
+};
-    barrier::dsb(barrier::ISHST);
-    let argument = page >> 12;
-    unsafe {
-        core::arch::asm!("tlbi vaae1, {argument}", argument = in(reg) argument);
-    }
-    barrier::dsb(barrier::ISH);
-    barrier::isb(barrier::SY);
-}
 
-pub fn at_s1e0r(input: usize) -> Option<u64> {
+pub unsafe fn setup_fixed_tables() {
-    barrier::dsb(barrier::ISHST);
+    let device_l2_physical = auto_lower_address(&raw const device::DEVICE_MEMORY.large);
-    unsafe {
+    FIXED_L1[DEVICE_MEMORY_L1] = PageEntry::table(
-        core::arch::asm!("at s1e0r, {address}", address = in(reg) input);
+        PhysicalAddress::from_usize(device_l2_physical),
-    }
+        PageAttributes::empty(),
-    barrier::dsb(barrier::ISH);
+    );
-    barrier::isb(barrier::SY);
+    for i in 0..DEVICE_MEMORY_L3_COUNT {
-    if PAR_EL1.matches_all(PAR_EL1::F::TranslationSuccessfull) {
+        let device_l3_physical = auto_lower_address(&raw const device::DEVICE_MEMORY.normal.0[i]);
-        Some(PAR_EL1.read(PAR_EL1::PA))
+        device::DEVICE_MEMORY.large.0[i] = PageEntry::table(
-    } else {
+            PhysicalAddress::from_usize(device_l3_physical),
-        None
+            PageAttributes::empty(),
-    }
-}
-
-pub fn at_s1e1r(input: usize) -> Option<u64> {
-    barrier::dsb(barrier::ISHST);
-    unsafe {
-        core::arch::asm!("at s1e1r, {address}", address = in(reg) input);
-    }
-    barrier::dsb(barrier::ISH);
-    barrier::isb(barrier::SY);
-    if PAR_EL1.matches_all(PAR_EL1::F::TranslationSuccessfull) {
-        Some(PAR_EL1.read(PAR_EL1::PA))
-    } else {
-        None
-    }
-}
-
-pub fn ic_iallu() {
-    atomic::compiler_fence(Ordering::SeqCst);
-    barrier::dsb(barrier::ISH);
-    barrier::isb(barrier::SY);
-    unsafe {
-        core::arch::asm!("ic iallu");
-    }
-    barrier::isb(barrier::SY);
-}
-
-pub fn dc_cvac(input: usize) {
-    barrier::dsb(barrier::ISHST);
-    unsafe {
-        core::arch::asm!("dc cvac, {address}", address = in(reg) input);
-    }
-}
-
-fn auto_address<T>(value: *const T) -> usize {
-    let addr = value.addr();
-    if addr < KERNEL_VIRT_OFFSET {
-        // Called from lower half
-        addr
-    } else {
-        // Called from higher-half
-        addr - KERNEL_VIRT_OFFSET
-    }
-}
-
-/// (BSP-early init) loads precomputed kernel mapping tables for the kernel to jump to "higher-half"
-///
-/// # Safety
-///
-/// Unsafe, must only be called by BSP during its early init while still in "lower-half"
-pub unsafe fn load_fixed_tables() {
-    let ttbr0 = auto_address(&raw const KERNEL_TABLES) as u64;
-
-    TTBR0_EL1.set(ttbr0);
-    TTBR1_EL1.set(ttbr0);
-}
-
-/// Sets up additional translation tables for kernel usage
-///
-/// # Safety
-///
-/// Unsafe, must only be called by BSP during its early init, must already be in "higher-half"
-pub unsafe fn init_fixed_tables() {
-    // TODO this could be built in compile-time too?
-    let mut tables = KERNEL_TABLES.grab();
-    let early_mapping_l3_phys = auto_address(&raw const EARLY_MAPPING_L3);
-    let device_mapping_l2_phys = auto_address(&raw const DEVICE_MAPPING_L2);
-
-    for i in 0..DEVICE_MAPPING_L3_COUNT {
-        let device_mapping_l3_phys = PhysicalAddress::from_usize(
-            &DEVICE_MAPPING_L3S[i] as *const _ as usize - KERNEL_VIRT_OFFSET,
         );
-        DEVICE_MAPPING_L2[i] = PageEntry::table(device_mapping_l3_phys, PageAttributes::empty());
     }
-
-    assert_eq!(tables.l2.data[EARLY_MAPPING_L2I], 0);
-    tables.l2.data[EARLY_MAPPING_L2I] =
-        (early_mapping_l3_phys as u64) | kernel_table_flags().bits();
-    tlb_flush_vaae1(EARLY_MAPPING_OFFSET);
-
-    assert_eq!(tables.l1.data[DEVICE_MAPPING_L1I], 0);
-    tables.l1.data[DEVICE_MAPPING_L1I] =
-        (device_mapping_l2_phys as u64) | kernel_table_flags().bits();
-
-    tlb_flush_all();
 }
 
-pub fn setup_memory_attributes() {
+pub fn load_fixed_tables() {
-    // TODO: Figure out why WriteBack_NonTransient_ReadWriteAlloc doesn't work on Pi 4B
+    let l1_physical = auto_lower_address(&raw const FIXED_L1);
+    TTBR0_EL1.set_baddr(l1_physical as _);
+    TTBR1_EL1.set_baddr(l1_physical as _);
+}
 
+pub fn configure_mmu() {
+    // TODO: Figure out why WriteBack_NonTransient_ReadWriteAlloc doesn't work on Pi 4B
     MAIR_EL1.write(
         //// Attribute 0 -- normal memory
         MAIR_EL1::Attr0_Normal_Inner::WriteBack_NonTransient +
@@ -504,6 +120,39 @@ pub fn setup_memory_attributes() {
         //// Attribute 2 -- device memory
         MAIR_EL1::Attr1_Device::nonGathering_nonReordering_EarlyWriteAck,
     );
+
+    TCR_EL1.write(
+        TCR_EL1::AS::ASID8Bits +
+             TCR_EL1::A1::TTBR0 +
+             TCR_EL1::HD::CLEAR +
+             // General
+             TCR_EL1::IPS::Bits_48 +
+             // TTBR0
+             TCR_EL1::TG0::KiB_4 + TCR_EL1::T0SZ.val(25) + TCR_EL1::SH0::Inner +
+             // TTBR1
+             TCR_EL1::TG1::KiB_4 + TCR_EL1::T1SZ.val(25) + TCR_EL1::SH1::Inner,
+    );
+    barrier::dsb(barrier::ISHST);
+    barrier::isb(barrier::SY);
+
+    SCTLR_EL1.modify(
+        SCTLR_EL1::E0E::LittleEndian
+            + SCTLR_EL1::EE::LittleEndian
+            + SCTLR_EL1::WXN::Disable
+            + SCTLR_EL1::SA0::Enable
+            + SCTLR_EL1::SA::Enable
+            + SCTLR_EL1::A::Enable
+            + SCTLR_EL1::I::NonCacheable
+            + SCTLR_EL1::C::NonCacheable,
+    );
+
+    barrier::dsb(barrier::ISH);
+    barrier::isb(barrier::SY);
+}
+
+pub fn enable_mmu() {
+    // Enable translation
+    SCTLR_EL1.modify(SCTLR_EL1::M::Enable);
 }
 
 /// Enables data cache.
@@ -549,3 +198,12 @@ pub unsafe fn disable_icache() {
     barrier::dsb(barrier::ISH);
     barrier::isb(barrier::SY);
 }
+
+fn auto_lower_address<T>(ptr: *const T) -> usize {
+    let address = ptr.addr();
+    if address < KERNEL_VIRT_OFFSET {
+        address
+    } else {
+        address - KERNEL_VIRT_OFFSET
+    }
+}

kernel/arch/aarch64/src/mem/table.rs

@@ -56,7 +56,7 @@ bitflags! {
 #[derive(Clone, Copy)]
 #[repr(C, align(0x1000))]
 pub struct PageTable<L: EntryLevel> {
-    entries: [PageEntry<L>; 512],
+    pub entries: [PageEntry<L>; 512],
 }
 
 #[derive(Clone, Copy)]
@@ -262,18 +262,13 @@ impl<L: NonTerminalEntryLevel> PageEntry<L> {
         )
     }
 
-    pub fn normal_block(phys: PhysicalAddress, attrs: PageAttributes) -> Self {
+    pub const fn normal_block(phys: PhysicalAddress, attrs: PageAttributes) -> Self {
-        Self(
+        const ATTR: u64 = PageAttributes::BLOCK.bits()
-            phys.into_u64()
+            | PageAttributes::PRESENT.bits()
-                | (PageAttributes::BLOCK
+            | PageAttributes::ACCESS.bits()
-                    | PageAttributes::PRESENT
+            | PageAttributes::SH_OUTER.bits()
-                    | PageAttributes::ACCESS
+            | PageAttributes::PAGE_ATTR_NORMAL.bits();
-                    | PageAttributes::SH_OUTER
+        Self(phys.into_u64() | attrs.bits() | ATTR, PhantomData)
-                    | PageAttributes::PAGE_ATTR_NORMAL
-                    | attrs)
-                    .bits(),
-            PhantomData,
-        )
     }
 
     pub fn device_block(phys: PhysicalAddress) -> Self {

kernel/arch/interface/src/sync.rs

@@ -42,7 +42,7 @@ impl<A: Architecture, T> Spinlock<A, T> {
         }
     }
 
-    pub fn lock(&self) -> SpinlockGuard<A, T> {
+    pub fn lock(&self) -> SpinlockGuard<'_, A, T> {
         // Loop until the lock can be acquired
         if LOCK_HACK.load(Ordering::Acquire) {
             return SpinlockGuard { lock: self };
@@ -103,7 +103,7 @@ impl<A: Architecture, T> IrqSafeSpinlock<A, T> {
     }
 
     /// Attempts to acquire a lock. IRQs will be disabled until the lock is released.
-    pub fn lock(&self) -> IrqSafeSpinlockGuard<A, T> {
+    pub fn lock(&self) -> IrqSafeSpinlockGuard<'_, A, T> {
         // Disable IRQs to avoid IRQ handler trying to acquire the same lock
         let irq_guard = IrqGuard::acquire();
 

kernel/arch/riscv64/Cargo.toml

@@ -7,7 +7,6 @@ edition = "2024"
 yggdrasil-abi.workspace = true
 kernel-arch-interface.workspace = true
 libk-mm-interface.workspace = true
-memtables.workspace = true
 device-api = { workspace = true, features = ["derive"] }
 
 tock-registers.workspace = true

kernel/arch/riscv64/src/context.rs

@@ -124,8 +124,7 @@ impl<K: KernelTableManager, PA: PhysicalMemoryAllocator<Address = PhysicalAddres
 
         // TODO stack is leaked
         let satp = InMemoryRegister::new(0);
-        let kernel_table_phys =
+        let kernel_table_phys = ((&raw const mem::FIXED_L1).addr() - KERNEL_VIRT_OFFSET) as u64;
-            ((&raw const mem::KERNEL_TABLES).addr() - KERNEL_VIRT_OFFSET) as u64;
         satp.write(SATP::MODE::Sv39 + SATP::ASID.val(0) + SATP::PPN.val(kernel_table_phys >> 12));
 
         Ok(Self {

kernel/arch/riscv64/src/mem/mod.rs

@@ -1,70 +1,38 @@
-use cfg_if::cfg_if;
+use kernel_arch_interface::mem::{
-use kernel_arch_interface::{
+    DeviceMemoryAttributes, KernelTableManager, RawDeviceMemoryMapping,
-    mem::{DeviceMemoryAttributes, KernelTableManager, RawDeviceMemoryMapping},
-    split_spinlock,
 };
 use libk_mm_interface::{
     address::PhysicalAddress,
     table::{page_index, EntryLevel, EntryLevelExt},
 };
-use memtables::riscv64::PageAttributes;
+use table::{PageEntry, PageTable, L1, L3};
-use static_assertions::{const_assert, const_assert_eq};
-use table::{PageEntry, PageTable, L1, L2, L3};
 use tock_registers::interfaces::Writeable;
 use yggdrasil_abi::error::Error;
 
-pub use memtables::riscv64::FixedTables;
-
 use crate::registers::SATP;
 
 pub mod process;
 pub mod table;
 
-split_spinlock! {
-    use crate::ArchitectureImpl;
-    use crate::mem::FixedTables;
-    use libk_mm_interface::KernelImageObject;
-
-    #[link_section = ".data.tables"]
-    #[used]
-    static KERNEL_TABLES: KernelImageObject<FixedTables> =
-        unsafe { KernelImageObject::new(FixedTables::zeroed()) };
-}
-
-cfg_if! {
-    if #[cfg(feature = "riscv64_board_virt")] {
-        pub const KERNEL_PHYS_BASE: usize = 0x80200000;
-    } else if #[cfg(feature = "riscv64_board_jh7110")] {
-        pub const KERNEL_PHYS_BASE: usize = 0x40200000;
-    } else if #[cfg(rust_analyzer)] {
-        pub const KERNEL_PHYS_BASE: usize = 0x80200000;
-    }
-}
-
 pub const KERNEL_VIRT_OFFSET: usize = kernel_arch_interface::KERNEL_VIRT_OFFSET;
 pub const SIGN_EXTEND_MASK: usize = 0xFFFFFF80_00000000;
 
-pub const KERNEL_START_L1I: usize = page_index::<L1>(KERNEL_VIRT_OFFSET + KERNEL_PHYS_BASE);
+pub const IDENTITY_SIZE: usize = 64 * L1::SIZE;
-pub const KERNEL_L2I: usize = page_index::<L2>(KERNEL_VIRT_OFFSET + KERNEL_PHYS_BASE);
+pub const IDENTITY_L1_START: usize = page_index::<L1>(KERNEL_VIRT_OFFSET);
-const_assert_eq!(KERNEL_L2I, 1);
 
-// Runtime mappings
+pub static mut FIXED_L1: PageTable<L1> = const {
-// 1GiB of device memory space
+    let mut table = PageTable::zeroed();
-const DEVICE_MAPPING_L1I: usize = KERNEL_START_L1I + 1;
-const DEVICE_MAPPING_L3_COUNT: usize = 4;
-// 32GiB of RAM space
-const RAM_MAPPING_START_L1I: usize = KERNEL_START_L1I + 2;
-const RAM_MAPPING_L1_COUNT: usize = 32;
-const_assert!(RAM_MAPPING_START_L1I + RAM_MAPPING_L1_COUNT <= 512);
-const_assert!(DEVICE_MAPPING_L1I < 512);
 
-const DEVICE_MAPPING_OFFSET: usize = (DEVICE_MAPPING_L1I << L1::SHIFT) | SIGN_EXTEND_MASK;
+    let mut i = 0;
-const RAM_MAPPING_OFFSET: usize = (RAM_MAPPING_START_L1I << L1::SHIFT) | SIGN_EXTEND_MASK;
+    while i < IDENTITY_SIZE / L1::SIZE {
+        let entry = PageEntry::identity_block(PhysicalAddress::from_usize(i * L1::SIZE));
+        table.entries[i] = entry;
+        table.entries[i + IDENTITY_L1_START] = entry;
+        i += 1;
+    }
 
-// Runtime tables
+    table
-static mut DEVICE_MAPPING_L2: PageTable<L2> = PageTable::zeroed();
+};
-static mut DEVICE_MAPPING_L3S: [PageTable<L3>; DEVICE_MAPPING_L3_COUNT] =
-    [const { PageTable::zeroed() }; DEVICE_MAPPING_L3_COUNT];
 
 /// Any VAs above this one are sign-extended
 pub const USER_BOUNDARY: usize = 0x40_00000000;
@@ -75,17 +43,17 @@ pub struct KernelTableManagerImpl;
 impl KernelTableManager for KernelTableManagerImpl {
     fn virtualize(address: u64) -> usize {
         let address = address as usize;
-        if address >= RAM_MAPPING_OFFSET {
+        if address >= IDENTITY_SIZE {
             panic!("Invalid physical address: {address:#x}");
         }
-        address + RAM_MAPPING_OFFSET
+        address + KERNEL_VIRT_OFFSET
     }
 
     fn physicalize(address: usize) -> u64 {
-        if address < RAM_MAPPING_OFFSET {
+        if address < KERNEL_VIRT_OFFSET || address - KERNEL_VIRT_OFFSET >= IDENTITY_SIZE {
             panic!("Invalid \"physicalized\" virtual address {address:#x}");
         }
-        (address - RAM_MAPPING_OFFSET) as u64
+        (address - KERNEL_VIRT_OFFSET) as u64
     }
 
     unsafe fn map_device_pages(
@@ -93,146 +61,31 @@ impl KernelTableManager for KernelTableManagerImpl {
         count: usize,
         attrs: DeviceMemoryAttributes,
     ) -> Result<RawDeviceMemoryMapping<Self>, Error> {
-        unsafe { map_device_memory(PhysicalAddress::from_u64(base), count, attrs) }
+        let _ = attrs;
+        let base = PhysicalAddress::from_u64(base);
+        let l3_aligned_base = base.page_align_down::<L3>();
+        let l3_aligned_end = base.add(count).page_align_up::<L3>();
+        let l3_offset = base - l3_aligned_base;
+        let l3_page_count = (l3_aligned_end - l3_aligned_base).page_count::<L3>();
+        let l3_aligned_virt = l3_aligned_base.add(KERNEL_VIRT_OFFSET).into_usize();
+
+        Ok(unsafe {
+            RawDeviceMemoryMapping::from_raw_parts(
+                l3_aligned_base.into_u64(),
+                l3_aligned_virt + l3_offset,
+                l3_aligned_virt,
+                l3_page_count,
+                L3::SIZE,
+            )
+        })
     }
 
     unsafe fn unmap_device_pages(mapping: &RawDeviceMemoryMapping<Self>) {
-        unsafe { unmap_device_memory(mapping) }
+        let _ = mapping;
     }
 }
 
-// Device mappings
+pub fn auto_lower_address<T>(x: *const T) -> usize {
-unsafe fn map_device_memory_l3(
-    base: PhysicalAddress,
-    count: usize,
-    _attrs: DeviceMemoryAttributes,
-) -> Result<usize, Error> {
-    // TODO don't map pages if already mapped
-
-    'l0: for i in 0..DEVICE_MAPPING_L3_COUNT * 512 {
-        for j in 0..count {
-            let l2i = (i + j) / 512;
-            let l3i = (i + j) % 512;
-
-            unsafe {
-                if DEVICE_MAPPING_L3S[l2i][l3i].is_present() {
-                    continue 'l0;
-                }
-            }
-        }
-
-        for j in 0..count {
-            let l2i = (i + j) / 512;
-            let l3i = (i + j) % 512;
-
-            unsafe {
-                DEVICE_MAPPING_L3S[l2i][l3i] =
-                    PageEntry::page(base.add(j * L3::SIZE), PageAttributes::W);
-            }
-        }
-
-        let start = DEVICE_MAPPING_OFFSET + i * L3::SIZE;
-        tlb_flush_range_va(start, count * L3::SIZE);
-        return Ok(start);
-    }
-
-    Err(Error::OutOfMemory)
-}
-
-#[allow(unused)]
-unsafe fn map_device_memory_l2(
-    base: PhysicalAddress,
-    count: usize,
-    _attrs: DeviceMemoryAttributes,
-) -> Result<usize, Error> {
-    'l0: for i in DEVICE_MAPPING_L3_COUNT..512 {
-        for j in 0..count {
-            unsafe {
-                if DEVICE_MAPPING_L2[i + j].is_present() {
-                    continue 'l0;
-                }
-            }
-        }
-
-        unsafe {
-            for j in 0..count {
-                DEVICE_MAPPING_L2[i + j] =
-                    PageEntry::<L2>::block(base.add(j * L2::SIZE), PageAttributes::W);
-            }
-        }
-
-        let start = DEVICE_MAPPING_OFFSET + i * L2::SIZE;
-        tlb_flush_range_va(start, count * L2::SIZE);
-        return Ok(start);
-    }
-
-    Err(Error::OutOfMemory)
-}
-
-pub(crate) unsafe fn map_device_memory(
-    base: PhysicalAddress,
-    size: usize,
-    attrs: DeviceMemoryAttributes,
-) -> Result<RawDeviceMemoryMapping<KernelTableManagerImpl>, Error> {
-    let l3_aligned = base.page_align_down::<L3>();
-    let l3_offset = base.page_offset::<L3>();
-    let page_count = (l3_offset + size).page_count::<L3>();
-
-    if page_count > 256 {
-        // Large mapping, use L2 mapping instead
-        let l2_aligned = base.page_align_down::<L2>();
-        let l2_offset = base.page_offset::<L2>();
-        let page_count = (l2_offset + size).page_count::<L2>();
-
-        unsafe {
-            let base_address = map_device_memory_l2(l2_aligned, page_count, attrs)?;
-            let address = base_address + l2_offset;
-
-            Ok(RawDeviceMemoryMapping::from_raw_parts(
-                l2_aligned.into_u64(),
-                address,
-                base_address,
-                page_count,
-                L2::SIZE,
-            ))
-        }
-    } else {
-        // Just map the pages directly
-        unsafe {
-            let base_address = map_device_memory_l3(l3_aligned, page_count, attrs)?;
-            let address = base_address + l3_offset;
-
-            Ok(RawDeviceMemoryMapping::from_raw_parts(
-                l3_aligned.into_u64(),
-                address,
-                base_address,
-                page_count,
-                L3::SIZE,
-            ))
-        }
-    }
-}
-
-pub(crate) unsafe fn unmap_device_memory(map: &RawDeviceMemoryMapping<KernelTableManagerImpl>) {
-    match map.page_size {
-        L3::SIZE => {
-            for i in 0..map.page_count {
-                let page = map.base_address + i * L3::SIZE;
-                let l2i = page.page_index::<L2>();
-                let l3i = page.page_index::<L3>();
-                unsafe {
-                    assert!(DEVICE_MAPPING_L3S[l2i][l3i].is_present());
-                    DEVICE_MAPPING_L3S[l2i][l3i] = PageEntry::INVALID;
-                }
-            }
-            tlb_flush_range_va(map.base_address, map.page_count * L3::SIZE);
-        }
-        L2::SIZE => todo!(),
-        _ => unimplemented!(),
-    }
-}
-
-pub fn auto_address<T>(x: *const T) -> usize {
     let x = x.addr();
     if x >= KERNEL_VIRT_OFFSET {
         x - KERNEL_VIRT_OFFSET
@@ -247,7 +100,7 @@ pub fn auto_address<T>(x: *const T) -> usize {
 ///
 /// Only meant to be called once per each HART during their early init.
 pub unsafe fn enable_mmu() {
-    let l1_phys = auto_address(&raw const KERNEL_TABLES) as u64;
+    let l1_phys = auto_lower_address(&raw const FIXED_L1) as u64;
     tlb_flush_full();
     SATP.write(SATP::PPN.val(l1_phys >> 12) + SATP::MODE::Sv39);
 }
@@ -258,49 +111,10 @@ pub unsafe fn enable_mmu() {
 ///
 /// Needs to be called once after secondary HARTs are initialized.
 pub unsafe fn unmap_lower_half() {
-    let mut tables = KERNEL_TABLES.lock();
+    // for i in 0..(IDENTITY_SIZE / L1::SIZE) {
-    let kernel_l1i_lower = page_index::<L1>(KERNEL_PHYS_BASE);
+    //     unsafe { FIXED_L1[i] = PageEntry::INVALID };
-    tables.l1.data[kernel_l1i_lower] = 0;
+    // }
-    tlb_flush_range_va(0x0, L1::SIZE);
+    // tlb_flush_full();
-}
-
-/// Sets up run-time kernel translation tables.
-///
-/// # Safety
-///
-/// The caller must ensure MMU is already enabled.
-pub unsafe fn setup_fixed_tables() {
-    let mut tables = KERNEL_TABLES.lock();
-
-    let device_mapping_l2_phys = auto_address(&raw const DEVICE_MAPPING_L2);
-
-    // Set up static runtime mappings
-    for i in 0..DEVICE_MAPPING_L3_COUNT {
-        unsafe {
-            let device_mapping_l3_phys = PhysicalAddress::from_usize(
-                (&raw const DEVICE_MAPPING_L3S[i]).addr() - KERNEL_VIRT_OFFSET,
-            );
-            DEVICE_MAPPING_L2[i] =
-                PageEntry::table(device_mapping_l3_phys, PageAttributes::empty());
-        }
-    }
-
-    assert_eq!(tables.l1.data[DEVICE_MAPPING_L1I], 0);
-    tables.l1.data[DEVICE_MAPPING_L1I] =
-        ((device_mapping_l2_phys as u64) >> 2) | PageAttributes::V.bits();
-
-    for l1i in 0..RAM_MAPPING_L1_COUNT {
-        let physical = (l1i as u64) << L1::SHIFT;
-        tables.l1.data[l1i + RAM_MAPPING_START_L1I] = (physical >> 2)
-            | (PageAttributes::R
-                | PageAttributes::W
-                | PageAttributes::A
-                | PageAttributes::D
-                | PageAttributes::V)
-                .bits();
-    }
-
-    tlb_flush_full();
 }
 
 pub fn tlb_flush_global_full() {
@@ -352,8 +166,9 @@ pub fn tlb_flush_va_asid(va: usize, asid: usize) {
 }
 
 pub fn clone_kernel_tables(dst: &mut PageTable<L1>) {
-    let tables = KERNEL_TABLES.lock();
+    // let tables = KERNEL_TABLES.lock();
     for l1i in page_index::<L1>(USER_BOUNDARY)..512 {
-        dst[l1i] = unsafe { PageEntry::from_raw(tables.l1.data[l1i]) };
+        // dst[l1i] = unsafe { PageEntry::from_raw(tables.l1.data[l1i]) };
+        dst[l1i] = unsafe { FIXED_L1[l1i] };
     }
 }

kernel/arch/riscv64/src/mem/process.rs

@@ -11,13 +11,12 @@ use libk_mm_interface::{
         EntryLevel, EntryLevelDrop, EntryLevelExt, MapAttributes, NextPageTable, TableAllocator,
     },
 };
-use memtables::riscv64::PageAttributes;
 use yggdrasil_abi::error::Error;
 
 use crate::mem::{clone_kernel_tables, table::PageEntry};
 
 use super::{
-    table::{DroppableRange, PageTable, L1, L2, L3},
+    table::{DroppableRange, PageAttributes, PageTable, L1, L2, L3},
     KernelTableManagerImpl, USER_BOUNDARY,
 };
 

kernel/arch/riscv64/src/mem/table.rs

@@ -3,6 +3,7 @@ use core::{
     ops::{Index, IndexMut, Range},
 };
 
+use bitflags::bitflags;
 use libk_mm_interface::{
     address::{AsPhysicalAddress, PhysicalAddress},
     pointer::{PhysicalRef, PhysicalRefMut},
@@ -16,7 +17,41 @@ use yggdrasil_abi::error::Error;
 
 use super::{KernelTableManagerImpl, USER_BOUNDARY};
 
-pub use memtables::riscv64::PageAttributes;
+// pub use memtables::riscv64::PageAttributes;
+bitflags! {
+    #[derive(Clone, Copy, PartialEq, Eq, Debug)]
+    pub struct PageAttributes: u64 {
+        const N = 1 << 63;
+        /// Software-tracked dirty bit (RSW[0])
+        const SW_DIRTY = 1 << 9;
+        /// Dirty bit
+        const D = 1 << 7;
+        /// Access bit
+        const A = 1 << 6;
+        /// Global mapping bit, implies all lower levels are also global
+        const G = 1 << 5;
+        /// U-mode access permission
+        const U = 1 << 4;
+        /// Execute permission
+        const X = 1 << 3;
+        /// Write permission
+        const W = 1 << 2;
+        /// Read-permission
+        const R = 1 << 1;
+        /// Valid bit
+        const V = 1 << 0;
+    }
+
+    // X W R    Meaning
+    // 0 0 0    Pointer to next level of page table
+    // 0 0 1    Read-only page
+    // 0 1 0    ---
+    // 0 1 1    Read-write page
+    // 1 0 0    Execute only
+    // 1 0 1    Read-execute page
+    // 1 1 0    ---
+    // 1 1 1    Read-write-execute page
+}
 
 /// L3 - entry is 4KiB
 #[derive(Debug, Clone, Copy)]
@@ -40,9 +75,10 @@ impl EntryLevel for L1 {
     const SHIFT: usize = 30;
 }
 
+#[derive(Clone, Copy)]
 #[repr(C, align(0x1000))]
 pub struct PageTable<L: EntryLevel> {
-    entries: [PageEntry<L>; 512],
+    pub entries: [PageEntry<L>; 512],
 }
 
 #[derive(Clone, Copy, Debug, PartialEq)]
@@ -204,6 +240,17 @@ impl<L: NonTerminalEntryLevel + 'static> NextPageTable for PageTable<L> {
 }
 
 impl<L: NonTerminalEntryLevel> PageEntry<L> {
+    pub const fn identity_block(address: PhysicalAddress) -> Self {
+        const ATTR: u64 = PageAttributes::R.bits()
+            | PageAttributes::W.bits()
+            | PageAttributes::X.bits()
+            | PageAttributes::A.bits()
+            | PageAttributes::D.bits()
+            | PageAttributes::V.bits();
+
+        Self((address.into_u64() >> 2) | ATTR, PhantomData)
+    }
+
     pub fn block(address: PhysicalAddress, attrs: PageAttributes) -> Self {
         // TODO validate address alignment
         Self(

kernel/arch/x86_64/Cargo.toml

@@ -7,7 +7,6 @@ edition = "2021"
 yggdrasil-abi.workspace = true
 kernel-arch-interface.workspace = true
 libk-mm-interface.workspace = true
-memtables.workspace = true
 device-api = { workspace = true, features = ["derive"] }
 kernel-arch-x86.workspace = true
 

kernel/arch/x86_64/src/context.rs

@@ -5,11 +5,14 @@ use kernel_arch_interface::{
     task::{ForkFrame, StackBuilder, TaskContext, TaskFrame, UserContextInfo},
 };
 use kernel_arch_x86::registers::{FpuContext, CR3, MSR_IA32_FS_BASE};
-use libk_mm_interface::address::{AsPhysicalAddress, PhysicalAddress};
+use libk_mm_interface::address::PhysicalAddress;
 use tock_registers::interfaces::Writeable;
 use yggdrasil_abi::{arch::SavedFrame, error::Error};
 
-use crate::{mem::KERNEL_TABLES, ArchitectureImpl};
+use crate::{
+    mem::{auto_lower_address, FIXED_PML4},
+    ArchitectureImpl,
+};
 
 /// Frame saved onto the stack when taking an IRQ
 #[derive(Debug)]
@@ -431,7 +434,7 @@ impl<K: KernelTableManager, PA: PhysicalMemoryAllocator<Address = PhysicalAddres
     fn kernel(entry: extern "C" fn(usize) -> !, arg: usize) -> Result<Self, Error> {
         const KERNEL_TASK_PAGES: usize = 32;
 
-        let cr3: usize = unsafe { KERNEL_TABLES.lock().as_physical_address() }.into();
+        let cr3 = auto_lower_address(&raw const FIXED_PML4);
 
         let stack_base_phys = PA::allocate_contiguous_pages(KERNEL_TASK_PAGES)?;
         let stack_base = stack_base_phys.raw_virtualize::<K>();

kernel/arch/x86_64/src/mem/device.rs

@@ -0,0 +1,87 @@
+use core::ops::Range;
+
+use kernel_arch_interface::mem::DeviceMemoryAttributes;
+use libk_mm_interface::{
+    address::PhysicalAddress,
+    table::{DevicePageManager, DevicePageManagerLevel},
+};
+
+use crate::mem::table::PageEntry;
+
+use super::{
+    table::{PageAttributes, PageTable, L2, L3},
+    DEVICE_MAPPING_OFFSET, DEVICE_MEMORY_L3_COUNT,
+};
+
+#[repr(transparent)]
+pub struct L2DeviceMemory(pub PageTable<L2>);
+#[repr(transparent)]
+pub struct L3DeviceMemory(pub [PageTable<L3>; DEVICE_MEMORY_L3_COUNT]);
+
+pub(super) static mut DEVICE_MEMORY: DevicePageManager<L3DeviceMemory, L2DeviceMemory> =
+    DevicePageManager::new(
+        L3DeviceMemory([PageTable::zeroed(); DEVICE_MEMORY_L3_COUNT]),
+        L2DeviceMemory(PageTable::zeroed()),
+    );
+
+impl DevicePageManagerLevel for L2DeviceMemory {
+    type Level = L2;
+    const VIRTUAL_BASE: usize = DEVICE_MAPPING_OFFSET;
+    const INDEX_RANGE: Range<usize> = DEVICE_MEMORY_L3_COUNT..512;
+
+    fn map_page(
+        &mut self,
+        index: usize,
+        physical: PhysicalAddress,
+        attrs: &DeviceMemoryAttributes,
+    ) {
+        let _ = attrs;
+        self.0[index] = PageEntry::<L2>::block(physical, PageAttributes::WRITABLE);
+    }
+
+    fn unmap_page(&mut self, index: usize) {
+        self.0[index] = PageEntry::INVALID;
+    }
+
+    fn is_mapped(&self, index: usize) -> bool {
+        self.0[index].is_present()
+    }
+
+    fn flush_range(range: Range<usize>) {
+        let _ = range;
+    }
+}
+
+impl DevicePageManagerLevel for L3DeviceMemory {
+    type Level = L3;
+    const VIRTUAL_BASE: usize = DEVICE_MAPPING_OFFSET;
+    const INDEX_RANGE: Range<usize> = 0..512 * DEVICE_MEMORY_L3_COUNT;
+
+    fn map_page(
+        &mut self,
+        index: usize,
+        physical: PhysicalAddress,
+        attrs: &DeviceMemoryAttributes,
+    ) {
+        let _ = attrs;
+        let l2i = index / 512;
+        let l3i = index % 512;
+        self.0[l2i][l3i] = PageEntry::page(physical, PageAttributes::WRITABLE);
+    }
+
+    fn unmap_page(&mut self, index: usize) {
+        let l2i = index / 512;
+        let l3i = index % 512;
+        self.0[l2i][l3i] = PageEntry::INVALID;
+    }
+
+    fn is_mapped(&self, index: usize) -> bool {
+        let l2i = index / 512;
+        let l3i = index % 512;
+        self.0[l2i][l3i].is_present()
+    }
+
+    fn flush_range(range: Range<usize>) {
+        let _ = range;
+    }
+}

kernel/arch/x86_64/src/mem/mod.rs

@@ -1,98 +1,39 @@
-use core::{
-    alloc::Layout,
-    ops::{Deref, DerefMut},
-    sync::atomic::{AtomicUsize, Ordering},
-};
-
 use kernel_arch_interface::{
     mem::{DeviceMemoryAttributes, KernelTableManager, RawDeviceMemoryMapping},
-    split_spinlock,
+    sync::IrqSafeSpinlock,
 };
 use kernel_arch_x86::registers::CR3;
 use libk_mm_interface::{
     address::PhysicalAddress,
-    table::{page_index, EntryLevel, EntryLevelExt},
+    table::{EntryLevel, EntryLevelExt},
 };
-use static_assertions::{const_assert_eq, const_assert_ne};
 use yggdrasil_abi::error::Error;
 
-use crate::KERNEL_VIRT_OFFSET;
+use crate::{ArchitectureImpl, KERNEL_VIRT_OFFSET};
 
-use self::table::{PageAttributes, PageEntry, PageTable, L0, L1, L2, L3};
+use self::table::{PageAttributes, PageEntry, PageTable, L0, L1, L2};
 
+pub mod device;
 pub mod process;
 pub mod table;
 
 #[derive(Debug)]
 pub struct KernelTableManagerImpl;
 
-const CANONICAL_ADDRESS_MASK: usize = 0xFFFF000000000000;
-const KERNEL_PHYS_BASE: usize = 0x200000;
-
-// Mapped at compile time
-const KERNEL_MAPPING_BASE: usize = KERNEL_VIRT_OFFSET + KERNEL_PHYS_BASE;
-const KERNEL_L0_INDEX: usize = page_index::<L0>(KERNEL_MAPPING_BASE);
-const KERNEL_L1_INDEX: usize = page_index::<L1>(KERNEL_MAPPING_BASE);
-const KERNEL_START_L2_INDEX: usize = page_index::<L2>(KERNEL_MAPPING_BASE);
-
-// Must not be zero, should be at 4MiB
-const_assert_ne!(KERNEL_START_L2_INDEX, 0);
-// From static mapping
-const_assert_eq!(KERNEL_L0_INDEX, 511);
-const_assert_eq!(KERNEL_L1_INDEX, 0);
-
-// Mapped at boot
-const EARLY_MAPPING_L2I: usize = KERNEL_START_L2_INDEX - 1;
-const DEVICE_MAPPING_L1I: usize = KERNEL_L1_INDEX + 2;
-const RAM_MAPPING_L0I: usize = KERNEL_L0_INDEX - 1;
-
-const DEVICE_MAPPING_L3_COUNT: usize = 4;
-
-split_spinlock! {
-    use libk_mm_interface::KernelImageObject;
-    use memtables::x86_64::FixedTables;
-    use crate::ArchitectureImpl;
-
-    #[link_section = ".data.tables"]
-    static KERNEL_TABLES: KernelImageObject<FixedTables> =
-        unsafe { KernelImageObject::new(FixedTables::zeroed()) };
-}
-
-// 2MiB for early mappings
-const EARLY_MAPPING_OFFSET: usize = CANONICAL_ADDRESS_MASK
-    | (KERNEL_L0_INDEX * L0::SIZE)
-    | (KERNEL_L1_INDEX * L1::SIZE)
-    | (EARLY_MAPPING_L2I * L2::SIZE);
-static mut EARLY_MAPPING_L3: PageTable<L3> = PageTable::zeroed();
-// 1GiB for device MMIO mapping
-const DEVICE_MAPPING_OFFSET: usize =
-    CANONICAL_ADDRESS_MASK | (KERNEL_L0_INDEX * L0::SIZE) | (DEVICE_MAPPING_L1I * L1::SIZE);
-static mut DEVICE_MAPPING_L2: PageTable<L2> = PageTable::zeroed();
-static mut DEVICE_MAPPING_L3S: [PageTable<L3>; DEVICE_MAPPING_L3_COUNT] =
-    [PageTable::zeroed(); DEVICE_MAPPING_L3_COUNT];
-// 512GiB for whole RAM mapping
-pub const RAM_MAPPING_OFFSET: usize = CANONICAL_ADDRESS_MASK | (RAM_MAPPING_L0I * L0::SIZE);
-pub static MEMORY_LIMIT: AtomicUsize = AtomicUsize::new(0);
-pub static mut RAM_MAPPING_L1: PageTable<L1> = PageTable::zeroed();
-
 impl KernelTableManager for KernelTableManagerImpl {
     fn virtualize(address: u64) -> usize {
         let address = address as usize;
-        if address < MEMORY_LIMIT.load(Ordering::Acquire) {
+        if address >= IDENTITY_SIZE {
-            address + RAM_MAPPING_OFFSET
+            panic!("Invalid physical address to virtualize: {address:#x}");
-        } else {
-            panic!("Invalid physical address: {:#x}", address);
         }
+        address + KERNEL_VIRT_OFFSET
     }
 
     fn physicalize(address: usize) -> u64 {
-        if address < RAM_MAPPING_OFFSET
+        if address < KERNEL_VIRT_OFFSET || address - KERNEL_VIRT_OFFSET >= IDENTITY_SIZE {
-            || address - RAM_MAPPING_OFFSET >= MEMORY_LIMIT.load(Ordering::Acquire)
+            panic!("Invalid virtualized address: {address:#x}");
-        {
-            panic!("Not a virtualized physical address: {:#x}", address);
         }
-
+        (address - KERNEL_VIRT_OFFSET) as _
-        (address - RAM_MAPPING_OFFSET) as _
     }
 
     unsafe fn map_device_pages(
@@ -100,245 +41,42 @@ impl KernelTableManager for KernelTableManagerImpl {
         count: usize,
         attrs: DeviceMemoryAttributes,
     ) -> Result<RawDeviceMemoryMapping<Self>, Error> {
-        map_device_memory(PhysicalAddress::from_u64(base), count, attrs)
+        let _guard = DEVICE_MEMORY_LOCK.lock();
+        #[allow(static_mut_refs)]
+        {
+            device::DEVICE_MEMORY.map_device_pages(PhysicalAddress::from_u64(base), count, attrs)
+        }
     }
 
     unsafe fn unmap_device_pages(mapping: &RawDeviceMemoryMapping<Self>) {
-        unmap_device_memory(mapping)
+        let _guard = DEVICE_MEMORY_LOCK.lock();
-    }
+        #[allow(static_mut_refs)]
-}
+        {
-
+            device::DEVICE_MEMORY.unmap_device_pages(mapping);
-// Early mappings
-unsafe fn map_early_pages(physical: PhysicalAddress, count: usize) -> Result<usize, Error> {
-    for l3i in 0..512 {
-        let mut taken = false;
-        for i in 0..count {
-            if EARLY_MAPPING_L3[i + l3i].is_present() {
-                taken = true;
-                break;
-            }
-        }
-
-        if taken {
-            continue;
-        }
-
-        for i in 0..count {
-            // TODO NX, NC
-            EARLY_MAPPING_L3[i + l3i] =
-                PageEntry::page(physical.add(i * L3::SIZE), PageAttributes::WRITABLE);
-            flush_tlb_entry(EARLY_MAPPING_OFFSET + (i + l3i) * L3::SIZE);
-        }
-
-        return Ok(EARLY_MAPPING_OFFSET + l3i * L3::SIZE);
-    }
-
-    Err(Error::OutOfMemory)
-}
-
-unsafe fn unmap_early_page(address: usize) {
-    if !(EARLY_MAPPING_OFFSET..EARLY_MAPPING_OFFSET + L2::SIZE).contains(&address) {
-        panic!("Tried to unmap invalid early mapping: {:#x}", address);
-    }
-
-    let l3i = (address - EARLY_MAPPING_OFFSET).page_index::<L3>();
-
-    assert!(EARLY_MAPPING_L3[l3i].is_present());
-    EARLY_MAPPING_L3[l3i] = PageEntry::INVALID;
-}
-
-// Device mappings
-unsafe fn map_device_memory_l3(
-    base: PhysicalAddress,
-    count: usize,
-    _attrs: DeviceMemoryAttributes,
-) -> Result<usize, Error> {
-    // TODO don't map pages if already mapped
-
-    'l0: for i in 0..DEVICE_MAPPING_L3_COUNT * 512 {
-        for j in 0..count {
-            let l2i = (i + j) / 512;
-            let l3i = (i + j) % 512;
-
-            if DEVICE_MAPPING_L3S[l2i][l3i].is_present() {
-                continue 'l0;
-            }
-        }
-
-        for j in 0..count {
-            let l2i = (i + j) / 512;
-            let l3i = (i + j) % 512;
-
-            // TODO NX, NC
-            DEVICE_MAPPING_L3S[l2i][l3i] =
-                PageEntry::page(base.add(j * L3::SIZE), PageAttributes::WRITABLE);
-        }
-
-        return Ok(DEVICE_MAPPING_OFFSET + i * L3::SIZE);
-    }
-
-    Err(Error::OutOfMemory)
-}
-
-unsafe fn map_device_memory_l2(
-    base: PhysicalAddress,
-    count: usize,
-    _attrs: DeviceMemoryAttributes,
-) -> Result<usize, Error> {
-    'l0: for i in DEVICE_MAPPING_L3_COUNT..512 {
-        for j in 0..count {
-            if DEVICE_MAPPING_L2[i + j].is_present() {
-                continue 'l0;
-            }
-        }
-
-        for j in 0..count {
-            DEVICE_MAPPING_L2[i + j] =
-                PageEntry::<L2>::block(base.add(j * L2::SIZE), PageAttributes::WRITABLE);
-        }
-
-        return Ok(DEVICE_MAPPING_OFFSET + i * L2::SIZE);
-    }
-
-    Err(Error::OutOfMemory)
-}
-
-unsafe fn map_device_memory(
-    base: PhysicalAddress,
-    size: usize,
-    attrs: DeviceMemoryAttributes,
-) -> Result<RawDeviceMemoryMapping<KernelTableManagerImpl>, Error> {
-    let l3_aligned = base.page_align_down::<L3>();
-    let l3_offset = base.page_offset::<L3>();
-    let page_count = (l3_offset + size).page_count::<L3>();
-
-    if page_count > 256 {
-        // Large mapping, use L2 mapping instead
-        let l2_aligned = base.page_align_down::<L2>();
-        let l2_offset = base.page_offset::<L2>();
-        let page_count = (l2_offset + size).page_count::<L2>();
-
-        let base_address = map_device_memory_l2(l2_aligned, page_count, attrs)?;
-        let address = base_address + l2_offset;
-
-        Ok(RawDeviceMemoryMapping::from_raw_parts(
-            l2_aligned.into_u64(),
-            address,
-            base_address,
-            page_count,
-            L2::SIZE,
-        ))
-    } else {
-        // Just map the pages directly
-        let base_address = map_device_memory_l3(l3_aligned, page_count, attrs)?;
-        let address = base_address + l3_offset;
-
-        Ok(RawDeviceMemoryMapping::from_raw_parts(
-            l3_aligned.into_u64(),
-            address,
-            base_address,
-            page_count,
-            L3::SIZE,
-        ))
-    }
-}
-
-unsafe fn unmap_device_memory(map: &RawDeviceMemoryMapping<KernelTableManagerImpl>) {
-    match map.page_size {
-        L3::SIZE => {
-            for i in 0..map.page_count {
-                let page = map.base_address + i * L3::SIZE;
-                let l2i = page.page_index::<L2>();
-                let l3i = page.page_index::<L3>();
-                assert!(DEVICE_MAPPING_L3S[l2i][l3i].is_present());
-                DEVICE_MAPPING_L3S[l2i][l3i] = PageEntry::INVALID;
-                flush_tlb_entry(page);
-            }
-        }
-        L2::SIZE => todo!(),
-        _ => unimplemented!(),
-    }
-}
-
-/// Memory mapping which may be used for performing early kernel initialization
-pub struct EarlyMapping<'a, T: ?Sized> {
-    value: &'a mut T,
-    page_count: usize,
-}
-
-impl<'a, T: Sized> EarlyMapping<'a, T> {
-    /// # Safety
-    ///
-    /// `physical` address provided must be a valid non-NULL address actually containing `T`.
-    pub unsafe fn map(physical: PhysicalAddress) -> Result<EarlyMapping<'a, T>, Error> {
-        let layout = Layout::new::<T>();
-        let aligned = physical.page_align_down::<L3>();
-        let offset = physical.page_offset::<L3>();
-        let page_count = (offset + layout.size()).div_ceil(L3::SIZE);
-
-        let virt = map_early_pages(aligned, page_count)?;
-        let value = &mut *((virt + offset) as *mut T);
-
-        Ok(EarlyMapping { value, page_count })
-    }
-
-    /// # Safety
-    ///
-    /// `physical` address provided must be a valid non-NULL address actually containing a `T`
-    /// slice of given `len`.
-    pub unsafe fn map_slice(
-        physical: PhysicalAddress,
-        len: usize,
-    ) -> Result<EarlyMapping<'a, [T]>, Error> {
-        let layout = Layout::array::<T>(len).unwrap();
-        let aligned = physical.page_align_down::<L3>();
-        let offset = physical.page_offset::<L3>();
-        let page_count = (offset + layout.size()).div_ceil(L3::SIZE);
-
-        let virt = map_early_pages(aligned, page_count)?;
-        let value = core::slice::from_raw_parts_mut((virt + offset) as *mut T, len);
-
-        Ok(EarlyMapping { value, page_count })
-    }
-}
-
-impl<T: ?Sized> Deref for EarlyMapping<'_, T> {
-    type Target = T;
-
-    fn deref(&self) -> &Self::Target {
-        self.value
-    }
-}
-
-impl<T: ?Sized> DerefMut for EarlyMapping<'_, T> {
-    fn deref_mut(&mut self) -> &mut Self::Target {
-        self.value
-    }
-}
-
-impl<T: ?Sized> Drop for EarlyMapping<'_, T> {
-    fn drop(&mut self) {
-        let address = (self.value as *mut T).addr() & !(L3::SIZE - 1);
-
-        for i in 0..self.page_count {
-            let page = address + i * L3::SIZE;
-
-            unsafe {
-                unmap_early_page(page);
-            }
         }
     }
 }
 
 pub fn clone_kernel_tables(dst: &mut PageTable<L0>) {
-    let tables = KERNEL_TABLES.lock();
     unsafe {
-        dst[KERNEL_L0_INDEX] = PageEntry::from_raw(tables.l0.data[KERNEL_L0_INDEX]);
+        let kernel_l0i = KERNEL_VIRT_OFFSET.page_index::<L0>();
-        dst[RAM_MAPPING_L0I] = PageEntry::from_raw(tables.l0.data[RAM_MAPPING_L0I]);
+        dst[kernel_l0i] = FIXED_PML4[kernel_l0i];
     }
 }
 
-pub fn auto_address<T>(pointer: *const T) -> usize {
+const FIXED_PD_COUNT: usize = 32;
+const IDENTITY_SIZE: usize = FIXED_PD_COUNT * L1::SIZE;
+
+const DEVICE_MAPPING_L1I: usize = FIXED_PD_COUNT;
+const DEVICE_MEMORY_L3_COUNT: usize = 16;
+const DEVICE_MAPPING_OFFSET: usize = KERNEL_VIRT_OFFSET + IDENTITY_SIZE;
+
+static DEVICE_MEMORY_LOCK: IrqSafeSpinlock<ArchitectureImpl, ()> = IrqSafeSpinlock::new(());
+static mut FIXED_PDS: [PageTable<L2>; FIXED_PD_COUNT] = [PageTable::zeroed(); FIXED_PD_COUNT];
+static mut FIXED_PDPT: PageTable<L1> = PageTable::zeroed();
+pub static mut FIXED_PML4: PageTable<L0> = PageTable::zeroed();
+
+pub fn auto_lower_address<T>(pointer: *const T) -> usize {
     let address = pointer.addr();
     if address < KERNEL_VIRT_OFFSET {
         address
@@ -347,53 +85,80 @@ pub fn auto_address<T>(pointer: *const T) -> usize {
     }
 }
 
-/// Sets up the following memory map:
+pub unsafe fn init_fixed_tables(have_1gib_pages: bool) {
-/// ...: KERNEL_TABLES.l0:
+    if have_1gib_pages {
-///     * 0xFFFFFF0000000000 .. 0xFFFFFFFF8000000000        : RAM_MAPPING_L1
+        for i in 0..IDENTITY_SIZE / L1::SIZE {
-///     * 0xFFFFFF8000000000 .. ...                         : KERNEL_TABLES.kernel_l1:
+            FIXED_PDPT[i] = PageEntry::<L1>::block(
-///         * 0xFFFFFF8000000000 .. 0xFFFFFF8040000000      : KERNEL_TABLES.kernel_l2
+                PhysicalAddress::from_usize(i * L1::SIZE),
-///             * 0xFFFFFF8000000000 .. 0xFFFFFF8000200000  : ---
+                PageAttributes::WRITABLE,
-///             * 0xFFFFFF8000200000 .. 0xFFFFFF8000400000  : EARLY_MAPPING_L3
+            );
-///             * 0xFFFFFF8000400000 .. ...                 : KERNEL_TABLES.kernel_l3s
+        }
-///         * 0xFFFFFF8040000000 .. 0xFFFFFF8080000000      : ---
+    } else {
-///         * 0xFFFFFF8080000000 .. 0xFFFFFF8100000000      : DEVICE_MAPPING_L2
+        for i in 0..IDENTITY_SIZE / L1::SIZE {
-///             * 0xFFFFFF8080000000 .. 0xFFFFFF8080800000  : DEVICE_MAPPING_L3S
+            for j in 0..512 {
-///             * 0xFFFFFF8080800000 .. 0xFFFFFF8100000000  : ...
+                FIXED_PDS[i][j] = PageEntry::<L2>::block(
-///
+                    PhysicalAddress::from_usize(i * L1::SIZE + j * L2::SIZE),
-/// # Safety
+                    PageAttributes::WRITABLE,
-///
+                );
-/// Unsafe, must only be called by BSP during its early init, must already be in "higher-half"
+            }
-pub unsafe fn init_fixed_tables() {
+            let pd_physical =
-    let mut tables = KERNEL_TABLES.lock();
+                PhysicalAddress::from_usize(auto_lower_address(&raw const FIXED_PDS[i]));
-
+            FIXED_PDPT[i] = PageEntry::table(pd_physical, PageAttributes::WRITABLE);
-    // TODO this could be built in compile-time too?
+        }
-    let early_mapping_l3_phys = auto_address(&raw const EARLY_MAPPING_L3);
-    let device_mapping_l2_phys = auto_address(&raw const DEVICE_MAPPING_L2);
-    let ram_mapping_l1_phys = auto_address(&raw const RAM_MAPPING_L1);
-
-    for i in 0..DEVICE_MAPPING_L3_COUNT {
-        let device_mapping_l3_phys =
-            PhysicalAddress::from_usize(auto_address(&raw const DEVICE_MAPPING_L3S[i]));
-        DEVICE_MAPPING_L2[i] = PageEntry::table(device_mapping_l3_phys, PageAttributes::WRITABLE);
     }
 
-    assert_eq!(tables.kernel_l2.data[EARLY_MAPPING_L2I], 0);
+    // Device memory
-    tables.kernel_l2.data[EARLY_MAPPING_L2I] = (early_mapping_l3_phys as u64)
+    let device_pd_physical =
-        | (PageAttributes::WRITABLE | PageAttributes::PRESENT).bits();
+        PhysicalAddress::from_usize(auto_lower_address(&raw const device::DEVICE_MEMORY.large));
 
-    assert_eq!(tables.kernel_l1.data[DEVICE_MAPPING_L1I], 0);
+    for i in 0..DEVICE_MEMORY_L3_COUNT {
-    tables.kernel_l1.data[DEVICE_MAPPING_L1I] = (device_mapping_l2_phys as u64)
+        let device_pt_physical = PhysicalAddress::from_usize(auto_lower_address(
-        | (PageAttributes::WRITABLE | PageAttributes::PRESENT).bits();
+            &raw const device::DEVICE_MEMORY.normal.0[i],
+        ));
+        device::DEVICE_MEMORY.large.0[i] =
+            PageEntry::table(device_pt_physical, PageAttributes::WRITABLE);
+    }
 
-    assert_eq!(tables.l0.data[RAM_MAPPING_L0I], 0);
+    FIXED_PDPT[DEVICE_MAPPING_L1I] =
-    tables.l0.data[RAM_MAPPING_L0I] =
+        PageEntry::<L1>::table(device_pd_physical, PageAttributes::WRITABLE);
-        (ram_mapping_l1_phys as u64) | (PageAttributes::WRITABLE | PageAttributes::PRESENT).bits();
 
-    // TODO ENABLE EFER.NXE
+    let pdpt_physical = PhysicalAddress::from_usize(auto_lower_address(&raw const FIXED_PDPT));
-    let cr3 = auto_address(&raw const tables.l0);
+
-    CR3.set_address(cr3);
+    FIXED_PML4[KERNEL_VIRT_OFFSET.page_index::<L0>()] =
+        PageEntry::table(pdpt_physical, PageAttributes::WRITABLE);
+
+    let pml4_physical = auto_lower_address(&raw const FIXED_PML4);
+    CR3.set_address(pml4_physical);
 }
-
+//     let mut tables = KERNEL_TABLES.lock();
+//
+//     // TODO this could be built in compile-time too?
+//     let early_mapping_l3_phys = auto_address(&raw const EARLY_MAPPING_L3);
+//     let device_mapping_l2_phys = auto_address(&raw const DEVICE_MAPPING_L2);
+//     let ram_mapping_l1_phys = auto_address(&raw const RAM_MAPPING_L1);
+//
+//     for i in 0..DEVICE_MAPPING_L3_COUNT {
+//         let device_mapping_l3_phys =
+//             PhysicalAddress::from_usize(auto_address(&raw const DEVICE_MAPPING_L3S[i]));
+//         DEVICE_MAPPING_L2[i] = PageEntry::table(device_mapping_l3_phys, PageAttributes::WRITABLE);
+//     }
+//
+//     assert_eq!(tables.kernel_l2.data[EARLY_MAPPING_L2I], 0);
+//     tables.kernel_l2.data[EARLY_MAPPING_L2I] = (early_mapping_l3_phys as u64)
+//         | (PageAttributes::WRITABLE | PageAttributes::PRESENT).bits();
+//
+//     assert_eq!(tables.kernel_l1.data[DEVICE_MAPPING_L1I], 0);
+//     tables.kernel_l1.data[DEVICE_MAPPING_L1I] = (device_mapping_l2_phys as u64)
+//         | (PageAttributes::WRITABLE | PageAttributes::PRESENT).bits();
+//
+//     assert_eq!(tables.l0.data[RAM_MAPPING_L0I], 0);
+//     tables.l0.data[RAM_MAPPING_L0I] =
+//         (ram_mapping_l1_phys as u64) | (PageAttributes::WRITABLE | PageAttributes::PRESENT).bits();
+//
+//     // TODO ENABLE EFER.NXE
+//     let cr3 = auto_address(&raw const tables.l0);
+// }
+//
 /// # Safety
 ///
 /// `address` must be page-aligned.

kernel/driver/block/ahci/src/port.rs

@@ -235,7 +235,7 @@ impl AhciPort {
         .await
     }
 
-    async fn submit<C: AtaCommand>(&self, command: &C) -> Result<SubmittedCommand, AhciError> {
+    async fn submit<C: AtaCommand>(&self, command: &C) -> Result<SubmittedCommand<'_>, AhciError> {
         if command.prd_count() > 2 {
             log::warn!("TODO: AHCI doesn't like 3+ PRD transfers");
             return Err(AhciError::RegionTooLarge);

kernel/driver/bus/pci/src/space/mod.rs

@@ -349,7 +349,7 @@ pub trait PciConfigurationSpace {
     }
 
     /// Returns an iterator over the PCI capabilities
-    fn capability_iter(&self) -> CapabilityIterator<Self> {
+    fn capability_iter(&self) -> CapabilityIterator<'_, Self> {
         let status = PciStatusRegister::from_bits_retain(self.status());
 
         let current = if status.contains(PciStatusRegister::CAPABILITIES_LIST) {

kernel/driver/bus/usb/src/address.rs

@@ -0,0 +1,58 @@
+use core::fmt;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
+pub struct UsbRoute {
+    bus: u16,
+    ports: [u8; 8],
+    len: u8,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
+pub struct UsbBusAddress {
+    pub bus: u16,
+    pub device: u8,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
+pub struct UsbInterfaceAddress {
+    pub device: UsbBusAddress,
+    pub interface: u8,
+}
+
+impl UsbBusAddress {
+    pub fn with_interface(self, interface: u8) -> UsbInterfaceAddress {
+        UsbInterfaceAddress {
+            device: self,
+            interface,
+        }
+    }
+}
+
+impl fmt::Display for UsbBusAddress {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "<Bus {} Device {}>", self.bus, self.device)
+    }
+}
+
+impl fmt::Display for UsbInterfaceAddress {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(
+            f,
+            "<Bus {} Device {} Interface {}>",
+            self.device.bus, self.device.device, self.interface
+        )
+    }
+}
+
+impl fmt::Display for UsbRoute {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}-", self.bus)?;
+        for (i, &port) in self.ports[..self.len as usize].iter().enumerate() {
+            if i != 0 {
+                write!(f, ".")?;
+            }
+            write!(f, "{port}")?;
+        }
+        Ok(())
+    }
+}

kernel/driver/bus/usb/src/bus.rs

@@ -1,34 +1,56 @@
 use core::sync::atomic::{AtomicU16, Ordering};
 
-use alloc::{collections::BTreeMap, sync::Arc};
+use alloc::{collections::BTreeMap, format, sync::Arc};
-use libk_util::{queue::UnboundedMpmcQueue, sync::spin_rwlock::IrqSafeRwLock};
+use libk_util::{queue::UnboundedMpmcQueue, sync::spin_rwlock::IrqSafeRwLock, OneTimeInit};
 
 use crate::{
+    address::UsbBusAddress,
     class_driver,
-    device::{UsbBusAddress, UsbDeviceAccess},
+    device::{UsbDevice, UsbDeviceAccess},
+    sysfs::{self, UsbBusKObject},
     UsbHostController,
 };
 
+pub struct UsbBusWrapper {
+    pub(crate) hc: Arc<dyn UsbHostController>,
+    pub(crate) index: u16,
+    kobject: OneTimeInit<UsbBusKObject>,
+}
+
 pub struct UsbBusManager {
-    busses: IrqSafeRwLock<BTreeMap<u16, Arc<dyn UsbHostController>>>,
+    busses: IrqSafeRwLock<BTreeMap<u16, Arc<UsbBusWrapper>>>,
     devices: IrqSafeRwLock<BTreeMap<UsbBusAddress, Arc<UsbDeviceAccess>>>,
 
     last_bus_address: AtomicU16,
 }
 
+impl UsbBusWrapper {
+    pub fn kobject(&self) -> &UsbBusKObject {
+        self.kobject.get()
+    }
+}
+
 impl UsbBusManager {
-    pub fn register_bus(hc: Arc<dyn UsbHostController>) -> u16 {
+    pub fn register_bus(hc: Arc<dyn UsbHostController>) -> (u16, Arc<UsbBusWrapper>) {
         let i = BUS_MANAGER.last_bus_address.fetch_add(1, Ordering::AcqRel);
-        BUS_MANAGER.busses.write().insert(i, hc);
+        let wrapper = Arc::new(UsbBusWrapper {
-        i
+            hc,
+            index: i,
+            kobject: OneTimeInit::new(),
+        });
+        BUS_MANAGER.busses.write().insert(i, wrapper.clone());
+        wrapper.kobject.init(sysfs::register_bus_kobject(&wrapper));
+        (i, wrapper)
     }
 
     pub fn register_device(device: Arc<UsbDeviceAccess>) {
         log::info!("usb: register device {}", device.bus_address());
+
         BUS_MANAGER
             .devices
             .write()
             .insert(device.bus_address(), device.clone());
+        device.kobject.init(sysfs::register_device_kobject(&device));
 
         QUEUE.push_back(device);
     }
@@ -51,7 +73,11 @@ pub async fn bus_handler() {
             new_device.bus_address()
         );
 
-        class_driver::spawn_driver(new_device).await.ok();
+        let address = new_device.bus_address();
+        if let Err(error) = class_driver::setup_device(new_device).await {
+            log::warn!("USB device {address} setup error: {error:?}",);
+        }
+        // class_driver::spawn_driver(new_device).await.ok();
     }
 }
 

kernel/driver/bus/usb/src/class_driver/hid_keyboard.rs

@@ -4,9 +4,12 @@ use alloc::{boxed::Box, sync::Arc};
 use async_trait::async_trait;
 use yggdrasil_abi::io::{KeyboardKey, KeyboardKeyEvent};
 
-use crate::{device::UsbDeviceAccess, error::UsbError, info::UsbDeviceClass};
+use crate::{
-
+    class_driver::{UsbInterfaceClass, UsbInterfaceDriver},
-use super::{UsbClassInfo, UsbDriver};
+    device::UsbDeviceAccess,
+    error::UsbError,
+    info::UsbInterfaceInfo,
+};
 
 pub struct UsbHidKeyboardDriver;
 
@@ -125,10 +128,14 @@ impl KeyboardState {
 }
 
 #[async_trait]
-impl UsbDriver for UsbHidKeyboardDriver {
+impl UsbInterfaceDriver for UsbHidKeyboardDriver {
-    async fn run(self: Arc<Self>, device: Arc<UsbDeviceAccess>) -> Result<(), UsbError> {
+    async fn run(
+        self: Arc<Self>,
+        device: Arc<UsbDeviceAccess>,
+        interface: UsbInterfaceInfo,
+    ) -> Result<(), UsbError> {
         // TODO not sure whether to use boot protocol (easy) or GetReport
-        let config = device.select_configuration(|_| true).await?.unwrap();
+        let config = device.current_configuration().unwrap();
 
         log::info!("Setup HID keyboard");
         let pipe = device
@@ -156,7 +163,7 @@ impl UsbDriver for UsbHidKeyboardDriver {
 
             for &event in events {
                 log::trace!("Generic Keyboard: {:?}", event);
-                ygg_driver_input::send_event(event);
+                ygg_driver_input::send_keyboard_event(event);
             }
         }
     }
@@ -165,12 +172,18 @@ impl UsbDriver for UsbHidKeyboardDriver {
         "USB HID Keyboard"
     }
 
-    fn probe(&self, class: &UsbClassInfo, _device: &UsbDeviceAccess) -> bool {
+    fn probe(
-        log::info!(
+        &self,
-            "class = {:?}, subclass = {:02x}",
+        device: &UsbDeviceAccess,
-            class.class,
+        interface: &UsbInterfaceInfo,
-            class.subclass
+        class: UsbInterfaceClass,
-        );
+    ) -> bool {
-        class.class == UsbDeviceClass::Hid && (class.subclass == 0x00 || class.subclass == 0x01)
+        let _ = (device, interface);
+        class
+            == UsbInterfaceClass {
+                class: 3,
+                subclass: 1,
+                protocol: 1,
+            }
     }
 }

kernel/driver/bus/usb/src/class_driver/hid_mouse.rs

@@ -0,0 +1,64 @@
+use alloc::{boxed::Box, sync::Arc};
+use async_trait::async_trait;
+use yggdrasil_abi::io::{ButtonMask, MouseEvent};
+
+use crate::{
+    class_driver::{UsbInterfaceClass, UsbInterfaceDriver},
+    device::UsbDeviceAccess,
+    error::UsbError,
+    info::UsbInterfaceInfo,
+};
+
+pub struct UsbHidMouseDriver;
+
+#[async_trait]
+impl UsbInterfaceDriver for UsbHidMouseDriver {
+    async fn run(
+        self: Arc<Self>,
+        device: Arc<UsbDeviceAccess>,
+        interface: UsbInterfaceInfo,
+    ) -> Result<(), UsbError> {
+        let config = device.current_configuration().unwrap();
+
+        log::info!("Setup HID mouse");
+        let pipe = device
+            .open_interrupt_in_pipe(1, config.endpoints[0].max_packet_size as u16)
+            .await?;
+
+        let mut buffer = [0; 16];
+        let mut button_state = 0;
+
+        loop {
+            let len = pipe.read(&mut buffer).await?;
+            if len < 4 {
+                continue;
+            }
+
+            let event = MouseEvent {
+                buttons: ButtonMask(buffer[0]),
+                dx: (buffer[1] as i8) as i32,
+                dy: (buffer[2] as i8) as i32,
+            };
+
+            ygg_driver_input::send_mouse_event(event);
+        }
+    }
+
+    fn name(&self) -> &'static str {
+        "USB HID Mouse"
+    }
+
+    fn probe(
+        &self,
+        device: &UsbDeviceAccess,
+        interface: &UsbInterfaceInfo,
+        class: UsbInterfaceClass,
+    ) -> bool {
+        class
+            == UsbInterfaceClass {
+                class: 3,
+                subclass: 1,
+                protocol: 2,
+            }
+    }
+}

kernel/driver/bus/usb/src/class_driver/mass_storage.rs

@@ -1,273 +1,273 @@
-use core::mem::MaybeUninit;
+// use core::mem::MaybeUninit;
-
+//
-use alloc::{boxed::Box, sync::Arc};
+// use alloc::{boxed::Box, sync::Arc};
-use async_trait::async_trait;
+// use async_trait::async_trait;
-use bytemuck::{Pod, Zeroable};
+// use bytemuck::{Pod, Zeroable};
-use libk::{
+// use libk::{
-    dma::{DmaBuffer, DmaSliceMut},
+//     dma::{DmaBuffer, DmaSliceMut},
-    error::Error,
+//     error::Error,
-};
+// };
-use ygg_driver_scsi::{transport::ScsiTransport, ScsiEnclosure};
+// use ygg_driver_scsi::{transport::ScsiTransport, ScsiEnclosure};
-
+//
-use crate::{
+// use crate::{
-    communication::UsbDirection,
+//     communication::UsbDirection,
-    device::{UsbDeviceAccess, UsbDeviceDetachHandler},
+//     device::{UsbDeviceAccess, UsbDeviceDetachHandler},
-    error::UsbError,
+//     error::UsbError,
-    info::{UsbDeviceClass, UsbEndpointType},
+//     info::{UsbDeviceClass, UsbEndpointType},
-    pipe::{
+//     pipe::{
-        control::{ControlTransferSetup, UsbClassSpecificRequest},
+//         control::{ControlTransferSetup, UsbClassSpecificRequest},
-        normal::{UsbBulkInPipeAccess, UsbBulkOutPipeAccess},
+//         normal::{UsbBulkInPipeAccess, UsbBulkOutPipeAccess},
-    },
+//     },
-};
+// };
-
+//
-use super::{UsbClassInfo, UsbDriver};
+// use super::{UsbClassInfo, UsbDriver};
-
+//
-pub struct UsbMassStorageDriverBulkOnly;
+// pub struct UsbMassStorageDriverBulkOnly;
-
+//
-#[derive(Debug, Clone, Copy, Zeroable, Pod)]
+// #[derive(Debug, Clone, Copy, Zeroable, Pod)]
-#[repr(C)]
+// #[repr(C)]
-struct Cbw {
+// struct Cbw {
-    signature: u32,       // 0x00
+//     signature: u32,       // 0x00
-    tag: u32,             // 0x04
+//     tag: u32,             // 0x04
-    transfer_length: u32, // 0x08
+//     transfer_length: u32, // 0x08
-    flags: u8,            // 0x0C
+//     flags: u8,            // 0x0C
-    lun: u8,              // 0x0D
+//     lun: u8,              // 0x0D
-    cb_length: u8,        // 0x0E
+//     cb_length: u8,        // 0x0E
-    cb_data: [u8; 16],    // 0x0F
+//     cb_data: [u8; 16],    // 0x0F
-    // Not sent
+//     // Not sent
-    _0: u8,
+//     _0: u8,
-}
+// }
-
+//
-#[derive(Debug, Clone, Copy, Zeroable, Pod)]
+// #[derive(Debug, Clone, Copy, Zeroable, Pod)]
-#[repr(C)]
+// #[repr(C)]
-struct Csw {
+// struct Csw {
-    signature: u32,
+//     signature: u32,
-    tag: u32,
+//     tag: u32,
-    data_residue: u32,
+//     data_residue: u32,
-    status: u8,
+//     status: u8,
-    _0: [u8; 3],
+//     _0: [u8; 3],
-}
+// }
-
+//
-struct Bbb {
+// struct Bbb {
-    #[allow(unused)]
+//     #[allow(unused)]
-    device: Arc<UsbDeviceAccess>,
+//     device: Arc<UsbDeviceAccess>,
-    in_pipe: UsbBulkInPipeAccess,
+//     in_pipe: UsbBulkInPipeAccess,
-    out_pipe: UsbBulkOutPipeAccess,
+//     out_pipe: UsbBulkOutPipeAccess,
-    last_tag: u32,
+//     last_tag: u32,
-}
+// }
-
+//
-struct DetachHandler(Arc<ScsiEnclosure>);
+// struct DetachHandler(Arc<ScsiEnclosure>);
-
+//
-impl Bbb {
+// impl Bbb {
-    pub fn new(
+//     pub fn new(
-        device: Arc<UsbDeviceAccess>,
+//         device: Arc<UsbDeviceAccess>,
-        in_pipe: UsbBulkInPipeAccess,
+//         in_pipe: UsbBulkInPipeAccess,
-        out_pipe: UsbBulkOutPipeAccess,
+//         out_pipe: UsbBulkOutPipeAccess,
-    ) -> Result<Self, UsbError> {
+//     ) -> Result<Self, UsbError> {
-        Ok(Self {
+//         Ok(Self {
-            device,
+//             device,
-            in_pipe,
+//             in_pipe,
-            out_pipe,
+//             out_pipe,
-            last_tag: 0,
+//             last_tag: 0,
-        })
+//         })
-    }
+//     }
-}
+// }
-
+//
-impl Bbb {
+// impl Bbb {
-    async fn send_cbw(
+//     async fn send_cbw(
-        &mut self,
+//         &mut self,
-        lun: u8,
+//         lun: u8,
-        host_to_dev: bool,
+//         host_to_dev: bool,
-        command: &[u8],
+//         command: &[u8],
-        response_len: usize,
+//         response_len: usize,
-    ) -> Result<u32, Error> {
+//     ) -> Result<u32, Error> {
-        self.last_tag = self.last_tag.wrapping_add(1);
+//         self.last_tag = self.last_tag.wrapping_add(1);
-
+//
-        let flags = if !host_to_dev { 1 << 7 } else { 0 };
+//         let flags = if !host_to_dev { 1 << 7 } else { 0 };
-        let tag = self.last_tag;
+//         let tag = self.last_tag;
-        let mut cbw_bytes = [0; 32];
+//         let mut cbw_bytes = [0; 32];
-        let cbw = bytemuck::from_bytes_mut::<Cbw>(&mut cbw_bytes);
+//         let cbw = bytemuck::from_bytes_mut::<Cbw>(&mut cbw_bytes);
-
+//
-        cbw.signature = 0x43425355;
+//         cbw.signature = 0x43425355;
-        cbw.transfer_length = response_len as u32;
+//         cbw.transfer_length = response_len as u32;
-        cbw.flags = flags;
+//         cbw.flags = flags;
-        cbw.tag = tag;
+//         cbw.tag = tag;
-        cbw.lun = lun;
+//         cbw.lun = lun;
-        cbw.cb_length = command.len() as u8;
+//         cbw.cb_length = command.len() as u8;
-        cbw.cb_data[..command.len()].copy_from_slice(command);
+//         cbw.cb_data[..command.len()].copy_from_slice(command);
-
+//
-        self.out_pipe
+//         self.out_pipe
-            .write(&cbw_bytes[..31])
+//             .write(&cbw_bytes[..31])
-            .await
+//             .await
-            .inspect_err(|error| log::error!("msc: CBW send error: {error:?}"))?;
+//             .inspect_err(|error| log::error!("msc: CBW send error: {error:?}"))?;
-
+//
-        Ok(tag)
+//         Ok(tag)
-    }
+//     }
-
+//
-    async fn read_csw(&mut self, tag: u32) -> Result<(), Error> {
+//     async fn read_csw(&mut self, tag: u32) -> Result<(), Error> {
-        let mut csw_bytes = [0; 16];
+//         let mut csw_bytes = [0; 16];
-        self.in_pipe
+//         self.in_pipe
-            .read_exact(&mut csw_bytes[..13])
+//             .read_exact(&mut csw_bytes[..13])
-            .await
+//             .await
-            .inspect_err(|error| log::error!("msc: CSW receive error: {error:?}"))?;
+//             .inspect_err(|error| log::error!("msc: CSW receive error: {error:?}"))?;
-        let csw = bytemuck::from_bytes::<Csw>(&csw_bytes);
+//         let csw = bytemuck::from_bytes::<Csw>(&csw_bytes);
-
+//
-        if csw.signature != 0x53425355 {
+//         if csw.signature != 0x53425355 {
-            log::warn!("msc: invalid csw signature");
+//             log::warn!("msc: invalid csw signature");
-            return Err(Error::InvalidArgument);
+//             return Err(Error::InvalidArgument);
-        }
+//         }
-        if csw.tag != tag {
+//         if csw.tag != tag {
-            let csw_tag = csw.tag;
+//             let csw_tag = csw.tag;
-            log::warn!("msc: invalid csw tag (got {}, expected {tag})", csw_tag);
+//             log::warn!("msc: invalid csw tag (got {}, expected {tag})", csw_tag);
-            return Err(Error::InvalidArgument);
+//             return Err(Error::InvalidArgument);
-        }
+//         }
-        if csw.status != 0x00 {
+//         if csw.status != 0x00 {
-            return Err(Error::InvalidArgument);
+//             return Err(Error::InvalidArgument);
-        }
+//         }
-        Ok(())
+//         Ok(())
-    }
+//     }
-
+//
-    async fn read_response_data(
+//     async fn read_response_data(
-        &mut self,
+//         &mut self,
-        buffer: DmaSliceMut<'_, MaybeUninit<u8>>,
+//         buffer: DmaSliceMut<'_, MaybeUninit<u8>>,
-    ) -> Result<usize, Error> {
+//     ) -> Result<usize, Error> {
-        if buffer.len() == 0 {
+//         if buffer.len() == 0 {
-            return Ok(0);
+//             return Ok(0);
-        }
+//         }
-        let len = self
+//         let len = self
-            .in_pipe
+//             .in_pipe
-            .read_dma(buffer)
+//             .read_dma(buffer)
-            .await
+//             .await
-            .inspect_err(|error| log::error!("msc: DMA read error: {error:?}"))?;
+//             .inspect_err(|error| log::error!("msc: DMA read error: {error:?}"))?;
-        Ok(len)
+//         Ok(len)
-    }
+//     }
-}
+// }
-
+//
-#[async_trait]
+// #[async_trait]
-impl ScsiTransport for Bbb {
+// impl ScsiTransport for Bbb {
-    fn allocate_buffer(&self, size: usize) -> Result<DmaBuffer<[MaybeUninit<u8>]>, Error> {
+//     fn allocate_buffer(&self, size: usize) -> Result<DmaBuffer<[MaybeUninit<u8>]>, Error> {
-        Ok(self.in_pipe.allocate_dma_buffer(size)?)
+//         Ok(self.in_pipe.allocate_dma_buffer(size)?)
-    }
+//     }
-
+//
-    async fn perform_request_raw(
+//     async fn perform_request_raw(
-        &mut self,
+//         &mut self,
-        lun: u8,
+//         lun: u8,
-        request_data: &[u8],
+//         request_data: &[u8],
-        response_buffer: DmaSliceMut<'_, MaybeUninit<u8>>,
+//         response_buffer: DmaSliceMut<'_, MaybeUninit<u8>>,
-    ) -> Result<usize, Error> {
+//     ) -> Result<usize, Error> {
-        if request_data.len() > 16 || response_buffer.len() > self.max_bytes_per_request() {
+//         if request_data.len() > 16 || response_buffer.len() > self.max_bytes_per_request() {
-            return Err(Error::InvalidArgument);
+//             return Err(Error::InvalidArgument);
-        }
+//         }
-
+//
-        let tag = self
+//         let tag = self
-            .send_cbw(lun, false, request_data, response_buffer.len())
+//             .send_cbw(lun, false, request_data, response_buffer.len())
-            .await?;
+//             .await?;
-        let response_len = self.read_response_data(response_buffer).await?;
+//         let response_len = self.read_response_data(response_buffer).await?;
-        self.read_csw(tag).await?;
+//         self.read_csw(tag).await?;
-        Ok(response_len)
+//         Ok(response_len)
-    }
+//     }
-
+//
-    fn max_bytes_per_request(&self) -> usize {
+//     fn max_bytes_per_request(&self) -> usize {
-        32768
+//         32768
-    }
+//     }
-}
+// }
-
+//
-impl UsbDeviceDetachHandler for DetachHandler {
+// impl UsbDeviceDetachHandler for DetachHandler {
-    fn handle_device_detach(&self) {
+//     fn handle_device_detach(&self) {
-        log::info!("Mass storage detached");
+//         log::info!("Mass storage detached");
-        self.0.detach();
+//         self.0.detach();
-    }
+//     }
-}
+// }
-
+//
-#[derive(Debug, Pod, Zeroable, Clone, Copy)]
+// #[derive(Debug, Pod, Zeroable, Clone, Copy)]
-#[repr(C)]
+// #[repr(C)]
-pub struct BulkOnlyMassStorageReset;
+// pub struct BulkOnlyMassStorageReset;
-
+//
-#[derive(Debug, Pod, Zeroable, Clone, Copy)]
+// #[derive(Debug, Pod, Zeroable, Clone, Copy)]
-#[repr(C)]
+// #[repr(C)]
-pub struct GetMaxLun;
+// pub struct GetMaxLun;
-
+//
-impl UsbClassSpecificRequest for BulkOnlyMassStorageReset {
+// impl UsbClassSpecificRequest for BulkOnlyMassStorageReset {
-    const BM_REQUEST_TYPE: u8 = 0b00100001;
+//     const BM_REQUEST_TYPE: u8 = 0b00100001;
-    const B_REQUEST: u8 = 0b11111111;
+//     const B_REQUEST: u8 = 0b11111111;
-}
+// }
-
+//
-impl UsbClassSpecificRequest for GetMaxLun {
+// impl UsbClassSpecificRequest for GetMaxLun {
-    const BM_REQUEST_TYPE: u8 = 0b10100001;
+//     const BM_REQUEST_TYPE: u8 = 0b10100001;
-    const B_REQUEST: u8 = 0b11111110;
+//     const B_REQUEST: u8 = 0b11111110;
-}
+// }
-
+//
-#[async_trait]
+// #[async_trait]
-impl UsbDriver for UsbMassStorageDriverBulkOnly {
+// impl UsbDriver for UsbMassStorageDriverBulkOnly {
-    async fn run(self: Arc<Self>, device: Arc<UsbDeviceAccess>) -> Result<(), UsbError> {
+//     async fn run(self: Arc<Self>, device: Arc<UsbDeviceAccess>) -> Result<(), UsbError> {
-        // TODO filter to only accept BBB config
+//         // TODO filter to only accept BBB config
-        let config = device.select_configuration(|_| true).await?.unwrap();
+//         let config = device.select_configuration(|_| true).await?.unwrap();
-        // Bulk-in, bulk-out
+//         // Bulk-in, bulk-out
-        assert_eq!(config.endpoints.len(), 2);
+//         assert_eq!(config.endpoints.len(), 2);
-        let control_pipe = device.control_pipe();
+//         let control_pipe = device.control_pipe();
-        let (in_index, in_info) = config
+//         let (in_index, in_info) = config
-            .find_endpoint(|ep| ep.is(UsbEndpointType::Bulk, UsbDirection::In))
+//             .find_endpoint(|ep| ep.is(UsbEndpointType::Bulk, UsbDirection::In))
-            .ok_or(UsbError::InvalidConfiguration)?;
+//             .ok_or(UsbError::InvalidConfiguration)?;
-        let (out_index, out_info) = config
+//         let (out_index, out_info) = config
-            .find_endpoint(|ep| ep.is(UsbEndpointType::Bulk, UsbDirection::Out))
+//             .find_endpoint(|ep| ep.is(UsbEndpointType::Bulk, UsbDirection::Out))
-            .ok_or(UsbError::InvalidConfiguration)?;
+//             .ok_or(UsbError::InvalidConfiguration)?;
-        let in_pipe = device
+//         let in_pipe = device
-            .open_bulk_in_pipe(in_index, in_info.max_packet_size as u16)
+//             .open_bulk_in_pipe(in_index, in_info.max_packet_size as u16)
-            .await?;
+//             .await?;
-        let out_pipe = device
+//         let out_pipe = device
-            .open_bulk_out_pipe(out_index, out_info.max_packet_size as u16)
+//             .open_bulk_out_pipe(out_index, out_info.max_packet_size as u16)
-            .await?;
+//             .await?;
-
+//
-        // Perform a Bulk-Only Mass Storage Reset
+//         // Perform a Bulk-Only Mass Storage Reset
-        // TODO interface id?
+//         // TODO interface id?
-        control_pipe
+//         control_pipe
-            .control_transfer(ControlTransferSetup {
+//             .control_transfer(ControlTransferSetup {
-                bm_request_type: BulkOnlyMassStorageReset::BM_REQUEST_TYPE,
+//                 bm_request_type: BulkOnlyMassStorageReset::BM_REQUEST_TYPE,
-                b_request: BulkOnlyMassStorageReset::B_REQUEST,
+//                 b_request: BulkOnlyMassStorageReset::B_REQUEST,
-                w_value: 0,
+//                 w_value: 0,
-                w_index: 0,
+//                 w_index: 0,
-                w_length: 0,
+//                 w_length: 0,
-            })
+//             })
-            .await?;
+//             .await?;
-
+//
-        // Get max LUN
+//         // Get max LUN
-        // TODO on devices which do not support multiple LUNs, this command may STALL
+//         // TODO on devices which do not support multiple LUNs, this command may STALL
-        let mut buffer = [MaybeUninit::uninit()];
+//         let mut buffer = [MaybeUninit::uninit()];
-        let len = control_pipe
+//         let len = control_pipe
-            .control_transfer_in(
+//             .control_transfer_in(
-                ControlTransferSetup {
+//                 ControlTransferSetup {
-                    bm_request_type: GetMaxLun::BM_REQUEST_TYPE,
+//                     bm_request_type: GetMaxLun::BM_REQUEST_TYPE,
-                    b_request: GetMaxLun::B_REQUEST,
+//                     b_request: GetMaxLun::B_REQUEST,
-                    w_value: 0,
+//                     w_value: 0,
-                    w_index: 0,
+//                     w_index: 0,
-                    w_length: 1,
+//                     w_length: 1,
-                },
+//                 },
-                &mut buffer,
+//                 &mut buffer,
-            )
+//             )
-            .await?;
+//             .await?;
-        let max_lun = if len < 1 {
+//         let max_lun = if len < 1 {
-            0
+//             0
-        } else {
+//         } else {
-            unsafe { buffer[0].assume_init() }
+//             unsafe { buffer[0].assume_init() }
-        };
+//         };
-
+//
-        let bbb = Bbb::new(device.clone(), in_pipe, out_pipe)?;
+//         let bbb = Bbb::new(device.clone(), in_pipe, out_pipe)?;
-        let scsi = ScsiEnclosure::setup(Box::new(bbb), max_lun as usize + 1)
+//         let scsi = ScsiEnclosure::setup(Box::new(bbb), max_lun as usize + 1)
-            .await
+//             .await
-            .inspect_err(|error| log::error!("msc: scsi error {error:?}"))
+//             .inspect_err(|error| log::error!("msc: scsi error {error:?}"))
-            .map_err(|_| UsbError::DriverError)?;
+//             .map_err(|_| UsbError::DriverError)?;
-        let detach = DetachHandler(scsi.clone());
+//         let detach = DetachHandler(scsi.clone());
-        device.set_detach_handler(Arc::new(detach));
+//         device.set_detach_handler(Arc::new(detach));
-
+//
-        Ok(())
+//         Ok(())
-    }
+//     }
-
+//
-    fn name(&self) -> &'static str {
+//     fn name(&self) -> &'static str {
-        "USB Mass Storage"
+//         "USB Mass Storage"
-    }
+//     }
-
+//
-    fn probe(&self, class: &UsbClassInfo, _device: &UsbDeviceAccess) -> bool {
+//     fn probe(&self, class: &UsbClassInfo, _device: &UsbDeviceAccess) -> bool {
-        // TODO support other protocols
+//         // TODO support other protocols
-        class.class == UsbDeviceClass::MassStorage && class.interface_protocol_number == 0x50
+//         class.class == UsbDeviceClass::MassStorage && class.interface_protocol_number == 0x50
-    }
+//     }
-}
+// }

kernel/driver/bus/usb/src/class_driver/mod.rs

@@ -1,117 +1,208 @@
+use core::mem::MaybeUninit;
+
 use alloc::{boxed::Box, sync::Arc, vec::Vec};
 use async_trait::async_trait;
 use libk::task::runtime;
 use libk_util::sync::spin_rwlock::IrqSafeRwLock;
 
 use crate::{
+    address::UsbInterfaceAddress,
     device::UsbDeviceAccess,
     error::UsbError,
-    info::{UsbDeviceClass, UsbDeviceProtocol},
+    info::{UsbInterfaceInfo, CLASS_FROM_INTERFACE},
+    pipe::control::{ControlTransferSetup, UsbClassSpecificRequest},
 };
 
+// use alloc::{boxed::Box, sync::Arc, vec::Vec};
+// use async_trait::async_trait;
+// use libk::task::runtime;
+// use libk_util::sync::spin_rwlock::IrqSafeRwLock;
+//
+// use crate::{
+//     device::UsbDeviceAccess,
+//     error::UsbError,
+//     info::{UsbDeviceClass, UsbDeviceProtocol},
+// };
+//
 pub mod hid_keyboard;
-pub mod mass_storage;
+pub mod hid_mouse;
-
+// pub mod mass_storage;
-#[derive(Debug)]
+//
-pub struct UsbClassInfo {
+// #[derive(Debug)]
-    pub class: UsbDeviceClass,
+// pub struct UsbClassInfo {
+//     pub class: UsbDeviceClass,
+//     pub subclass: u8,
+//     pub protocol: UsbDeviceProtocol,
+//     pub device_protocol_number: u8,
+//     pub interface_protocol_number: u8,
+// }
+//
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct UsbInterfaceClass {
+    pub class: u8,
     pub subclass: u8,
-    pub protocol: UsbDeviceProtocol,
+    pub protocol: u8,
-    pub device_protocol_number: u8,
-    pub interface_protocol_number: u8,
 }
 
 #[async_trait]
-pub trait UsbDriver: Send + Sync {
+pub trait UsbInterfaceDriver: Send + Sync {
-    async fn run(self: Arc<Self>, device: Arc<UsbDeviceAccess>) -> Result<(), UsbError>;
+    async fn run(
+        self: Arc<Self>,
+        device: Arc<UsbDeviceAccess>,
+        interface: UsbInterfaceInfo,
+    ) -> Result<(), UsbError>;
 
     fn name(&self) -> &'static str;
-    fn probe(&self, class: &UsbClassInfo, device: &UsbDeviceAccess) -> bool;
+    fn probe(
+        &self,
+        device: &UsbDeviceAccess,
+        interface: &UsbInterfaceInfo,
+        class: UsbInterfaceClass,
+    ) -> bool;
 }
 
-async fn extract_class_info(device: &UsbDeviceAccess) -> Result<Option<UsbClassInfo>, UsbError> {
+// async fn extract_class_info(device: &UsbDeviceAccess) -> Result<Option<UsbClassInfo>, UsbError> {
-    if device.info.num_configurations != 1 {
+//     if device.info.num_configurations != 1 {
-        return Ok(None);
+//         return Ok(None);
-    }
+//     }
-    let device_info = &device.info;
+//     let device_info = &device.info;
-    let config_info = device.query_configuration_info(0).await?;
+//     let config_info = device.query_configuration_info(0).await?;
+//
+//     if config_info.interfaces.len() >= 1 {
+//         let if_info = &config_info.interfaces[0];
+//
+//         let class = if device_info.device_class == UsbDeviceClass::FromInterface {
+//             if_info.interface_class
+//         } else {
+//             device_info.device_class
+//         };
+//         let subclass = if device_info.device_subclass == 0 {
+//             if_info.interface_subclass
+//         } else {
+//             device_info.device_subclass
+//         };
+//         let protocol = if device_info.device_protocol == UsbDeviceProtocol::FromInterface {
+//             if_info.interface_protocol
+//         } else {
+//             device_info.device_protocol
+//         };
+//
+//         Ok(Some(UsbClassInfo {
+//             class,
+//             subclass,
+//             protocol,
+//             interface_protocol_number: if_info.interface_protocol_number,
+//             device_protocol_number: device_info.device_protocol_number,
+//         }))
+//     } else {
+//         Ok(None)
+//     }
+// }
+//
+// async fn pick_driver(
+//     device: &UsbDeviceAccess,
+// ) -> Result<Option<Arc<dyn UsbDriver + 'static>>, UsbError> {
+//     let Some(class) = extract_class_info(device).await? else {
+//         return Ok(None);
+//     };
+//
+//     for driver in USB_DEVICE_DRIVERS.read().iter() {
+//         if driver.probe(&class, device) {
+//             return Ok(Some(driver.clone()));
+//         }
+//     }
+//     Ok(None)
+// }
+//
+// pub async fn spawn_driver(device: Arc<UsbDeviceAccess>) -> Result<(), UsbError> {
+//     // if let Some(driver) = pick_driver(&device).await? {
+//     //     runtime::spawn(async move {
+//     //         let name = driver.name();
+//     //         match driver.run(device).await {
+//     //             e @ Err(UsbError::DeviceDisconnected) => {
+//     //                 log::warn!(
+//     //                     "Driver {:?} did not exit cleanly: device disconnected",
+//     //                     name,
+//     //                 );
+//
+//     //                 e
+//     //             }
+//     //             e => e,
+//     //         }
+//     //     })
+//     //     .map_err(UsbError::SystemError)?;
+//     // }
+//     Ok(())
+// }
 
-    if config_info.interfaces.len() >= 1 {
+async fn setup_interface(
-        let if_info = &config_info.interfaces[0];
+    device: &Arc<UsbDeviceAccess>,
-
+    address: UsbInterfaceAddress,
-        let class = if device_info.device_class == UsbDeviceClass::FromInterface {
+    interface: &UsbInterfaceInfo,
-            if_info.interface_class
+) -> Result<(), UsbError> {
-        } else {
+    let class = UsbInterfaceClass {
-            device_info.device_class
+        class: interface.interface_class,
-        };
+        subclass: interface.interface_subclass,
-        let subclass = if device_info.device_subclass == 0 {
+        protocol: interface.interface_protocol,
-            if_info.interface_subclass
-        } else {
-            device_info.device_subclass
-        };
-        let protocol = if device_info.device_protocol == UsbDeviceProtocol::FromInterface {
-            if_info.interface_protocol
-        } else {
-            device_info.device_protocol
-        };
-
-        Ok(Some(UsbClassInfo {
-            class,
-            subclass,
-            protocol,
-            interface_protocol_number: if_info.interface_protocol_number,
-            device_protocol_number: device_info.device_protocol_number,
-        }))
-    } else {
-        Ok(None)
-    }
-}
-
-async fn pick_driver(
-    device: &UsbDeviceAccess,
-) -> Result<Option<Arc<dyn UsbDriver + 'static>>, UsbError> {
-    let Some(class) = extract_class_info(device).await? else {
-        return Ok(None);
     };
-
+    let drivers = USB_INTERFACE_DRIVERS.read();
-    for driver in USB_DEVICE_DRIVERS.read().iter() {
+    for driver in drivers.iter() {
-        if driver.probe(&class, device) {
+        if driver.probe(device, interface, class) {
-            return Ok(Some(driver.clone()));
+            let driver = driver.clone();
-        }
+            let device = device.clone();
-    }
+            let interface = interface.clone();
-    Ok(None)
+            runtime::spawn(async move {
-}
+                let name = driver.name();
-
+                match driver.run(device, interface).await {
-pub async fn spawn_driver(device: Arc<UsbDeviceAccess>) -> Result<(), UsbError> {
+                    e @ Err(UsbError::DeviceDisconnected) => {
-    if let Some(driver) = pick_driver(&device).await? {
+                        log::warn!("{address} did not exit cleanly: device disconnected ({name})");
-        runtime::spawn(async move {
+                        e
-            let name = driver.name();
+                    }
-            match driver.run(device).await {
+                    e => e,
-                e @ Err(UsbError::DeviceDisconnected) => {
-                    log::warn!(
-                        "Driver {:?} did not exit cleanly: device disconnected",
-                        name,
-                    );
-
-                    e
                 }
-                e => e,
+            })
-            }
+            .map_err(UsbError::SystemError)?;
-        })
+            break;
-        .map_err(UsbError::SystemError)?;
+        }
     }
     Ok(())
 }
 
-pub fn register_driver(driver: Arc<dyn UsbDriver + 'static>) {
+pub async fn setup_device(device: Arc<UsbDeviceAccess>) -> Result<(), UsbError> {
+    // If device has only one configuration available, use it
+    // TODO support devices with multiple configurations
+    let address = device.bus_address();
+    log::info!("Setup USB device @ {address}");
+
+    let Some(config_info) = device.use_default_configuration().await? else {
+        log::warn!("{address} has multiple configurations, not supported yet",);
+        return Ok(());
+    };
+
+    // Setup drivers for interfaces
+    log::info!("{address}: {config_info:#?}");
+
+    // TODO device-level drivers
+    for interface in config_info.interfaces.iter() {
+        let address = address.with_interface(interface.number);
+        if let Err(error) = setup_interface(&device, address, interface).await {
+            log::error!("{}: {:?}", address, error);
+        }
+    }
+
+    Ok(())
+}
+
+pub fn register_driver(driver: Arc<dyn UsbInterfaceDriver + 'static>) {
     // TODO check for duplicates
-    USB_DEVICE_DRIVERS.write().push(driver);
+    USB_INTERFACE_DRIVERS.write().push(driver);
 }
 
 pub fn register_default_class_drivers() {
     register_driver(Arc::new(hid_keyboard::UsbHidKeyboardDriver));
-    register_driver(Arc::new(mass_storage::UsbMassStorageDriverBulkOnly));
+    register_driver(Arc::new(hid_mouse::UsbHidMouseDriver));
+    // register_driver(Arc::new(mass_storage::UsbMassStorageDriverBulkOnly));
 }
 
-static USB_DEVICE_DRIVERS: IrqSafeRwLock<Vec<Arc<dyn UsbDriver + 'static>>> =
+static USB_INTERFACE_DRIVERS: IrqSafeRwLock<Vec<Arc<dyn UsbInterfaceDriver + 'static>>> =
     IrqSafeRwLock::new(Vec::new());

kernel/driver/bus/usb/src/descriptor.rs

@@ -2,9 +2,9 @@ use bytemuck::{Pod, Zeroable};
 
 use crate::{
     communication::UsbDirection,
-    device::UsbSpeed,
+    device::{self, UsbSpeed},
     error::UsbError,
-    info::{UsbDeviceClass, UsbDeviceProtocol, UsbEndpointType, UsbVersion},
+    info::UsbEndpointType,
 };
 
 #[derive(Clone, Copy, Debug, Default, Pod, Zeroable)]
@@ -91,15 +91,15 @@ pub struct UsbOtherSpeedConfiguration {
     pub max_power: u8,
 }
 
-impl UsbInterfaceDescriptor {
+// impl UsbInterfaceDescriptor {
-    pub fn class(&self) -> UsbDeviceClass {
+//     pub fn class(&self) -> UsbDeviceClass {
-        UsbDeviceClass::try_from(self.interface_class).unwrap_or(UsbDeviceClass::Unknown)
+//         UsbDeviceClass::try_from(self.interface_class).unwrap_or(UsbDeviceClass::Unknown)
-    }
+//     }
-
+//
-    pub fn protocol(&self) -> UsbDeviceProtocol {
+//     pub fn protocol(&self) -> UsbDeviceProtocol {
-        UsbDeviceProtocol::try_from(self.interface_protocol).unwrap_or(UsbDeviceProtocol::Unknown)
+//         UsbDeviceProtocol::try_from(self.interface_protocol).unwrap_or(UsbDeviceProtocol::Unknown)
-    }
+//     }
-}
+// }
 
 impl UsbEndpointDescriptor {
     pub fn direction(&self) -> UsbDirection {
@@ -127,16 +127,16 @@ impl UsbEndpointDescriptor {
 }
 
 impl UsbDeviceDescriptor {
-    pub fn class(&self) -> UsbDeviceClass {
+    // pub fn class(&self) -> UsbDeviceClass {
-        UsbDeviceClass::try_from(self.device_class).unwrap_or(UsbDeviceClass::Unknown)
+    //     UsbDeviceClass::try_from(self.device_class).unwrap_or(UsbDeviceClass::Unknown)
-    }
+    // }
 
-    pub fn protocol(&self) -> UsbDeviceProtocol {
+    // pub fn protocol(&self) -> UsbDeviceProtocol {
-        UsbDeviceProtocol::try_from(self.device_protocol).unwrap_or(UsbDeviceProtocol::Unknown)
+    //     UsbDeviceProtocol::try_from(self.device_protocol).unwrap_or(UsbDeviceProtocol::Unknown)
-    }
+    // }
 
-    pub fn max_packet_size(&self, version: UsbVersion, speed: UsbSpeed) -> Result<usize, UsbError> {
+    pub fn max_packet_size(&self, version: u16, speed: UsbSpeed) -> Result<usize, UsbError> {
-        match (version.is_version_3(), speed, self.max_packet_size_0) {
+        match (is_version_3(version), speed, self.max_packet_size_0) {
             (true, UsbSpeed::Super, 9) => Ok(1 << 9),
             (true, _, _) => todo!("Non-GenX speed USB3+ maxpacketsize0"),
             (false, _, 8) => Ok(8),
@@ -147,3 +147,7 @@ impl UsbDeviceDescriptor {
         }
     }
 }
+
+pub fn is_version_3(version: u16) -> bool {
+    version & 0xFF00 == 0x300
+}

kernel/driver/bus/usb/src/device.rs

@@ -1,14 +1,28 @@
-use core::{fmt, ops::Deref};
+use core::{
+    fmt,
+    ops::{Deref, Sub},
+};
 
-use alloc::{boxed::Box, sync::Arc, vec::Vec};
+use alloc::{
+    boxed::Box,
+    format,
+    string::{String, ToString},
+    sync::Arc,
+    vec::Vec,
+};
 use async_trait::async_trait;
-use libk_util::sync::spin_rwlock::{IrqSafeRwLock, IrqSafeRwLockReadGuard};
+use libk::error::Error;
+use libk_util::{
+    sync::spin_rwlock::{IrqSafeRwLock, IrqSafeRwLockReadGuard},
+    OneTimeInit,
+};
 
 use crate::{
+    address::UsbBusAddress,
+    bus::UsbBusWrapper,
     error::UsbError,
     info::{
         UsbConfigurationInfo, UsbDeviceInfo, UsbEndpointInfo, UsbEndpointType, UsbInterfaceInfo,
-        UsbVersion,
     },
     pipe::{
         control::{ConfigurationDescriptorEntry, UsbControlPipeAccess},
@@ -17,21 +31,26 @@ use crate::{
             UsbNormalPipeOut,
         },
     },
+    sysfs::UsbDeviceKObject,
     UsbHostController,
 };
 
 // High-level structures for info provided through descriptors
 
-#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)]
+// #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)]
-pub struct UsbBusAddress {
+// pub struct UsbBusAddress {
-    pub bus: u16,
+//     pub bus: u16,
-    pub device: u8,
+//     pub device: u8,
-}
+// }
 
 pub struct UsbDeviceAccess {
     pub device: Arc<dyn UsbDevice>,
+    pub bus: Arc<UsbBusWrapper>,
     pub info: UsbDeviceInfo,
-    pub current_configuration: IrqSafeRwLock<Option<UsbConfigurationInfo>>,
+    pub configurations: Vec<UsbConfigurationInfo>,
+    current_configuration: IrqSafeRwLock<Option<usize>>,
+
+    pub(crate) kobject: OneTimeInit<UsbDeviceKObject>,
 }
 
 #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)]
@@ -69,7 +88,7 @@ pub trait UsbDevice: Send + Sync {
     fn port_number(&self) -> u8;
     fn bus_address(&self) -> UsbBusAddress;
     fn speed(&self) -> UsbSpeed;
-    fn controller_ref(&self) -> &dyn UsbHostController;
+    fn host_controller(&self) -> Arc<dyn UsbHostController>;
 
     fn set_detach_handler(&self, handler: Arc<dyn UsbDeviceDetachHandler>);
     fn handle_detach(&self);
@@ -84,47 +103,43 @@ impl UsbDeviceAccess {
     /// * Device is not yet configured
     /// * Control pipe for the device has been properly set up
     /// * Device has been assigned a bus address
-    pub async fn setup(raw: Arc<dyn UsbDevice>) -> Result<Self, UsbError> {
+    pub async fn setup(bus: Arc<UsbBusWrapper>, raw: Arc<dyn UsbDevice>) -> Result<Self, UsbError> {
         let control = raw.control_pipe();
 
         let device_desc = control.query_device_descriptor().await?;
 
-        let bcd_usb = device_desc.bcd_usb;
-        let usb_version = UsbVersion::from_bcd_usb(device_desc.bcd_usb)
-            .ok_or(UsbError::InvalidDescriptorField)
-            .inspect_err(|_| {
-                log::error!(
-                    "{}: unsupported/invalid USB version: {:#x}",
-                    raw.bus_address(),
-                    bcd_usb
-                )
-            })?;
-
         let manufacturer = control.query_string(device_desc.manufacturer_str).await?;
         let product = control.query_string(device_desc.product_str).await?;
 
+        // Query device
         let info = UsbDeviceInfo {
             manufacturer,
             product,
-            usb_version,
+            usb_version: device_desc.bcd_usb,
 
             id_vendor: device_desc.id_vendor,
             id_product: device_desc.id_product,
 
-            device_class: device_desc.class(),
+            device_class: device_desc.device_class,
             device_subclass: device_desc.device_subclass,
-            device_protocol: device_desc.protocol(),
+            device_protocol: device_desc.device_protocol,
-            device_protocol_number: device_desc.device_protocol,
 
             num_configurations: device_desc.num_configurations,
 
-            max_packet_size: device_desc.max_packet_size(usb_version, raw.speed())?,
+            max_packet_size: device_desc.max_packet_size(device_desc.bcd_usb, raw.speed())?,
         };
 
+        let configurations =
+            Self::query_configurations(control, device_desc.num_configurations).await?;
+
         Ok(Self {
             device: raw,
+            bus,
             info,
             current_configuration: IrqSafeRwLock::new(None),
+            configurations,
+
+            kobject: OneTimeInit::new(),
         })
     }
 
@@ -164,45 +179,54 @@ impl UsbDeviceAccess {
         Ok(UsbBulkOutPipeAccess(pipe))
     }
 
-    pub fn read_current_configuration(
+    pub fn current_configuration(&self) -> Option<&UsbConfigurationInfo> {
-        &self,
+        let index = (*self.current_configuration.read())?;
-    ) -> IrqSafeRwLockReadGuard<Option<UsbConfigurationInfo>> {
+        Some(&self.configurations[index])
-        self.current_configuration.read()
     }
 
-    pub async fn select_configuration<F: Fn(&UsbConfigurationInfo) -> bool>(
+    pub async fn use_default_configuration(
         &self,
-        predicate: F,
     ) -> Result<Option<UsbConfigurationInfo>, UsbError> {
-        let mut current_config = self.current_configuration.write();
+        if self.configurations.len() != 1 {
-        let control_pipe = self.control_pipe();
+            return Ok(None);
-
-        for i in 0..self.info.num_configurations {
-            let info = self.query_configuration_info(i).await?;
-
-            if predicate(&info) {
-                log::debug!("Selected configuration: {:#?}", info);
-                let config = current_config.insert(info);
-
-                control_pipe
-                    .set_configuration(config.config_value as _)
-                    .await?;
-
-                return Ok(Some(config.clone()));
-            }
         }
 
-        Ok(None)
+        self.set_configuration(0).await.map(Some)
     }
 
-    pub async fn query_configuration_info(
+    pub async fn set_configuration(&self, index: usize) -> Result<UsbConfigurationInfo, UsbError> {
-        &self,
+        if index >= self.configurations.len() {
-        index: u8,
-    ) -> Result<UsbConfigurationInfo, UsbError> {
-        if index >= self.info.num_configurations {
             return Err(UsbError::InvalidConfiguration);
         }
+
+        let mut current = self.current_configuration.write();
         let control_pipe = self.control_pipe();
+        let info = self.configurations[index].clone();
+
+        control_pipe
+            .set_configuration(info.config_value as _)
+            .await?;
+        *current = Some(index);
+
+        Ok(info)
+    }
+
+    async fn query_configurations(
+        control_pipe: &UsbControlPipeAccess,
+        num_configurations: u8,
+    ) -> Result<Vec<UsbConfigurationInfo>, UsbError> {
+        let mut configurations = Vec::new();
+        for i in 0..num_configurations {
+            let configuration = Self::query_configuration(control_pipe, i).await?;
+            configurations.push(configuration);
+        }
+        Ok(configurations)
+    }
+
+    async fn query_configuration(
+        control_pipe: &UsbControlPipeAccess,
+        index: u8,
+    ) -> Result<UsbConfigurationInfo, UsbError> {
         let query = control_pipe.query_configuration_descriptor(index).await?;
 
         let configuration_name = control_pipe
@@ -228,10 +252,9 @@ impl UsbDeviceAccess {
                         name,
                         number: iface.interface_number,
 
-                        interface_class: iface.class(),
+                        interface_class: iface.interface_class,
                         interface_subclass: iface.interface_subclass,
-                        interface_protocol: iface.protocol(),
+                        interface_protocol: iface.interface_protocol,
-                        interface_protocol_number: iface.interface_protocol,
                     });
                 }
                 _ => (),
@@ -248,6 +271,13 @@ impl UsbDeviceAccess {
         Ok(info)
     }
 
+    // pub async fn query_configuration_info(
+    //     &self,
+    //     index: u8,
+    // ) -> Result<UsbConfigurationInfo, UsbError> {
+    //     let control_pipe = self.control_pipe();
+    // }
+
     pub fn set_detach_handler(&self, handler: Arc<dyn UsbDeviceDetachHandler>) {
         self.device.set_detach_handler(handler);
     }
@@ -260,9 +290,3 @@ impl Deref for UsbDeviceAccess {
         &*self.device
     }
 }
-
-impl fmt::Display for UsbBusAddress {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "{}:{}", self.bus, self.device)
-    }
-}

kernel/driver/bus/usb/src/info.rs

@@ -29,41 +29,51 @@ pub enum UsbUsageType {
     Reserved,
 }
 
-#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)]
+// #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)]
-pub enum UsbVersion {
+// pub enum UsbVersion {
-    Usb11,
+//     Usb11,
-    Usb20,
+//     Usb20,
-    Usb21,
+//     Usb21,
-    Usb30,
+//     Usb30,
-    Usb31,
+//     Usb31,
-    Usb32,
+//     Usb32,
-}
+// }
 
-primitive_enum! {
+pub const CLASS_FROM_INTERFACE: u8 = 0x00;
-    pub enum UsbDeviceClass: u8 {
+pub const CLASS_HID: u8 = 0x03;
-        FromInterface = 0x00,
+pub const CLASS_MASS_STORAGE: u8 = 0x08;
-        Hid = 0x03,
-        MassStorage = 0x08,
-        Unknown = 0xFF,
-    }
-}
 
-primitive_enum! {
+// primitive_enum! {
-    pub enum UsbDeviceProtocol: u8 {
+//     pub enum UsbDeviceClass: u8 {
-        FromInterface = 0x00,
+//         FromInterface = 0x00,
-        Unknown = 0xFF,
+//         Hid = 0x03,
-    }
+//         MassStorage = 0x08,
-}
+//         Unknown = 0xFF,
+//     }
+// }
+//
+// primitive_enum! {
+//     pub enum UsbDeviceProtocol: u8 {
+//         FromInterface = 0x00,
+//         Unknown = 0xFF,
+//     }
+// }
 
 #[derive(Debug, Clone)]
 pub struct UsbInterfaceInfo {
     pub name: String,
     pub number: u8,
 
-    pub interface_class: UsbDeviceClass,
+    pub interface_class: u8,
     pub interface_subclass: u8,
-    pub interface_protocol: UsbDeviceProtocol,
+    pub interface_protocol: u8,
-    pub interface_protocol_number: u8,
+    // pub name: String,
+    // pub number: u8,
+
+    // pub interface_class: UsbDeviceClass,
+    // pub interface_subclass: u8,
+    // pub interface_protocol: UsbDeviceProtocol,
+    // pub interface_protocol_number: u8,
 }
 
 #[derive(Debug, Clone)]
@@ -87,15 +97,14 @@ pub struct UsbDeviceInfo {
     pub manufacturer: String,
     pub product: String,
 
-    pub usb_version: UsbVersion,
+    pub usb_version: u16,
 
     pub id_vendor: u16,
     pub id_product: u16,
 
-    pub device_class: UsbDeviceClass,
+    pub device_class: u8,
     pub device_subclass: u8,
-    pub device_protocol: UsbDeviceProtocol,
+    pub device_protocol: u8,
-    pub device_protocol_number: u8,
 
     /// Max packet size for endpoint zero
     pub max_packet_size: usize,
@@ -103,37 +112,37 @@ pub struct UsbDeviceInfo {
     pub num_configurations: u8,
 }
 
-impl UsbVersion {
+// impl UsbVersion {
-    pub fn is_version_3(&self) -> bool {
+//     pub fn is_version_3(&self) -> bool {
-        matches!(self, Self::Usb30 | Self::Usb31 | Self::Usb32)
+//         matches!(self, Self::Usb30 | Self::Usb31 | Self::Usb32)
-    }
+//     }
-
+//
-    pub fn from_bcd_usb(value: u16) -> Option<Self> {
+//     pub fn from_bcd_usb(value: u16) -> Option<Self> {
-        match value {
+//         match value {
-            0x110 => Some(UsbVersion::Usb11),
+//             0x110 => Some(UsbVersion::Usb11),
-            0x200..=0x20F => Some(UsbVersion::Usb20),
+//             0x200..=0x20F => Some(UsbVersion::Usb20),
-            0x210..=0x21F => Some(UsbVersion::Usb21),
+//             0x210..=0x21F => Some(UsbVersion::Usb21),
-            0x300 => Some(UsbVersion::Usb30),
+//             0x300 => Some(UsbVersion::Usb30),
-            0x310 => Some(UsbVersion::Usb31),
+//             0x310 => Some(UsbVersion::Usb31),
-            0x320 => Some(UsbVersion::Usb32),
+//             0x320 => Some(UsbVersion::Usb32),
-            _ => None,
+//             _ => None,
-        }
+//         }
-    }
+//     }
-}
+// }
-
+//
-impl fmt::Display for UsbVersion {
+// impl fmt::Display for UsbVersion {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+//     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        let string = match self {
+//         let string = match self {
-            Self::Usb11 => "USB1.1",
+//             Self::Usb11 => "USB1.1",
-            Self::Usb20 => "USB2.0",
+//             Self::Usb20 => "USB2.0",
-            Self::Usb21 => "USB2.1",
+//             Self::Usb21 => "USB2.1",
-            Self::Usb30 => "USB3.0",
+//             Self::Usb30 => "USB3.0",
-            Self::Usb31 => "USB3.1",
+//             Self::Usb31 => "USB3.1",
-            Self::Usb32 => "USB3.2",
+//             Self::Usb32 => "USB3.2",
-        };
+//         };
-        f.write_str(string)
+//         f.write_str(string)
-    }
+//     }
-}
+// }
 
 impl UsbEndpointInfo {
     pub fn is(&self, ty: UsbEndpointType, dir: UsbDirection) -> bool {

kernel/driver/bus/usb/src/interface.rs

@@ -0,0 +1 @@
+pub struct UsbInterface {}

kernel/driver/bus/usb/src/lib.rs

@@ -8,15 +8,20 @@
     maybe_uninit_fill
 )]
 
+use crate::sysfs::UsbBusKObject;
+
 extern crate alloc;
 
+pub mod address;
 pub mod bus;
 pub mod communication;
 pub mod descriptor;
 pub mod device;
 pub mod error;
 pub mod info;
+pub mod interface;
 pub mod pipe;
+pub mod sysfs;
 pub mod util;
 
 pub mod class_driver;
@@ -25,4 +30,8 @@ pub mod class_driver;
 
 pub trait UsbEndpoint: Sync {}
 
-pub trait UsbHostController: Sync + Send {}
+pub trait UsbHostController: Sync + Send {
+    fn register_sysfs_properties(&self, kobject: &UsbBusKObject) {
+        let _ = kobject;
+    }
+}

kernel/driver/bus/usb/src/pipe/control.rs

@@ -37,6 +37,7 @@ pub trait UsbDeviceRequest: Sized + Pod {
 pub trait UsbClassSpecificRequest: Sized + Pod {
     const BM_REQUEST_TYPE: u8;
     const B_REQUEST: u8;
+    const W_VALUE: u16 = 0;
 }
 
 pub trait UsbDescriptorRequest: UsbDeviceRequest {

kernel/driver/bus/usb/src/sysfs.rs

@@ -0,0 +1,129 @@
+use alloc::{format, sync::Arc};
+use libk::{
+    error::Error,
+    fs::sysfs::{
+        self,
+        attribute::{IntegerAttribute, IntegerAttributeFormat, IntegerAttributeOps},
+        object::KObject,
+    },
+};
+use libk_util::OneTimeInit;
+
+use crate::{bus::UsbBusWrapper, device::UsbDeviceAccess};
+
+pub type UsbBusKObject = Arc<KObject<Arc<UsbBusWrapper>>>;
+pub type UsbDeviceKObject = Arc<KObject<Arc<UsbDeviceAccess>>>;
+
+pub(crate) fn register_bus_kobject(bus: &Arc<UsbBusWrapper>) -> UsbBusKObject {
+    let root = sysfs_usb_root();
+    let bus_kobject = KObject::new(bus.clone());
+    bus.hc.register_sysfs_properties(&bus_kobject);
+    root.add_object(format!("{}", bus.index), bus_kobject.clone())
+        .ok();
+    bus_kobject
+}
+
+pub(crate) fn register_device_kobject(device: &Arc<UsbDeviceAccess>) -> UsbDeviceKObject {
+    struct Class;
+    struct Subclass;
+    struct Protocol;
+    struct Version;
+    struct IdVendor;
+    struct IdProduct;
+
+    impl IntegerAttributeOps<u8> for Class {
+        type Data = Arc<UsbDeviceAccess>;
+        const NAME: &'static str = "class";
+        const FORMAT: IntegerAttributeFormat = IntegerAttributeFormat::Hex;
+
+        fn read(state: &Self::Data) -> Result<u8, Error> {
+            Ok(state.info.device_class)
+        }
+    }
+    impl IntegerAttributeOps<u8> for Subclass {
+        type Data = Arc<UsbDeviceAccess>;
+        const NAME: &'static str = "subclass";
+        const FORMAT: IntegerAttributeFormat = IntegerAttributeFormat::Hex;
+
+        fn read(state: &Self::Data) -> Result<u8, Error> {
+            Ok(state.info.device_subclass)
+        }
+    }
+    impl IntegerAttributeOps<u8> for Protocol {
+        type Data = Arc<UsbDeviceAccess>;
+        const NAME: &'static str = "protocol";
+        const FORMAT: IntegerAttributeFormat = IntegerAttributeFormat::Hex;
+
+        fn read(state: &Self::Data) -> Result<u8, Error> {
+            Ok(state.info.device_protocol)
+        }
+    }
+    impl IntegerAttributeOps<u16> for Version {
+        type Data = Arc<UsbDeviceAccess>;
+        const NAME: &'static str = "version";
+        const FORMAT: IntegerAttributeFormat = IntegerAttributeFormat::Hex;
+
+        fn read(state: &Self::Data) -> Result<u16, Error> {
+            Ok(state.info.usb_version)
+        }
+    }
+    impl IntegerAttributeOps<u16> for IdVendor {
+        type Data = Arc<UsbDeviceAccess>;
+        const NAME: &'static str = "vendor";
+        const FORMAT: IntegerAttributeFormat = IntegerAttributeFormat::Hex;
+
+        fn read(state: &Self::Data) -> Result<u16, Error> {
+            Ok(state.info.id_vendor)
+        }
+    }
+    impl IntegerAttributeOps<u16> for IdProduct {
+        type Data = Arc<UsbDeviceAccess>;
+        const NAME: &'static str = "product";
+        const FORMAT: IntegerAttributeFormat = IntegerAttributeFormat::Hex;
+
+        fn read(state: &Self::Data) -> Result<u16, Error> {
+            Ok(state.info.id_product)
+        }
+    }
+
+    let bus_kobject = device.bus.kobject();
+    let device_kobject = KObject::new(device.clone());
+
+    device_kobject
+        .add_attribute(IntegerAttribute::from(Class))
+        .ok();
+    device_kobject
+        .add_attribute(IntegerAttribute::from(Subclass))
+        .ok();
+    device_kobject
+        .add_attribute(IntegerAttribute::from(Protocol))
+        .ok();
+    device_kobject
+        .add_attribute(IntegerAttribute::from(Version))
+        .ok();
+    device_kobject
+        .add_attribute(IntegerAttribute::from(IdVendor))
+        .ok();
+    device_kobject
+        .add_attribute(IntegerAttribute::from(IdProduct))
+        .ok();
+
+    let address = device.bus_address();
+
+    bus_kobject
+        .add_object(format!("{}", address.device), device_kobject.clone())
+        .ok();
+
+    device_kobject
+}
+
+fn sysfs_usb_root() -> &'static Arc<KObject<()>> {
+    static USB_ROOT: OneTimeInit<Arc<KObject<()>>> = OneTimeInit::new();
+
+    USB_ROOT.or_init_with(|| {
+        let bus_object = sysfs::bus().expect("bus object");
+        let usb_object = KObject::new(());
+        bus_object.add_object("usb", usb_object.clone()).ok();
+        usb_object
+    })
+}

kernel/driver/fs/ext2/src/dir/walk.rs

@@ -127,7 +127,7 @@ impl<'a> DirentIter<'a> {
         self.offset
     }
 
-    pub fn next_record(&mut self) -> Option<Record<&[u8]>> {
+    pub fn next_record(&mut self) -> Option<Record<'_, &[u8]>> {
         if self.offset + size_of::<Dirent>() > self.block.len() {
             return None;
         }
@@ -183,7 +183,7 @@ impl<'a> DirentIterMut<'a> {
         Self { fs, block, offset }
     }
 
-    pub fn next_record(&mut self) -> Option<Record<&mut [u8]>> {
+    pub fn next_record(&mut self) -> Option<Record<'_, &mut [u8]>> {
         if self.offset + size_of::<Dirent>() > self.block.len() {
             return None;
         }

kernel/driver/fs/ext2/src/lib.rs

@@ -1,6 +1,6 @@
 #![feature(if_let_guard, async_drop, impl_trait_in_assoc_type)]
 #![cfg_attr(not(test), no_std)]
-#![allow(clippy::new_ret_no_self)]
+#![allow(clippy::new_ret_no_self, incomplete_features)]
 
 extern crate alloc;
 

kernel/driver/input/src/lib.rs

@@ -9,14 +9,20 @@ use async_trait::async_trait;
 use device_api::device::Device;
 use libk::{device::char::CharDevice, vfs::FileReadiness};
 use libk_util::{ring::LossyRingQueue, OneTimeInit};
-use yggdrasil_abi::{error::Error, io::KeyboardKeyEvent};
+use yggdrasil_abi::{
+    abi_serde::wire,
+    error::Error,
+    io::{KeyboardKeyEvent, MouseEvent},
+};
 
 #[derive(Clone, Copy)]
 pub struct KeyboardDevice;
+#[derive(Clone, Copy)]
+pub struct MouseDevice;
 
 impl FileReadiness for KeyboardDevice {
     fn poll_read(&self, cx: &mut Context<'_>) -> Poll<Result<(), Error>> {
-        INPUT_QUEUE.poll_readable(cx).map(Ok)
+        KEYBOARD_INPUT_QUEUE.poll_readable(cx).map(Ok)
     }
 }
 
@@ -33,7 +39,7 @@ impl CharDevice for KeyboardDevice {
             return Ok(0);
         }
 
-        let ev = INPUT_QUEUE.read().await;
+        let ev = KEYBOARD_INPUT_QUEUE.read().await;
 
         buf[..4].copy_from_slice(&ev.as_bytes());
 
@@ -45,7 +51,7 @@ impl CharDevice for KeyboardDevice {
             return Ok(0);
         }
 
-        let ev = INPUT_QUEUE.try_read().ok_or(Error::WouldBlock)?;
+        let ev = KEYBOARD_INPUT_QUEUE.try_read().ok_or(Error::WouldBlock)?;
 
         buf[..4].copy_from_slice(&ev.as_bytes());
 
@@ -68,15 +74,68 @@ impl CharDevice for KeyboardDevice {
     }
 }
 
-static INPUT_QUEUE: LossyRingQueue<KeyboardKeyEvent> = LossyRingQueue::with_capacity(32);
+impl FileReadiness for MouseDevice {
+    fn poll_read(&self, cx: &mut Context<'_>) -> Poll<Result<(), Error>> {
+        MOUSE_INPUT_QUEUE.poll_readable(cx).map(Ok)
+    }
+}
+
+impl Device for MouseDevice {
+    fn display_name(&self) -> &str {
+        "Mouse input pseudo-device"
+    }
+}
+
+#[async_trait]
+impl CharDevice for MouseDevice {
+    async fn read(&self, buf: &mut [u8]) -> Result<usize, Error> {
+        let ev = MOUSE_INPUT_QUEUE.read().await;
+        let len = wire::to_slice(&ev, buf)?;
+        Ok(len)
+    }
+
+    fn read_nonblocking(&self, buf: &mut [u8]) -> Result<usize, Error> {
+        let ev = MOUSE_INPUT_QUEUE.try_read().ok_or(Error::WouldBlock)?;
+        let len = wire::to_slice(&ev, buf)?;
+        Ok(len)
+    }
+
+    fn is_writeable(&self) -> bool {
+        false
+    }
+
+    fn device_request(&self, option: u32, buffer: &mut [u8], len: usize) -> Result<usize, Error> {
+        let _ = option;
+        let _ = buffer;
+        let _ = len;
+        Err(Error::InvalidOperation)
+    }
+
+    fn is_terminal(&self) -> bool {
+        false
+    }
+}
+
+static KEYBOARD_INPUT_QUEUE: LossyRingQueue<KeyboardKeyEvent> = LossyRingQueue::with_capacity(32);
 static KEYBOARD_DEVICE: OneTimeInit<Arc<KeyboardDevice>> = OneTimeInit::new();
 
-pub fn setup() -> Arc<KeyboardDevice> {
+static MOUSE_INPUT_QUEUE: LossyRingQueue<MouseEvent> = LossyRingQueue::with_capacity(32);
+static MOUSE_DEVICE: OneTimeInit<Arc<MouseDevice>> = OneTimeInit::new();
+
+pub fn setup_keyboard() -> Arc<KeyboardDevice> {
     KEYBOARD_DEVICE
         .or_init_with(|| Arc::new(KeyboardDevice))
         .clone()
 }
 
-pub fn send_event(ev: KeyboardKeyEvent) {
+pub fn setup_mouse() -> Arc<MouseDevice> {
-    INPUT_QUEUE.write(ev);
+    MOUSE_DEVICE.or_init_with(|| Arc::new(MouseDevice)).clone()
+}
+
+pub fn send_keyboard_event(ev: KeyboardKeyEvent) {
+    KEYBOARD_INPUT_QUEUE.write(ev);
+}
+
+pub fn send_mouse_event(ev: MouseEvent) {
+    MOUSE_INPUT_QUEUE.write(ev);
 }

kernel/driver/net/core/src/ephy.rs

@@ -125,24 +125,31 @@ impl<'a, M: MdioBus> PhyAccess<'a, M> {
         })
     }
 
-    pub fn setup_link(&self, have_pause: bool, force_gbesr: GBESR) -> Result<(), Error> {
+    pub fn setup_link(&self, have_pause: bool, force_gbesr: Option<GBESR>) -> Result<(), Error> {
         let bmsr = BMSR::from(self.read_reg(REG_BMSR)?);
-        let mut gbesr = if bmsr.contains(BMSR::EXT_STATUS_1000BASET) {
+        let gbesr = if let Some(force_gbesr) = force_gbesr {
-            GBESR::from(self.read_reg(REG_GBESR)?)
+            let mut gbesr = if bmsr.contains(BMSR::EXT_STATUS_1000BASET) {
+                GBESR::from(self.read_reg(REG_GBESR)?)
+            } else {
+                GBESR::empty()
+            };
+            gbesr |= force_gbesr;
+            Some(gbesr)
         } else {
-            GBESR::empty()
+            None
         };
-        gbesr |= force_gbesr;
         let mut anar = ANAR::from_capabilities(bmsr);
         if have_pause {
             anar |= ANAR::HAVE_PAUSE | ANAR::ASM_DIR;
         }
         let mut gbcr = GBCR::empty();
-        if gbesr.contains(GBESR::HAVE_1000BASET_HALF) {
+        if let Some(gbesr) = gbesr {
-            gbcr |= GBCR::HAVE_1000BASET_HALF;
+            if gbesr.contains(GBESR::HAVE_1000BASET_HALF) {
-        }
+                gbcr |= GBCR::HAVE_1000BASET_HALF;
-        if gbesr.contains(GBESR::HAVE_1000BASET_FULL) {
+            }
-            gbcr |= GBCR::HAVE_1000BASET_FULL;
+            if gbesr.contains(GBESR::HAVE_1000BASET_FULL) {
+                gbcr |= GBCR::HAVE_1000BASET_FULL;
+            }
         }
 
         self.write_reg(REG_ANAR, anar.bits())?;

kernel/driver/net/core/src/lib.rs

@@ -1,4 +1,4 @@
-#![feature(map_try_insert, let_chains, result_flattening)]
+#![feature(map_try_insert, let_chains)]
 #![allow(clippy::type_complexity, clippy::new_without_default)]
 #![no_std]
 

kernel/driver/net/core/src/socket/local/mod.rs

@@ -15,7 +15,7 @@ enum OwnedAddress {
     Anonymous(u64),
 }
 
-pub fn load_address(bytes: &[u8]) -> Result<LocalSocketAddress, Error> {
+pub fn load_address(bytes: &[u8]) -> Result<LocalSocketAddress<'_>, Error> {
     Ok(wire::from_slice(bytes)?)
 }
 
@@ -76,7 +76,7 @@ pub fn write_ancillary(
 }
 
 impl OwnedAddress {
-    fn to_borrowed(&self) -> LocalSocketAddress {
+    fn to_borrowed(&self) -> LocalSocketAddress<'_> {
         match self {
             Self::Path(path) => LocalSocketAddress::Path(path.as_ref()),
             Self::Anonymous(anon) => LocalSocketAddress::Anonymous(*anon),

kernel/driver/net/core/src/socket/tcp/listener.rs

@@ -70,7 +70,7 @@ impl TcpListener {
     pub(super) fn poll_accept(
         &self,
         cx: &mut Context<'_>,
-    ) -> Poll<IrqSafeSpinlockGuard<Vec<Arc<TcpStream>>>> {
+    ) -> Poll<IrqSafeSpinlockGuard<'_, Vec<Arc<TcpStream>>>> {
         let lock = self.pending_accept.lock();
         self.accept_notify.register(cx.waker());
         if !lock.is_empty() {

kernel/driver/net/core/src/socket/tcp/stream.rs

@@ -214,7 +214,7 @@ impl TcpStream {
     pub(super) fn poll_receive(
         &self,
         cx: &mut Context<'_>,
-    ) -> Poll<Result<IrqSafeRwLockWriteGuard<TcpConnection>, Error>> {
+    ) -> Poll<Result<IrqSafeRwLockWriteGuard<'_, TcpConnection>, Error>> {
         let lock = self.connection.write();
         match lock.poll_receive(cx) {
             Poll::Ready(Ok(())) => Poll::Ready(Ok(lock)),

kernel/driver/net/igbe/src/lib.rs

@@ -26,12 +26,15 @@ use ygg_driver_pci::{
 };
 use yggdrasil_abi::net::{link::LinkState, MacAddress};
 
+use crate::regs::Revision;
+
 extern crate alloc;
 
 mod regs;
 mod ring;
 
 struct Igbe {
+    chip: Revision,
     regs: IrqSafeSpinlock<Regs>,
     dma: Arc<dyn DmaAllocator>,
     pci: PciDeviceInfo,
@@ -43,8 +46,9 @@ struct Igbe {
 }
 
 impl Igbe {
-    pub fn new(dma: Arc<dyn DmaAllocator>, regs: Regs, pci: PciDeviceInfo) -> Self {
+    pub fn new(dma: Arc<dyn DmaAllocator>, regs: Regs, chip: Revision, pci: PciDeviceInfo) -> Self {
         Self {
+            chip,
             dma,
             pci,
             mac: OneTimeInit::new(),
@@ -74,7 +78,7 @@ impl Device for Igbe {
         regs.reset(Duration::from_millis(200))?;
         // Intel 8257x manuals say an additional interrupt disable is needed after a global reset
         regs.disable_interrupts();
-        regs.set_link_up()?;
+        regs.set_link_up(self.chip)?;
 
         // Initialize Rx
         regs.initialize_receiver(&rx_ring);
@@ -175,6 +179,10 @@ impl NetworkDevice for Igbe {
 
 pci_driver! {
     matches: [
+        device (0x8086:0x100E), // 82540EM (E1000)
+        device (0x8086:0x100C), // 82544GC (E1000)
+        device (0x8086:0x100F), // 82545EM (E1000)
+        device (0x8086:0x10D3), // 82574L (E1000E) [[BROKEN]]
         device (0x8086:0x10C9), // 82576 GbE
         device (0x8086:0x1502), // 82579LM GbE (Lewisville)
     ],
@@ -197,11 +205,22 @@ pci_driver! {
                 }
             };
 
+            let chip = match info.device_id {
+                0x100E | 0x100C | 0x100F => Revision::I8254x,
+                0x10D3 => Revision::I82574L,
+                0x10C9 => Revision::I82576,
+                0x1502 => Revision::I82579LM,
+                id => {
+                    log::error!("Invalid igbe chip variant: {id:#04x}");
+                    return Err(Error::InvalidOperation)
+                },
+            };
+
             info.init_interrupts(PreferredInterruptMode::Msi(true))?;
             info.set_command(true, use_mmio, !use_mmio, true);
 
             let regs = unsafe { Regs::map(base) }?;
-            let device = Igbe::new(dma.clone(), regs, info.clone());
+            let device = Igbe::new(dma.clone(), regs, chip, info.clone());
             Ok(Arc::new(device))
         }
 

kernel/driver/net/igbe/src/regs.rs

@@ -42,6 +42,14 @@ pub trait Reg {
     const OFFSET: u16;
 }
 
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub enum Revision {
+    I8254x,
+    I82574L,
+    I82576,
+    I82579LM,
+}
+
 register_bitfields! {
     u32,
     pub CTRL [
@@ -329,6 +337,7 @@ impl MdioBus for Regs {
 
         let mdic = self.inner.extract();
         if mdic.matches_all(MDIC::E::SET) {
+            log::warn!("MDIO read error: phyaddr={phyaddr:#x}, regaddr={regaddr:#x}");
             return Err(Error::InvalidOperation);
         }
 
@@ -350,6 +359,9 @@ impl MdioBus for Regs {
         )?;
 
         if self.inner.matches_all(MDIC::E::SET) {
+            log::warn!(
+                "MDIO write error: phyaddr={phyaddr:#x}, regaddr={regaddr:#x}, value={value:#x}"
+            );
             return Err(Error::InvalidOperation);
         }
 
@@ -403,7 +415,7 @@ impl Regs {
         })
     }
 
-    pub fn set_link_up(&mut self) -> Result<(), Error> {
+    pub fn set_link_up(&mut self, chip: Revision) -> Result<(), Error> {
         self.inner
             .modify(CTRL::SLU::SET + CTRL::RFCE::SET + CTRL::TFCE::SET);
 
@@ -412,8 +424,14 @@ impl Regs {
         let (id0, id1) = phy.id()?;
         log::info!("PHY {:04x}:{:04x}", id0, id1);
 
-        phy.reset(Duration::from_millis(200))?;
+        phy.reset(Duration::from_millis(200))
-        phy.setup_link(true, GBESR::empty())?;
+            .inspect_err(|e| log::error!("PHY reset error {e:?}"))?;
+        let force_gbesr = match chip {
+            Revision::I82576 | Revision::I82579LM => Some(GBESR::empty()),
+            _ => None,
+        };
+        phy.setup_link(true, force_gbesr)
+            .inspect_err(|e| log::error!("PHY setup error: {e:?}"))?;
 
         Ok(())
     }

kernel/driver/net/rtl81xx/src/rtl8168.rs

@@ -498,7 +498,7 @@ impl Regs {
         phy.write_reg(0x0E, 0x00)?;
 
         phy.reset(timeout)?;
-        phy.setup_link(true, GBESR::empty())?;
+        phy.setup_link(true, Some(GBESR::empty()))?;
 
         psleep(Duration::from_millis(100));
 

kernel/driver/net/stmmac/src/lib.rs

@@ -337,7 +337,7 @@ impl Device for Stmmac {
         let (id0, id1) = phy.id()?;
         log::info!("stmmac: PHY {id0:04x}:{id1:04x}");
         phy.reset(Duration::from_millis(100))?;
-        phy.setup_link(true, GBESR::empty())?;
+        phy.setup_link(true, Some(GBESR::empty()))?;
 
         self.inner.init(Inner {
             regs: IrqSafeSpinlock::new(regs),

kernel/driver/usb/xhci/src/controller.rs

@@ -25,10 +25,11 @@ use tock_registers::{
 };
 use ygg_driver_pci::{device::PciDeviceInfo, PciConfigurationSpace};
 use ygg_driver_usb::{
-    bus::UsbBusManager,
+    address::UsbBusAddress,
-    device::{UsbBusAddress, UsbDeviceAccess, UsbSpeed},
+    bus::{UsbBusManager, UsbBusWrapper},
+    descriptor,
+    device::{UsbDeviceAccess, UsbSpeed},
     error::UsbError,
-    info::UsbVersion,
     pipe::control::UsbControlPipeAccess,
     UsbHostController,
 };
@@ -67,7 +68,7 @@ struct ScratchpadArray {
 }
 
 struct RootHubPort {
-    version: UsbVersion,
+    version: u16,
     slot_type: u8,
 }
 
@@ -90,6 +91,7 @@ pub struct Xhci {
     pub(crate) slots: Vec<IrqSafeRwLock<Option<Arc<XhciBusDevice>>>>,
     pub(crate) port_slot_map: Vec<AtomicU8>,
     bus_index: OneTimeInit<u16>,
+    bus: OneTimeInit<Arc<UsbBusWrapper>>,
     port_event_map: EventBitmap,
 }
 
@@ -146,9 +148,7 @@ impl Xhci {
         for cap in regs.extended_capabilities.iter() {
             match cap {
                 ExtendedCapability::ProtocolSupport(support) => {
-                    let Some(version) = support.usb_revision() else {
+                    let version = support.usb_revision();
-                        continue;
-                    };
 
                     for port in support.port_range() {
                         log::info!("* Port {port}: {version}");
@@ -195,6 +195,7 @@ impl Xhci {
 
             root_hub_ports,
             bus_index: OneTimeInit::new(),
+            bus: OneTimeInit::new(),
             endpoints: IrqSafeRwLock::new(BTreeMap::new()),
             slots,
             port_slot_map,
@@ -282,7 +283,7 @@ impl Xhci {
             .as_ref()
             .ok_or(UsbError::PortInitFailed)?;
 
-        let need_reset = !root_hub_port.version.is_version_3();
+        let need_reset = !descriptor::is_version_3(root_hub_port.version);
 
         if need_reset {
             self.reset_port(regs).await?;
@@ -341,7 +342,8 @@ impl Xhci {
             device: bus_address,
         });
 
-        let device = UsbDeviceAccess::setup(slot).await?;
+        let bus = self.bus.get();
+        let device = UsbDeviceAccess::setup(bus.clone(), slot).await?;
         UsbBusManager::register_device(device.into());
 
         Ok(())
@@ -523,8 +525,9 @@ impl Device for Xhci {
 
         op.wait_usbsts_bit(USBSTS::CNR::CLEAR, 100000000)?;
 
-        let bus = UsbBusManager::register_bus(self.clone());
+        let (bus_index, bus) = UsbBusManager::register_bus(self.clone());
-        self.bus_index.init(bus);
+        self.bus_index.init(bus_index);
+        self.bus.init(bus);
 
         runtime::spawn(self.clone().port_handler_task()).ok();
 

kernel/driver/usb/xhci/src/device.rs

@@ -6,8 +6,9 @@ use libk_util::{
 };
 use xhci_lib::context;
 use ygg_driver_usb::{
+    address::UsbBusAddress,
     communication::UsbDirection,
-    device::{UsbBusAddress, UsbDevice, UsbDeviceDetachHandler, UsbSpeed},
+    device::{UsbDevice, UsbDeviceDetachHandler, UsbSpeed},
     error::UsbError,
     info::UsbEndpointType,
     pipe::{
@@ -63,8 +64,8 @@ impl UsbDevice for XhciBusDevice {
         *self.detach_handler.lock() = Some(handler);
     }
 
-    fn controller_ref(&self) -> &dyn UsbHostController {
+    fn host_controller(&self) -> Arc<dyn UsbHostController> {
-        self.xhci.as_ref()
+        self.xhci.clone()
     }
 
     fn debug(&self) {}

kernel/driver/usb/xhci/src/regs/extended.rs

@@ -7,7 +7,7 @@ use alloc::vec::Vec;
 use libk::error::Error;
 use libk_mm::{address::PhysicalAddress, device::DeviceMemoryIo};
 use libk_util::sync::spin_rwlock::IrqSafeRwLock;
-use ygg_driver_usb::{error::UsbError, info::UsbVersion};
+use ygg_driver_usb::error::UsbError;
 
 pub struct ProtocolSupport {
     words: [u32; 4],
@@ -69,8 +69,8 @@ impl ExtendedCapability {
 }
 
 impl ProtocolSupport {
-    pub fn usb_revision(&self) -> Option<UsbVersion> {
+    pub fn usb_revision(&self) -> u16 {
-        UsbVersion::from_bcd_usb((self.words[0] >> 16) as u16)
+        (self.words[0] >> 16) as u16
     }
 
     pub fn slot_type(&self) -> u8 {

kernel/driver/usb/xhci/src/ring/transfer.rs

@@ -94,7 +94,7 @@ impl TransferRing {
         })
     }
 
-    pub fn transaction_builder(self: &Arc<Self>) -> Result<TransactionBuilder, UsbError> {
+    pub fn transaction_builder(self: &Arc<Self>) -> Result<TransactionBuilder<'_>, UsbError> {
         if self.shutdown.load(Ordering::Acquire) {
             return Err(UsbError::DeviceDisconnected);
         }

kernel/driver/virtio/core/src/transport/pci.rs

@@ -40,7 +40,7 @@ impl Transport for PciTransport {
         true
     }
 
-    fn device_cfg(&self) -> Option<&DeviceMemoryIo<[u8]>> {
+    fn device_cfg(&self) -> Option<&DeviceMemoryIo<'_, [u8]>> {
         Some(&self.device_cfg)
     }
 

kernel/driver/virtio/gpu/src/lib.rs

@@ -161,7 +161,7 @@ impl<T: Transport + 'static> VirtioGpu<T> {
         Ok(())
     }
 
-    fn begin_command(&self) -> CommandExecution<T> {
+    fn begin_command(&self) -> CommandExecution<'_, T> {
         CommandExecution {
             transport: &self.transport,
             control: &self.queues.get().control,

kernel/lib/device-tree/src/tree.rs

@@ -78,7 +78,7 @@ impl<'a> DeviceTree<'a> {
     }
 
     /// Returns the root node of this device tree
-    pub fn root(&self) -> TNode {
+    pub fn root(&self) -> TNode<'_> {
         self.index.root()
     }
 
@@ -172,12 +172,12 @@ impl<'a> DeviceTree<'a> {
     }
 
     /// Returns an iterator over the memory regions specified by this device tree
-    pub fn memory_regions(&self) -> DeviceTreeMemoryRegionIter {
+    pub fn memory_regions(&self) -> DeviceTreeMemoryRegionIter<'_> {
         DeviceTreeMemoryRegionIter::new(self)
     }
 
     /// Returns an iterator over the reserved memory regions specified by this device tree
-    pub fn reserved_regions(&self) -> DeviceTreeReservedRegionIter {
+    pub fn reserved_regions(&self) -> DeviceTreeReservedRegionIter<'_> {
         DeviceTreeReservedRegionIter::new(self)
     }
 

kernel/lib/memtables/Cargo.toml

@@ -1,13 +0,0 @@
-[package]
-name = "memtables"
-version = "0.1.0"
-edition = "2021"
-authors = ["Mark Poliakov <mark@alnyan.me>"]
-
-[dependencies]
-bitflags.workspace = true
-bytemuck.workspace = true
-
-[features]
-default = []
-all = []

kernel/lib/memtables/src/aarch64.rs

@@ -1,26 +0,0 @@
-use bytemuck::{Pod, Zeroable};
-
-use crate::RawTable;
-
-pub const KERNEL_L3_COUNT: usize = 8;
-
-#[derive(Clone, Copy, Pod, Zeroable)]
-#[repr(C)]
-pub struct FixedTables {
-    // 1GiB entries
-    pub l1: RawTable,
-
-    // 2MiB entries
-    pub l2: RawTable,
-    pub l3s: [RawTable; KERNEL_L3_COUNT],
-}
-
-impl FixedTables {
-    pub const fn zeroed() -> Self {
-        Self {
-            l1: RawTable::zeroed(),
-            l2: RawTable::zeroed(),
-            l3s: [RawTable::zeroed(); KERNEL_L3_COUNT],
-        }
-    }
-}

kernel/lib/memtables/src/any.rs

@@ -1,35 +0,0 @@
-use crate::{aarch64, riscv64, x86_64};
-
-pub enum AnyTables {
-    X86_64(x86_64::FixedTables),
-    AArch64(aarch64::FixedTables),
-    Riscv64(riscv64::FixedTables),
-}
-
-impl AnyTables {
-    pub fn as_bytes(&self) -> &[u8] {
-        match self {
-            Self::X86_64(tables) => bytemuck::bytes_of(tables),
-            Self::AArch64(tables) => bytemuck::bytes_of(tables),
-            Self::Riscv64(tables) => bytemuck::bytes_of(tables),
-        }
-    }
-}
-
-impl From<x86_64::FixedTables> for AnyTables {
-    fn from(value: x86_64::FixedTables) -> Self {
-        Self::X86_64(value)
-    }
-}
-
-impl From<aarch64::FixedTables> for AnyTables {
-    fn from(value: aarch64::FixedTables) -> Self {
-        Self::AArch64(value)
-    }
-}
-
-impl From<riscv64::FixedTables> for AnyTables {
-    fn from(value: riscv64::FixedTables) -> Self {
-        Self::Riscv64(value)
-    }
-}

kernel/lib/memtables/src/lib.rs

@@ -1,36 +0,0 @@
-#![no_std]
-
-use bytemuck::{Pod, Zeroable};
-
-// AArch64
-#[cfg(any(feature = "all", target_arch = "aarch64"))]
-pub mod aarch64;
-#[cfg(all(not(feature = "all"), target_arch = "aarch64"))]
-pub use aarch64::FixedTables;
-
-// x86-64
-#[cfg(any(feature = "all", target_arch = "x86_64"))]
-pub mod x86_64;
-#[cfg(all(not(feature = "all"), target_arch = "x86_64"))]
-pub use x86_64::FixedTables;
-
-// RISC-V 64-bit
-#[cfg(any(feature = "all", target_arch = "riscv64"))]
-pub mod riscv64;
-#[cfg(all(not(feature = "all"), target_arch = "riscv64"))]
-pub use riscv64::FixedTables;
-
-#[cfg(feature = "all")]
-pub mod any;
-
-#[derive(Clone, Copy, Pod, Zeroable)]
-#[repr(C, align(0x1000))]
-pub struct RawTable {
-    pub data: [u64; 512],
-}
-
-impl RawTable {
-    pub const fn zeroed() -> Self {
-        Self { data: [0; 512] }
-    }
-}

kernel/lib/memtables/src/riscv64.rs

@@ -1,84 +0,0 @@
-use core::fmt;
-
-use bitflags::bitflags;
-use bytemuck::{Pod, Zeroable};
-
-use crate::RawTable;
-
-pub const KERNEL_L3_COUNT: usize = 8;
-
-bitflags! {
-    #[derive(Clone, Copy, PartialEq, Eq, Debug)]
-    pub struct PageAttributes: u64 {
-        const N = 1 << 63;
-        /// Software-tracked dirty bit (RSW[0])
-        const SW_DIRTY = 1 << 9;
-        /// Dirty bit
-        const D = 1 << 7;
-        /// Access bit
-        const A = 1 << 6;
-        /// Global mapping bit, implies all lower levels are also global
-        const G = 1 << 5;
-        /// U-mode access permission
-        const U = 1 << 4;
-        /// Execute permission
-        const X = 1 << 3;
-        /// Write permission
-        const W = 1 << 2;
-        /// Read-permission
-        const R = 1 << 1;
-        /// Valid bit
-        const V = 1 << 0;
-    }
-
-    // X W R    Meaning
-    // 0 0 0    Pointer to next level of page table
-    // 0 0 1    Read-only page
-    // 0 1 0    ---
-    // 0 1 1    Read-write page
-    // 1 0 0    Execute only
-    // 1 0 1    Read-execute page
-    // 1 1 0    ---
-    // 1 1 1    Read-write-execute page
-}
-
-#[derive(Clone, Copy, Pod, Zeroable)]
-#[repr(C)]
-pub struct FixedTables {
-    pub l1: RawTable,
-    pub kernel_l2: RawTable,
-    pub kernel_l3s: [RawTable; KERNEL_L3_COUNT],
-}
-
-impl FixedTables {
-    pub const fn zeroed() -> Self {
-        Self {
-            l1: RawTable::zeroed(),
-            kernel_l2: RawTable::zeroed(),
-            kernel_l3s: [RawTable::zeroed(); KERNEL_L3_COUNT],
-        }
-    }
-}
-
-impl fmt::Display for PageAttributes {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        use fmt::Write;
-
-        macro_rules! bit {
-            ($self:ident, $field:expr, $letter:literal) => {
-                if $self.contains($field) {
-                    f.write_char($letter)
-                } else {
-                    f.write_char('-')
-                }
-            };
-        }
-
-        bit!(self, Self::R, 'r')?;
-        bit!(self, Self::W, 'w')?;
-        bit!(self, Self::X, 'x')?;
-        bit!(self, Self::U, 'u')?;
-
-        Ok(())
-    }
-}

kernel/lib/memtables/src/x86_64.rs

@@ -1,27 +0,0 @@
-use bytemuck::{Pod, Zeroable};
-
-use crate::RawTable;
-
-pub const KERNEL_L3_COUNT: usize = 16;
-
-#[derive(Clone, Copy, Pod, Zeroable)]
-#[repr(C)]
-pub struct FixedTables {
-    pub l0: RawTable,
-
-    pub kernel_l1: RawTable,
-    pub kernel_l2: RawTable,
-    pub kernel_l3s: [RawTable; KERNEL_L3_COUNT],
-}
-
-impl FixedTables {
-    pub const fn zeroed() -> Self {
-        Self {
-            l0: RawTable::zeroed(),
-
-            kernel_l1: RawTable::zeroed(),
-            kernel_l2: RawTable::zeroed(),
-            kernel_l3s: [RawTable::zeroed(); KERNEL_L3_COUNT],
-        }
-    }
-}

kernel/libk/libk-mm/interface/src/table.rs

@@ -1,6 +1,9 @@
 use core::ops::{Deref, DerefMut, Range};
 
 use bitflags::bitflags;
+use kernel_arch_interface::mem::{
+    DeviceMemoryAttributes, KernelTableManager, RawDeviceMemoryMapping,
+};
 use yggdrasil_abi::error::Error;
 
 use super::address::PhysicalAddress;
@@ -184,3 +187,124 @@ impl<T: AddressLike> EntryLevelExt for T {
         self.page_offset::<L>() == 0
     }
 }
+
+pub trait DevicePageManagerLevel {
+    type Level: EntryLevel;
+    const INDEX_RANGE: Range<usize>;
+    const VIRTUAL_BASE: usize;
+
+    fn map_page(&mut self, index: usize, physical: PhysicalAddress, attrs: &DeviceMemoryAttributes);
+    fn unmap_page(&mut self, index: usize);
+    fn is_mapped(&self, index: usize) -> bool;
+    fn flush_range(range: Range<usize>);
+
+    fn allocate_mapping(
+        &mut self,
+        base: PhysicalAddress,
+        page_count: usize,
+        attrs: &DeviceMemoryAttributes,
+    ) -> Option<usize> {
+        'l0: for i in Self::INDEX_RANGE {
+            for j in 0..page_count {
+                if self.is_mapped(i + j) {
+                    continue 'l0;
+                }
+            }
+
+            for j in 0..page_count {
+                self.map_page(i + j, base.add(j * Self::Level::SIZE), attrs);
+            }
+
+            let start = Self::VIRTUAL_BASE + i * Self::Level::SIZE;
+            Self::flush_range(i..i + page_count);
+            return Some(start);
+        }
+        None
+    }
+
+    unsafe fn remove_mapping(&mut self, base: usize, page_count: usize) {
+        let base_index = (base - Self::VIRTUAL_BASE) / Self::Level::SIZE;
+        for i in 0..page_count {
+            self.unmap_page(base_index + i);
+        }
+    }
+}
+
+pub struct DevicePageManager<N: DevicePageManagerLevel, L: DevicePageManagerLevel> {
+    pub normal: N,
+    pub large: L,
+}
+
+impl<N: DevicePageManagerLevel, L: DevicePageManagerLevel> DevicePageManager<N, L> {
+    pub const fn new(normal: N, large: L) -> Self {
+        Self { normal, large }
+    }
+
+    pub unsafe fn map_device_pages<A: KernelTableManager>(
+        &mut self,
+        base: PhysicalAddress,
+        size: usize,
+        attrs: DeviceMemoryAttributes,
+    ) -> Result<RawDeviceMemoryMapping<A>, Error> {
+        let small_aligned_base = base.page_align_down::<N::Level>();
+        let small_aligned_end = base.add(size).page_align_up::<N::Level>();
+        let small_offset = base - small_aligned_base;
+        let small_page_count = (small_aligned_end - small_aligned_base).page_count::<N::Level>();
+
+        if small_page_count > 128 {
+            // Allocate from large page pool
+            let large_aligned_base = base.page_align_down::<L::Level>();
+            let large_aligned_end = base.add(size).page_align_up::<L::Level>();
+            let large_offset = base - large_aligned_base;
+            let large_page_count =
+                (large_aligned_end - large_aligned_base).page_count::<L::Level>();
+
+            let mapped_base = self
+                .large
+                .allocate_mapping(large_aligned_base, large_page_count, &attrs)
+                .ok_or(Error::OutOfMemory)?;
+            let mapped_address = mapped_base + large_offset;
+
+            Ok(RawDeviceMemoryMapping::from_raw_parts(
+                large_aligned_base.into_u64(),
+                mapped_address,
+                mapped_base,
+                large_page_count,
+                L::Level::SIZE,
+            ))
+        } else {
+            // Allocate from small page pool
+            let mapped_base = self
+                .normal
+                .allocate_mapping(small_aligned_base, small_page_count, &attrs)
+                .ok_or(Error::OutOfMemory)?;
+            let mapped_address = mapped_base + small_offset;
+
+            Ok(RawDeviceMemoryMapping::from_raw_parts(
+                small_aligned_base.into_u64(),
+                mapped_address,
+                mapped_base,
+                small_page_count,
+                N::Level::SIZE,
+            ))
+        }
+    }
+
+    pub unsafe fn unmap_device_pages<A: KernelTableManager>(
+        &mut self,
+        mapping: &RawDeviceMemoryMapping<A>,
+    ) {
+        if mapping.page_size == N::Level::SIZE {
+            self.normal
+                .remove_mapping(mapping.base_address, mapping.page_count);
+        } else if mapping.page_size == L::Level::SIZE {
+            self.large
+                .remove_mapping(mapping.base_address, mapping.page_count);
+        } else {
+            unreachable!(
+                "Invalid device memory mapping with page size {:#x}",
+                mapping.page_size
+            )
+        }
+    }
+}

kernel/libk/libk-mm/src/phys/mod.rs

@@ -1,6 +1,8 @@
 use core::ops::Range;
 
-use kernel_arch::{mem::PhysicalMemoryAllocator, Architecture, ArchitectureImpl};
+use kernel_arch::{
+    mem::PhysicalMemoryAllocator, Architecture, ArchitectureImpl, KERNEL_VIRT_OFFSET,
+};
 use libk_mm_interface::address::PhysicalAddress;
 use libk_util::{sync::IrqSafeSpinlock, OneTimeInit};
 use yggdrasil_abi::{error::Error, system::SystemMemoryStats};
@@ -178,20 +180,14 @@ pub fn find_contiguous_region<I: Iterator<Item = PhysicalMemoryRegion>>(
 ///
 /// The caller must ensure this function has not been called before and that the regions
 /// are valid and actually available.
-pub unsafe fn init_from_iter<
+pub unsafe fn init_from_iter<I: Iterator<Item = PhysicalMemoryRegion> + Clone>(
-    I: Iterator<Item = PhysicalMemoryRegion> + Clone,
-    Map: FnOnce(I, PhysicalAddress, PhysicalAddress) -> Result<(), Error>,
->(
     it: I,
-    map_physical_memory: Map,
 ) -> Result<(), Error> {
     // Map the physical memory
     let (phys_start, phys_end) = physical_memory_range(it.clone()).unwrap();
 
     reserve_region("kernel", kernel_physical_memory_region());
 
-    map_physical_memory(it.clone(), phys_start, phys_end)?;
-
     let total_count = (phys_end - phys_start) / L3_PAGE_SIZE;
     let page_bitmap_size = (total_count + BITMAP_WORD_SIZE - 1) / (BITMAP_WORD_SIZE / 8);
     let page_bitmap_page_count = page_bitmap_size.div_ceil(L3_PAGE_SIZE);
@@ -245,18 +241,16 @@ pub unsafe fn init_from_iter<
 }
 
 fn kernel_physical_memory_region() -> PhysicalMemoryRegion {
-    use core::ptr::addr_of;
-
     extern "C" {
         static __kernel_start: u8;
         static __kernel_end: u8;
     }
 
-    let start = addr_of!(__kernel_start);
+    let start = (&raw const __kernel_start).addr();
-    let end = addr_of!(__kernel_end);
+    let end = (&raw const __kernel_end).addr();
 
-    let base = PhysicalAddress::from_usize(start.addr() - kernel_arch::KERNEL_VIRT_OFFSET);
+    let base = PhysicalAddress::from_usize(start - KERNEL_VIRT_OFFSET);
-    let size = end.addr() - start.addr();
+    let size = end - start;
 
     PhysicalMemoryRegion { base, size }
 }

kernel/libk/libk-mm/src/process.rs

@@ -524,7 +524,7 @@ impl<TA: TableAllocator> ProcessAddressSpace<TA> {
 
     /// Same as [ProcessAddressSpace::translate], except the lock on the address space is held
     /// until the resulting [TranslateGuard] is dropped.
-    pub fn translate_lock(&self, address: usize) -> Result<TranslateGuard<TA>, Error> {
+    pub fn translate_lock(&self, address: usize) -> Result<TranslateGuard<'_, TA>, Error> {
         let guard = self.inner.lock();
         let address = guard.table.translate(address).map(|e| e.0)?;
         Ok(TranslateGuard {

kernel/libk/libk-util/src/lru_hash_table.rs

@@ -51,7 +51,7 @@ impl<K, V> LruCacheBucket<K, V> {
 }
 
 impl<K: Eq, V> LruCacheBucket<K, V> {
-    pub fn entry_mut(&mut self, key: &K) -> EntryMut<K, V> {
+    pub fn entry_mut(&mut self, key: &K) -> EntryMut<'_, K, V> {
         let mut cursor = self.data.cursor_front_mut();
 
         loop {
@@ -125,7 +125,7 @@ impl<K, V, H: BuildHasher> LruCache<K, V, H> {
             .find_map(|bucket| bucket.pop_front())
     }
 
-    pub fn flush(&mut self) -> FlushIter<K, V, H> {
+    pub fn flush(&mut self) -> FlushIter<'_, K, V, H> {
         FlushIter { cache: self }
     }
 }

kernel/libk/libk-util/src/ring.rs

@@ -203,7 +203,7 @@ impl<T: Copy> LossyRingQueue<T> {
     }
 
     #[inline]
-    pub fn try_read_lock(&self) -> Option<IrqSafeSpinlockGuard<RingBuffer<T>>> {
+    pub fn try_read_lock(&self) -> Option<IrqSafeSpinlockGuard<'_, RingBuffer<T>>> {
         let lock = self.ring.lock();
         lock.is_readable().then_some(lock)
     }
@@ -214,12 +214,12 @@ impl<T: Copy> LossyRingQueue<T> {
         unsafe { lock.read_single_unchecked() }
     }
 
-    pub async fn read_lock(&self) -> IrqSafeSpinlockGuard<RingBuffer<T>> {
+    pub async fn read_lock(&self) -> IrqSafeSpinlockGuard<'_, RingBuffer<T>> {
         poll_fn(|cx| self.poll_lock(cx)).await
     }
 
     #[inline]
-    pub fn poll_lock(&self, cx: &mut Context<'_>) -> Poll<IrqSafeSpinlockGuard<RingBuffer<T>>> {
+    pub fn poll_lock(&self, cx: &mut Context<'_>) -> Poll<IrqSafeSpinlockGuard<'_, RingBuffer<T>>> {
         self.read_notify.register(cx.waker());
         if let Some(lock) = self.try_read_lock() {
             self.read_notify.remove(cx.waker());
@@ -284,7 +284,7 @@ impl<T: Copy> BlockingRingQueue<T> {
         true
     }
 
-    pub async fn read_lock(&self) -> IrqSafeSpinlockGuard<RingBuffer<T>> {
+    pub async fn read_lock(&self) -> IrqSafeSpinlockGuard<'_, RingBuffer<T>> {
         poll_fn(|cx| self.poll_read_lock(cx)).await
     }
 
@@ -292,7 +292,7 @@ impl<T: Copy> BlockingRingQueue<T> {
     pub fn poll_read_lock(
         &self,
         cx: &mut Context<'_>,
-    ) -> Poll<IrqSafeSpinlockGuard<RingBuffer<T>>> {
+    ) -> Poll<IrqSafeSpinlockGuard<'_, RingBuffer<T>>> {
         if let Some(lock) = self.try_read_lock() {
             self.read_notify.remove(cx.waker());
             Poll::Ready(lock)
@@ -303,7 +303,7 @@ impl<T: Copy> BlockingRingQueue<T> {
     }
 
     #[inline]
-    pub fn try_read_lock(&self) -> Option<IrqSafeSpinlockGuard<RingBuffer<T>>> {
+    pub fn try_read_lock(&self) -> Option<IrqSafeSpinlockGuard<'_, RingBuffer<T>>> {
         let lock = self.ring.lock();
         lock.is_readable().then_some(lock)
     }

kernel/libk/libk-util/src/string.rs

@@ -28,7 +28,7 @@ impl Utf16LeStr {
         core::mem::transmute(raw)
     }
 
-    pub fn chars(&self) -> Utf16LeIter {
+    pub fn chars(&self) -> Utf16LeIter<'_> {
         Utf16LeIter {
             chars: self.raw.chunks_exact(size_of::<u16>()),
         }

kernel/libk/libk-util/src/sync/spin_rwlock.rs

@@ -131,7 +131,7 @@ impl<T> IrqSafeRwLock<T> {
         }
     }
 
-    pub fn read(&self) -> IrqSafeRwLockReadGuard<T> {
+    pub fn read(&self) -> IrqSafeRwLockReadGuard<'_, T> {
         let guard = IrqGuard::acquire();
         self.inner.acquire_read();
         IrqSafeRwLockReadGuard {
@@ -140,7 +140,7 @@ impl<T> IrqSafeRwLock<T> {
         }
     }
 
-    pub fn write(&self) -> IrqSafeRwLockWriteGuard<T> {
+    pub fn write(&self) -> IrqSafeRwLockWriteGuard<'_, T> {
         let guard = IrqGuard::acquire();
         self.inner.acquire_write();
         IrqSafeRwLockWriteGuard {

kernel/libk/src/device/display/console.rs

@@ -284,7 +284,7 @@ impl ConsoleBuffer {
     }
 
     /// Returns an iterator over dirty rows, while clearing dirty flag for them
-    pub fn flush_rows(&mut self) -> RowIter {
+    pub fn flush_rows(&mut self) -> RowIter<'_> {
         RowIter {
             buffer: self,
             index: 0,

kernel/libk/src/dma.rs

@@ -198,7 +198,7 @@ impl<T> DmaBuffer<[T]> {
         assert!(range.start <= range.end, "Invalid DMA slice range");
     }
 
-    pub fn slice(&self, range: Range<usize>) -> DmaSlice<T> {
+    pub fn slice(&self, range: Range<usize>) -> DmaSlice<'_, T> {
         self.slice_range_check(&range);
         DmaSlice {
             buffer: self,
@@ -206,7 +206,7 @@ impl<T> DmaBuffer<[T]> {
         }
     }
 
-    pub fn slice_mut(&mut self, range: Range<usize>) -> DmaSliceMut<T> {
+    pub fn slice_mut(&mut self, range: Range<usize>) -> DmaSliceMut<'_, T> {
         self.slice_range_check(&range);
         DmaSliceMut {
             buffer: self,

kernel/libk/src/fs/sysfs/attribute/integer.rs

@@ -0,0 +1,239 @@
+use core::{
+    any::Any,
+    marker::PhantomData,
+    sync::atomic::{AtomicBool, Ordering},
+};
+
+use alloc::{
+    string::{String, ToString},
+    sync::Arc,
+    vec::Vec,
+};
+use libk_util::sync::spin_rwlock::IrqSafeRwLock;
+use yggdrasil_abi::{
+    error::Error,
+    io::{FileMode, OpenOptions},
+};
+
+use crate::{
+    fs::sysfs::object::KObject,
+    vfs::{CommonImpl, Filename, InstanceData, Metadata, Node, NodeFlags, NodeRef, RegularImpl},
+};
+
+use super::Attribute;
+
+pub trait IntegerAttributeValue: Copy + Sync + Send + 'static {
+    fn to_bytes(&self, format: IntegerAttributeFormat) -> Vec<u8>;
+}
+
+pub enum IntegerAttributeFormat {
+    Decimal,
+    Octal,
+    Hex,
+}
+
+macro_rules! impl_integer_value {
+    ($($ty:ty),+) => {
+        $(
+            impl IntegerAttributeValue for $ty {
+                fn to_bytes(&self, format: IntegerAttributeFormat) -> Vec<u8> {
+                    match format {
+                        IntegerAttributeFormat::Decimal => alloc::format!("{self}"),
+                        IntegerAttributeFormat::Octal => alloc::format!("{self:o}"),
+                        IntegerAttributeFormat::Hex => alloc::format!("{self:x}"),
+                    }.into_bytes()
+                }
+            }
+        )+
+    };
+}
+
+impl_integer_value!(u8, u16, u32, u64);
+
+pub trait IntegerAttributeOps<T: IntegerAttributeValue>: Sync + Send + 'static {
+    type Data: Send + 'static = ();
+
+    const WRITEABLE: bool = false;
+    const NAME: &'static str;
+    const FORMAT: IntegerAttributeFormat = IntegerAttributeFormat::Decimal;
+
+    fn read(state: &Self::Data) -> Result<T, Error> {
+        let _ = state;
+        Err(Error::NotImplemented)
+    }
+    fn write(state: &Self::Data, value: T) -> Result<(), Error> {
+        let _ = state;
+        let _ = value;
+        Err(Error::ReadOnly)
+    }
+}
+
+pub struct IntegerAttribute<T: IntegerAttributeValue, V: IntegerAttributeOps<T>>(
+    PhantomData<(T, V)>,
+);
+
+struct IntegerAttributeNode<T: IntegerAttributeValue, V: IntegerAttributeOps<T>> {
+    object: Arc<KObject<V::Data>>,
+    _pd: PhantomData<V>,
+}
+
+struct IntegerAttributeState<T: IntegerAttributeValue> {
+    value: IrqSafeRwLock<Vec<u8>>,
+    modified: AtomicBool,
+    _pd: PhantomData<T>,
+}
+
+impl<T: IntegerAttributeValue, V: IntegerAttributeOps<T>> CommonImpl
+    for IntegerAttributeNode<T, V>
+{
+    fn size(&self, _node: &NodeRef) -> Result<u64, Error> {
+        Ok(0)
+    }
+
+    fn as_any(&self) -> &dyn Any {
+        self as _
+    }
+}
+
+impl<T: IntegerAttributeValue, V: IntegerAttributeOps<T>> RegularImpl
+    for IntegerAttributeNode<T, V>
+{
+    fn open(
+        &self,
+        _node: &NodeRef,
+        opts: OpenOptions,
+    ) -> Result<(u64, Option<InstanceData>), Error> {
+        if opts.contains(OpenOptions::WRITE) && !V::WRITEABLE {
+            return Err(Error::ReadOnly);
+        }
+
+        let mut value = V::read(self.object.data())?.to_bytes(V::FORMAT);
+        value.push(b'\n');
+
+        let instance = IntegerAttributeState {
+            value: IrqSafeRwLock::new(value),
+            modified: AtomicBool::new(false),
+            _pd: PhantomData::<T>,
+        };
+
+        Ok((0, Some(Arc::new(instance))))
+    }
+
+    fn close(&self, _node: &NodeRef, instance: Option<&InstanceData>) -> Result<(), Error> {
+        if V::WRITEABLE {
+            todo!()
+            // let instance = instance.ok_or(Error::InvalidFile)?;
+            // let instance = instance
+            //     .downcast_ref::<StringAttributeState>()
+            //     .ok_or(Error::InvalidFile)?;
+
+            // if instance.modified.load(Ordering::Acquire) {
+            //     let value = instance.value.read();
+            //     let value_str =
+            //         core::str::from_utf8(&value[..]).map_err(|_| Error::InvalidArgument)?;
+
+            //     // Trim whitespace and newlines
+            //     V::write(&self.object.data, value_str.trim())?;
+            // }
+        }
+
+        Ok(())
+    }
+
+    fn read(
+        &self,
+        _node: &NodeRef,
+        instance: Option<&InstanceData>,
+        pos: u64,
+        buf: &mut [u8],
+    ) -> Result<usize, Error> {
+        let instance = instance.ok_or(Error::InvalidFile)?;
+        let instance = instance
+            .downcast_ref::<IntegerAttributeState<T>>()
+            .ok_or(Error::InvalidFile)?;
+
+        let value = instance.value.read();
+        let len = value.len();
+        if pos >= len as u64 {
+            return Ok(0);
+        }
+        let pos = pos as usize;
+        let amount = (len - pos).min(buf.len());
+        buf[..amount].copy_from_slice(&value[pos..pos + amount]);
+
+        Ok(amount)
+    }
+
+    fn write(
+        &self,
+        _node: &NodeRef,
+        instance: Option<&InstanceData>,
+        pos: u64,
+        buf: &[u8],
+    ) -> Result<usize, Error> {
+        todo!()
+        // if !V::WRITEABLE {
+        //     return Err(Error::InvalidFile);
+        // }
+        // let instance = instance.ok_or(Error::InvalidFile)?;
+        // let instance = instance
+        //     .downcast_ref::<StringAttributeState>()
+        //     .ok_or(Error::InvalidFile)?;
+
+        // let mut value = instance.value.write();
+
+        // let pos: usize = pos.try_into().map_err(|_| Error::InvalidFile)?;
+        // if pos > value.len() {
+        //     return Err(Error::InvalidArgument);
+        // }
+        // if pos + buf.len() > V::LIMIT {
+        //     return Err(Error::InvalidArgument);
+        // }
+
+        // let amount_copy = (value.len() - pos).min(buf.len());
+
+        // value[pos..pos + amount_copy].copy_from_slice(&buf[..amount_copy]);
+        // if amount_copy < buf.len() {
+        //     value.extend_from_slice(&buf[amount_copy..]);
+        // }
+        // instance.modified.store(true, Ordering::Release);
+
+        // Ok(buf.len())
+    }
+
+    fn truncate(&self, _node: &NodeRef, _new_size: u64) -> Result<(), Error> {
+        Ok(())
+    }
+}
+
+impl<T: IntegerAttributeValue, V: IntegerAttributeOps<T>> From<V> for IntegerAttribute<T, V> {
+    fn from(_value: V) -> Self {
+        Self(PhantomData)
+    }
+}
+
+impl<T: IntegerAttributeValue, V: IntegerAttributeOps<T>> Attribute<V::Data>
+    for IntegerAttribute<T, V>
+{
+    fn instantiate(&self, parent: &Arc<KObject<V::Data>>) -> Result<Arc<Node>, Error> {
+        let mode = match V::WRITEABLE {
+            false => FileMode::new(0o444),
+            true => FileMode::new(0o644),
+        };
+
+        Ok(Node::regular(
+            IntegerAttributeNode {
+                object: parent.clone(),
+                _pd: PhantomData::<V>,
+            },
+            NodeFlags::IN_MEMORY_PROPS,
+            Some(Metadata::now_root(mode, 0)),
+            None,
+        ))
+    }
+
+    // TODO implement this properly
+    fn name(&self) -> &Filename {
+        unsafe { Filename::from_str_unchecked(V::NAME) }
+    }
+}

kernel/libk/src/fs/sysfs/attribute/mod.rs

@@ -6,6 +6,7 @@ use crate::vfs::{Filename, NodeRef};
 use super::object::KObject;
 
 mod bytes;
+mod integer;
 mod string;
 
 pub trait Attribute<D>: Sync + Send {
@@ -14,4 +15,7 @@ pub trait Attribute<D>: Sync + Send {
 }
 
 pub use bytes::{BytesAttribute, BytesAttributeOps};
+pub use integer::{
+    IntegerAttribute, IntegerAttributeFormat, IntegerAttributeOps, IntegerAttributeValue,
+};
 pub use string::{StringAttribute, StringAttributeOps};

kernel/libk/src/lib.rs

@@ -18,7 +18,6 @@
     arbitrary_self_types,
     slice_split_once,
     arbitrary_self_types_pointers,
-    result_flattening,
     negative_impls,
     decl_macro,
     optimize_attribute

kernel/libk/src/task/process.rs

@@ -464,7 +464,9 @@ impl Process {
         }
     }
 
-    fn poll_any_child_exit(&self) -> Option<(Arc<Process>, IrqSafeRwLockWriteGuard<ProcessInner>)> {
+    fn poll_any_child_exit(
+        &self,
+    ) -> Option<(Arc<Process>, IrqSafeRwLockWriteGuard<'_, ProcessInner>)> {
         let read = self.inner.read();
         if let Some(child) = read.any_exited_child() {
             let write = IrqSafeRwLockReadGuard::upgrade(read);

kernel/libk/src/task/sync.rs

@@ -59,7 +59,7 @@ impl<T> AsyncMutex<T> {
             .is_ok()
     }
 
-    pub fn poll_lock(&self, cx: &mut Context<'_>) -> Poll<AsyncMutexGuard<T>> {
+    pub fn poll_lock(&self, cx: &mut Context<'_>) -> Poll<AsyncMutexGuard<'_, T>> {
         self.waker.register(cx.waker());
 
         if self.try_lock() {
@@ -70,7 +70,7 @@ impl<T> AsyncMutex<T> {
         Poll::Pending
     }
 
-    pub async fn lock(&self) -> AsyncMutexGuard<T> {
+    pub async fn lock(&self) -> AsyncMutexGuard<'_, T> {
         poll_fn(|cx| self.poll_lock(cx)).await
     }
 

kernel/libk/src/task/thread.rs

@@ -552,7 +552,7 @@ impl Thread {
         self.enqueue();
     }
 
-    pub fn hold_suspended(&self) -> Result<SuspendGuard, Error> {
+    pub fn hold_suspended(&self) -> Result<SuspendGuard<'_>, Error> {
         let sched = self.sched.lock();
         if sched.state == ThreadState::Running {
             return Err(Error::InvalidOperation);

kernel/libk/src/vfs/filesystem.rs

@@ -23,7 +23,9 @@ pub enum FilesystemMountOption<'a> {
     FsSpecific(&'a str),
 }
 
-pub fn parse_mount_options(options: Option<&str>) -> impl Iterator<Item = FilesystemMountOption> {
+pub fn parse_mount_options(
+    options: Option<&str>,
+) -> impl Iterator<Item = FilesystemMountOption<'_>> {
     options
         .unwrap_or("")
         .trim()

kernel/libk/src/vfs/node/impls.rs

@@ -396,6 +396,7 @@ impl SymlinkImpl for FixedPathSymlink {
             buf[..self.target.len()].copy_from_slice(self.target.as_bytes());
             Ok(self.target.len())
         } else {
+            log::warn!("FixedPathSymlink: BufferTooSmall");
             Err(Error::BufferTooSmall)
         }
     }

kernel/libk/src/vfs/pid.rs

@@ -94,6 +94,11 @@ impl PidFile {
 
     pub fn read(&self, buf: &mut [u8], _non_blocking: bool) -> Result<usize, Error> {
         if buf.len() < size_of::<u32>() + size_of::<i32>() {
+            log::warn!(
+                "PidFd: BufferTooSmall (need {}, got {})",
+                size_of::<u32>() + size_of::<i32>(),
+                buf.len()
+            );
             return Err(Error::BufferTooSmall);
         }
         match self {

kernel/libk/src/vfs/terminal.rs

@@ -162,7 +162,9 @@ impl<O: TerminalOutput> Terminal<O> {
                 if config.line.contains(TerminalLineOptions::SIGNAL) {
                     self.output.notify_readers();
 
-                    if let Some(group_id) = *self.input.signal_pgroup.read() {
+                    let pgrp = *self.input.signal_pgroup.read();
+                    log::info!("Send terminal SIGINT to {pgrp:?}");
+                    if let Some(group_id) = pgrp {
                         Process::signal_group(None, group_id, Signal::Interrupted);
                         self.input.ready_ring.notify_all();
                         return;

kernel/libk/src/vfs/timer.rs

@@ -35,6 +35,7 @@ impl TimerFile {
 
     pub fn read(&self, buf: &mut [u8], non_blocking: bool) -> Result<usize, Error> {
         if buf.len() < size_of::<u8>() {
+            log::warn!("TimerFile: BufferTooSmall");
             return Err(Error::BufferTooSmall);
         }
         if non_blocking {

kernel/src/arch/aarch64/boot/entry.S

@@ -1,187 +1,91 @@
-.set CNTHCTL_EL2_EL1PCTEN, 1 << 0
+// vi:set ft=asm:
-.set CNTHCTL_EL2_EL1PCEN,  1 << 1
-
-.set HCR_EL2_RW_EL1IsAArch64, 1 << 31
-
-.set SCTLR_EL2_RES1, 0x30C50830
-
-.set SPSR_EL2_EL1h, 0x5
-.set SPSR_EL2_MASK_DAIF, 0xF << 6
-
-.macro MOV_L reg, value
-    mov \reg, #((\value) & 0xFFFF)
-    movk \reg, #((\value) >> 16), lsl #16
-.endm
-
-.macro MOV_ABS reg, sym
-    movz \reg, #:abs_g2:\sym
-    movk \reg, #:abs_g1_nc:\sym
-    movk \reg, #:abs_g0_nc:\sym
-.endm
-
-// clobbers: x1
-.macro LEAVE_EL2 destination
-    // Setup EL1 physical timer
-    mrs x1, cnthctl_el2
-    orr x1, x1, #(1 << 0)       // EL0PCTEN = 1
-    orr x1, x1, #(1 << 1)       // EL0VCTEN = 1
-    orr x1, x1, #(1 << 10)      // EL1PCTEN = 1
-    orr x1, x1, #(1 << 11)      // EL1PTEN = 1
-    msr cnthctl_el2, x1
-    mov x1, #1
-    msr cnthp_ctl_el2, x1
-    msr cntkctl_el1, xzr
-    msr cntvoff_el2, xzr
-
-    mov x1, #(1 << 31)          // RW = 1, EL1 is AArch64
-    orr x1, x1, #(1 << 1)       // SWIO = 1 (Pi has this hardwired)
-    msr hcr_el2, x1
-
-    // Return to EL1
-    mov x1, #0x3C5
-    msr spsr_el2, x1
-    adr x1, \destination
-    msr elr_el2, x1
-    eret
-.endm
 
 .global __aarch64_entry
 .global __aarch64_ap_entry
 .global __aarch64_ap_spin_table_entry
 
-.section .text.entry
+.macro ADR_WIDE reg, sym
+    adrp \reg, \sym
+    add \reg, \reg, :lo12:\sym
+.endm
+
+.pushsection .text.entry
+    // TODO insert image header here
 __aarch64_entry:
-    // x0 -- dtb_phys
+    adr x29, __aarch64_entry
+    mov x30, x0
 
-    //////////////////////////////
+    // Park AP cores in a spin loop
-    // Check CPU index          //
+    mrs x0, mpidr_el1
-    //////////////////////////////
+    ands x0, x0, #0xF
-    mrs x1, mpidr_el1
+    bne .park_ap_core
-    ands x1, x1, #3
-    bne .spin_wait
-
-    //////////////////////////////
-    // Check CurrentEL          //
-    //////////////////////////////
-    mrs x1, CurrentEL
-    lsr x1, x1, #2
-    and x1, x1, #3
-    MOV_ABS x2, .bsp_el_table
-    ldr x2, [x2, x1, lsl #3]
-    br x2
-
-// TODO code for leaving EL3
-.bsp_el3:
-    b .
-
-.bsp_el2:
-    //////////////////////////////
-    // Leave EL2                //
-    //////////////////////////////
-
-    LEAVE_EL2 .bsp_el1
-
-.bsp_el1:
-    //////////////////////////////
-    // Setup EL1 for Rust entry //
-    //////////////////////////////
-
-    // Load proper SP
-    MOV_ABS x1, {stack_bottom} + {stack_size} - {kernel_virt_offset}
-    mov sp, x1
 
     // Zero .bss
-    MOV_ABS x1, __bss_start_phys
+    adrp x0, __bss_start
-    MOV_ABS x2, __bss_size
+    adrp x1, __bss_end
-    lsr x2, x2, #3
+1:
-    cbz x2, 11f
+    cmp x0, x1
-10:
+    beq 2f
-    str xzr, [x1]
+    str xzr, [x0], #8
-    add x1, x1, #8
+    b 1b
-    subs x2, x2, #1
+2:
-    bne 10b
-11:
 
-    // Perform Rust code entry
+    // Store image base + dtb address
-    bl {bsp_el1_entry} - {kernel_virt_offset}
+    ADR_WIDE x0, {dtb_address}
+    str x30, [x0]
+    ADR_WIDE x0, {load_base}
+    str x29, [x0]
+
+    // Load sp and perform relocation
+    ADR_WIDE x0, {stack_bottom}
+    add x0, x0, #{stack_size}
+    mov sp, x0
+
+    mov x0, x29
+    ADR_WIDE x1, __rela_start
+    ADR_WIDE x2, __rela_end
+    bl {relocate_function}
+
+    // Jump to corresponding ELx entry
+    mrs x0, currentel
+    adr x1, .bsp_el_branch_table
+    add x0, x0, x1
+    br x0
+
+.bsp_el_branch_table:
+    b .         // EL0 impossible
+    b {bsp_el1_entry}
+    b {bsp_el2_entry}
+    b .         // EL3 TODO
+
+.park_ap_core:
     b .
 
-// EL0 impossible
+.popsection // .text.entry
-.bsp_el0:
-    udf #0
-
-.spin_wait:
-    // AP in a SMP system
-    // TODO spin loop for this method of init
-    wfe
-    b .spin_wait
-
-.p2align 3
-.bsp_el_table:
-    .dword .bsp_el0
-    .dword .bsp_el1
-    .dword .bsp_el2
-    .dword .bsp_el3
-
-//////////////////////////////
-// AP startup code          //
-//////////////////////////////
-
-.section .text
-__aarch64_ap_spin_table_entry:
-    // Spin-table entry, expected state:
-    //  * MMU is not yet enabled
-    //  * In lower half
-    //  * x0 is uninitialized
-    MOV_ABS x1, {spin_table_stack} - {kernel_virt_offset}
-
-    // Atomic load-acquire the stack address
-    ldar x0, [x1]
-
-    dsb ish
-    isb sy
-
-    // Fall through to __aarch64_ap_entry
 
+.pushsection .text
 __aarch64_ap_entry:
-    // Application processsor entry, expected state:
+    // Application processor entry
-    //  * MMU is not yet enabled
+    // Expected state:
-    //  * In lower half
+    // * MMU is not yet enabled
-    //  * x0 -- physical stack pointer
+    // * In lower half
-
+    // * x0 -- physical stack pointer
-    mrs x1, CurrentEL
-    lsr x1, x1, #2
-    and x1, x1, #3
-    adr x2, .el_table_ap
-    ldr x2, [x2, x1, lsl #3]
-    br x2
-
-    b .
-
-// TODO code to leave EL3 on AP
-.ap_el3:
-    b .
-
-.ap_el2:
-    LEAVE_EL2 .ap_el1
-
-.ap_el1:
-    dsb ish
-    isb sy
 
     mov sp, x0
-    MOV_ABS x0, {ap_el1_entry} - {kernel_virt_offset}
-    blr x0
 
+    mrs x1, currentel
+    adr x2, .ap_el_branch_table
+    add x1, x1, x2
+    br x1
+.ap_el_branch_table:
     b .
+    b {ap_el1_entry}
+    b {ap_el2_entry}
+    b .                 // EL3 TODO
 
-// EL0 impossible
+// TODO
-.ap_el0:
+__aarch64_ap_spin_table_entry:
-    udf #0
+1:
-
+    wfe
-.p2align 3
+    b 1b
-.el_table_ap:
+.popsection // .text
-    .dword .ap_el0 - {kernel_virt_offset}
-    .dword .ap_el1 - {kernel_virt_offset}
-    .dword .ap_el2 - {kernel_virt_offset}
-    .dword .ap_el3 - {kernel_virt_offset}

kernel/src/arch/aarch64/boot/mod.rs

@@ -1,121 +1,73 @@
 //! AArch64 boot and entry implementation
-use core::{
+use core::sync::atomic::{AtomicUsize, Ordering};
-    arch::global_asm,
-    sync::atomic::{AtomicUsize, Ordering},
-};
 
 use aarch64_cpu::{
     asm::barrier,
-    registers::{CPACR_EL1, ID_AA64MMFR0_EL1, SCTLR_EL1, TCR_EL1, TTBR0_EL1},
+    registers::{
-};
+        CNTHCTL_EL2, CNTHP_CTL_EL2, CNTKCTL_EL1, CNTVOFF_EL2, CPACR_EL1, ELR_EL2, HCR_EL2,
-use kernel_arch::{absolute_address, Architecture, ArchitectureImpl};
+        SPSR_EL2, SP_EL1, TTBR0_EL1,
-use kernel_arch_aarch64::{
+    },
-    mem::{self, table::L3},
-    CPU_COUNT,
 };
+use elf::relocation::Elf64_Rela;
+use kernel_arch::{Architecture, ArchitectureImpl, KERNEL_VIRT_OFFSET};
+use kernel_arch_aarch64::{mem, CPU_COUNT};
 use libk::{
     fs::{devfs, sysfs},
     task::runtime,
 };
-use libk_mm::{
+use libk_mm::address::PhysicalAddress;
-    address::{PhysicalAddress, Virtualize},
+use tock_registers::interfaces::{ReadWriteable, Writeable};
-    phys,
-    table::EntryLevel,
-};
-use tock_registers::interfaces::{ReadWriteable, Readable, Writeable};
 
-use super::{exception, BootStack, PLATFORM};
 use crate::{
-    arch::aarch64::BOOT_STACK_SIZE, kernel_main, kernel_secondary_main, mem::KERNEL_VIRT_OFFSET,
+    arch::{aarch64::exception, PLATFORM},
+    kernel_main, kernel_secondary_main,
 };
 
-unsafe fn check_mmu_features() {
+use super::{BootStack, BOOT_STACK_SIZE};
-    if ID_AA64MMFR0_EL1.matches_all(ID_AA64MMFR0_EL1::TGran4::NotSupported) {
+
-        // TODO early panic
+#[link_section = ".bss"]
-        ArchitectureImpl::halt();
+static BSP_STACK: BootStack<BOOT_STACK_SIZE> = BootStack::zeroed();
+static mut DTB_ADDRESS: u64 = 0;
+static mut LOAD_BASE: u64 = 0;
+
+pub(crate) static SPIN_TABLE_STACK: AtomicUsize = AtomicUsize::new(0);
+
+unsafe extern "C" fn relocate_function(image_base: u64, rela_start: usize, rela_end: usize) {
+    let rela_count = (rela_end - rela_start) / size_of::<Elf64_Rela>();
+    let rela_ptr = core::ptr::with_exposed_provenance::<Elf64_Rela>(rela_start);
+    let rela_array = core::slice::from_raw_parts(rela_ptr, rela_count);
+
+    for rela in rela_array {
+        let qword =
+            core::ptr::with_exposed_provenance_mut::<i64>((rela.r_offset + image_base) as usize);
+
+        match rela.r_info as u32 {
+            elf::abi::R_AARCH64_RELATIVE => {
+                let value = rela.r_addend + image_base as i64;
+                qword.write_volatile(value);
+            }
+            _ => loop {},
+        }
     }
 }
 
-unsafe fn pre_init_mmu() {
+unsafe extern "C" fn bsp_el1_upper_entry() -> ! {
-    mem::setup_memory_attributes();
+    // Relocate the kernel to upper half
-
+    extern "C" {
-    TCR_EL1.write(
+        static __rela_start: u8;
-        TCR_EL1::AS::ASID8Bits +
+        static __rela_end: u8;
-        TCR_EL1::A1::TTBR0 +
-        TCR_EL1::HD::CLEAR +
-        // General
-        TCR_EL1::IPS::Bits_48 +
-        // TTBR0
-        TCR_EL1::TG0::KiB_4 + TCR_EL1::T0SZ.val(25) + TCR_EL1::SH0::Inner +
-        // TTBR1
-        TCR_EL1::TG1::KiB_4 + TCR_EL1::T1SZ.val(25) + TCR_EL1::SH1::Inner,
-    );
-}
-
-unsafe fn enable_mmu() {
-    barrier::dsb(barrier::ISHST);
-    barrier::isb(barrier::SY);
-
-    SCTLR_EL1.modify(
-        SCTLR_EL1::E0E::LittleEndian
-            + SCTLR_EL1::EE::LittleEndian
-            + SCTLR_EL1::WXN::Disable
-            + SCTLR_EL1::SA0::Enable
-            + SCTLR_EL1::SA::Enable
-            + SCTLR_EL1::A::Enable
-            + SCTLR_EL1::I::NonCacheable
-            + SCTLR_EL1::C::NonCacheable,
-    );
-
-    barrier::dsb(barrier::ISH);
-    barrier::isb(barrier::SY);
-
-    // Enable translation
-    SCTLR_EL1.modify(SCTLR_EL1::M::Enable);
-
-    // Enable caches
-    mem::enable_icache();
-    mem::enable_dcache();
-}
-
-unsafe fn enter_higher_half(sp: usize, elr: usize, x0: usize) -> ! {
-    unsafe {
-        core::arch::asm!(r#"
-            mov sp, {sp}
-            mov x0, {x0}
-            mov lr, xzr
-            br {elr}
-        "#, elr = in(reg) elr, sp = in(reg) sp, x0 = in(reg) x0, options(noreturn));
     }
-}
+    let rela_start = (&raw const __rela_start).addr();
+    let rela_end = (&raw const __rela_end).addr();
+    relocate_function(KERNEL_VIRT_OFFSET as u64 + LOAD_BASE, rela_start, rela_end);
 
-// NOTE executes in "lower-half" address space, MMU not yet enabled
+    // Unmap lower half
-unsafe extern "C" fn __aarch64_el1_bsp_lower_entry(dtb: PhysicalAddress) -> ! {
-    ArchitectureImpl::set_interrupt_mask(true);
-
-    // Don't trap FP operations
-    CPACR_EL1.modify(CPACR_EL1::FPEN::TrapNothing);
-
-    // Setup MMU to jump to "higher-half" address space
-    check_mmu_features();
-    pre_init_mmu();
-    kernel_arch_aarch64::mem::load_fixed_tables();
-    enable_mmu();
-
-    // SP points to the .bss section, so it's +offset mapped
-    let sp = BSP_STACK.top_addr() + KERNEL_VIRT_OFFSET;
-    let elr = absolute_address!(__aarch64_bsp_upper_entry);
-
-    enter_higher_half(sp, elr, dtb.into_usize());
-}
-
-unsafe extern "C" fn __aarch64_bsp_upper_entry(dtb: PhysicalAddress) -> ! {
-    // Remove the "lower-half" mapping, no longer needed
     TTBR0_EL1.set(0);
-
     mem::tlb_flush_all();
 
     // Setup the "runtime" part of the kernel tables
+    let dtb = PhysicalAddress::from_u64(core::ptr::read_volatile(&raw const DTB_ADDRESS));
+
     if PLATFORM.init_memory_management(dtb).is_err() {
         ArchitectureImpl::halt();
     }
@@ -135,69 +87,114 @@ unsafe extern "C" fn __aarch64_bsp_upper_entry(dtb: PhysicalAddress) -> ! {
         ArchitectureImpl::halt();
     }
 
+    #[allow(static_mut_refs)]
+    {
+        log::info!("Kernel load base: {LOAD_BASE:#x}");
+    }
+
     kernel_main()
 }
 
-// TODO re-implement for Pi 4B
+unsafe extern "C" fn ap_el1_upper_entry() -> ! {
-unsafe extern "C" fn __aarch64_el1_ap_lower_entry() -> ! {
+    barrier::dsb(barrier::ISH);
-    const AP_STACK_PAGES: usize = 8;
+    barrier::isb(barrier::SY);
-    ArchitectureImpl::set_interrupt_mask(true);
 
-    // Unmask FP operations
+    // Unmap lower half
-    CPACR_EL1.modify(CPACR_EL1::FPEN::TrapNothing);
+    TTBR0_EL1.set(0);
-
+    mem::tlb_flush_all();
-    check_mmu_features();
-    pre_init_mmu();
-    kernel_arch_aarch64::mem::load_fixed_tables();
-    enable_mmu();
-
-    let stack_pages = phys::alloc_pages_contiguous(AP_STACK_PAGES).unwrap();
-    let stack_base = stack_pages.virtualize();
-    let sp = stack_base + L3::SIZE * AP_STACK_PAGES;
-
-    let elr = absolute_address!(__aarch64_ap_upper_entry);
 
     barrier::dsb(barrier::ISH);
     barrier::isb(barrier::SY);
 
-    enter_higher_half(sp, elr, 0);
+    CPU_COUNT.fetch_add(1, Ordering::SeqCst);
-}
-
-extern "C" fn __aarch64_ap_upper_entry() -> ! {
-    // Flush lower half
-    TTBR0_EL1.set(0);
-
-    mem::tlb_flush_all();
-
-    let cpu_id = CPU_COUNT.fetch_add(1, Ordering::SeqCst);
-    barrier::dsb(barrier::ISHST);
-    barrier::isb(barrier::SY);
     aarch64_cpu::asm::sev();
-    barrier::isb(barrier::SY);
-
-    log::info!("cpu{} initializing", cpu_id);
 
     exception::init_exceptions();
-
+    // Initialize the AP CPU
-    unsafe {
+    if PLATFORM.init_platform(false).is_err() {
-        PLATFORM
+        ArchitectureImpl::halt();
-            .init_platform(false)
-            .expect("Could not initialize the AP");
     }
 
     kernel_secondary_main()
 }
 
-#[link_section = ".bss"]
+unsafe fn common_early_init() {
-static BSP_STACK: BootStack<BOOT_STACK_SIZE> = BootStack::zeroed();
+    // Don't trap FP operations
-pub(crate) static SPIN_TABLE_STACK: AtomicUsize = AtomicUsize::new(0);
+    CPACR_EL1.modify(CPACR_EL1::FPEN::TrapNothing);
 
-global_asm!(
+    // Configure and enable MMU
+    mem::configure_mmu();
+    mem::load_fixed_tables();
+    mem::enable_mmu();
+
+    // Enable caches
+    mem::enable_icache();
+    mem::enable_dcache();
+}
+
+unsafe extern "C" fn bsp_el1_entry() -> ! {
+    mem::setup_fixed_tables();
+    common_early_init();
+
+    // Jump to higher half
+    let pc = bsp_el1_upper_entry as usize + KERNEL_VIRT_OFFSET;
+    let sp = BSP_STACK.top_addr() + KERNEL_VIRT_OFFSET;
+    long_jump(pc, sp);
+}
+
+unsafe fn drop_to_el1(pc: usize, sp: usize, x0: usize) -> ! {
+    CNTHCTL_EL2.write(CNTHCTL_EL2::EL1PCEN::SET + CNTHCTL_EL2::EL1PCTEN::SET);
+    CNTHP_CTL_EL2.write(CNTHP_CTL_EL2::ENABLE::SET);
+    CNTKCTL_EL1.set(0);
+    CNTVOFF_EL2.set(0);
+
+    HCR_EL2.write(HCR_EL2::RW::EL1IsAarch64 + HCR_EL2::SWIO::SET);
+
+    SPSR_EL2.write(
+        SPSR_EL2::D::SET
+            + SPSR_EL2::A::SET
+            + SPSR_EL2::I::SET
+            + SPSR_EL2::F::SET
+            + SPSR_EL2::M::EL1h,
+    );
+    ELR_EL2.set(pc as u64);
+    SP_EL1.set(sp as u64);
+    core::arch::asm!("eret", in("x0") x0, options(noreturn));
+}
+
+unsafe extern "C" fn bsp_el2_entry() -> ! {
+    drop_to_el1(bsp_el1_entry as usize, BSP_STACK.top_addr(), 0)
+}
+
+unsafe extern "C" fn ap_el1_entry(sp: usize) -> ! {
+    common_early_init();
+
+    // Jump to higher half
+    let pc = ap_el1_upper_entry as usize + KERNEL_VIRT_OFFSET;
+    let sp = sp + KERNEL_VIRT_OFFSET;
+    long_jump(pc, sp);
+}
+
+unsafe extern "C" fn ap_el2_entry(sp: usize) -> ! {
+    drop_to_el1(ap_el1_entry as usize, sp, sp)
+}
+
+#[inline]
+unsafe fn long_jump(pc: usize, sp: usize) -> ! {
+    core::arch::asm!("mov sp, {sp}; br {pc}", sp = in(reg) sp, pc = in(reg) pc, options(noreturn));
+}
+
+core::arch::global_asm!(
     include_str!("entry.S"),
-    bsp_el1_entry = sym __aarch64_el1_bsp_lower_entry,
+    relocate_function = sym relocate_function,
-    ap_el1_entry = sym __aarch64_el1_ap_lower_entry,
+    bsp_el1_entry = sym bsp_el1_entry,
-    spin_table_stack = sym SPIN_TABLE_STACK,
+    bsp_el2_entry = sym bsp_el2_entry,
+    ap_el1_entry = sym ap_el1_entry,
+    ap_el2_entry = sym ap_el2_entry,
+
     stack_bottom = sym BSP_STACK,
-    kernel_virt_offset = const KERNEL_VIRT_OFFSET,
+    stack_size = const BOOT_STACK_SIZE,
-    stack_size = const BOOT_STACK_SIZE
+
+    load_base = sym LOAD_BASE,
+    dtb_address = sym DTB_ADDRESS,
 );

kernel/src/arch/aarch64/mod.rs

@@ -14,11 +14,7 @@ use device_tree::{
     DeviceTree, DeviceTreeNodeExt,
 };
 use kernel_arch_aarch64::{
-    mem::{
+    mem::{self, table::L3},
-        self,
-        table::{L1, L3},
-        EarlyMapping, MEMORY_LIMIT, RAM_MAPPING_L1_COUNT,
-    },
     ArchitectureImpl, PerCpuData,
 };
 use libk::{arch::Cpu, config, debug, device::external_interrupt_controller};
@@ -119,36 +115,10 @@ impl AArch64 {
         GIC.init(gic);
     }
 
-    fn map_physical_memory<I: Iterator<Item = PhysicalMemoryRegion> + Clone>(
-        _it: I,
-        _memory_start: PhysicalAddress,
-        memory_end: PhysicalAddress,
-    ) -> Result<(), Error> {
-        let end_l1i = memory_end.page_align_up::<L1>().page_index::<L1>();
-        if end_l1i > RAM_MAPPING_L1_COUNT {
-            panic!("TODO: partial physical memory mapping");
-        }
-
-        // Map 1GiB chunks
-        for index in 0..end_l1i {
-            unsafe {
-                kernel_arch_aarch64::mem::map_ram_l1(index);
-            }
-        }
-
-        MEMORY_LIMIT.store(memory_end.into_usize(), Ordering::Release);
-
-        Ok(())
-    }
-
     unsafe fn init_memory_management(&'static self, dtb: PhysicalAddress) -> Result<(), Error> {
-        // Initialize the runtime mappings
-        kernel_arch_aarch64::mem::init_fixed_tables();
-
-        // Extract the size of the device tree
         let dtb_size = {
-            let dtb_header = EarlyMapping::<u8>::map_slice(dtb, DeviceTree::MIN_HEADER_SIZE)?;
+            let bytes = PhysicalRef::map_slice(dtb, DeviceTree::MIN_HEADER_SIZE);
-            DeviceTree::read_totalsize(dtb_header.as_ref()).map_err(|_| Error::InvalidArgument)?
+            DeviceTree::read_totalsize(&bytes[..])?
         };
 
         reserve_region(
@@ -159,7 +129,7 @@ impl AArch64 {
             },
         );
 
-        let dtb_slice = EarlyMapping::<u8>::map_slice(dtb, dtb_size)?;
+        let dtb_slice = PhysicalRef::<u8>::map_slice(dtb, dtb_size);
         let dt = DeviceTree::from_raw(dtb_slice.as_ptr() as usize)?;
 
         // Setup initrd from the dt
@@ -180,12 +150,7 @@ impl AArch64 {
         }
 
         // Initialize the physical memory
-        phys::init_from_iter(dt.memory_regions(), Self::map_physical_memory)?;
+        phys::init_from_iter(dt.memory_regions())?;
-
-        // EarlyMapping for DTB no longer needed, it lives in physical memory and can be obtained
-        // through PhysicalRef
-        let dtb_slice: PhysicalRef<'static, [u8]> = PhysicalRef::map_slice(dtb, dtb_size);
-        let dt = DeviceTree::from_raw(dtb_slice.as_ptr().addr())?;
 
         self.dt.init(dt);
 

kernel/src/arch/aarch64/vectors.S

@@ -1,4 +1,4 @@
-// vi:ft=a64asm:
+// vi:ft=asm:
 
 .macro EXC_VECTOR el, ht, bits, kind
 .p2align 7
@@ -93,7 +93,7 @@ __aa\bits\()_el\el\ht\()_\kind:
     isb sy
 .endm
 
-.section .text.vectors
+.pushsection .text
 .global __aarch64_el1_vectors
 .p2align 12
 __aarch64_el1_vectors:
@@ -117,7 +117,6 @@ __aarch64_el1_vectors:
     EXC_VECTOR 0, t, 32, fiq
     EXC_VECTOR 0, t, 32, serror
 
-.section .text
 .p2align 7
 EXC_HANDLER 1, t, 64, sync
 EXC_HANDLER 1, t, 64, irq
@@ -138,3 +137,5 @@ EXC_HANDLER 0, t, 32, sync
 EXC_HANDLER 0, t, 32, irq
 EXC_HANDLER 0, t, 32, fiq
 EXC_HANDLER 0, t, 32, serror
+
+.popsection // .text

kernel/src/arch/i686/mod.rs

@@ -140,7 +140,7 @@ impl I686 {
             TerminalInput::with_capacity(256)?,
             ConsoleWrapper(textfb),
         ));
-        let keyboard_input = ygg_driver_input::setup();
+        let keyboard_input = ygg_driver_input::setup_keyboard();
 
         runtime::spawn(
             textfb_console

kernel/src/arch/riscv64/boot/entry.S

@@ -23,7 +23,7 @@ __rv64_entry:
     j       __rv64_real_entry   // Jump to real entry (if entered by non-Linux bootloader)
     .long   0
     .quad   0x200000            // Offset from RAM start
-    .quad   2000000             // Image size TODO fill this by post-tooling
+    .quad   8388608             // Image size TODO fill this by post-tooling
     .quad   0                   // Kernel flags
     .long   0x2                 // Header version
     .long   0
@@ -41,27 +41,42 @@ __rv64_entry:
 
 .type __rv64_real_entry, @function
 __rv64_real_entry:
-    // a0 - bootstrap HART ID
-    // a1 - device tree blob
-    // mhartid == a0
     csrw satp, zero
     mv tp, zero
 
-    // Zero the .bss
+    // a0 - bootstrap HART ID
-    LOAD_PCREL .L00, t0, __bss_start_phys
+    // a1 - device tree blob
-    LOAD_PCREL .L01, t1, __bss_end_phys
+    LOAD_PCREL .L00, s0, __kernel_start
+    mv s1, a0
+    mv s2, a1
 
+    // Zero the .bss
+    LOAD_PCREL .L01, t0, __bss_start
+    LOAD_PCREL .L02, t1, __bss_end
 1:  bgeu t0, t1, 2f
     sd zero, (t0)
     addi t0, t0, 4
     j 1b
 2:
 
-    // Setup boot stack and entry point
+    // Calculate load address
-    LOAD_PCREL .L02, sp, {boot_stack_bottom} + {boot_stack_size} - {kernel_virt_offset}
+    LOAD_PCREL .L03, t0, {load_base}
-    LOAD_PCREL .L03, t0, {entry_smode_lower} - {kernel_virt_offset}
+    sd s0, (t0)
+    LOAD_PCREL .L04, t0, {boot_hart_id}
+    sd s1, (t0)
+    LOAD_PCREL .L05, t0, {dtb_address}
+    sd s2, (t0)
 
-    jr t0
+    LOAD_PCREL .L06, sp, {boot_stack_bottom} + {boot_stack_size}
+
+    // Perform first kernel relocation
+    mv a0, s0
+    LOAD_PCREL .L07, a1, __rela_start
+    LOAD_PCREL .L08, a2, __rela_end
+    jal {relocate_function}
+
+    // Enter the kernel
+    j {bsp_smode_entry}
 .size __rv64_real_entry, . - __rv64_real_entry
 
 .type __rv64_secondary_entry, @function
@@ -74,8 +89,9 @@ __rv64_secondary_entry:
     ld sp, (a1)
     mv a0, a1
     mv a1, sp
-    LOAD_PCREL .L04, t0, {entry_smode_secondary_lower} - {kernel_virt_offset}
+    j {ap_smode_entry}
-    jr t0
+    // LOAD_PCREL .L04, t0, entry_smode_secondary_lower - kernel_virt_offset
+    // jr t0
 .size __rv64_secondary_entry, . - __rv64_secondary_entry
 
 .option pop

kernel/src/arch/riscv64/boot/mod.rs

@@ -3,6 +3,7 @@ use core::{
     sync::atomic::{compiler_fence, Ordering},
 };
 
+use elf::relocation::Elf64_Rela;
 use kernel_arch::Architecture;
 use kernel_arch_riscv64::{
     mem::{self, KERNEL_VIRT_OFFSET},
@@ -36,24 +37,28 @@ impl<const N: usize> BootStack<N> {
 #[link_section = ".bss"]
 static mut BOOT_STACK: BootStack<BOOT_STACK_SIZE> = BootStack::zeroed();
 
-unsafe fn long_jump(pc: usize, sp: usize, a0: usize, a1: usize) -> ! {
+static mut BOOT_HART_ID: u64 = 0;
-    core::arch::asm!(r#"
+static mut LOAD_BASE: u64 = 0;
-        mv sp, {sp}
+static mut DTB_ADDRESS: u64 = 0;
-        jr {pc}
+
-    "#,
+unsafe extern "C" fn bsp_smode_upper_entry() -> ! {
-        in("a0") a0,
+    // Relocate the kernel to upper half
-        in("a1") a1,
+    extern "C" {
-        pc = in(reg) pc,
+        static __rela_start: u8;
-        sp = in(reg) sp,
+        static __rela_end: u8;
-        options(noreturn)
+    }
-    );
+    let rela_start = (&raw const __rela_start).addr();
-}
+    let rela_end = (&raw const __rela_end).addr();
+    relocate_function(KERNEL_VIRT_OFFSET as u64 + LOAD_BASE, rela_start, rela_end);
+
+    // Unmap lower half
+    mem::unmap_lower_half();
 
-unsafe extern "C" fn __rv64_bsp_entry_upper(bsp_hart_id: u64, dtb_physical: PhysicalAddress) -> ! {
     debug::init_logger();
     super::debug::register_sbi_debug();
 
     log::info!("Starting riscv64 upper half");
+    let dtb_physical = PhysicalAddress::from_u64(unsafe { DTB_ADDRESS });
 
     if dtb_physical.is_zero() {
         log::error!("No device tree provided");
@@ -71,7 +76,7 @@ unsafe extern "C" fn __rv64_bsp_entry_upper(bsp_hart_id: u64, dtb_physical: Phys
 
     runtime::init_task_queue();
 
-    if let Err(error) = PLATFORM.init_platform(bsp_hart_id as _, 0, true) {
+    if let Err(error) = PLATFORM.init_platform(BOOT_HART_ID as _, 0, true) {
         log::error!("Failed to initialize the platform: {error:?}");
         ArchitectureImpl::halt();
     }
@@ -79,7 +84,7 @@ unsafe extern "C" fn __rv64_bsp_entry_upper(bsp_hart_id: u64, dtb_physical: Phys
     kernel_main()
 }
 
-unsafe extern "C" fn __rv64_secondary_entry_upper(context: PhysicalAddress) -> ! {
+unsafe extern "C" fn ap_smode_upper_entry(context: PhysicalAddress) -> ! {
     let hart_id = {
         let context = PageBox::<SecondaryContext>::from_physical_raw(context);
         context.hart_id
@@ -94,22 +99,19 @@ unsafe extern "C" fn __rv64_secondary_entry_upper(context: PhysicalAddress) -> !
     kernel_secondary_main()
 }
 
-unsafe extern "C" fn __rv64_bsp_smode_entry_lower(a0: usize, a1: usize) -> ! {
+unsafe extern "C" fn bsp_smode_entry() -> ! {
     ArchitectureImpl::set_interrupt_mask(true);
 
     mem::enable_mmu();
 
     let stack = (&raw const BOOT_STACK).addr() + KERNEL_VIRT_OFFSET;
-    let pc = __rv64_bsp_entry_upper as usize + KERNEL_VIRT_OFFSET;
+    let pc = bsp_smode_upper_entry as usize + KERNEL_VIRT_OFFSET;
     let sp = stack + BOOT_STACK_SIZE;
 
-    long_jump(pc, sp, a0, a1)
+    long_jump(pc, sp, 0, 0);
 }
 
-unsafe extern "C" fn __rv64_secondary_smode_entry_lower(
+unsafe extern "C" fn ap_smode_entry(context: PhysicalAddress, sp: PhysicalAddress) -> ! {
-    context: PhysicalAddress,
-    sp: PhysicalAddress,
-) -> ! {
     let sp = sp.virtualize();
 
     ArchitectureImpl::set_interrupt_mask(true);
@@ -118,15 +120,46 @@ unsafe extern "C" fn __rv64_secondary_smode_entry_lower(
     mem::enable_mmu();
     compiler_fence(Ordering::Acquire);
 
-    let pc = __rv64_secondary_entry_upper as usize + KERNEL_VIRT_OFFSET;
+    let pc = ap_smode_upper_entry as usize + KERNEL_VIRT_OFFSET;
     long_jump(pc, sp, context.into_usize(), 0)
 }
 
+unsafe fn long_jump(pc: usize, sp: usize, a0: usize, a1: usize) -> ! {
+    core::arch::asm!("mv sp, {sp}; jr {pc}", pc = in(reg) pc, sp = in(reg) sp, in("a0") a0, in("a1") a1, options(noreturn));
+}
+
+unsafe extern "C" fn relocate_function(image_base: u64, rela_start: usize, rela_end: usize) {
+    let rela_count = (rela_end - rela_start) / size_of::<Elf64_Rela>();
+    let rela_ptr = core::ptr::with_exposed_provenance::<Elf64_Rela>(rela_start);
+    let rela_array = core::slice::from_raw_parts(rela_ptr, rela_count);
+
+    for rela in rela_array {
+        let qword =
+            core::ptr::with_exposed_provenance_mut::<i64>((rela.r_offset + image_base) as usize);
+
+        match rela.r_info as u32 {
+            elf::abi::R_RISCV_RELATIVE => {
+                let value = rela.r_addend + image_base as i64;
+                qword.write_volatile(value);
+            }
+            _ => loop {},
+        }
+    }
+}
+
 global_asm!(
     include_str!("entry.S"),
-    entry_smode_lower = sym __rv64_bsp_smode_entry_lower,
+    load_base = sym LOAD_BASE,
-    entry_smode_secondary_lower = sym __rv64_secondary_smode_entry_lower,
+    boot_hart_id = sym BOOT_HART_ID,
+    dtb_address = sym DTB_ADDRESS,
+
+    bsp_smode_entry = sym bsp_smode_entry,
+    ap_smode_entry = sym ap_smode_entry,
+    relocate_function = sym relocate_function,
+
+    // entry_smode_lower = sym __rv64_bsp_smode_entry_lower,
+    // entry_smode_secondary_lower = sym __rv64_secondary_smode_entry_lower,
     boot_stack_bottom = sym BOOT_STACK,
-    kernel_virt_offset = const KERNEL_VIRT_OFFSET,
     boot_stack_size = const BOOT_STACK_SIZE,
+    // kernel_virt_offset = const KERNEL_VIRT_OFFSET,
 );

kernel/src/arch/riscv64/mod.rs

@@ -13,7 +13,7 @@ use device_tree::{
 };
 use kernel_arch::Architecture;
 use kernel_arch_riscv64::{
-    mem::{self, KERNEL_VIRT_OFFSET},
+    mem::KERNEL_VIRT_OFFSET,
     registers::{SIE, SSTATUS},
     ArchitectureImpl, PerCpuData,
 };
@@ -91,9 +91,6 @@ impl Platform for Riscv64 {
 
 impl Riscv64 {
     unsafe fn init_memory_management(&'static self, dtb: PhysicalAddress) -> Result<(), Error> {
-        // Unmap the lower half
-        mem::setup_fixed_tables();
-
         // Extract the size of the device tree
         let dtb_size = {
             let dtb_header = PhysicalRef::<u8>::map_slice(dtb, DeviceTree::MIN_HEADER_SIZE);
@@ -139,7 +136,7 @@ impl Riscv64 {
         }
 
         // Initialize the physical memory
-        phys::init_from_iter(dt.memory_regions(), |_, _, _| Ok(()))?;
+        phys::init_from_iter(dt.memory_regions())?;
 
         self.dt.init(dt);
 

kernel/src/arch/riscv64/smp.rs

@@ -29,7 +29,7 @@ fn start_secondary_hart(hart_id: u64) -> Result<(), Error> {
         fn __rv64_secondary_entry();
     }
 
-    let start_addr = __rv64_secondary_entry as usize;
+    let start_addr = mem::auto_lower_address(__rv64_secondary_entry as *const fn());
     let stack = PageBox::<u8>::new_uninit_slice(SECONDARY_STACK_SIZE)?;
     let stack_top = unsafe { PageBox::as_physical_address(&stack).add(SECONDARY_STACK_SIZE) };
     let context = PageBox::new(SecondaryContext {