dev/pci: MSI-X capability and APIC support for it

lib/device-api/src/interrupt.rs

@@ -33,10 +33,23 @@ pub struct IrqOptions {
     pub trigger: IrqTrigger,
 }
 
+#[derive(Clone, Copy, Debug)]
+pub struct MsiInfo {
+    pub address: usize,
+    pub value: u32,
+    pub vector: usize,
+}
+
 pub trait InterruptTable {
     fn handler(&self, index: usize) -> Option<&'static dyn InterruptHandler>;
 }
 
+pub trait MessageInterruptController {
+    fn register_msi(&self, handler: &'static dyn MsiHandler) -> Result<MsiInfo, Error>;
+
+    fn handle_msi(&self, #[allow(unused)] vector: usize) {}
+}
+
 pub trait ExternalInterruptController {
     type IrqNumber;
 
@@ -80,6 +93,10 @@ pub trait InterruptHandler: Device {
     fn handle_irq(&self) -> bool;
 }
 
+pub trait MsiHandler: Device {
+    fn handle_msi(&self, vector: usize) -> bool;
+}
+
 pub struct FixedInterruptTable<const SIZE: usize> {
     entries: [Option<&'static dyn InterruptHandler>; SIZE],
 }

src/arch/mod.rs

@@ -21,7 +21,10 @@ macro_rules! absolute_address {
 
 use cfg_if::cfg_if;
 use device_api::{
-    interrupt::{ExternalInterruptController, IpiDeliveryTarget, LocalInterruptController},
+    interrupt::{
+        ExternalInterruptController, IpiDeliveryTarget, LocalInterruptController,
+        MessageInterruptController,
+    },
     timer::MonotonicTimestampProviderDevice,
     ResetDevice,
 };
@@ -157,6 +160,14 @@ pub trait Architecture {
         Err(Error::NotImplemented)
     }
 
+    /// Adds a message-signalled interrupt (MSI/MSI-X) controller to the system
+    fn register_message_interrupt_controller(
+        &self,
+        intc: &'static dyn MessageInterruptController,
+    ) -> Result<(), Error> {
+        Err(Error::NotImplemented)
+    }
+
     /// Adds a monotonic timer to the system
     fn register_monotonic_timer(
         &self,
@@ -185,6 +196,11 @@ pub trait Architecture {
         unimplemented!()
     }
 
+    /// Returns the MSI/MSI-X-capable interrupt controller
+    fn message_interrupt_controller(&'static self) -> &'static dyn MessageInterruptController {
+        unimplemented!()
+    }
+
     /// Returns the monotonic timer
     fn monotonic_timer(&'static self) -> &'static dyn MonotonicTimestampProviderDevice {
         unimplemented!()

src/arch/x86_64/apic/ioapic.rs

@@ -20,7 +20,7 @@ use crate::{
     mem::{address::FromRaw, device::DeviceMemoryIo, PhysicalAddress},
 };
 
-use super::{APIC_EXTERNAL_OFFSET, MAX_EXTERNAL_VECTORS};
+use super::{APIC_EXTERNAL_OFFSET, POPULATED_EXTERNAL_VECTORS};
 
 // IRQ 0 is timer, IRQ 1 reserved (for now?), +32 offset for exception entries
 const IO_APIC_VECTOR_OFFSET: u32 = 32 + APIC_EXTERNAL_OFFSET;
@@ -64,7 +64,7 @@ pub struct IoApic {
     inner: IrqSafeSpinlock<Inner>,
     isa_redirections: [Option<IsaRedirection>; 16],
 
-    table: IrqSafeSpinlock<FixedInterruptTable<{ MAX_EXTERNAL_VECTORS as usize }>>,
+    table: IrqSafeSpinlock<FixedInterruptTable<{ POPULATED_EXTERNAL_VECTORS as usize }>>,
 }
 
 impl Regs {

src/arch/x86_64/apic/local.rs

@@ -1,11 +1,16 @@
 //! x86-64 Local APIC driver implementation
 use core::sync::atomic::Ordering;
 
+use abi::error::Error;
+use alloc::vec::Vec;
 use device_api::{
-    interrupt::{IpiDeliveryTarget, LocalInterruptController},
+    interrupt::{
+        IpiDeliveryTarget, LocalInterruptController, MessageInterruptController, MsiHandler,
+        MsiInfo,
+    },
     Device,
 };
-use kernel_util::util::OneTimeInit;
+use kernel_util::{sync::IrqSafeSpinlock, util::OneTimeInit};
 use tock_registers::{
     interfaces::{ReadWriteable, Readable, Writeable},
     register_bitfields, register_structs,
@@ -14,10 +19,16 @@ use tock_registers::{
 
 use crate::{
     arch::{
-        x86_64::{mem::table::L3, registers::MSR_IA32_APIC_BASE, smp::CPU_COUNT},
+        x86_64::{
+            apic::APIC_MSI_OFFSET, mem::table::L3, registers::MSR_IA32_APIC_BASE, smp::CPU_COUNT,
+        },
         CpuMessage,
     },
-    mem::{address::FromRaw, device::DeviceMemoryIo, PhysicalAddress},
+    mem::{
+        address::{FromRaw, IntoRaw},
+        device::DeviceMemoryIo,
+        PhysicalAddress,
+    },
     task::Cpu,
 };
 
@@ -131,6 +142,7 @@ register_structs! {
 /// Per-processor local APIC interface
 pub struct LocalApic {
     regs: DeviceMemoryIo<'static, Regs>,
+    msi_vectors: IrqSafeSpinlock<Vec<&'static dyn MsiHandler>>,
 }
 
 unsafe impl Send for LocalApic {}
@@ -142,14 +154,51 @@ impl Device for LocalApic {
     }
 }
 
+impl MessageInterruptController for LocalApic {
+    fn handle_msi(&self, vector: usize) {
+        // TODO this is ugly
+        let mut i = 0;
+
+        loop {
+            let table = self.msi_vectors.lock();
+            let Some(&handler) = table.get(i) else {
+                break;
+            };
+            drop(table);
+
+            if handler.handle_msi(vector) {
+                break;
+            }
+
+            i += 1;
+        }
+    }
+
+    fn register_msi(&self, handler: &'static dyn MsiHandler) -> Result<MsiInfo, Error> {
+        // TODO only 1 ISR vector allocated for MSIs
+        let vector = 0;
+        let mut table = self.msi_vectors.lock();
+
+        table.push(handler);
+
+        // TODO magic numbers
+        let apic_vector = 32 + APIC_MSI_OFFSET + vector;
+
+        let value = apic_vector;
+        let address = Self::base();
+
+        Ok(MsiInfo {
+            address: address.into_raw(),
+            value,
+            vector: vector as _,
+        })
+    }
+}
+
 impl LocalInterruptController for LocalApic {
     type IpiMessage = CpuMessage;
 
-    fn send_ipi(
+    fn send_ipi(&self, target: IpiDeliveryTarget, msg: Self::IpiMessage) -> Result<(), Error> {
-        &self,
-        target: device_api::interrupt::IpiDeliveryTarget,
-        msg: Self::IpiMessage,
-    ) -> Result<(), abi::error::Error> {
         while self.regs.ICR0.matches_all(ICR0::DeliveryStatus::SET) {
             core::hint::spin_loop();
         }
@@ -178,7 +227,7 @@ impl LocalInterruptController for LocalApic {
         }
     }
 
-    unsafe fn init_ap(&self) -> Result<(), abi::error::Error> {
+    unsafe fn init_ap(&self) -> Result<(), Error> {
         todo!()
     }
 }
@@ -228,7 +277,10 @@ impl LocalApic {
             LocalVectorEntry::Mask::Masked + LocalVectorEntry::Vector.val(APIC_LINT1_VECTOR + 32),
         );
 
-        Self { regs }
+        Self {
+            regs,
+            msi_vectors: IrqSafeSpinlock::new(Vec::new()),
+        }
     }
 
     /// Signals local APIC that we've handled the IRQ

src/arch/x86_64/apic/mod.rs

@@ -2,6 +2,8 @@
 
 use core::arch::global_asm;
 
+use static_assertions::{const_assert, const_assert_eq};
+
 use crate::{
     arch::{x86_64::cpu::Cpu, Architecture},
     task::thread::Thread,
@@ -29,8 +31,17 @@ pub const APIC_IPI_VECTOR: u32 = 0x03;
 pub const APIC_SPURIOUS_VECTOR: u32 = 0xDF;
 /// Start of the I/O APIC IRQ range
 pub const APIC_EXTERNAL_OFFSET: u32 = 4;
+/// Start of the MSI range
+pub const APIC_MSI_OFFSET: u32 = APIC_EXTERNAL_OFFSET + MAX_EXTERNAL_VECTORS;
 /// Maximum number of APIC vectors allocated for handling IRQs from I/O APIC
-pub const MAX_EXTERNAL_VECTORS: u32 = 16;
+pub const MAX_EXTERNAL_VECTORS: u32 = APIC_SPURIOUS_VECTOR - APIC_EXTERNAL_OFFSET - MAX_MSI_VECTORS;
+/// Number of I/O APIC IRQ vectors that are actually populated
+pub const POPULATED_EXTERNAL_VECTORS: u32 = 16;
+/// Maximum number of APIC vectors allocated for handling MSIs
+pub const MAX_MSI_VECTORS: u32 = 16;
+
+const_assert!(POPULATED_EXTERNAL_VECTORS <= MAX_EXTERNAL_VECTORS);
+const_assert_eq!(APIC_MSI_OFFSET + MAX_MSI_VECTORS, APIC_SPURIOUS_VECTOR);
 
 /// Fills the IDT with interrupt vectors for this APIC
 pub fn setup_vectors(idt: &mut [exception::Entry]) {
@@ -49,6 +60,10 @@ pub fn setup_vectors(idt: &mut [exception::Entry]) {
 }
 
 unsafe extern "C" fn irq_handler(vector: usize, frame: *mut IrqFrame) {
+    if vector >= POPULATED_EXTERNAL_VECTORS as _ {
+        todo!("Got a weird IRQ with vector {}", vector);
+    }
+
     let cpu = Cpu::local();
     let frame = &mut *frame;
 
@@ -62,6 +77,24 @@ unsafe extern "C" fn irq_handler(vector: usize, frame: *mut IrqFrame) {
     }
 }
 
+unsafe extern "C" fn msi_handler(vector: usize, frame: *mut IrqFrame) {
+    if vector != 0 {
+        todo!("Got a weird MSI with vector {}", vector);
+    }
+
+    let cpu = Cpu::local();
+    let frame = &mut *frame;
+
+    ARCHITECTURE
+        .message_interrupt_controller()
+        .handle_msi(vector);
+    cpu.local_apic().clear_interrupt();
+
+    if let Some(thread) = Thread::get_current() {
+        thread.handle_pending_signals(frame);
+    }
+}
+
 unsafe extern "C" fn local_timer_irq_handler(frame: *mut IrqFrame) {
     let frame = &mut *frame;
     let cpu = Cpu::local();
@@ -87,7 +120,12 @@ global_asm!(
     include_str!("vectors.S"),
     local_timer_irq_handler = sym local_timer_irq_handler,
     irq_handler = sym irq_handler,
+    msi_handler = sym msi_handler,
     ipi_handler = sym ipi_handler,
     dummy_irq_handler = sym dummy_irq_handler,
+    irq_vector_offset = const APIC_EXTERNAL_OFFSET,
+    irq_vector_count = const MAX_EXTERNAL_VECTORS,
+    msi_vector_offset = const APIC_MSI_OFFSET,
+    msi_vector_count = const MAX_MSI_VECTORS,
     options(att_syntax)
 );

src/arch/x86_64/apic/vectors.S

@@ -123,6 +123,56 @@ irq_vector_\n:
 .endr
 .endm
 
+.macro MSI_VECTOR, n
+msi_vector_\n:
+    // %rsp + 0: %rip
+    // %rsp + 8: %cs
+    IRQ_SWAPGS_IF_NEEDED 8
+    IRQ_SAVE_STATE
+
+    // Force correct segment registers
+    mov $0x10, %ax
+    mov %ax, %ss
+    mov %ax, %ds
+    mov %ax, %es
+
+    mov $\n, %rdi
+    mov %rbp, %rsi
+    call {msi_handler}
+
+    IRQ_RESTORE_STATE
+    IRQ_SWAPGS_IF_NEEDED 8
+
+    iretq
+.endm
+
+.macro MSI_VECTOR_ENTRY, n
+.quad msi_vector_\n
+.endm
+
+.macro MSI_VECTORS, start, end
+.set i, 0
+.rept \end - \start
+    MSI_VECTOR %i
+    .set i, i+1
+.endr
+.endm
+
+.macro MSI_VECTOR_ENTRIES, start, end
+.set i, 0
+.rept \end - \start
+    MSI_VECTOR_ENTRY %i
+    .set i, i+1
+.endr
+.endm
+
+.macro FILL_EMPTY_SPACE, start, end
+.set i, 0
+.rept \end - \start
+.quad dummy_vector
+.endr
+.endm
+
 .section .text
 local_timer_vector:
     IRQ_SWAPGS_IF_NEEDED 8
@@ -149,7 +199,8 @@ dummy_vector:
     call {dummy_irq_handler}
     jmp .
 
-IRQ_VECTORS 4, 255
+IRQ_VECTORS {irq_vector_offset}, {irq_vector_offset} + {irq_vector_count}
+MSI_VECTORS {msi_vector_offset}, {msi_vector_offset} + {msi_vector_count}
 
 .section .rodata
 // 224 vectors: 256 - 32 (exceptions)
@@ -163,8 +214,10 @@ __x86_64_apic_vectors:
     .quad dummy_vector
     // IPI vector: 3
     .quad ipi_vector
-    // Regular IRQ vectors: 4..=222
+    // Regular IRQ vectors: 4..207
-    IRQ_VECTOR_ENTRIES 4, 223
+    IRQ_VECTOR_ENTRIES {irq_vector_offset}, {irq_vector_offset} + {irq_vector_count}
+    // MSI vectors: 207..223
+    MSI_VECTOR_ENTRIES {msi_vector_offset}, {msi_vector_offset} + {msi_vector_count}
     // Spurious interrupt vector: 223
     .quad dummy_vector
 .size __x86_64_apic_vectors, . - __x86_64_apic_vectors

src/arch/x86_64/mod.rs

@@ -5,8 +5,10 @@ use abi::error::Error;
 use acpi_lib::{mcfg::Mcfg, AcpiTables, InterruptModel};
 use alloc::boxed::Box;
 use device_api::{
-    input::KeyboardProducer, interrupt::ExternalInterruptController,
+    input::KeyboardProducer,
-    timer::MonotonicTimestampProviderDevice, Device,
+    interrupt::{ExternalInterruptController, MessageInterruptController},
+    timer::MonotonicTimestampProviderDevice,
+    Device,
 };
 use git_version::git_version;
 use kernel_util::{sync::SpinFence, util::OneTimeInit};
@@ -281,6 +283,10 @@ impl Architecture for X86_64 {
         self.ioapic.get()
     }
 
+    fn message_interrupt_controller(&'static self) -> &'static dyn MessageInterruptController {
+        Cpu::local().local_apic()
+    }
+
     fn monotonic_timer(&'static self) -> &'static dyn MonotonicTimestampProviderDevice {
         self.timer.get()
     }

src/device/bus/pci/capability.rs

@@ -0,0 +1,105 @@
+//! PCI capability structures and queries
+
+use abi::error::Error;
+use device_api::interrupt::{MessageInterruptController, MsiHandler};
+
+use crate::{
+    device::bus::pci::PciBaseAddress,
+    mem::{address::FromRaw, device::DeviceMemoryIoMut, PhysicalAddress},
+};
+
+use super::{PciCapability, PciCapabilityId, PciConfigurationSpace};
+
+/// MSI-X capability query
+pub struct MsiXCapability;
+
+/// Represents an entry in MSI-X vector table
+#[repr(C)]
+pub struct MsiXEntry {
+    /// Address to which the value is written on interrupt
+    pub address: u64,
+    /// Value which is written to trigger an interrupt
+    pub data: u32,
+    /// Vector control word
+    pub control: u32,
+}
+
+/// MSI-X capability data structure
+pub struct MsiXData<'s, S: PciConfigurationSpace + ?Sized + 's> {
+    space: &'s S,
+    offset: usize,
+}
+
+impl PciCapability for MsiXCapability {
+    const ID: PciCapabilityId = PciCapabilityId::MsiX;
+    type CapabilityData<'a, S: PciConfigurationSpace + ?Sized + 'a> = MsiXData<'a, S>;
+
+    fn data<'s, S: PciConfigurationSpace + ?Sized + 's>(
+        space: &'s S,
+        offset: usize,
+    ) -> Self::CapabilityData<'s, S> {
+        MsiXData { space, offset }
+    }
+}
+
+impl<'s, S: PciConfigurationSpace + ?Sized + 's> MsiXData<'s, S> {
+    // TODO use pending bits as well
+    /// Maps and returns the vector table associated with the device's MSI-X capability
+    pub fn vector_table<'a>(&self) -> Result<DeviceMemoryIoMut<'a, [MsiXEntry]>, Error> {
+        let w0 = self.space.read_u16(self.offset + 2);
+        let dw1 = self.space.read_u32(self.offset + 4);
+
+        let table_size = (w0 as usize & 0x3FF) + 1;
+        let bir = dw1 as usize & 0x3;
+        let table_offset = dw1 as usize & !0x3;
+
+        let Some(base) = self.space.bar(bir) else {
+            return Err(Error::DoesNotExist);
+        };
+        let PciBaseAddress::Memory(base) = base else {
+            return Err(Error::InvalidOperation);
+        };
+
+        debugln!("MSI-X table address: {:#x}", base + table_offset);
+
+        unsafe {
+            DeviceMemoryIoMut::map_slice(PhysicalAddress::from_raw(base + table_offset), table_size)
+        }
+    }
+
+    /// Changes the global enable status for the device's MSI-X capability. If set, regular IRQs
+    /// are not generated.
+    pub fn set_enabled(&mut self, enabled: bool) {
+        let mut w0 = self.space.read_u32(self.offset);
+        if enabled {
+            w0 |= 1 << 31;
+        } else {
+            w0 &= !(1 << 31);
+        }
+        self.space.write_u32(self.offset, w0);
+    }
+}
+
+impl MsiXEntry {
+    /// If set, prevents the MSI-X interrupt from being delivered
+    pub fn set_masked(&mut self, masked: bool) {
+        if masked {
+            self.control |= 1;
+        } else {
+            self.control &= !1;
+        }
+    }
+
+    /// Registers the MSI-X vector with the interrupt controller and enables it
+    pub fn register<C: MessageInterruptController + ?Sized>(
+        &mut self,
+        ic: &C,
+        handler: &'static dyn MsiHandler,
+    ) -> Result<(), Error> {
+        let info = ic.register_msi(handler)?;
+        self.address = info.address as _;
+        self.data = info.value as _;
+        self.set_masked(false);
+        Ok(())
+    }
+}

src/device/bus/pci/mod.rs

@@ -8,6 +8,7 @@ use device_api::Device;
 use kernel_util::sync::IrqSafeSpinlock;
 use yggdrasil_abi::error::Error;
 
+pub mod capability;
 mod space;
 
 pub use space::{
@@ -33,6 +34,14 @@ bitflags! {
     }
 }
 
+bitflags! {
+    /// Status register of the PCI configuration space
+    pub struct PciStatusRegister: u16 {
+        /// Read-only. If set, the configuration space has a pointer to the capabilities list.
+        const CAPABILITIES_LIST = 1 << 4;
+    }
+}
+
 /// Represents the address of a single object on a bus (or the bus itself)
 #[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)]
 pub struct PciAddress {
@@ -55,6 +64,33 @@ pub enum PciBaseAddress {
     Io(u16),
 }
 
+/// Unique ID assigned to PCI capability structures
+#[derive(Clone, Copy, PartialEq, Eq, Debug)]
+#[non_exhaustive]
+#[repr(u8)]
+pub enum PciCapabilityId {
+    /// MSI (32-bit or 64-bit)
+    Msi = 0x05,
+    /// MSI-X
+    MsiX = 0x11,
+    /// Unknown capability missing from this list
+    Unknown,
+}
+
+/// Interface used for querying PCI capabilities
+pub trait PciCapability {
+    /// Capability ID
+    const ID: PciCapabilityId;
+    /// Wrapper for accessing the capability data structure
+    type CapabilityData<'a, S: PciConfigurationSpace + ?Sized + 'a>;
+
+    /// Constructs an access wrapper for this capability with given offset
+    fn data<'s, S: PciConfigurationSpace + ?Sized + 's>(
+        space: &'s S,
+        offset: usize,
+    ) -> Self::CapabilityData<'s, S>;
+}
+
 /// Describes a PCI device
 #[derive(Debug)]
 pub struct PciDeviceInfo {

src/device/bus/pci/space/mod.rs

@@ -1,4 +1,6 @@
-use super::{PciAddress, PciBaseAddress, PciEcam};
+use crate::device::bus::pci::PciStatusRegister;
+
+use super::{PciAddress, PciBaseAddress, PciCapability, PciCapabilityId, PciEcam};
 
 pub(super) mod ecam;
 
@@ -72,6 +74,30 @@ pub enum PciConfigSpace {
     Ecam(PciEcam),
 }
 
+pub struct CapabilityIterator<'s, S: PciConfigurationSpace + ?Sized> {
+    space: &'s S,
+    current: Option<usize>,
+}
+
+impl<'s, S: PciConfigurationSpace + ?Sized> Iterator for CapabilityIterator<'s, S> {
+    type Item = (PciCapabilityId, usize);
+
+    fn next(&mut self) -> Option<Self::Item> {
+        let offset = self.current? & !0x3;
+
+        let id = unsafe { core::mem::transmute(self.space.read_u8(offset)) };
+        let next_pointer = self.space.read_u8(offset + 1);
+
+        self.current = if next_pointer != 0 {
+            Some(next_pointer as usize)
+        } else {
+            None
+        };
+
+        Some((id, offset))
+    }
+}
+
 /// Interface for accessing the configuration space of a device
 pub trait PciConfigurationSpace {
     /// Reads a 32-bit value from the device configuration space.
@@ -120,7 +146,7 @@ pub trait PciConfigurationSpace {
     }
 
     /// Writes a byte to the device configuration space
-    fn write_u8(&self, offset: usize, value: u16) {
+    fn write_u8(&self, _offset: usize, _value: u16) {
         todo!()
     }
 
@@ -185,6 +211,14 @@ pub trait PciConfigurationSpace {
         "#]
         secondary_bus
     );
+    pci_config_field!(
+        0x34 => u8,
+        #[doc =
+            r"Returns the offset within the configuration space where the Capabilities List
+            is located. Only valid if the corresponding Status Register bit is set"
+        ]
+        capability_pointer
+    );
 
     /// Returns the value of the Base Address Register with given index.
     ///
@@ -242,4 +276,38 @@ pub trait PciConfigurationSpace {
             }
         }
     }
+
+    /// Returns an iterator over the PCI capabilities
+    fn capability_iter(&self) -> CapabilityIterator<Self> {
+        let status = PciStatusRegister::from_bits_retain(self.status());
+
+        let current = if status.contains(PciStatusRegister::CAPABILITIES_LIST) {
+            let ptr = self.capability_pointer() as usize;
+
+            if ptr != 0 {
+                Some(self.capability_pointer() as usize)
+            } else {
+                None
+            }
+        } else {
+            // Return an empty iterator
+            None
+        };
+
+        CapabilityIterator {
+            space: self,
+            current,
+        }
+    }
+
+    /// Locates a capability within this configuration space
+    fn capability<'s, C: PciCapability>(&'s self) -> Option<C::CapabilityData<'s, Self>> {
+        self.capability_iter().find_map(|(id, offset)| {
+            if id == C::ID {
+                Some(C::data(self, offset))
+            } else {
+                None
+            }
+        })
+    }
 }

src/device/nvme/command.rs

@@ -1,3 +1,5 @@
+#![allow(unused)]
+
 use core::fmt::{self, Write};
 
 use crate::{
@@ -47,6 +49,7 @@ pub struct IdentifyControllerRequest {
 pub struct CreateIoCompletionQueue {
     pub id: u32,
     pub size: usize,
+    pub vector: u32,
     pub data: PhysicalAddress,
 }
 
@@ -110,9 +113,7 @@ impl Command for CreateIoCompletionQueue {
         sqe.command.set_opcode(0x05);
         sqe.data_pointer[1] = PhysicalRegionPage::with_addr(self.data.into_raw());
         sqe.command_specific[0] = ((self.size as u32 - 1) << 16) | self.id;
-        sqe.command_specific[1] = 1;
+        sqe.command_specific[1] = (self.vector << 16) | 3;
-        // TODO ENABLE IRQS HERE
-        sqe.command_specific[2] = 0;
     }
 }
 

src/device/nvme/mod.rs

@@ -3,7 +3,7 @@ use core::{mem::size_of, time::Duration};
 
 use abi::error::Error;
 use alloc::vec::Vec;
-use device_api::Device;
+use device_api::{interrupt::MsiHandler, Device};
 use kernel_util::{sync::IrqSafeSpinlock, util::OneTimeInit};
 use tock_registers::{
     interfaces::{ReadWriteable, Readable, Writeable},
@@ -12,8 +12,11 @@ use tock_registers::{
 };
 
 use crate::{
+    arch::{Architecture, ARCHITECTURE},
     device::{
-        bus::pci::{PciBaseAddress, PciCommandRegister, PciConfigurationSpace},
+        bus::pci::{
+            capability::MsiXCapability, PciBaseAddress, PciCommandRegister, PciConfigurationSpace,
+        },
         nvme::{
             command::IdentifyControllerRequest,
             queue::{CompletionQueueEntry, SubmissionQueueEntry},
@@ -21,7 +24,7 @@ use crate::{
     },
     mem::{
         address::{FromRaw, IntoRaw},
-        device::DeviceMemoryIo,
+        device::{DeviceMemoryIo, DeviceMemoryIoMut},
         PhysicalAddress,
     },
     task::runtime,
@@ -33,7 +36,7 @@ use self::{
     queue::QueuePair,
 };
 
-use super::bus::pci::{FromPciBus, PciDeviceInfo};
+use super::bus::pci::{capability::MsiXEntry, FromPciBus, PciDeviceInfo};
 
 mod command;
 mod error;
@@ -109,6 +112,7 @@ pub struct NvmeController {
     regs: IrqSafeSpinlock<DeviceMemoryIo<'static, Regs>>,
     admin_q: OneTimeInit<QueuePair<'static>>,
     ioqs: OneTimeInit<Vec<QueuePair<'static>>>,
+    vector_table: IrqSafeSpinlock<DeviceMemoryIoMut<'static, [MsiXEntry]>>,
 
     doorbell_shift: usize,
 }
@@ -123,9 +127,11 @@ impl Regs {
 
 impl NvmeController {
     async fn late_init(&'static self) -> Result<(), NvmeError> {
-        runtime::spawn(self.poll_task()).expect("Couldn't spawn NVMe poll task");
+        // runtime::spawn(self.poll_task()).expect("Couldn't spawn NVMe poll task");
         let admin_q = self.admin_q.get();
 
+        infoln!("SETUP");
+
         // Request a CQ/SQ pair for I/O
         admin_q
             .request_no_data(SetFeatureRequest::NumberOfQueues(1, 1))
@@ -133,7 +139,8 @@ impl NvmeController {
 
         // Allocate the queue
         let (sq_doorbell, cq_doorbell) = unsafe { self.doorbell_pair(1) };
-        let io_q = QueuePair::new(32, sq_doorbell, cq_doorbell).map_err(NvmeError::MemoryError)?;
+        let io_q =
+            QueuePair::new(0, 32, sq_doorbell, cq_doorbell).map_err(NvmeError::MemoryError)?;
 
         // Identify the controller
         let identify = admin_q
@@ -145,6 +152,7 @@ impl NvmeController {
             .request_no_data(CreateIoCompletionQueue {
                 id: 1,
                 size: 32,
+                vector: 0,
                 data: io_q.cq_physical_pointer(),
             })
             .await?;
@@ -160,14 +168,6 @@ impl NvmeController {
         loop {}
     }
 
-    // TODO MSI(-X) or IRQ (ACPI currently broken) support for PCIe-based NVMe
-    async fn poll_task(&'static self) {
-        loop {
-            self.admin_q.get().process_completions();
-            runtime::sleep(Duration::from_millis(100)).await;
-        }
-    }
-
     unsafe fn doorbell_pair(&self, idx: usize) -> (*mut u32, *mut u32) {
         let regs = self.regs.lock();
         let sq_ptr = regs.doorbell_ptr(self.doorbell_shift, false, idx);
@@ -176,12 +176,18 @@ impl NvmeController {
     }
 }
 
+impl MsiHandler for NvmeController {
+    fn handle_msi(&self, vector: usize) -> bool {
+        debugln!("handle_msi {}", vector);
+        self.admin_q.get().process_completions() != 0
+    }
+}
+
 impl Device for NvmeController {
     unsafe fn init(&'static self) -> Result<(), Error> {
         let regs = self.regs.lock();
 
         let min_page_size = 1usize << (12 + regs.CAP.read(CAP::MPSMIN));
-        let max_page_size = 1usize << (12 + regs.CAP.read(CAP::MPSMAX));
 
         if min_page_size > 4096 {
             panic!();
@@ -205,8 +211,13 @@ impl Device for NvmeController {
         // Setup the admin queue (index 0)
         let admin_sq_doorbell = unsafe { regs.doorbell_ptr(self.doorbell_shift, false, 0) };
         let admin_cq_doorbell = unsafe { regs.doorbell_ptr(self.doorbell_shift, true, 0) };
-        let admin_q =
+        let admin_q = QueuePair::new(
-            QueuePair::new(queue_slots as usize, admin_sq_doorbell, admin_cq_doorbell).unwrap();
+            0,
+            queue_slots as usize,
+            admin_sq_doorbell,
+            admin_cq_doorbell,
+        )
+        .unwrap();
 
         regs.AQA
             .modify(AQA::ASQS.val(queue_slots as u32 - 1) + AQA::ACQS.val(queue_slots as u32 - 1));
@@ -239,6 +250,16 @@ impl Device for NvmeController {
 
         self.admin_q.init(admin_q);
 
+        // Register the IRQs (TODO: use multiple)
+        {
+            let mut vt = self.vector_table.lock();
+
+            // Register vector 0
+            vt[0]
+                .register(ARCHITECTURE.message_interrupt_controller(), self)
+                .unwrap();
+        }
+
         // Schedule late_init task
         runtime::spawn(self.late_init())?;
 
@@ -256,6 +277,15 @@ impl FromPciBus for NvmeController {
             panic!();
         };
 
+        // TODO also support MSI
+        let mut msix = info.config_space.capability::<MsiXCapability>().unwrap();
+        let mut vt = msix.vector_table()?;
+
+        for vector in vt.iter_mut() {
+            vector.set_masked(true);
+        }
+        msix.set_enabled(true);
+
         let mut cmd = PciCommandRegister::from_bits_retain(info.config_space.command());
         cmd &= !(PciCommandRegister::DISABLE_INTERRUPTS | PciCommandRegister::ENABLE_IO);
         cmd |= PciCommandRegister::ENABLE_MEMORY | PciCommandRegister::BUS_MASTER;
@@ -272,6 +302,7 @@ impl FromPciBus for NvmeController {
             regs: IrqSafeSpinlock::new(regs),
             admin_q: OneTimeInit::new(),
             ioqs: OneTimeInit::new(),
+            vector_table: IrqSafeSpinlock::new(vt),
             doorbell_shift,
         })
     }

src/device/nvme/queue.rs

@@ -1,6 +1,5 @@
 use core::{
     mem::size_of,
-    ops::DerefMut,
     pin::Pin,
     ptr::null_mut,
     task::{Context, Poll},
@@ -94,6 +93,8 @@ pub struct QueuePair<'a> {
     base: PhysicalAddress,
     page_count: usize,
 
+    vector: usize,
+
     sq_base: PhysicalAddress,
     cq_base: PhysicalAddress,
 
@@ -246,6 +247,7 @@ impl<'a, T> Queue<'a, T> {
 
 impl<'a> QueuePair<'a> {
     pub fn new(
+        vector: usize,
         capacity: usize,
         sq_doorbell: *mut u32,
         cq_doorbell: *mut u32,
@@ -279,6 +281,7 @@ impl<'a> QueuePair<'a> {
         Ok(Self {
             completion_notify: QueueWaker::new(),
 
+            vector,
             base,
             page_count,
             sq_base,

src/mem/device.rs

@@ -1,12 +1,19 @@
 //! Facilities for mapping devices to virtual address space
-use core::{mem::size_of, ops::Deref};
+use core::{
+    alloc::Layout,
+    mem::size_of,
+    ops::{Deref, DerefMut},
+};
 
 use abi::error::Error;
 use alloc::sync::Arc;
 
 use crate::arch::{Architecture, ARCHITECTURE};
 
-use super::PhysicalAddress;
+use super::{
+    address::{AsPhysicalAddress, FromRaw},
+    PhysicalAddress,
+};
 
 /// Describes a single device memory mapping
 #[derive(Debug)]
@@ -37,6 +44,14 @@ pub struct DeviceMemoryIo<'a, T: ?Sized> {
     value: &'a T,
 }
 
+/// Describes a single typed and mutable device memory mapping
+#[derive(Debug)]
+pub struct DeviceMemoryIoMut<'a, T: ?Sized> {
+    #[allow(unused)]
+    inner: RawDeviceMemoryMapping,
+    value: &'a mut T,
+}
+
 impl RawDeviceMemoryMapping {
     /// Maps a region of physical memory as device memory of given size.
     ///
@@ -130,3 +145,42 @@ impl<'a, T: ?Sized> Deref for DeviceMemoryIo<'a, T> {
         self.value
     }
 }
+
+impl<T: ?Sized> AsPhysicalAddress for DeviceMemoryIo<'_, T> {
+    unsafe fn as_physical_address(&self) -> PhysicalAddress {
+        PhysicalAddress::from_raw(self.inner.base_address)
+    }
+}
+
+impl<'a, T: Sized> DeviceMemoryIoMut<'a, T> {
+    /// Maps a physical address as device memory to a slice `[T; len]`
+    ///
+    /// # Safety
+    ///
+    /// The caller must ensure the address actually points to a value of type `T`, as well as
+    /// proper access synchronization. The caller must also ensure the `len` is valid.
+    pub unsafe fn map_slice(
+        base: PhysicalAddress,
+        len: usize,
+    ) -> Result<DeviceMemoryIoMut<'a, [T]>, Error> {
+        let layout = Layout::array::<T>(len).unwrap();
+        let inner = RawDeviceMemoryMapping::map(base, layout.size())?;
+        let value = core::slice::from_raw_parts_mut(inner.address as *mut T, len);
+
+        Ok(DeviceMemoryIoMut { inner, value })
+    }
+}
+
+impl<'a, T: ?Sized> Deref for DeviceMemoryIoMut<'a, T> {
+    type Target = T;
+
+    fn deref(&self) -> &Self::Target {
+        self.value
+    }
+}
+
+impl<'a, T: ?Sized> DerefMut for DeviceMemoryIoMut<'a, T> {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        self.value
+    }
+}