From 506476e9c352d21c68f715f3fef6001f6e0ae0b3 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Sat, 9 Dec 2023 15:27:09 +0200 Subject: [PATCH] dev/pci: MSI-X capability and APIC support for it --- lib/device-api/src/interrupt.rs | 17 +++++ src/arch/mod.rs | 18 +++++- src/arch/x86_64/apic/ioapic.rs | 4 +- src/arch/x86_64/apic/local.rs | 74 ++++++++++++++++++---- src/arch/x86_64/apic/mod.rs | 40 +++++++++++- src/arch/x86_64/apic/vectors.S | 59 ++++++++++++++++- src/arch/x86_64/mod.rs | 10 ++- src/device/bus/pci/capability.rs | 105 +++++++++++++++++++++++++++++++ src/device/bus/pci/mod.rs | 36 +++++++++++ src/device/bus/pci/space/mod.rs | 72 ++++++++++++++++++++- src/device/nvme/command.rs | 7 ++- src/device/nvme/mod.rs | 65 ++++++++++++++----- src/device/nvme/queue.rs | 5 +- src/mem/device.rs | 58 ++++++++++++++++- 14 files changed, 525 insertions(+), 45 deletions(-) create mode 100644 src/device/bus/pci/capability.rs diff --git a/lib/device-api/src/interrupt.rs b/lib/device-api/src/interrupt.rs index e8d9cf2a..f64e026e 100644 --- a/lib/device-api/src/interrupt.rs +++ b/lib/device-api/src/interrupt.rs @@ -33,10 +33,23 @@ pub struct IrqOptions { pub trigger: IrqTrigger, } +#[derive(Clone, Copy, Debug)] +pub struct MsiInfo { + pub address: usize, + pub value: u32, + pub vector: usize, +} + pub trait InterruptTable { fn handler(&self, index: usize) -> Option<&'static dyn InterruptHandler>; } +pub trait MessageInterruptController { + fn register_msi(&self, handler: &'static dyn MsiHandler) -> Result; + + fn handle_msi(&self, #[allow(unused)] vector: usize) {} +} + pub trait ExternalInterruptController { type IrqNumber; @@ -80,6 +93,10 @@ pub trait InterruptHandler: Device { fn handle_irq(&self) -> bool; } +pub trait MsiHandler: Device { + fn handle_msi(&self, vector: usize) -> bool; +} + pub struct FixedInterruptTable { entries: [Option<&'static dyn InterruptHandler>; SIZE], } diff --git a/src/arch/mod.rs b/src/arch/mod.rs index fe5241e6..c021f3be 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -21,7 +21,10 @@ macro_rules! absolute_address { use cfg_if::cfg_if; use device_api::{ - interrupt::{ExternalInterruptController, IpiDeliveryTarget, LocalInterruptController}, + interrupt::{ + ExternalInterruptController, IpiDeliveryTarget, LocalInterruptController, + MessageInterruptController, + }, timer::MonotonicTimestampProviderDevice, ResetDevice, }; @@ -157,6 +160,14 @@ pub trait Architecture { Err(Error::NotImplemented) } + /// Adds a message-signalled interrupt (MSI/MSI-X) controller to the system + fn register_message_interrupt_controller( + &self, + intc: &'static dyn MessageInterruptController, + ) -> Result<(), Error> { + Err(Error::NotImplemented) + } + /// Adds a monotonic timer to the system fn register_monotonic_timer( &self, @@ -185,6 +196,11 @@ pub trait Architecture { unimplemented!() } + /// Returns the MSI/MSI-X-capable interrupt controller + fn message_interrupt_controller(&'static self) -> &'static dyn MessageInterruptController { + unimplemented!() + } + /// Returns the monotonic timer fn monotonic_timer(&'static self) -> &'static dyn MonotonicTimestampProviderDevice { unimplemented!() diff --git a/src/arch/x86_64/apic/ioapic.rs b/src/arch/x86_64/apic/ioapic.rs index 9e5c39af..a6be3d91 100644 --- a/src/arch/x86_64/apic/ioapic.rs +++ b/src/arch/x86_64/apic/ioapic.rs @@ -20,7 +20,7 @@ use crate::{ mem::{address::FromRaw, device::DeviceMemoryIo, PhysicalAddress}, }; -use super::{APIC_EXTERNAL_OFFSET, MAX_EXTERNAL_VECTORS}; +use super::{APIC_EXTERNAL_OFFSET, POPULATED_EXTERNAL_VECTORS}; // IRQ 0 is timer, IRQ 1 reserved (for now?), +32 offset for exception entries const IO_APIC_VECTOR_OFFSET: u32 = 32 + APIC_EXTERNAL_OFFSET; @@ -64,7 +64,7 @@ pub struct IoApic { inner: IrqSafeSpinlock, isa_redirections: [Option; 16], - table: IrqSafeSpinlock>, + table: IrqSafeSpinlock>, } impl Regs { diff --git a/src/arch/x86_64/apic/local.rs b/src/arch/x86_64/apic/local.rs index 3cb014e0..6158a4c6 100644 --- a/src/arch/x86_64/apic/local.rs +++ b/src/arch/x86_64/apic/local.rs @@ -1,11 +1,16 @@ //! x86-64 Local APIC driver implementation use core::sync::atomic::Ordering; +use abi::error::Error; +use alloc::vec::Vec; use device_api::{ - interrupt::{IpiDeliveryTarget, LocalInterruptController}, + interrupt::{ + IpiDeliveryTarget, LocalInterruptController, MessageInterruptController, MsiHandler, + MsiInfo, + }, Device, }; -use kernel_util::util::OneTimeInit; +use kernel_util::{sync::IrqSafeSpinlock, util::OneTimeInit}; use tock_registers::{ interfaces::{ReadWriteable, Readable, Writeable}, register_bitfields, register_structs, @@ -14,10 +19,16 @@ use tock_registers::{ use crate::{ arch::{ - x86_64::{mem::table::L3, registers::MSR_IA32_APIC_BASE, smp::CPU_COUNT}, + x86_64::{ + apic::APIC_MSI_OFFSET, mem::table::L3, registers::MSR_IA32_APIC_BASE, smp::CPU_COUNT, + }, CpuMessage, }, - mem::{address::FromRaw, device::DeviceMemoryIo, PhysicalAddress}, + mem::{ + address::{FromRaw, IntoRaw}, + device::DeviceMemoryIo, + PhysicalAddress, + }, task::Cpu, }; @@ -131,6 +142,7 @@ register_structs! { /// Per-processor local APIC interface pub struct LocalApic { regs: DeviceMemoryIo<'static, Regs>, + msi_vectors: IrqSafeSpinlock>, } unsafe impl Send for LocalApic {} @@ -142,14 +154,51 @@ impl Device for LocalApic { } } +impl MessageInterruptController for LocalApic { + fn handle_msi(&self, vector: usize) { + // TODO this is ugly + let mut i = 0; + + loop { + let table = self.msi_vectors.lock(); + let Some(&handler) = table.get(i) else { + break; + }; + drop(table); + + if handler.handle_msi(vector) { + break; + } + + i += 1; + } + } + + fn register_msi(&self, handler: &'static dyn MsiHandler) -> Result { + // TODO only 1 ISR vector allocated for MSIs + let vector = 0; + let mut table = self.msi_vectors.lock(); + + table.push(handler); + + // TODO magic numbers + let apic_vector = 32 + APIC_MSI_OFFSET + vector; + + let value = apic_vector; + let address = Self::base(); + + Ok(MsiInfo { + address: address.into_raw(), + value, + vector: vector as _, + }) + } +} + impl LocalInterruptController for LocalApic { type IpiMessage = CpuMessage; - fn send_ipi( - &self, - target: device_api::interrupt::IpiDeliveryTarget, - msg: Self::IpiMessage, - ) -> Result<(), abi::error::Error> { + fn send_ipi(&self, target: IpiDeliveryTarget, msg: Self::IpiMessage) -> Result<(), Error> { while self.regs.ICR0.matches_all(ICR0::DeliveryStatus::SET) { core::hint::spin_loop(); } @@ -178,7 +227,7 @@ impl LocalInterruptController for LocalApic { } } - unsafe fn init_ap(&self) -> Result<(), abi::error::Error> { + unsafe fn init_ap(&self) -> Result<(), Error> { todo!() } } @@ -228,7 +277,10 @@ impl LocalApic { LocalVectorEntry::Mask::Masked + LocalVectorEntry::Vector.val(APIC_LINT1_VECTOR + 32), ); - Self { regs } + Self { + regs, + msi_vectors: IrqSafeSpinlock::new(Vec::new()), + } } /// Signals local APIC that we've handled the IRQ diff --git a/src/arch/x86_64/apic/mod.rs b/src/arch/x86_64/apic/mod.rs index 29a03f15..c1c4f8cb 100644 --- a/src/arch/x86_64/apic/mod.rs +++ b/src/arch/x86_64/apic/mod.rs @@ -2,6 +2,8 @@ use core::arch::global_asm; +use static_assertions::{const_assert, const_assert_eq}; + use crate::{ arch::{x86_64::cpu::Cpu, Architecture}, task::thread::Thread, @@ -29,8 +31,17 @@ pub const APIC_IPI_VECTOR: u32 = 0x03; pub const APIC_SPURIOUS_VECTOR: u32 = 0xDF; /// Start of the I/O APIC IRQ range pub const APIC_EXTERNAL_OFFSET: u32 = 4; +/// Start of the MSI range +pub const APIC_MSI_OFFSET: u32 = APIC_EXTERNAL_OFFSET + MAX_EXTERNAL_VECTORS; /// Maximum number of APIC vectors allocated for handling IRQs from I/O APIC -pub const MAX_EXTERNAL_VECTORS: u32 = 16; +pub const MAX_EXTERNAL_VECTORS: u32 = APIC_SPURIOUS_VECTOR - APIC_EXTERNAL_OFFSET - MAX_MSI_VECTORS; +/// Number of I/O APIC IRQ vectors that are actually populated +pub const POPULATED_EXTERNAL_VECTORS: u32 = 16; +/// Maximum number of APIC vectors allocated for handling MSIs +pub const MAX_MSI_VECTORS: u32 = 16; + +const_assert!(POPULATED_EXTERNAL_VECTORS <= MAX_EXTERNAL_VECTORS); +const_assert_eq!(APIC_MSI_OFFSET + MAX_MSI_VECTORS, APIC_SPURIOUS_VECTOR); /// Fills the IDT with interrupt vectors for this APIC pub fn setup_vectors(idt: &mut [exception::Entry]) { @@ -49,6 +60,10 @@ pub fn setup_vectors(idt: &mut [exception::Entry]) { } unsafe extern "C" fn irq_handler(vector: usize, frame: *mut IrqFrame) { + if vector >= POPULATED_EXTERNAL_VECTORS as _ { + todo!("Got a weird IRQ with vector {}", vector); + } + let cpu = Cpu::local(); let frame = &mut *frame; @@ -62,6 +77,24 @@ unsafe extern "C" fn irq_handler(vector: usize, frame: *mut IrqFrame) { } } +unsafe extern "C" fn msi_handler(vector: usize, frame: *mut IrqFrame) { + if vector != 0 { + todo!("Got a weird MSI with vector {}", vector); + } + + let cpu = Cpu::local(); + let frame = &mut *frame; + + ARCHITECTURE + .message_interrupt_controller() + .handle_msi(vector); + cpu.local_apic().clear_interrupt(); + + if let Some(thread) = Thread::get_current() { + thread.handle_pending_signals(frame); + } +} + unsafe extern "C" fn local_timer_irq_handler(frame: *mut IrqFrame) { let frame = &mut *frame; let cpu = Cpu::local(); @@ -87,7 +120,12 @@ global_asm!( include_str!("vectors.S"), local_timer_irq_handler = sym local_timer_irq_handler, irq_handler = sym irq_handler, + msi_handler = sym msi_handler, ipi_handler = sym ipi_handler, dummy_irq_handler = sym dummy_irq_handler, + irq_vector_offset = const APIC_EXTERNAL_OFFSET, + irq_vector_count = const MAX_EXTERNAL_VECTORS, + msi_vector_offset = const APIC_MSI_OFFSET, + msi_vector_count = const MAX_MSI_VECTORS, options(att_syntax) ); diff --git a/src/arch/x86_64/apic/vectors.S b/src/arch/x86_64/apic/vectors.S index 6b90c713..2ff7a27b 100644 --- a/src/arch/x86_64/apic/vectors.S +++ b/src/arch/x86_64/apic/vectors.S @@ -123,6 +123,56 @@ irq_vector_\n: .endr .endm +.macro MSI_VECTOR, n +msi_vector_\n: + // %rsp + 0: %rip + // %rsp + 8: %cs + IRQ_SWAPGS_IF_NEEDED 8 + IRQ_SAVE_STATE + + // Force correct segment registers + mov $0x10, %ax + mov %ax, %ss + mov %ax, %ds + mov %ax, %es + + mov $\n, %rdi + mov %rbp, %rsi + call {msi_handler} + + IRQ_RESTORE_STATE + IRQ_SWAPGS_IF_NEEDED 8 + + iretq +.endm + +.macro MSI_VECTOR_ENTRY, n +.quad msi_vector_\n +.endm + +.macro MSI_VECTORS, start, end +.set i, 0 +.rept \end - \start + MSI_VECTOR %i + .set i, i+1 +.endr +.endm + +.macro MSI_VECTOR_ENTRIES, start, end +.set i, 0 +.rept \end - \start + MSI_VECTOR_ENTRY %i + .set i, i+1 +.endr +.endm + +.macro FILL_EMPTY_SPACE, start, end +.set i, 0 +.rept \end - \start +.quad dummy_vector +.endr +.endm + .section .text local_timer_vector: IRQ_SWAPGS_IF_NEEDED 8 @@ -149,7 +199,8 @@ dummy_vector: call {dummy_irq_handler} jmp . -IRQ_VECTORS 4, 255 +IRQ_VECTORS {irq_vector_offset}, {irq_vector_offset} + {irq_vector_count} +MSI_VECTORS {msi_vector_offset}, {msi_vector_offset} + {msi_vector_count} .section .rodata // 224 vectors: 256 - 32 (exceptions) @@ -163,8 +214,10 @@ __x86_64_apic_vectors: .quad dummy_vector // IPI vector: 3 .quad ipi_vector - // Regular IRQ vectors: 4..=222 - IRQ_VECTOR_ENTRIES 4, 223 + // Regular IRQ vectors: 4..207 + IRQ_VECTOR_ENTRIES {irq_vector_offset}, {irq_vector_offset} + {irq_vector_count} + // MSI vectors: 207..223 + MSI_VECTOR_ENTRIES {msi_vector_offset}, {msi_vector_offset} + {msi_vector_count} // Spurious interrupt vector: 223 .quad dummy_vector .size __x86_64_apic_vectors, . - __x86_64_apic_vectors diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index 14b58c2e..0ca37092 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -5,8 +5,10 @@ use abi::error::Error; use acpi_lib::{mcfg::Mcfg, AcpiTables, InterruptModel}; use alloc::boxed::Box; use device_api::{ - input::KeyboardProducer, interrupt::ExternalInterruptController, - timer::MonotonicTimestampProviderDevice, Device, + input::KeyboardProducer, + interrupt::{ExternalInterruptController, MessageInterruptController}, + timer::MonotonicTimestampProviderDevice, + Device, }; use git_version::git_version; use kernel_util::{sync::SpinFence, util::OneTimeInit}; @@ -281,6 +283,10 @@ impl Architecture for X86_64 { self.ioapic.get() } + fn message_interrupt_controller(&'static self) -> &'static dyn MessageInterruptController { + Cpu::local().local_apic() + } + fn monotonic_timer(&'static self) -> &'static dyn MonotonicTimestampProviderDevice { self.timer.get() } diff --git a/src/device/bus/pci/capability.rs b/src/device/bus/pci/capability.rs new file mode 100644 index 00000000..73132c90 --- /dev/null +++ b/src/device/bus/pci/capability.rs @@ -0,0 +1,105 @@ +//! PCI capability structures and queries + +use abi::error::Error; +use device_api::interrupt::{MessageInterruptController, MsiHandler}; + +use crate::{ + device::bus::pci::PciBaseAddress, + mem::{address::FromRaw, device::DeviceMemoryIoMut, PhysicalAddress}, +}; + +use super::{PciCapability, PciCapabilityId, PciConfigurationSpace}; + +/// MSI-X capability query +pub struct MsiXCapability; + +/// Represents an entry in MSI-X vector table +#[repr(C)] +pub struct MsiXEntry { + /// Address to which the value is written on interrupt + pub address: u64, + /// Value which is written to trigger an interrupt + pub data: u32, + /// Vector control word + pub control: u32, +} + +/// MSI-X capability data structure +pub struct MsiXData<'s, S: PciConfigurationSpace + ?Sized + 's> { + space: &'s S, + offset: usize, +} + +impl PciCapability for MsiXCapability { + const ID: PciCapabilityId = PciCapabilityId::MsiX; + type CapabilityData<'a, S: PciConfigurationSpace + ?Sized + 'a> = MsiXData<'a, S>; + + fn data<'s, S: PciConfigurationSpace + ?Sized + 's>( + space: &'s S, + offset: usize, + ) -> Self::CapabilityData<'s, S> { + MsiXData { space, offset } + } +} + +impl<'s, S: PciConfigurationSpace + ?Sized + 's> MsiXData<'s, S> { + // TODO use pending bits as well + /// Maps and returns the vector table associated with the device's MSI-X capability + pub fn vector_table<'a>(&self) -> Result, Error> { + let w0 = self.space.read_u16(self.offset + 2); + let dw1 = self.space.read_u32(self.offset + 4); + + let table_size = (w0 as usize & 0x3FF) + 1; + let bir = dw1 as usize & 0x3; + let table_offset = dw1 as usize & !0x3; + + let Some(base) = self.space.bar(bir) else { + return Err(Error::DoesNotExist); + }; + let PciBaseAddress::Memory(base) = base else { + return Err(Error::InvalidOperation); + }; + + debugln!("MSI-X table address: {:#x}", base + table_offset); + + unsafe { + DeviceMemoryIoMut::map_slice(PhysicalAddress::from_raw(base + table_offset), table_size) + } + } + + /// Changes the global enable status for the device's MSI-X capability. If set, regular IRQs + /// are not generated. + pub fn set_enabled(&mut self, enabled: bool) { + let mut w0 = self.space.read_u32(self.offset); + if enabled { + w0 |= 1 << 31; + } else { + w0 &= !(1 << 31); + } + self.space.write_u32(self.offset, w0); + } +} + +impl MsiXEntry { + /// If set, prevents the MSI-X interrupt from being delivered + pub fn set_masked(&mut self, masked: bool) { + if masked { + self.control |= 1; + } else { + self.control &= !1; + } + } + + /// Registers the MSI-X vector with the interrupt controller and enables it + pub fn register( + &mut self, + ic: &C, + handler: &'static dyn MsiHandler, + ) -> Result<(), Error> { + let info = ic.register_msi(handler)?; + self.address = info.address as _; + self.data = info.value as _; + self.set_masked(false); + Ok(()) + } +} diff --git a/src/device/bus/pci/mod.rs b/src/device/bus/pci/mod.rs index abb3f8c1..d699e412 100644 --- a/src/device/bus/pci/mod.rs +++ b/src/device/bus/pci/mod.rs @@ -8,6 +8,7 @@ use device_api::Device; use kernel_util::sync::IrqSafeSpinlock; use yggdrasil_abi::error::Error; +pub mod capability; mod space; pub use space::{ @@ -33,6 +34,14 @@ bitflags! { } } +bitflags! { + /// Status register of the PCI configuration space + pub struct PciStatusRegister: u16 { + /// Read-only. If set, the configuration space has a pointer to the capabilities list. + const CAPABILITIES_LIST = 1 << 4; + } +} + /// Represents the address of a single object on a bus (or the bus itself) #[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] pub struct PciAddress { @@ -55,6 +64,33 @@ pub enum PciBaseAddress { Io(u16), } +/// Unique ID assigned to PCI capability structures +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +#[non_exhaustive] +#[repr(u8)] +pub enum PciCapabilityId { + /// MSI (32-bit or 64-bit) + Msi = 0x05, + /// MSI-X + MsiX = 0x11, + /// Unknown capability missing from this list + Unknown, +} + +/// Interface used for querying PCI capabilities +pub trait PciCapability { + /// Capability ID + const ID: PciCapabilityId; + /// Wrapper for accessing the capability data structure + type CapabilityData<'a, S: PciConfigurationSpace + ?Sized + 'a>; + + /// Constructs an access wrapper for this capability with given offset + fn data<'s, S: PciConfigurationSpace + ?Sized + 's>( + space: &'s S, + offset: usize, + ) -> Self::CapabilityData<'s, S>; +} + /// Describes a PCI device #[derive(Debug)] pub struct PciDeviceInfo { diff --git a/src/device/bus/pci/space/mod.rs b/src/device/bus/pci/space/mod.rs index 72a5018d..7a2e0977 100644 --- a/src/device/bus/pci/space/mod.rs +++ b/src/device/bus/pci/space/mod.rs @@ -1,4 +1,6 @@ -use super::{PciAddress, PciBaseAddress, PciEcam}; +use crate::device::bus::pci::PciStatusRegister; + +use super::{PciAddress, PciBaseAddress, PciCapability, PciCapabilityId, PciEcam}; pub(super) mod ecam; @@ -72,6 +74,30 @@ pub enum PciConfigSpace { Ecam(PciEcam), } +pub struct CapabilityIterator<'s, S: PciConfigurationSpace + ?Sized> { + space: &'s S, + current: Option, +} + +impl<'s, S: PciConfigurationSpace + ?Sized> Iterator for CapabilityIterator<'s, S> { + type Item = (PciCapabilityId, usize); + + fn next(&mut self) -> Option { + let offset = self.current? & !0x3; + + let id = unsafe { core::mem::transmute(self.space.read_u8(offset)) }; + let next_pointer = self.space.read_u8(offset + 1); + + self.current = if next_pointer != 0 { + Some(next_pointer as usize) + } else { + None + }; + + Some((id, offset)) + } +} + /// Interface for accessing the configuration space of a device pub trait PciConfigurationSpace { /// Reads a 32-bit value from the device configuration space. @@ -120,7 +146,7 @@ pub trait PciConfigurationSpace { } /// Writes a byte to the device configuration space - fn write_u8(&self, offset: usize, value: u16) { + fn write_u8(&self, _offset: usize, _value: u16) { todo!() } @@ -185,6 +211,14 @@ pub trait PciConfigurationSpace { "#] secondary_bus ); + pci_config_field!( + 0x34 => u8, + #[doc = + r"Returns the offset within the configuration space where the Capabilities List + is located. Only valid if the corresponding Status Register bit is set" + ] + capability_pointer + ); /// Returns the value of the Base Address Register with given index. /// @@ -242,4 +276,38 @@ pub trait PciConfigurationSpace { } } } + + /// Returns an iterator over the PCI capabilities + fn capability_iter(&self) -> CapabilityIterator { + let status = PciStatusRegister::from_bits_retain(self.status()); + + let current = if status.contains(PciStatusRegister::CAPABILITIES_LIST) { + let ptr = self.capability_pointer() as usize; + + if ptr != 0 { + Some(self.capability_pointer() as usize) + } else { + None + } + } else { + // Return an empty iterator + None + }; + + CapabilityIterator { + space: self, + current, + } + } + + /// Locates a capability within this configuration space + fn capability<'s, C: PciCapability>(&'s self) -> Option> { + self.capability_iter().find_map(|(id, offset)| { + if id == C::ID { + Some(C::data(self, offset)) + } else { + None + } + }) + } } diff --git a/src/device/nvme/command.rs b/src/device/nvme/command.rs index 9f6231f4..10fb324a 100644 --- a/src/device/nvme/command.rs +++ b/src/device/nvme/command.rs @@ -1,3 +1,5 @@ +#![allow(unused)] + use core::fmt::{self, Write}; use crate::{ @@ -47,6 +49,7 @@ pub struct IdentifyControllerRequest { pub struct CreateIoCompletionQueue { pub id: u32, pub size: usize, + pub vector: u32, pub data: PhysicalAddress, } @@ -110,9 +113,7 @@ impl Command for CreateIoCompletionQueue { sqe.command.set_opcode(0x05); sqe.data_pointer[1] = PhysicalRegionPage::with_addr(self.data.into_raw()); sqe.command_specific[0] = ((self.size as u32 - 1) << 16) | self.id; - sqe.command_specific[1] = 1; - // TODO ENABLE IRQS HERE - sqe.command_specific[2] = 0; + sqe.command_specific[1] = (self.vector << 16) | 3; } } diff --git a/src/device/nvme/mod.rs b/src/device/nvme/mod.rs index f11928da..cbaae53d 100644 --- a/src/device/nvme/mod.rs +++ b/src/device/nvme/mod.rs @@ -3,7 +3,7 @@ use core::{mem::size_of, time::Duration}; use abi::error::Error; use alloc::vec::Vec; -use device_api::Device; +use device_api::{interrupt::MsiHandler, Device}; use kernel_util::{sync::IrqSafeSpinlock, util::OneTimeInit}; use tock_registers::{ interfaces::{ReadWriteable, Readable, Writeable}, @@ -12,8 +12,11 @@ use tock_registers::{ }; use crate::{ + arch::{Architecture, ARCHITECTURE}, device::{ - bus::pci::{PciBaseAddress, PciCommandRegister, PciConfigurationSpace}, + bus::pci::{ + capability::MsiXCapability, PciBaseAddress, PciCommandRegister, PciConfigurationSpace, + }, nvme::{ command::IdentifyControllerRequest, queue::{CompletionQueueEntry, SubmissionQueueEntry}, @@ -21,7 +24,7 @@ use crate::{ }, mem::{ address::{FromRaw, IntoRaw}, - device::DeviceMemoryIo, + device::{DeviceMemoryIo, DeviceMemoryIoMut}, PhysicalAddress, }, task::runtime, @@ -33,7 +36,7 @@ use self::{ queue::QueuePair, }; -use super::bus::pci::{FromPciBus, PciDeviceInfo}; +use super::bus::pci::{capability::MsiXEntry, FromPciBus, PciDeviceInfo}; mod command; mod error; @@ -109,6 +112,7 @@ pub struct NvmeController { regs: IrqSafeSpinlock>, admin_q: OneTimeInit>, ioqs: OneTimeInit>>, + vector_table: IrqSafeSpinlock>, doorbell_shift: usize, } @@ -123,9 +127,11 @@ impl Regs { impl NvmeController { async fn late_init(&'static self) -> Result<(), NvmeError> { - runtime::spawn(self.poll_task()).expect("Couldn't spawn NVMe poll task"); + // runtime::spawn(self.poll_task()).expect("Couldn't spawn NVMe poll task"); let admin_q = self.admin_q.get(); + infoln!("SETUP"); + // Request a CQ/SQ pair for I/O admin_q .request_no_data(SetFeatureRequest::NumberOfQueues(1, 1)) @@ -133,7 +139,8 @@ impl NvmeController { // Allocate the queue let (sq_doorbell, cq_doorbell) = unsafe { self.doorbell_pair(1) }; - let io_q = QueuePair::new(32, sq_doorbell, cq_doorbell).map_err(NvmeError::MemoryError)?; + let io_q = + QueuePair::new(0, 32, sq_doorbell, cq_doorbell).map_err(NvmeError::MemoryError)?; // Identify the controller let identify = admin_q @@ -145,6 +152,7 @@ impl NvmeController { .request_no_data(CreateIoCompletionQueue { id: 1, size: 32, + vector: 0, data: io_q.cq_physical_pointer(), }) .await?; @@ -160,14 +168,6 @@ impl NvmeController { loop {} } - // TODO MSI(-X) or IRQ (ACPI currently broken) support for PCIe-based NVMe - async fn poll_task(&'static self) { - loop { - self.admin_q.get().process_completions(); - runtime::sleep(Duration::from_millis(100)).await; - } - } - unsafe fn doorbell_pair(&self, idx: usize) -> (*mut u32, *mut u32) { let regs = self.regs.lock(); let sq_ptr = regs.doorbell_ptr(self.doorbell_shift, false, idx); @@ -176,12 +176,18 @@ impl NvmeController { } } +impl MsiHandler for NvmeController { + fn handle_msi(&self, vector: usize) -> bool { + debugln!("handle_msi {}", vector); + self.admin_q.get().process_completions() != 0 + } +} + impl Device for NvmeController { unsafe fn init(&'static self) -> Result<(), Error> { let regs = self.regs.lock(); let min_page_size = 1usize << (12 + regs.CAP.read(CAP::MPSMIN)); - let max_page_size = 1usize << (12 + regs.CAP.read(CAP::MPSMAX)); if min_page_size > 4096 { panic!(); @@ -205,8 +211,13 @@ impl Device for NvmeController { // Setup the admin queue (index 0) let admin_sq_doorbell = unsafe { regs.doorbell_ptr(self.doorbell_shift, false, 0) }; let admin_cq_doorbell = unsafe { regs.doorbell_ptr(self.doorbell_shift, true, 0) }; - let admin_q = - QueuePair::new(queue_slots as usize, admin_sq_doorbell, admin_cq_doorbell).unwrap(); + let admin_q = QueuePair::new( + 0, + queue_slots as usize, + admin_sq_doorbell, + admin_cq_doorbell, + ) + .unwrap(); regs.AQA .modify(AQA::ASQS.val(queue_slots as u32 - 1) + AQA::ACQS.val(queue_slots as u32 - 1)); @@ -239,6 +250,16 @@ impl Device for NvmeController { self.admin_q.init(admin_q); + // Register the IRQs (TODO: use multiple) + { + let mut vt = self.vector_table.lock(); + + // Register vector 0 + vt[0] + .register(ARCHITECTURE.message_interrupt_controller(), self) + .unwrap(); + } + // Schedule late_init task runtime::spawn(self.late_init())?; @@ -256,6 +277,15 @@ impl FromPciBus for NvmeController { panic!(); }; + // TODO also support MSI + let mut msix = info.config_space.capability::().unwrap(); + let mut vt = msix.vector_table()?; + + for vector in vt.iter_mut() { + vector.set_masked(true); + } + msix.set_enabled(true); + let mut cmd = PciCommandRegister::from_bits_retain(info.config_space.command()); cmd &= !(PciCommandRegister::DISABLE_INTERRUPTS | PciCommandRegister::ENABLE_IO); cmd |= PciCommandRegister::ENABLE_MEMORY | PciCommandRegister::BUS_MASTER; @@ -272,6 +302,7 @@ impl FromPciBus for NvmeController { regs: IrqSafeSpinlock::new(regs), admin_q: OneTimeInit::new(), ioqs: OneTimeInit::new(), + vector_table: IrqSafeSpinlock::new(vt), doorbell_shift, }) } diff --git a/src/device/nvme/queue.rs b/src/device/nvme/queue.rs index afab4c02..31a8ee14 100644 --- a/src/device/nvme/queue.rs +++ b/src/device/nvme/queue.rs @@ -1,6 +1,5 @@ use core::{ mem::size_of, - ops::DerefMut, pin::Pin, ptr::null_mut, task::{Context, Poll}, @@ -94,6 +93,8 @@ pub struct QueuePair<'a> { base: PhysicalAddress, page_count: usize, + vector: usize, + sq_base: PhysicalAddress, cq_base: PhysicalAddress, @@ -246,6 +247,7 @@ impl<'a, T> Queue<'a, T> { impl<'a> QueuePair<'a> { pub fn new( + vector: usize, capacity: usize, sq_doorbell: *mut u32, cq_doorbell: *mut u32, @@ -279,6 +281,7 @@ impl<'a> QueuePair<'a> { Ok(Self { completion_notify: QueueWaker::new(), + vector, base, page_count, sq_base, diff --git a/src/mem/device.rs b/src/mem/device.rs index 7a190134..45f5776b 100644 --- a/src/mem/device.rs +++ b/src/mem/device.rs @@ -1,12 +1,19 @@ //! Facilities for mapping devices to virtual address space -use core::{mem::size_of, ops::Deref}; +use core::{ + alloc::Layout, + mem::size_of, + ops::{Deref, DerefMut}, +}; use abi::error::Error; use alloc::sync::Arc; use crate::arch::{Architecture, ARCHITECTURE}; -use super::PhysicalAddress; +use super::{ + address::{AsPhysicalAddress, FromRaw}, + PhysicalAddress, +}; /// Describes a single device memory mapping #[derive(Debug)] @@ -37,6 +44,14 @@ pub struct DeviceMemoryIo<'a, T: ?Sized> { value: &'a T, } +/// Describes a single typed and mutable device memory mapping +#[derive(Debug)] +pub struct DeviceMemoryIoMut<'a, T: ?Sized> { + #[allow(unused)] + inner: RawDeviceMemoryMapping, + value: &'a mut T, +} + impl RawDeviceMemoryMapping { /// Maps a region of physical memory as device memory of given size. /// @@ -130,3 +145,42 @@ impl<'a, T: ?Sized> Deref for DeviceMemoryIo<'a, T> { self.value } } + +impl AsPhysicalAddress for DeviceMemoryIo<'_, T> { + unsafe fn as_physical_address(&self) -> PhysicalAddress { + PhysicalAddress::from_raw(self.inner.base_address) + } +} + +impl<'a, T: Sized> DeviceMemoryIoMut<'a, T> { + /// Maps a physical address as device memory to a slice `[T; len]` + /// + /// # Safety + /// + /// The caller must ensure the address actually points to a value of type `T`, as well as + /// proper access synchronization. The caller must also ensure the `len` is valid. + pub unsafe fn map_slice( + base: PhysicalAddress, + len: usize, + ) -> Result, Error> { + let layout = Layout::array::(len).unwrap(); + let inner = RawDeviceMemoryMapping::map(base, layout.size())?; + let value = core::slice::from_raw_parts_mut(inner.address as *mut T, len); + + Ok(DeviceMemoryIoMut { inner, value }) + } +} + +impl<'a, T: ?Sized> Deref for DeviceMemoryIoMut<'a, T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + self.value + } +} + +impl<'a, T: ?Sized> DerefMut for DeviceMemoryIoMut<'a, T> { + fn deref_mut(&mut self) -> &mut Self::Target { + self.value + } +}