alnyan/yggdrasil
1
0
Fork 0
Code Issues 33 Pull Requests Packages Projects Releases Wiki Activity

dev/pci: MSI-X capability and APIC support for it

Browse Source
This commit is contained in:
Mark Poliakov 2023-12-09 15:27:09 +02:00
parent f166968e57
commit 506476e9c3
14 changed files with 525 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
lib/device-api/src/interrupt.rs
View File

@ -33,10 +33,23 @@ pub struct IrqOptions {
pub trigger: IrqTrigger,
}
#[derive(Clone, Copy, Debug)]
pub struct MsiInfo {
pub address: usize,
pub value: u32,
pub vector: usize,
}
pub trait InterruptTable {
fn handler(&self, index: usize) -> Option<&'static dyn InterruptHandler>;
}
pub trait MessageInterruptController {
fn register_msi(&self, handler: &'static dyn MsiHandler) -> Result<MsiInfo, Error>;
fn handle_msi(&self, #[allow(unused)] vector: usize) {}
}
pub trait ExternalInterruptController {
type IrqNumber;
@ -80,6 +93,10 @@ pub trait InterruptHandler: Device {
fn handle_irq(&self) -> bool;
}
pub trait MsiHandler: Device {
fn handle_msi(&self, vector: usize) -> bool;
}
pub struct FixedInterruptTable<const SIZE: usize> {
entries: [Option<&'static dyn InterruptHandler>; SIZE],
}

18
src/arch/mod.rs
View File

@ -21,7 +21,10 @@ macro_rules! absolute_address {
use cfg_if::cfg_if;
use device_api::{
interrupt::{ExternalInterruptController, IpiDeliveryTarget, LocalInterruptController},
interrupt::{
ExternalInterruptController, IpiDeliveryTarget, LocalInterruptController,
MessageInterruptController,
},
timer::MonotonicTimestampProviderDevice,
ResetDevice,
};
@ -157,6 +160,14 @@ pub trait Architecture {
Err(Error::NotImplemented)
}
/// Adds a message-signalled interrupt (MSI/MSI-X) controller to the system
fn register_message_interrupt_controller(
&self,
intc: &'static dyn MessageInterruptController,
) -> Result<(), Error> {
Err(Error::NotImplemented)
}
/// Adds a monotonic timer to the system
fn register_monotonic_timer(
&self,
@ -185,6 +196,11 @@ pub trait Architecture {
unimplemented!()
}
/// Returns the MSI/MSI-X-capable interrupt controller
fn message_interrupt_controller(&'static self) -> &'static dyn MessageInterruptController {
unimplemented!()
}
/// Returns the monotonic timer
fn monotonic_timer(&'static self) -> &'static dyn MonotonicTimestampProviderDevice {
unimplemented!()

4
src/arch/x86_64/apic/ioapic.rs
View File

@ -20,7 +20,7 @@ use crate::{
mem::{address::FromRaw, device::DeviceMemoryIo, PhysicalAddress},
};
use super::{APIC_EXTERNAL_OFFSET, MAX_EXTERNAL_VECTORS};
use super::{APIC_EXTERNAL_OFFSET, POPULATED_EXTERNAL_VECTORS};
// IRQ 0 is timer, IRQ 1 reserved (for now?), +32 offset for exception entries
const IO_APIC_VECTOR_OFFSET: u32 = 32 + APIC_EXTERNAL_OFFSET;
@ -64,7 +64,7 @@ pub struct IoApic {
inner: IrqSafeSpinlock<Inner>,
isa_redirections: [Option<IsaRedirection>; 16],
table: IrqSafeSpinlock<FixedInterruptTable<{ MAX_EXTERNAL_VECTORS as usize }>>,
table: IrqSafeSpinlock<FixedInterruptTable<{ POPULATED_EXTERNAL_VECTORS as usize }>>,
}
impl Regs {

74
src/arch/x86_64/apic/local.rs
View File

@ -1,11 +1,16 @@
//! x86-64 Local APIC driver implementation
use core::sync::atomic::Ordering;
use abi::error::Error;
use alloc::vec::Vec;
use device_api::{
interrupt::{IpiDeliveryTarget, LocalInterruptController},
interrupt::{
IpiDeliveryTarget, LocalInterruptController, MessageInterruptController, MsiHandler,
MsiInfo,
},
Device,
};
use kernel_util::util::OneTimeInit;
use kernel_util::{sync::IrqSafeSpinlock, util::OneTimeInit};
use tock_registers::{
interfaces::{ReadWriteable, Readable, Writeable},
register_bitfields, register_structs,
@ -14,10 +19,16 @@ use tock_registers::{
use crate::{
arch::{
x86_64::{mem::table::L3, registers::MSR_IA32_APIC_BASE, smp::CPU_COUNT},
x86_64::{
apic::APIC_MSI_OFFSET, mem::table::L3, registers::MSR_IA32_APIC_BASE, smp::CPU_COUNT,
},
CpuMessage,
},
mem::{address::FromRaw, device::DeviceMemoryIo, PhysicalAddress},
mem::{
address::{FromRaw, IntoRaw},
device::DeviceMemoryIo,
PhysicalAddress,
},
task::Cpu,
};
@ -131,6 +142,7 @@ register_structs! {
/// Per-processor local APIC interface
pub struct LocalApic {
regs: DeviceMemoryIo<'static, Regs>,
msi_vectors: IrqSafeSpinlock<Vec<&'static dyn MsiHandler>>,
}
unsafe impl Send for LocalApic {}
@ -142,14 +154,51 @@ impl Device for LocalApic {
}
}
impl MessageInterruptController for LocalApic {
fn handle_msi(&self, vector: usize) {
// TODO this is ugly
let mut i = 0;
loop {
let table = self.msi_vectors.lock();
let Some(&handler) = table.get(i) else {
break;
};
drop(table);
if handler.handle_msi(vector) {
break;
}
i += 1;
}
}
fn register_msi(&self, handler: &'static dyn MsiHandler) -> Result<MsiInfo, Error> {
// TODO only 1 ISR vector allocated for MSIs
let vector = 0;
let mut table = self.msi_vectors.lock();
table.push(handler);
// TODO magic numbers
let apic_vector = 32 + APIC_MSI_OFFSET + vector;
let value = apic_vector;
let address = Self::base();
Ok(MsiInfo {
address: address.into_raw(),
value,
vector: vector as _,
})
}
}
impl LocalInterruptController for LocalApic {
type IpiMessage = CpuMessage;
fn send_ipi(
&self,
target: device_api::interrupt::IpiDeliveryTarget,
msg: Self::IpiMessage,
) -> Result<(), abi::error::Error> {
fn send_ipi(&self, target: IpiDeliveryTarget, msg: Self::IpiMessage) -> Result<(), Error> {
while self.regs.ICR0.matches_all(ICR0::DeliveryStatus::SET) {
core::hint::spin_loop();
}
@ -178,7 +227,7 @@ impl LocalInterruptController for LocalApic {
}
}
unsafe fn init_ap(&self) -> Result<(), abi::error::Error> {
unsafe fn init_ap(&self) -> Result<(), Error> {
todo!()
}
}
@ -228,7 +277,10 @@ impl LocalApic {
LocalVectorEntry::Mask::Masked + LocalVectorEntry::Vector.val(APIC_LINT1_VECTOR + 32),
);
Self { regs }
Self {
regs,
msi_vectors: IrqSafeSpinlock::new(Vec::new()),
}
}
/// Signals local APIC that we've handled the IRQ

40
src/arch/x86_64/apic/mod.rs
View File

@ -2,6 +2,8 @@
use core::arch::global_asm;
use static_assertions::{const_assert, const_assert_eq};
use crate::{
arch::{x86_64::cpu::Cpu, Architecture},
task::thread::Thread,
@ -29,8 +31,17 @@ pub const APIC_IPI_VECTOR: u32 = 0x03;
pub const APIC_SPURIOUS_VECTOR: u32 = 0xDF;
/// Start of the I/O APIC IRQ range
pub const APIC_EXTERNAL_OFFSET: u32 = 4;
/// Start of the MSI range
pub const APIC_MSI_OFFSET: u32 = APIC_EXTERNAL_OFFSET + MAX_EXTERNAL_VECTORS;
/// Maximum number of APIC vectors allocated for handling IRQs from I/O APIC
pub const MAX_EXTERNAL_VECTORS: u32 = 16;
pub const MAX_EXTERNAL_VECTORS: u32 = APIC_SPURIOUS_VECTOR - APIC_EXTERNAL_OFFSET - MAX_MSI_VECTORS;
/// Number of I/O APIC IRQ vectors that are actually populated
pub const POPULATED_EXTERNAL_VECTORS: u32 = 16;
/// Maximum number of APIC vectors allocated for handling MSIs
pub const MAX_MSI_VECTORS: u32 = 16;
const_assert!(POPULATED_EXTERNAL_VECTORS <= MAX_EXTERNAL_VECTORS);
const_assert_eq!(APIC_MSI_OFFSET + MAX_MSI_VECTORS, APIC_SPURIOUS_VECTOR);
/// Fills the IDT with interrupt vectors for this APIC
pub fn setup_vectors(idt: &mut [exception::Entry]) {
@ -49,6 +60,10 @@ pub fn setup_vectors(idt: &mut [exception::Entry]) {
}
unsafe extern "C" fn irq_handler(vector: usize, frame: *mut IrqFrame) {
if vector >= POPULATED_EXTERNAL_VECTORS as _ {
todo!("Got a weird IRQ with vector {}", vector);
}
let cpu = Cpu::local();
let frame = &mut *frame;
@ -62,6 +77,24 @@ unsafe extern "C" fn irq_handler(vector: usize, frame: *mut IrqFrame) {
}
}
unsafe extern "C" fn msi_handler(vector: usize, frame: *mut IrqFrame) {
if vector != 0 {
todo!("Got a weird MSI with vector {}", vector);
}
let cpu = Cpu::local();
let frame = &mut *frame;
ARCHITECTURE
.message_interrupt_controller()
.handle_msi(vector);
cpu.local_apic().clear_interrupt();
if let Some(thread) = Thread::get_current() {
thread.handle_pending_signals(frame);
}
}
unsafe extern "C" fn local_timer_irq_handler(frame: *mut IrqFrame) {
let frame = &mut *frame;
let cpu = Cpu::local();
@ -87,7 +120,12 @@ global_asm!(
include_str!("vectors.S"),
local_timer_irq_handler = sym local_timer_irq_handler,
irq_handler = sym irq_handler,
msi_handler = sym msi_handler,
ipi_handler = sym ipi_handler,
dummy_irq_handler = sym dummy_irq_handler,
irq_vector_offset = const APIC_EXTERNAL_OFFSET,
irq_vector_count = const MAX_EXTERNAL_VECTORS,
msi_vector_offset = const APIC_MSI_OFFSET,
msi_vector_count = const MAX_MSI_VECTORS,
options(att_syntax)
);

59
src/arch/x86_64/apic/vectors.S
View File

@ -123,6 +123,56 @@ irq_vector_\n:
.endr
.endm
.macro MSI_VECTOR, n
msi_vector_\n:
// %rsp + 0: %rip
// %rsp + 8: %cs
IRQ_SWAPGS_IF_NEEDED 8
IRQ_SAVE_STATE
// Force correct segment registers
mov $0x10, %ax
mov %ax, %ss
mov %ax, %ds
mov %ax, %es
mov $\n, %rdi
mov %rbp, %rsi
call {msi_handler}
IRQ_RESTORE_STATE
IRQ_SWAPGS_IF_NEEDED 8
iretq
.endm
.macro MSI_VECTOR_ENTRY, n
.quad msi_vector_\n
.endm
.macro MSI_VECTORS, start, end
.set i, 0
.rept \end - \start
MSI_VECTOR %i
.set i, i+1
.endr
.endm
.macro MSI_VECTOR_ENTRIES, start, end
.set i, 0
.rept \end - \start
MSI_VECTOR_ENTRY %i
.set i, i+1
.endr
.endm
.macro FILL_EMPTY_SPACE, start, end
.set i, 0
.rept \end - \start
.quad dummy_vector
.endr
.endm
.section .text
local_timer_vector:
IRQ_SWAPGS_IF_NEEDED 8
@ -149,7 +199,8 @@ dummy_vector:
call {dummy_irq_handler}
jmp .
IRQ_VECTORS 4, 255
IRQ_VECTORS {irq_vector_offset}, {irq_vector_offset} + {irq_vector_count}
MSI_VECTORS {msi_vector_offset}, {msi_vector_offset} + {msi_vector_count}
.section .rodata
// 224 vectors: 256 - 32 (exceptions)
@ -163,8 +214,10 @@ __x86_64_apic_vectors:
.quad dummy_vector
// IPI vector: 3
.quad ipi_vector
// Regular IRQ vectors: 4..=222
IRQ_VECTOR_ENTRIES 4, 223
// Regular IRQ vectors: 4..207
IRQ_VECTOR_ENTRIES {irq_vector_offset}, {irq_vector_offset} + {irq_vector_count}
// MSI vectors: 207..223
MSI_VECTOR_ENTRIES {msi_vector_offset}, {msi_vector_offset} + {msi_vector_count}
// Spurious interrupt vector: 223
.quad dummy_vector
.size __x86_64_apic_vectors, . - __x86_64_apic_vectors

10
src/arch/x86_64/mod.rs
View File

@ -5,8 +5,10 @@ use abi::error::Error;
use acpi_lib::{mcfg::Mcfg, AcpiTables, InterruptModel};
use alloc::boxed::Box;
use device_api::{
input::KeyboardProducer, interrupt::ExternalInterruptController,
timer::MonotonicTimestampProviderDevice, Device,
input::KeyboardProducer,
interrupt::{ExternalInterruptController, MessageInterruptController},
timer::MonotonicTimestampProviderDevice,
Device,
};
use git_version::git_version;
use kernel_util::{sync::SpinFence, util::OneTimeInit};
@ -281,6 +283,10 @@ impl Architecture for X86_64 {
self.ioapic.get()
}
fn message_interrupt_controller(&'static self) -> &'static dyn MessageInterruptController {
Cpu::local().local_apic()
}
fn monotonic_timer(&'static self) -> &'static dyn MonotonicTimestampProviderDevice {
self.timer.get()
}

105
src/device/bus/pci/capability.rs Normal file
View File

@ -0,0 +1,105 @@
//! PCI capability structures and queries
use abi::error::Error;
use device_api::interrupt::{MessageInterruptController, MsiHandler};
use crate::{
device::bus::pci::PciBaseAddress,
mem::{address::FromRaw, device::DeviceMemoryIoMut, PhysicalAddress},
};
use super::{PciCapability, PciCapabilityId, PciConfigurationSpace};
/// MSI-X capability query
pub struct MsiXCapability;
/// Represents an entry in MSI-X vector table
#[repr(C)]
pub struct MsiXEntry {
/// Address to which the value is written on interrupt
pub address: u64,
/// Value which is written to trigger an interrupt
pub data: u32,
/// Vector control word
pub control: u32,
}
/// MSI-X capability data structure
pub struct MsiXData<'s, S: PciConfigurationSpace + ?Sized + 's> {
space: &'s S,
offset: usize,
}
impl PciCapability for MsiXCapability {
const ID: PciCapabilityId = PciCapabilityId::MsiX;
type CapabilityData<'a, S: PciConfigurationSpace + ?Sized + 'a> = MsiXData<'a, S>;
fn data<'s, S: PciConfigurationSpace + ?Sized + 's>(
space: &'s S,
offset: usize,
) -> Self::CapabilityData<'s, S> {
MsiXData { space, offset }
}
}
impl<'s, S: PciConfigurationSpace + ?Sized + 's> MsiXData<'s, S> {
// TODO use pending bits as well
/// Maps and returns the vector table associated with the device's MSI-X capability
pub fn vector_table<'a>(&self) -> Result<DeviceMemoryIoMut<'a, [MsiXEntry]>, Error> {
let w0 = self.space.read_u16(self.offset + 2);
let dw1 = self.space.read_u32(self.offset + 4);
let table_size = (w0 as usize & 0x3FF) + 1;
let bir = dw1 as usize & 0x3;
let table_offset = dw1 as usize & !0x3;
let Some(base) = self.space.bar(bir) else {
return Err(Error::DoesNotExist);
};
let PciBaseAddress::Memory(base) = base else {
return Err(Error::InvalidOperation);
};
debugln!("MSI-X table address: {:#x}", base + table_offset);
unsafe {
DeviceMemoryIoMut::map_slice(PhysicalAddress::from_raw(base + table_offset), table_size)
}
}
/// Changes the global enable status for the device's MSI-X capability. If set, regular IRQs
/// are not generated.
pub fn set_enabled(&mut self, enabled: bool) {
let mut w0 = self.space.read_u32(self.offset);
if enabled {
w0 |= 1 << 31;
} else {
w0 &= !(1 << 31);
}
self.space.write_u32(self.offset, w0);
}
}
impl MsiXEntry {
/// If set, prevents the MSI-X interrupt from being delivered
pub fn set_masked(&mut self, masked: bool) {
if masked {
self.control |= 1;
} else {
self.control &= !1;
}
}
/// Registers the MSI-X vector with the interrupt controller and enables it
pub fn register<C: MessageInterruptController + ?Sized>(
&mut self,
ic: &C,
handler: &'static dyn MsiHandler,
) -> Result<(), Error> {
let info = ic.register_msi(handler)?;
self.address = info.address as _;
self.data = info.value as _;
self.set_masked(false);
Ok(())
}
}

36
src/device/bus/pci/mod.rs
View File

@ -8,6 +8,7 @@ use device_api::Device;
use kernel_util::sync::IrqSafeSpinlock;
use yggdrasil_abi::error::Error;
pub mod capability;
mod space;
pub use space::{
@ -33,6 +34,14 @@ bitflags! {
}
}
bitflags! {
/// Status register of the PCI configuration space
pub struct PciStatusRegister: u16 {
/// Read-only. If set, the configuration space has a pointer to the capabilities list.
const CAPABILITIES_LIST = 1 << 4;
}
}
/// Represents the address of a single object on a bus (or the bus itself)
#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)]
pub struct PciAddress {
@ -55,6 +64,33 @@ pub enum PciBaseAddress {
Io(u16),
}
/// Unique ID assigned to PCI capability structures
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
#[non_exhaustive]
#[repr(u8)]
pub enum PciCapabilityId {
/// MSI (32-bit or 64-bit)
Msi = 0x05,
/// MSI-X
MsiX = 0x11,
/// Unknown capability missing from this list
Unknown,
}
/// Interface used for querying PCI capabilities
pub trait PciCapability {
/// Capability ID
const ID: PciCapabilityId;
/// Wrapper for accessing the capability data structure
type CapabilityData<'a, S: PciConfigurationSpace + ?Sized + 'a>;
/// Constructs an access wrapper for this capability with given offset
fn data<'s, S: PciConfigurationSpace + ?Sized + 's>(
space: &'s S,
offset: usize,
) -> Self::CapabilityData<'s, S>;
}
/// Describes a PCI device
#[derive(Debug)]
pub struct PciDeviceInfo {

72
src/device/bus/pci/space/mod.rs
View File

@ -1,4 +1,6 @@
use super::{PciAddress, PciBaseAddress, PciEcam};
use crate::device::bus::pci::PciStatusRegister;
use super::{PciAddress, PciBaseAddress, PciCapability, PciCapabilityId, PciEcam};
pub(super) mod ecam;
@ -72,6 +74,30 @@ pub enum PciConfigSpace {
Ecam(PciEcam),
}
pub struct CapabilityIterator<'s, S: PciConfigurationSpace + ?Sized> {
space: &'s S,
current: Option<usize>,
}
impl<'s, S: PciConfigurationSpace + ?Sized> Iterator for CapabilityIterator<'s, S> {
type Item = (PciCapabilityId, usize);
fn next(&mut self) -> Option<Self::Item> {
let offset = self.current? & !0x3;
let id = unsafe { core::mem::transmute(self.space.read_u8(offset)) };
let next_pointer = self.space.read_u8(offset + 1);
self.current = if next_pointer != 0 {
Some(next_pointer as usize)
} else {
None
};
Some((id, offset))
}
}
/// Interface for accessing the configuration space of a device
pub trait PciConfigurationSpace {
/// Reads a 32-bit value from the device configuration space.
@ -120,7 +146,7 @@ pub trait PciConfigurationSpace {
}
/// Writes a byte to the device configuration space
fn write_u8(&self, offset: usize, value: u16) {
fn write_u8(&self, _offset: usize, _value: u16) {
todo!()
}
@ -185,6 +211,14 @@ pub trait PciConfigurationSpace {
"#]
secondary_bus
);
pci_config_field!(
0x34 => u8,
#[doc =
r"Returns the offset within the configuration space where the Capabilities List
is located. Only valid if the corresponding Status Register bit is set"
]
capability_pointer
);
/// Returns the value of the Base Address Register with given index.
///
@ -242,4 +276,38 @@ pub trait PciConfigurationSpace {
}
}
}
/// Returns an iterator over the PCI capabilities
fn capability_iter(&self) -> CapabilityIterator<Self> {
let status = PciStatusRegister::from_bits_retain(self.status());
let current = if status.contains(PciStatusRegister::CAPABILITIES_LIST) {
let ptr = self.capability_pointer() as usize;
if ptr != 0 {
Some(self.capability_pointer() as usize)
} else {
None
}
} else {
// Return an empty iterator
None
};
CapabilityIterator {
space: self,
current,
}
}
/// Locates a capability within this configuration space
fn capability<'s, C: PciCapability>(&'s self) -> Option<C::CapabilityData<'s, Self>> {
self.capability_iter().find_map(|(id, offset)| {
if id == C::ID {
Some(C::data(self, offset))
} else {
None
}
})
}
}

7
src/device/nvme/command.rs
View File

@ -1,3 +1,5 @@
#![allow(unused)]
use core::fmt::{self, Write};
use crate::{
@ -47,6 +49,7 @@ pub struct IdentifyControllerRequest {
pub struct CreateIoCompletionQueue {
pub id: u32,
pub size: usize,
pub vector: u32,
pub data: PhysicalAddress,
}
@ -110,9 +113,7 @@ impl Command for CreateIoCompletionQueue {
sqe.command.set_opcode(0x05);
sqe.data_pointer[1] = PhysicalRegionPage::with_addr(self.data.into_raw());
sqe.command_specific[0] = ((self.size as u32 - 1) << 16) | self.id;
sqe.command_specific[1] = 1;
// TODO ENABLE IRQS HERE
sqe.command_specific[2] = 0;
sqe.command_specific[1] = (self.vector << 16) | 3;
}
}

65
src/device/nvme/mod.rs
View File

@ -3,7 +3,7 @@ use core::{mem::size_of, time::Duration};
use abi::error::Error;
use alloc::vec::Vec;
use device_api::Device;
use device_api::{interrupt::MsiHandler, Device};
use kernel_util::{sync::IrqSafeSpinlock, util::OneTimeInit};
use tock_registers::{
interfaces::{ReadWriteable, Readable, Writeable},
@ -12,8 +12,11 @@ use tock_registers::{
};
use crate::{
arch::{Architecture, ARCHITECTURE},
device::{
bus::pci::{PciBaseAddress, PciCommandRegister, PciConfigurationSpace},
bus::pci::{
capability::MsiXCapability, PciBaseAddress, PciCommandRegister, PciConfigurationSpace,
},
nvme::{
command::IdentifyControllerRequest,
queue::{CompletionQueueEntry, SubmissionQueueEntry},
@ -21,7 +24,7 @@ use crate::{
},
mem::{
address::{FromRaw, IntoRaw},
device::DeviceMemoryIo,
device::{DeviceMemoryIo, DeviceMemoryIoMut},
PhysicalAddress,
},
task::runtime,
@ -33,7 +36,7 @@ use self::{
queue::QueuePair,
};
use super::bus::pci::{FromPciBus, PciDeviceInfo};
use super::bus::pci::{capability::MsiXEntry, FromPciBus, PciDeviceInfo};
mod command;
mod error;
@ -109,6 +112,7 @@ pub struct NvmeController {
regs: IrqSafeSpinlock<DeviceMemoryIo<'static, Regs>>,
admin_q: OneTimeInit<QueuePair<'static>>,
ioqs: OneTimeInit<Vec<QueuePair<'static>>>,
vector_table: IrqSafeSpinlock<DeviceMemoryIoMut<'static, [MsiXEntry]>>,
doorbell_shift: usize,
}
@ -123,9 +127,11 @@ impl Regs {
impl NvmeController {
async fn late_init(&'static self) -> Result<(), NvmeError> {
runtime::spawn(self.poll_task()).expect("Couldn't spawn NVMe poll task");
// runtime::spawn(self.poll_task()).expect("Couldn't spawn NVMe poll task");
let admin_q = self.admin_q.get();
infoln!("SETUP");
// Request a CQ/SQ pair for I/O
admin_q
.request_no_data(SetFeatureRequest::NumberOfQueues(1, 1))
@ -133,7 +139,8 @@ impl NvmeController {
// Allocate the queue
let (sq_doorbell, cq_doorbell) = unsafe { self.doorbell_pair(1) };
let io_q = QueuePair::new(32, sq_doorbell, cq_doorbell).map_err(NvmeError::MemoryError)?;
let io_q =
QueuePair::new(0, 32, sq_doorbell, cq_doorbell).map_err(NvmeError::MemoryError)?;
// Identify the controller
let identify = admin_q
@ -145,6 +152,7 @@ impl NvmeController {
.request_no_data(CreateIoCompletionQueue {
id: 1,
size: 32,
vector: 0,
data: io_q.cq_physical_pointer(),
})
.await?;
@ -160,14 +168,6 @@ impl NvmeController {
loop {}
}
// TODO MSI(-X) or IRQ (ACPI currently broken) support for PCIe-based NVMe
async fn poll_task(&'static self) {
loop {
self.admin_q.get().process_completions();
runtime::sleep(Duration::from_millis(100)).await;
}
}
unsafe fn doorbell_pair(&self, idx: usize) -> (*mut u32, *mut u32) {
let regs = self.regs.lock();
let sq_ptr = regs.doorbell_ptr(self.doorbell_shift, false, idx);
@ -176,12 +176,18 @@ impl NvmeController {
}
}
impl MsiHandler for NvmeController {
fn handle_msi(&self, vector: usize) -> bool {
debugln!("handle_msi {}", vector);
self.admin_q.get().process_completions() != 0
}
}
impl Device for NvmeController {
unsafe fn init(&'static self) -> Result<(), Error> {
let regs = self.regs.lock();
let min_page_size = 1usize << (12 + regs.CAP.read(CAP::MPSMIN));
let max_page_size = 1usize << (12 + regs.CAP.read(CAP::MPSMAX));
if min_page_size > 4096 {
panic!();
@ -205,8 +211,13 @@ impl Device for NvmeController {
// Setup the admin queue (index 0)
let admin_sq_doorbell = unsafe { regs.doorbell_ptr(self.doorbell_shift, false, 0) };
let admin_cq_doorbell = unsafe { regs.doorbell_ptr(self.doorbell_shift, true, 0) };
let admin_q =
QueuePair::new(queue_slots as usize, admin_sq_doorbell, admin_cq_doorbell).unwrap();
let admin_q = QueuePair::new(
0,
queue_slots as usize,
admin_sq_doorbell,
admin_cq_doorbell,
)
.unwrap();
regs.AQA
.modify(AQA::ASQS.val(queue_slots as u32 - 1) + AQA::ACQS.val(queue_slots as u32 - 1));
@ -239,6 +250,16 @@ impl Device for NvmeController {
self.admin_q.init(admin_q);
// Register the IRQs (TODO: use multiple)
{
let mut vt = self.vector_table.lock();
// Register vector 0
vt[0]
.register(ARCHITECTURE.message_interrupt_controller(), self)
.unwrap();
}
// Schedule late_init task
runtime::spawn(self.late_init())?;
@ -256,6 +277,15 @@ impl FromPciBus for NvmeController {
panic!();
};
// TODO also support MSI
let mut msix = info.config_space.capability::<MsiXCapability>().unwrap();
let mut vt = msix.vector_table()?;
for vector in vt.iter_mut() {
vector.set_masked(true);
}
msix.set_enabled(true);
let mut cmd = PciCommandRegister::from_bits_retain(info.config_space.command());
cmd &= !(PciCommandRegister::DISABLE_INTERRUPTS | PciCommandRegister::ENABLE_IO);
cmd |= PciCommandRegister::ENABLE_MEMORY | PciCommandRegister::BUS_MASTER;
@ -272,6 +302,7 @@ impl FromPciBus for NvmeController {
regs: IrqSafeSpinlock::new(regs),
admin_q: OneTimeInit::new(),
ioqs: OneTimeInit::new(),
vector_table: IrqSafeSpinlock::new(vt),
doorbell_shift,
})
}

5
src/device/nvme/queue.rs
View File

@ -1,6 +1,5 @@
use core::{
mem::size_of,
ops::DerefMut,
pin::Pin,
ptr::null_mut,
task::{Context, Poll},
@ -94,6 +93,8 @@ pub struct QueuePair<'a> {
base: PhysicalAddress,
page_count: usize,
vector: usize,
sq_base: PhysicalAddress,
cq_base: PhysicalAddress,
@ -246,6 +247,7 @@ impl<'a, T> Queue<'a, T> {
impl<'a> QueuePair<'a> {
pub fn new(
vector: usize,
capacity: usize,
sq_doorbell: *mut u32,
cq_doorbell: *mut u32,
@ -279,6 +281,7 @@ impl<'a> QueuePair<'a> {
Ok(Self {
completion_notify: QueueWaker::new(),
vector,
base,
page_count,
sq_base,

58
src/mem/device.rs
View File

@ -1,12 +1,19 @@
//! Facilities for mapping devices to virtual address space
use core::{mem::size_of, ops::Deref};
use core::{
alloc::Layout,
mem::size_of,
ops::{Deref, DerefMut},
};
use abi::error::Error;
use alloc::sync::Arc;
use crate::arch::{Architecture, ARCHITECTURE};
use super::PhysicalAddress;
use super::{
address::{AsPhysicalAddress, FromRaw},
PhysicalAddress,
};
/// Describes a single device memory mapping
#[derive(Debug)]
@ -37,6 +44,14 @@ pub struct DeviceMemoryIo<'a, T: ?Sized> {
value: &'a T,
}
/// Describes a single typed and mutable device memory mapping
#[derive(Debug)]
pub struct DeviceMemoryIoMut<'a, T: ?Sized> {
#[allow(unused)]
inner: RawDeviceMemoryMapping,
value: &'a mut T,
}
impl RawDeviceMemoryMapping {
/// Maps a region of physical memory as device memory of given size.
///
@ -130,3 +145,42 @@ impl<'a, T: ?Sized> Deref for DeviceMemoryIo<'a, T> {
self.value
}
}
impl<T: ?Sized> AsPhysicalAddress for DeviceMemoryIo<'_, T> {
unsafe fn as_physical_address(&self) -> PhysicalAddress {
PhysicalAddress::from_raw(self.inner.base_address)
}
}
impl<'a, T: Sized> DeviceMemoryIoMut<'a, T> {
/// Maps a physical address as device memory to a slice `[T; len]`
///
/// # Safety
///
/// The caller must ensure the address actually points to a value of type `T`, as well as
/// proper access synchronization. The caller must also ensure the `len` is valid.
pub unsafe fn map_slice(
base: PhysicalAddress,
len: usize,
) -> Result<DeviceMemoryIoMut<'a, [T]>, Error> {
let layout = Layout::array::<T>(len).unwrap();
let inner = RawDeviceMemoryMapping::map(base, layout.size())?;
let value = core::slice::from_raw_parts_mut(inner.address as *mut T, len);
Ok(DeviceMemoryIoMut { inner, value })
}
}
impl<'a, T: ?Sized> Deref for DeviceMemoryIoMut<'a, T> {
type Target = T;
fn deref(&self) -> &Self::Target {
self.value
}
}
impl<'a, T: ?Sized> DerefMut for DeviceMemoryIoMut<'a, T> {
fn deref_mut(&mut self) -> &mut Self::Target {
self.value
}
}