alnyan/yggdrasil
1
0
Fork 0
Code Issues 33 Pull Requests Packages Projects Releases Wiki Activity

dev/block: better nvme completion await

Browse Source
This commit is contained in:
Mark Poliakov 2023-12-08 17:23:03 +02:00
parent d1fe89c134
commit 148acca561
3 changed files with 249 additions and 121 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

74
src/device/nvme/command.rs
View File

@ -1,36 +1,78 @@
use core::fmt::{self, Write};
use bytemuck::{Pod, Zeroable};
use super::queue::SubmissionQueueEntry;
pub trait Command {
fn fill_sqe(&self, sqe: &mut SubmissionQueueEntry);
}
#[derive(Clone, Copy, Debug)]
#[repr(u32)]
pub enum IdentifyCommand {
Controller = 0x01,
pub trait Request: Command {
type Response;
}
#[derive(Clone, Copy)]
#[repr(transparent)]
pub struct String<const N: usize> {
data: [u8; N],
}
#[derive(Clone, Copy, Debug)]
#[non_exhaustive]
#[repr(u8)]
pub enum ControllerType {
Reserved,
Io,
Discovery,
Administrative,
}
#[derive(Clone, Copy, Debug)]
#[repr(C)]
pub struct IdentifyControllerResponse {
pub pci_vid: u16,
pub pci_ssvid: u16,
pub serial_number: [u8; 20],
pub model_number: [u8; 40],
pub serial_number: String<20>,
pub model_number: String<40>,
pub firmware_rev: u64,
_0: [u8; 5], // 72..77
pub mdts: u8,
pub cntlid: u16,
pub ver: u32,
_1: [u8; 12], // 84..96
pub ctratt: u32,
_2: [u8; 11], // 100..111
pub cntrltype: ControllerType,
}
pub enum AdminCommand {
Identify { nsid: u32, cns: IdentifyCommand },
#[derive(Clone, Copy, Debug)]
pub struct IdentifyControllerRequest {
pub nsid: u32,
}
impl Command for AdminCommand {
impl Command for IdentifyControllerRequest {
fn fill_sqe(&self, sqe: &mut SubmissionQueueEntry) {
match self {
AdminCommand::Identify { nsid, cns } => {
sqe.command.set_opcode(0x06);
sqe.nsid = *nsid;
sqe.command_specific[0] = *cns as u32;
}
}
sqe.command.set_opcode(0x06);
sqe.command_specific[0] = 0x01;
sqe.nsid = self.nsid;
}
}
impl Request for IdentifyControllerRequest {
type Response = IdentifyControllerResponse;
}
impl<const N: usize> fmt::Debug for String<N> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_char('"')?;
for ch in self.data {
if ch == b' ' || ch == 0 {
break;
}
f.write_char(ch as _)?;
}
f.write_char('"')?;
Ok(())
}
}

59
src/device/nvme/mod.rs
View File

@ -18,7 +18,7 @@ use crate::{
device::{
bus::pci::{PciBaseAddress, PciCommandRegister, PciConfigurationSpace},
nvme::{
command::{AdminCommand, IdentifyCommand},
command::IdentifyControllerRequest,
queue::{CompletionQueueEntry, SubmissionQueueEntry},
},
},
@ -108,7 +108,7 @@ register_structs! {
pub struct NvmeController {
regs: IrqSafeSpinlock<DeviceMemoryIo<'static, Regs>>,
admin_q: OneTimeInit<IrqSafeSpinlock<QueuePair<'static>>>,
admin_q: OneTimeInit<QueuePair<'static>>,
}
impl Regs {
@ -121,41 +121,30 @@ impl Regs {
impl NvmeController {
async fn late_init(&'static self) {
let mut admin_q = self.admin_q.get().lock();
runtime::spawn(self.poll_task()).expect("Couldn't spawn NVMe poll task");
// Identify
let page = phys::alloc_page().unwrap();
let mut ranges = [page];
let admin_q = self.admin_q.get();
debugln!("output range = {:#x}", ranges[0]);
admin_q
.perform_request(
AdminCommand::Identify {
nsid: 0,
cns: IdentifyCommand::Controller,
},
&ranges,
)
.await;
let response = admin_q
.request(IdentifyControllerRequest { nsid: 0 })
.unwrap()
.await
.unwrap();
let pref = unsafe { PhysicalRef::<command::IdentifyControllerResponse>::map(page) };
fn cstr(s: &[u8]) -> &str {
let i = s
.iter()
.position(|x| *x == 0 || *x == b' ')
.unwrap_or_else(|| s.len());
core::str::from_utf8(&s[..i]).unwrap()
}
let sn = cstr(&pref.serial_number);
let mn = cstr(&pref.model_number);
debugln!("Serial Number: {:?}", sn);
debugln!("Model Number: {:?}", mn);
infoln!("Model: {:#?}", response.model_number);
infoln!("Serial: {:#?}", response.serial_number);
infoln!("Type: {:?}", response.cntrltype);
loop {}
}
// TODO MSI(-X) or IRQ (ACPI currently broken) support for PCIe-based NVMe
async fn poll_task(&'static self) {
loop {
self.admin_q.get().process_completions();
runtime::sleep(Duration::from_millis(100)).await;
}
}
}
impl Device for NvmeController {
@ -169,10 +158,6 @@ impl Device for NvmeController {
panic!();
}
if regs.CAP.read(CAP::CSS_IO_COMMANDS) != 1 {
panic!();
}
let timeout = Duration::from_millis(regs.CAP.read(CAP::TO) * 500);
debugln!("Worst-case timeout: {:?}", timeout);
@ -192,7 +177,7 @@ impl Device for NvmeController {
let doorbell_shift = regs.CAP.read(CAP::DSTRD) + 2;
let admin_sq_doorbell = unsafe { regs.doorbell_ptr(doorbell_shift, false, 0) };
let admin_cq_doorbell = unsafe { regs.doorbell_ptr(doorbell_shift, true, 0) };
let mut admin_q =
let admin_q =
QueuePair::new(queue_slots as usize, admin_sq_doorbell, admin_cq_doorbell).unwrap();
regs.AQA
@ -224,7 +209,7 @@ impl Device for NvmeController {
todo!("CFS set after reset!");
}
self.admin_q.init(IrqSafeSpinlock::new(admin_q));
self.admin_q.init(admin_q);
// Schedule late_init task
runtime::spawn(self.late_init())?;

237
src/device/nvme/queue.rs
View File

@ -1,13 +1,19 @@
use core::{
mem::size_of,
ops::DerefMut,
pin::Pin,
ptr::null_mut,
task::{Context, Poll},
};
use abi::error::Error;
use alloc::{
collections::{BTreeMap, BTreeSet},
vec::Vec,
};
use bytemuck::{Pod, Zeroable};
use futures_util::Future;
use kernel_util::sync::IrqSafeSpinlock;
use static_assertions::const_assert;
use crate::{
@ -20,9 +26,10 @@ use crate::{
PhysicalAddress,
},
proc,
task::runtime::QueueWaker,
};
use super::command::Command;
use super::command::{Command, Request};
#[derive(Zeroable, Pod, Clone, Copy, Debug)]
#[repr(C)]
@ -54,6 +61,13 @@ pub struct SubmissionQueueEntry {
pub command_specific: [u32; 6], // 10, 11, 12, 13, 14, 15
}
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
pub struct CommandError {
sct: u8,
sc: u8,
}
#[derive(Zeroable, Pod, Clone, Copy, Debug)]
#[repr(C)]
pub struct CompletionQueueEntry {
dw: [u32; 4],
@ -71,16 +85,25 @@ pub struct Queue<'a, T> {
tail_doorbell: *mut u32,
}
unsafe impl<'a, T> Sync for Queue<'a, T> {}
unsafe impl<'a, T> Send for Queue<'a, T> {}
struct Inner<'a> {
sq: Queue<'a, SubmissionQueueEntry>,
cq: Queue<'a, CompletionQueueEntry>,
completed: BTreeMap<u32, CompletionQueueEntry>,
pending: BTreeSet<u32>,
}
// TODO PageBox<T>?
pub struct QueuePair<'a> {
base: PhysicalAddress,
page_count: usize,
sq: Queue<'a, SubmissionQueueEntry>,
cq: Queue<'a, CompletionQueueEntry>,
sq_base: PhysicalAddress,
cq_base: PhysicalAddress,
completion_notify: QueueWaker,
inner: IrqSafeSpinlock<Inner<'a>>,
}
const_assert!(size_of::<CompletionQueueEntry>().is_power_of_two());
@ -96,9 +119,10 @@ impl PhysicalRegionPage {
}
impl CommandDword0 {
pub fn set_command_id(&mut self, id: u16) {
pub fn set_command_id(&mut self, id: u32) {
debug_assert!(id & 0xFFFF0000 == 0);
self.0 &= !(0xFFFF << 16);
self.0 |= (id as u32) << 16;
self.0 |= id << 16;
}
pub fn set_opcode(&mut self, opcode: u8) {
@ -112,13 +136,30 @@ impl CompletionQueueEntry {
self.dw[3] & (1 << 16) != 0
}
pub fn subqueue_id(&self) -> u32 {
pub fn sub_queue_id(&self) -> u32 {
self.dw[2] >> 16
}
pub fn subqueue_head(&self) -> usize {
pub fn sub_queue_head(&self) -> usize {
(self.dw[2] & 0xFFFF) as _
}
pub fn command_id(&self) -> u32 {
self.dw[3] & 0xFFFF
}
pub fn error(&self) -> Option<CommandError> {
let status = (self.dw[3] >> 17) as u16;
if status != 0 {
Some(CommandError {
sct: ((status >> 8) & 0x7) as u8,
sc: status as u8,
})
} else {
None
}
}
}
impl<'a, T> Queue<'a, T> {
@ -146,10 +187,6 @@ impl<'a, T> Queue<'a, T> {
}
}
pub fn physical_pointer(&self) -> PhysicalAddress {
unsafe { self.data.as_physical_address() }
}
pub fn enqueue(&mut self, item: T) -> usize {
let index = self.tail;
self.data[self.tail] = item;
@ -236,99 +273,163 @@ impl<'a> QueuePair<'a> {
let sq = unsafe { Queue::from_raw_parts(sq_base, capacity, null_mut(), sq_doorbell, true) };
let cq = unsafe { Queue::from_raw_parts(cq_base, capacity, cq_doorbell, null_mut(), true) };
Ok(Self {
base,
page_count,
let inner = IrqSafeSpinlock::new(Inner {
sq,
cq,
pending: BTreeSet::new(),
completed: BTreeMap::new(),
});
Ok(Self {
completion_notify: QueueWaker::new(),
base,
page_count,
sq_base,
cq_base,
inner,
})
}
#[inline]
pub fn sq_physical_pointer(&self) -> PhysicalAddress {
self.sq.physical_pointer()
self.sq_base
}
#[inline]
pub fn cq_physical_pointer(&self) -> PhysicalAddress {
self.cq.physical_pointer()
self.cq_base
}
pub fn wait_for_completion<'r, T: Unpin + 'r>(
&'r self,
command_id: u32,
result: T,
) -> impl Future<Output = Result<T, CommandError>> + 'r
where
'r: 'a,
{
struct Fut<'r, R: Unpin + 'r> {
this: &'r QueuePair<'r>,
response: Option<R>,
command_id: u32,
};
impl<'r, R: Unpin + 'r> Future for Fut<'r, R> {
type Output = Result<R, CommandError>;
fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
self.this.completion_notify.register(cx.waker());
let mut inner = self.this.inner.lock();
if let Some(entry) = inner.completed.remove(&self.command_id) {
self.this.completion_notify.remove(cx.waker());
let result = if let Some(error) = entry.error() {
Err(error)
} else {
Ok(self.response.take().unwrap())
};
Poll::Ready(result)
} else {
Poll::Pending
}
}
}
Fut {
this: self,
response: Some(result),
command_id,
}
}
pub fn submit<C: Command>(
&mut self,
command: C,
&self,
cmd: C,
ranges: &[PhysicalAddress],
) -> Result<(), Error> {
let index = self.sq.tail;
set_pending: bool,
) -> Result<u32, Error> {
let mut inner = self.inner.lock();
let mut sqe = SubmissionQueueEntry::zeroed();
sqe.command.set_command_id(index.try_into().unwrap());
if ranges.len() != 1 {
todo!();
cmd.fill_sqe(&mut sqe);
let command_id = inner.sq.tail.try_into().unwrap();
sqe.command.set_command_id(command_id);
if set_pending {
inner.pending.insert(command_id);
}
sqe.data_pointer[0] = PhysicalRegionPage::with_addr(ranges[0].into_raw());
sqe.data_pointer[1] = PhysicalRegionPage::null();
command.fill_sqe(&mut sqe);
inner.sq.enqueue(sqe);
self.sq.enqueue(sqe);
Ok(())
Ok(command_id)
}
pub fn advance_head(&mut self, new_head: usize) {
self.sq.take_until(new_head);
pub fn request<'r, R: Request>(
&'r self,
req: R,
) -> Result<impl Future<Output = Result<PhysicalRefMut<'r, R::Response>, CommandError>>, Error>
where
R::Response: 'r,
'r: 'a,
{
assert!(size_of::<R::Response>() < 0x1000);
let page = phys::alloc_page()?;
// TODO PageBox
let response = unsafe { PhysicalRefMut::map(page) };
let command_id = self.submit(req, &[page], true)?;
Ok(self.wait_for_completion(command_id, response))
}
pub fn process_completions(&mut self) -> usize {
let mut i = 0;
pub fn process_completions(&self) -> usize {
let mut inner = self.inner.lock();
let mut n = 0;
let mut completion_list = Vec::new();
loop {
let (packet, expected_phase) = self.cq.at_head(i);
let packet_phase = packet.phase();
let (cmp, expected_phase) = inner.cq.at_head(n);
let cmp_phase = cmp.phase();
if packet_phase != expected_phase {
if cmp_phase != expected_phase {
break;
}
i += 1;
n += 1;
let sqid = packet.subqueue_id();
// TODO
assert_eq!(sqid, 0);
let sub_queue_id = cmp.sub_queue_id();
// TODO support queues other than admin q
assert_eq!(sub_queue_id, 0);
let sqhd = packet.subqueue_head();
self.advance_head(sqhd);
let sub_queue_head = cmp.sub_queue_head();
let cmp = *cmp;
inner.sq.take_until(sub_queue_head);
completion_list.push(cmp);
}
if i != 0 {
self.cq.take(i);
if n != 0 {
inner.cq.take(n);
}
i
}
for cmp in completion_list {
let command_id = cmp.command_id();
pub async fn perform_request<C: Command>(&mut self, command: C, ranges: &[PhysicalAddress]) {
self.submit(command, ranges).unwrap();
while self.process_completions() == 0 {
core::hint::spin_loop();
}
}
pub fn wait_all_complete(&'a self, index: usize) -> impl Future<Output = ()> + 'a {
struct F<'q> {
queue: &'q QueuePair<'q>,
index: usize,
}
impl<'q> Future for F<'q> {
type Output = ();
fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
todo!()
if inner.pending.remove(&command_id) {
debugln!("Insert completion: {}", command_id);
inner.completed.insert(command_id, cmp);
}
}
F { queue: self, index }
if n != 0 {
self.completion_notify.wake_all();
}
n
}
}