From 2770281213a75e8df83f05743dac86b46e391da0 Mon Sep 17 00:00:00 2001
From: Mark Poliakov <mark@alnyan.me>
Date: Fri, 8 Dec 2023 22:49:33 +0200
Subject: [PATCH] dev/block: create io cq/sq pair for nvme
---
src/device/nvme/command.rs | 75 +++++++++++++++++++++++++++++++++---
src/device/nvme/mod.rs | 78 ++++++++++++++++++++++++++++----------
src/device/nvme/queue.rs | 35 ++++++++++++-----
3 files changed, 154 insertions(+), 34 deletions(-)
diff --git a/src/device/nvme/command.rs b/src/device/nvme/command.rs
index 893ee927..9f6231f4 100644
--- a/src/device/nvme/command.rs
+++ b/src/device/nvme/command.rs
@@ -1,6 +1,9 @@
use core::fmt::{self, Write};
-use bytemuck::{Pod, Zeroable};
+use crate::{
+ device::nvme::queue::PhysicalRegionPage,
+ mem::{address::IntoRaw, PhysicalAddress},
+};
use super::queue::SubmissionQueueEntry;
@@ -28,6 +31,35 @@ pub enum ControllerType {
Administrative,
}
+// Requests
+
+#[derive(Clone, Copy, Debug)]
+pub enum SetFeatureRequest {
+ NumberOfQueues(u32, u32),
+}
+
+#[derive(Clone, Copy, Debug)]
+pub struct IdentifyControllerRequest {
+ pub nsid: u32,
+}
+
+#[derive(Clone, Copy, Debug)]
+pub struct CreateIoCompletionQueue {
+ pub id: u32,
+ pub size: usize,
+ pub data: PhysicalAddress,
+}
+
+#[derive(Clone, Copy, Debug)]
+pub struct CreateIoSubmissionQueue {
+ pub id: u32,
+ pub cq_id: u32,
+ pub size: usize,
+ pub data: PhysicalAddress,
+}
+
+// Replies
+
#[derive(Clone, Copy, Debug)]
#[repr(C)]
pub struct IdentifyControllerResponse {
@@ -46,11 +78,6 @@ pub struct IdentifyControllerResponse {
pub cntrltype: ControllerType,
}
-#[derive(Clone, Copy, Debug)]
-pub struct IdentifyControllerRequest {
- pub nsid: u32,
-}
-
impl Command for IdentifyControllerRequest {
fn fill_sqe(&self, sqe: &mut SubmissionQueueEntry) {
sqe.command.set_opcode(0x06);
@@ -63,6 +90,42 @@ impl Request for IdentifyControllerRequest {
type Response = IdentifyControllerResponse;
}
+impl Command for SetFeatureRequest {
+ fn fill_sqe(&self, sqe: &mut SubmissionQueueEntry) {
+ sqe.command.set_opcode(0x09);
+
+ match self {
+ Self::NumberOfQueues(cq, sq) => {
+ let dw11 = (cq << 16) | sq;
+
+ sqe.command_specific[0] = 0x07;
+ sqe.command_specific[1] = dw11;
+ }
+ }
+ }
+}
+
+impl Command for CreateIoCompletionQueue {
+ fn fill_sqe(&self, sqe: &mut SubmissionQueueEntry) {
+ sqe.command.set_opcode(0x05);
+ sqe.data_pointer[1] = PhysicalRegionPage::with_addr(self.data.into_raw());
+ sqe.command_specific[0] = ((self.size as u32 - 1) << 16) | self.id;
+ sqe.command_specific[1] = 1;
+ // TODO ENABLE IRQS HERE
+ sqe.command_specific[2] = 0;
+ }
+}
+
+impl Command for CreateIoSubmissionQueue {
+ fn fill_sqe(&self, sqe: &mut SubmissionQueueEntry) {
+ sqe.command.set_opcode(0x01);
+ sqe.data_pointer[1] = PhysicalRegionPage::with_addr(self.data.into_raw());
+ sqe.command_specific[0] = ((self.size as u32 - 1) << 16) | self.id;
+ // Medium priority
+ sqe.command_specific[1] = (self.cq_id << 16) | 1;
+ }
+}
+
impl<const N: usize> fmt::Debug for String<N> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_char('"')?;
diff --git a/src/device/nvme/mod.rs b/src/device/nvme/mod.rs
index f76cae88..67548164 100644
--- a/src/device/nvme/mod.rs
+++ b/src/device/nvme/mod.rs
@@ -1,11 +1,10 @@
#![allow(missing_docs)]
-use core::{mem::size_of, ptr::null_mut, time::Duration};
+use core::{mem::size_of, time::Duration};
use abi::error::Error;
-use bytemuck::{Pod, Zeroable};
+use alloc::vec::Vec;
use device_api::Device;
use kernel_util::{sync::IrqSafeSpinlock, util::OneTimeInit};
-use static_assertions::{const_assert, const_assert_eq};
use tock_registers::{
interfaces::{ReadWriteable, Readable, Writeable},
register_bitfields, register_structs,
@@ -13,8 +12,6 @@ use tock_registers::{
};
use crate::{
- arch::{x86_64::mem::table::L3, Architecture, ARCHITECTURE},
- debug,
device::{
bus::pci::{PciBaseAddress, PciCommandRegister, PciConfigurationSpace},
nvme::{
@@ -23,17 +20,17 @@ use crate::{
},
},
mem::{
- address::{AsPhysicalAddress, FromRaw, IntoRaw},
+ address::{FromRaw, IntoRaw},
device::DeviceMemoryIo,
- phys,
- pointer::{PhysicalRef, PhysicalRefMut},
- table::EntryLevel,
PhysicalAddress,
},
task::runtime,
};
-use self::queue::QueuePair;
+use self::{
+ command::{CreateIoCompletionQueue, CreateIoSubmissionQueue, SetFeatureRequest},
+ queue::QueuePair,
+};
use super::bus::pci::{FromPciBus, PciDeviceInfo};
@@ -109,10 +106,13 @@ register_structs! {
pub struct NvmeController {
regs: IrqSafeSpinlock<DeviceMemoryIo<'static, Regs>>,
admin_q: OneTimeInit<QueuePair<'static>>,
+ ioqs: OneTimeInit<Vec<QueuePair<'static>>>,
+
+ doorbell_shift: usize,
}
impl Regs {
- unsafe fn doorbell_ptr(&self, shift: u64, completion: bool, queue_index: usize) -> *mut u32 {
+ unsafe fn doorbell_ptr(&self, shift: usize, completion: bool, queue_index: usize) -> *mut u32 {
let doorbell_base = (self as *const Regs as *mut Regs).addr() + 0x1000;
let offset = (queue_index << shift) + completion as usize * 4;
(doorbell_base + offset) as *mut u32
@@ -121,19 +121,49 @@ impl Regs {
impl NvmeController {
async fn late_init(&'static self) {
- runtime::spawn(self.poll_task()).expect("Couldn't spawn NVMe poll task");
+ // let ioq = QueuePair::new(capacity, sq_doorbell, cq_doorbell).unwrap();
+ runtime::spawn(self.poll_task()).expect("Couldn't spawn NVMe poll task");
let admin_q = self.admin_q.get();
- let response = admin_q
+ // Request a CQ/SQ pair for I/O
+ admin_q
+ .request_no_data(SetFeatureRequest::NumberOfQueues(1, 1))
+ .unwrap()
+ .await
+ .unwrap();
+
+ // Allocate the queue
+ let (sq_doorbell, cq_doorbell) = unsafe { self.doorbell_pair(1) };
+ let io_q = QueuePair::new(32, sq_doorbell, cq_doorbell).unwrap();
+
+ // Identify the controller
+ let identify = admin_q
.request(IdentifyControllerRequest { nsid: 0 })
.unwrap()
.await
.unwrap();
- infoln!("Model: {:#?}", response.model_number);
- infoln!("Serial: {:#?}", response.serial_number);
- infoln!("Type: {:?}", response.cntrltype);
+ // Create the queue on the device side
+ admin_q
+ .request_no_data(CreateIoCompletionQueue {
+ id: 1,
+ size: 32,
+ data: io_q.cq_physical_pointer(),
+ })
+ .unwrap()
+ .await
+ .unwrap();
+ admin_q
+ .request_no_data(CreateIoSubmissionQueue {
+ id: 1,
+ cq_id: 1,
+ size: 32,
+ data: io_q.sq_physical_pointer(),
+ })
+ .unwrap()
+ .await
+ .unwrap();
loop {}
}
@@ -145,6 +175,13 @@ impl NvmeController {
runtime::sleep(Duration::from_millis(100)).await;
}
}
+
+ unsafe fn doorbell_pair(&self, idx: usize) -> (*mut u32, *mut u32) {
+ let regs = self.regs.lock();
+ let sq_ptr = regs.doorbell_ptr(self.doorbell_shift, false, idx);
+ let cq_ptr = regs.doorbell_ptr(self.doorbell_shift, true, idx);
+ (sq_ptr, cq_ptr)
+ }
}
impl Device for NvmeController {
@@ -174,9 +211,8 @@ impl Device for NvmeController {
}
// Setup the admin queue (index 0)
- let doorbell_shift = regs.CAP.read(CAP::DSTRD) + 2;
- let admin_sq_doorbell = unsafe { regs.doorbell_ptr(doorbell_shift, false, 0) };
- let admin_cq_doorbell = unsafe { regs.doorbell_ptr(doorbell_shift, true, 0) };
+ let admin_sq_doorbell = unsafe { regs.doorbell_ptr(self.doorbell_shift, false, 0) };
+ let admin_cq_doorbell = unsafe { regs.doorbell_ptr(self.doorbell_shift, true, 0) };
let admin_q =
QueuePair::new(queue_slots as usize, admin_sq_doorbell, admin_cq_doorbell).unwrap();
@@ -238,9 +274,13 @@ impl FromPciBus for NvmeController {
// Disable the controller
regs.CC.modify(CC::ENABLE::CLEAR);
+ let doorbell_shift = regs.CAP.read(CAP::DSTRD) as usize + 2;
+
Ok(Self {
regs: IrqSafeSpinlock::new(regs),
admin_q: OneTimeInit::new(),
+ ioqs: OneTimeInit::new(),
+ doorbell_shift,
})
}
}
diff --git a/src/device/nvme/queue.rs b/src/device/nvme/queue.rs
index 06c7ec67..7cece0c1 100644
--- a/src/device/nvme/queue.rs
+++ b/src/device/nvme/queue.rs
@@ -18,14 +18,7 @@ use static_assertions::const_assert;
use crate::{
arch::x86_64::mem::table::L3,
- mem::{
- address::{AsPhysicalAddress, IntoRaw},
- phys,
- pointer::PhysicalRefMut,
- table::EntryLevel,
- PhysicalAddress,
- },
- proc,
+ mem::{address::IntoRaw, phys, pointer::PhysicalRefMut, table::EntryLevel, PhysicalAddress},
task::runtime::QueueWaker,
};
@@ -313,7 +306,7 @@ impl<'a> QueuePair<'a> {
this: &'r QueuePair<'r>,
response: Option<R>,
command_id: u32,
- };
+ }
impl<'r, R: Unpin + 'r> Future for Fut<'r, R> {
type Output = Result<R, CommandError>;
@@ -354,6 +347,18 @@ impl<'a> QueuePair<'a> {
let mut inner = self.inner.lock();
let mut sqe = SubmissionQueueEntry::zeroed();
+ match ranges.len() {
+ 1 => {
+ sqe.data_pointer[0] = PhysicalRegionPage::with_addr(ranges[0].into_raw());
+ sqe.data_pointer[1] = PhysicalRegionPage::null();
+ }
+ 0 => {
+ sqe.data_pointer[0] = PhysicalRegionPage::null();
+ sqe.data_pointer[1] = PhysicalRegionPage::null();
+ }
+ _ => todo!(),
+ }
+
cmd.fill_sqe(&mut sqe);
let command_id = inner.sq.tail.try_into().unwrap();
@@ -368,6 +373,17 @@ impl<'a> QueuePair<'a> {
Ok(command_id)
}
+ pub fn request_no_data<'r, C: Command>(
+ &'r self,
+ req: C,
+ ) -> Result<impl Future<Output = Result<(), CommandError>> + 'r, Error>
+ where
+ 'r: 'a,
+ {
+ let command_id = self.submit(req, &[], true)?;
+ Ok(self.wait_for_completion(command_id, ()))
+ }
+
pub fn request<'r, R: Request>(
&'r self,
req: R,
@@ -376,6 +392,7 @@ impl<'a> QueuePair<'a> {
R::Response: 'r,
'r: 'a,
{
+ assert_ne!(size_of::<R::Response>(), 0);
assert!(size_of::<R::Response>() < 0x1000);
let page = phys::alloc_page()?;