alnyan/yggdrasil
1
0
Fork 0
Code Issues 33 Pull Requests Packages Projects Releases Wiki Activity

dev/block: create io cq/sq pair for nvme

Browse Source
This commit is contained in:
Mark Poliakov 2023-12-08 22:49:33 +02:00
parent 148acca561
commit 2770281213
3 changed files with 154 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

75
src/device/nvme/command.rs
View File

@ -1,6 +1,9 @@
use core::fmt::{self, Write};
use bytemuck::{Pod, Zeroable};
use crate::{
device::nvme::queue::PhysicalRegionPage,
mem::{address::IntoRaw, PhysicalAddress},
};
use super::queue::SubmissionQueueEntry;
@ -28,6 +31,35 @@ pub enum ControllerType {
Administrative,
}
// Requests
#[derive(Clone, Copy, Debug)]
pub enum SetFeatureRequest {
NumberOfQueues(u32, u32),
}
#[derive(Clone, Copy, Debug)]
pub struct IdentifyControllerRequest {
pub nsid: u32,
}
#[derive(Clone, Copy, Debug)]
pub struct CreateIoCompletionQueue {
pub id: u32,
pub size: usize,
pub data: PhysicalAddress,
}
#[derive(Clone, Copy, Debug)]
pub struct CreateIoSubmissionQueue {
pub id: u32,
pub cq_id: u32,
pub size: usize,
pub data: PhysicalAddress,
}
// Replies
#[derive(Clone, Copy, Debug)]
#[repr(C)]
pub struct IdentifyControllerResponse {
@ -46,11 +78,6 @@ pub struct IdentifyControllerResponse {
pub cntrltype: ControllerType,
}
#[derive(Clone, Copy, Debug)]
pub struct IdentifyControllerRequest {
pub nsid: u32,
}
impl Command for IdentifyControllerRequest {
fn fill_sqe(&self, sqe: &mut SubmissionQueueEntry) {
sqe.command.set_opcode(0x06);
@ -63,6 +90,42 @@ impl Request for IdentifyControllerRequest {
type Response = IdentifyControllerResponse;
}
impl Command for SetFeatureRequest {
fn fill_sqe(&self, sqe: &mut SubmissionQueueEntry) {
sqe.command.set_opcode(0x09);
match self {
Self::NumberOfQueues(cq, sq) => {
let dw11 = (cq << 16) | sq;
sqe.command_specific[0] = 0x07;
sqe.command_specific[1] = dw11;
}
}
}
}
impl Command for CreateIoCompletionQueue {
fn fill_sqe(&self, sqe: &mut SubmissionQueueEntry) {
sqe.command.set_opcode(0x05);
sqe.data_pointer[1] = PhysicalRegionPage::with_addr(self.data.into_raw());
sqe.command_specific[0] = ((self.size as u32 - 1) << 16) | self.id;
sqe.command_specific[1] = 1;
// TODO ENABLE IRQS HERE
sqe.command_specific[2] = 0;
}
}
impl Command for CreateIoSubmissionQueue {
fn fill_sqe(&self, sqe: &mut SubmissionQueueEntry) {
sqe.command.set_opcode(0x01);
sqe.data_pointer[1] = PhysicalRegionPage::with_addr(self.data.into_raw());
sqe.command_specific[0] = ((self.size as u32 - 1) << 16) | self.id;
// Medium priority
sqe.command_specific[1] = (self.cq_id << 16) | 1;
}
}
impl<const N: usize> fmt::Debug for String<N> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_char('"')?;

78
src/device/nvme/mod.rs
View File

@ -1,11 +1,10 @@
#![allow(missing_docs)]
use core::{mem::size_of, ptr::null_mut, time::Duration};
use core::{mem::size_of, time::Duration};
use abi::error::Error;
use bytemuck::{Pod, Zeroable};
use alloc::vec::Vec;
use device_api::Device;
use kernel_util::{sync::IrqSafeSpinlock, util::OneTimeInit};
use static_assertions::{const_assert, const_assert_eq};
use tock_registers::{
interfaces::{ReadWriteable, Readable, Writeable},
register_bitfields, register_structs,
@ -13,8 +12,6 @@ use tock_registers::{
};
use crate::{
arch::{x86_64::mem::table::L3, Architecture, ARCHITECTURE},
debug,
device::{
bus::pci::{PciBaseAddress, PciCommandRegister, PciConfigurationSpace},
nvme::{
@ -23,17 +20,17 @@ use crate::{
},
},
mem::{
address::{AsPhysicalAddress, FromRaw, IntoRaw},
address::{FromRaw, IntoRaw},
device::DeviceMemoryIo,
phys,
pointer::{PhysicalRef, PhysicalRefMut},
table::EntryLevel,
PhysicalAddress,
},
task::runtime,
};
use self::queue::QueuePair;
use self::{
command::{CreateIoCompletionQueue, CreateIoSubmissionQueue, SetFeatureRequest},
queue::QueuePair,
};
use super::bus::pci::{FromPciBus, PciDeviceInfo};
@ -109,10 +106,13 @@ register_structs! {
pub struct NvmeController {
regs: IrqSafeSpinlock<DeviceMemoryIo<'static, Regs>>,
admin_q: OneTimeInit<QueuePair<'static>>,
ioqs: OneTimeInit<Vec<QueuePair<'static>>>,
doorbell_shift: usize,
}
impl Regs {
unsafe fn doorbell_ptr(&self, shift: u64, completion: bool, queue_index: usize) -> *mut u32 {
unsafe fn doorbell_ptr(&self, shift: usize, completion: bool, queue_index: usize) -> *mut u32 {
let doorbell_base = (self as *const Regs as *mut Regs).addr() + 0x1000;
let offset = (queue_index << shift) + completion as usize * 4;
(doorbell_base + offset) as *mut u32
@ -121,19 +121,49 @@ impl Regs {
impl NvmeController {
async fn late_init(&'static self) {
runtime::spawn(self.poll_task()).expect("Couldn't spawn NVMe poll task");
// let ioq = QueuePair::new(capacity, sq_doorbell, cq_doorbell).unwrap();
runtime::spawn(self.poll_task()).expect("Couldn't spawn NVMe poll task");
let admin_q = self.admin_q.get();
let response = admin_q
// Request a CQ/SQ pair for I/O
admin_q
.request_no_data(SetFeatureRequest::NumberOfQueues(1, 1))
.unwrap()
.await
.unwrap();
// Allocate the queue
let (sq_doorbell, cq_doorbell) = unsafe { self.doorbell_pair(1) };
let io_q = QueuePair::new(32, sq_doorbell, cq_doorbell).unwrap();
// Identify the controller
let identify = admin_q
.request(IdentifyControllerRequest { nsid: 0 })
.unwrap()
.await
.unwrap();
infoln!("Model: {:#?}", response.model_number);
infoln!("Serial: {:#?}", response.serial_number);
infoln!("Type: {:?}", response.cntrltype);
// Create the queue on the device side
admin_q
.request_no_data(CreateIoCompletionQueue {
id: 1,
size: 32,
data: io_q.cq_physical_pointer(),
})
.unwrap()
.await
.unwrap();
admin_q
.request_no_data(CreateIoSubmissionQueue {
id: 1,
cq_id: 1,
size: 32,
data: io_q.sq_physical_pointer(),
})
.unwrap()
.await
.unwrap();
loop {}
}
@ -145,6 +175,13 @@ impl NvmeController {
runtime::sleep(Duration::from_millis(100)).await;
}
}
unsafe fn doorbell_pair(&self, idx: usize) -> (*mut u32, *mut u32) {
let regs = self.regs.lock();
let sq_ptr = regs.doorbell_ptr(self.doorbell_shift, false, idx);
let cq_ptr = regs.doorbell_ptr(self.doorbell_shift, true, idx);
(sq_ptr, cq_ptr)
}
}
impl Device for NvmeController {
@ -174,9 +211,8 @@ impl Device for NvmeController {
}
// Setup the admin queue (index 0)
let doorbell_shift = regs.CAP.read(CAP::DSTRD) + 2;
let admin_sq_doorbell = unsafe { regs.doorbell_ptr(doorbell_shift, false, 0) };
let admin_cq_doorbell = unsafe { regs.doorbell_ptr(doorbell_shift, true, 0) };
let admin_sq_doorbell = unsafe { regs.doorbell_ptr(self.doorbell_shift, false, 0) };
let admin_cq_doorbell = unsafe { regs.doorbell_ptr(self.doorbell_shift, true, 0) };
let admin_q =
QueuePair::new(queue_slots as usize, admin_sq_doorbell, admin_cq_doorbell).unwrap();
@ -238,9 +274,13 @@ impl FromPciBus for NvmeController {
// Disable the controller
regs.CC.modify(CC::ENABLE::CLEAR);
let doorbell_shift = regs.CAP.read(CAP::DSTRD) as usize + 2;
Ok(Self {
regs: IrqSafeSpinlock::new(regs),
admin_q: OneTimeInit::new(),
ioqs: OneTimeInit::new(),
doorbell_shift,
})
}
}

35
src/device/nvme/queue.rs
View File

@ -18,14 +18,7 @@ use static_assertions::const_assert;
use crate::{
arch::x86_64::mem::table::L3,
mem::{
address::{AsPhysicalAddress, IntoRaw},
phys,
pointer::PhysicalRefMut,
table::EntryLevel,
PhysicalAddress,
},
proc,
mem::{address::IntoRaw, phys, pointer::PhysicalRefMut, table::EntryLevel, PhysicalAddress},
task::runtime::QueueWaker,
};
@ -313,7 +306,7 @@ impl<'a> QueuePair<'a> {
this: &'r QueuePair<'r>,
response: Option<R>,
command_id: u32,
};
}
impl<'r, R: Unpin + 'r> Future for Fut<'r, R> {
type Output = Result<R, CommandError>;
@ -354,6 +347,18 @@ impl<'a> QueuePair<'a> {
let mut inner = self.inner.lock();
let mut sqe = SubmissionQueueEntry::zeroed();
match ranges.len() {
1 => {
sqe.data_pointer[0] = PhysicalRegionPage::with_addr(ranges[0].into_raw());
sqe.data_pointer[1] = PhysicalRegionPage::null();
}
0 => {
sqe.data_pointer[0] = PhysicalRegionPage::null();
sqe.data_pointer[1] = PhysicalRegionPage::null();
}
_ => todo!(),
}
cmd.fill_sqe(&mut sqe);
let command_id = inner.sq.tail.try_into().unwrap();
@ -368,6 +373,17 @@ impl<'a> QueuePair<'a> {
Ok(command_id)
}
pub fn request_no_data<'r, C: Command>(
&'r self,
req: C,
) -> Result<impl Future<Output = Result<(), CommandError>> + 'r, Error>
where
'r: 'a,
{
let command_id = self.submit(req, &[], true)?;
Ok(self.wait_for_completion(command_id, ()))
}
pub fn request<'r, R: Request>(
&'r self,
req: R,
@ -376,6 +392,7 @@ impl<'a> QueuePair<'a> {
R::Response: 'r,
'r: 'a,
{
assert_ne!(size_of::<R::Response>(), 0);
assert!(size_of::<R::Response>() < 0x1000);
let page = phys::alloc_page()?;