From e812453a97a3f893b16b8b95d7ef0c560e89e893 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Thu, 6 Feb 2025 21:05:53 +0200 Subject: [PATCH] dev: use DmaBuffer instead of PageBox where needed --- Cargo.lock | 4 + kernel/driver/block/ahci/src/command.rs | 32 +-- kernel/driver/block/ahci/src/data.rs | 14 +- kernel/driver/block/ahci/src/lib.rs | 20 +- kernel/driver/block/ahci/src/port.rs | 58 ++-- kernel/driver/block/ahci/src/regs.rs | 10 +- kernel/driver/block/nvme/src/command.rs | 6 +- kernel/driver/block/nvme/src/drive.rs | 28 +- kernel/driver/block/nvme/src/lib.rs | 44 ++- kernel/driver/block/nvme/src/queue.rs | 54 ++-- kernel/driver/block/scsi/src/lib.rs | 45 ++- kernel/driver/block/scsi/src/transport.rs | 21 +- kernel/driver/bus/pci/src/driver.rs | 8 +- kernel/driver/bus/pci/src/lib.rs | 8 +- .../bus/usb/src/class_driver/hid_keyboard.rs | 5 +- .../bus/usb/src/class_driver/mass_storage.rs | 58 ++-- kernel/driver/bus/usb/src/device.rs | 17 +- kernel/driver/bus/usb/src/error.rs | 5 +- kernel/driver/bus/usb/src/lib.rs | 9 +- kernel/driver/bus/usb/src/pipe/control.rs | 147 ++++------ kernel/driver/bus/usb/src/pipe/mod.rs | 10 + kernel/driver/bus/usb/src/pipe/normal.rs | 37 ++- kernel/driver/net/core/src/ethernet.rs | 4 +- kernel/driver/net/core/src/interface.rs | 17 +- kernel/driver/net/core/src/l3/mod.rs | 8 +- kernel/driver/net/core/src/lib.rs | 18 +- kernel/driver/net/core/src/socket/raw.rs | 10 +- kernel/driver/net/loopback/Cargo.toml | 2 + kernel/driver/net/loopback/src/lib.rs | 25 +- kernel/driver/net/rtl81xx/src/lib.rs | 10 +- kernel/driver/net/rtl81xx/src/rtl8139.rs | 56 ++-- kernel/driver/net/rtl81xx/src/rtl8168.rs | 100 ++++--- kernel/driver/usb/xhci/src/context.rs | 51 ++-- kernel/driver/usb/xhci/src/controller.rs | 77 ++--- kernel/driver/usb/xhci/src/device.rs | 14 +- kernel/driver/usb/xhci/src/lib.rs | 8 +- kernel/driver/usb/xhci/src/pipe.rs | 115 ++++++-- .../driver/usb/xhci/src/regs/operational.rs | 7 +- kernel/driver/usb/xhci/src/regs/runtime.rs | 10 +- kernel/driver/usb/xhci/src/ring/command.rs | 62 ++--- kernel/driver/usb/xhci/src/ring/event.rs | 60 ++-- kernel/driver/usb/xhci/src/ring/mod.rs | 15 +- kernel/driver/usb/xhci/src/ring/transfer.rs | 129 +++++---- kernel/driver/virtio/core/Cargo.toml | 1 + kernel/driver/virtio/core/src/queue.rs | 60 ++-- .../driver/virtio/core/src/transport/mod.rs | 15 +- kernel/driver/virtio/gpu/src/command.rs | 58 ++-- kernel/driver/virtio/gpu/src/lib.rs | 96 ++++--- kernel/driver/virtio/net/Cargo.toml | 1 + kernel/driver/virtio/net/src/lib.rs | 48 ++-- kernel/lib/device-api/src/dma.rs | 16 +- kernel/libk/src/device/manager.rs | 25 +- kernel/libk/src/dma.rs | 263 +++++++++++++++++- kernel/libk/src/lib.rs | 1 + kernel/src/arch/x86/mod.rs | 21 +- kernel/src/arch/x86/peripherals/hpet.rs | 14 +- kernel/src/arch/x86/peripherals/i8259.rs | 4 +- kernel/src/arch/x86/peripherals/ps2/mod.rs | 4 +- kernel/src/arch/x86/peripherals/serial.rs | 13 +- kernel/src/arch/x86_64/apic/ioapic.rs | 4 +- kernel/src/device/interrupt/riscv_plic.rs | 4 +- kernel/src/device/serial/ns16550a.rs | 4 +- kernel/src/device/serial/snps_dw_apb_uart.rs | 4 +- 63 files changed, 1328 insertions(+), 766 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a1ea5b22..20e9de71 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2681,6 +2681,8 @@ name = "ygg_driver_net_loopback" version = "0.1.0" dependencies = [ "bytemuck", + "device-api", + "libk", "libk-mm", "libk-util", "ygg_driver_net_core", @@ -2794,6 +2796,7 @@ version = "0.1.0" dependencies = [ "bitflags 2.8.0", "device-api", + "libk", "libk-mm", "libk-util", "log", @@ -2824,6 +2827,7 @@ dependencies = [ "bitflags 2.8.0", "bytemuck", "device-api", + "libk", "libk-mm", "libk-util", "log", diff --git a/kernel/driver/block/ahci/src/command.rs b/kernel/driver/block/ahci/src/command.rs index 2c9f84d2..a6f524a4 100644 --- a/kernel/driver/block/ahci/src/command.rs +++ b/kernel/driver/block/ahci/src/command.rs @@ -1,9 +1,7 @@ use core::mem::{size_of, MaybeUninit}; -use libk_mm::{ - address::{AsPhysicalAddress, PhysicalAddress}, - PageBox, PageSlice, -}; +use device_api::dma::DmaAllocator; +use libk::dma::{BusAddress, DmaBuffer}; use tock_registers::register_structs; use crate::{data::AtaString, error::AhciError, MAX_PRD_SIZE, SECTOR_SIZE}; @@ -22,7 +20,7 @@ pub trait AtaCommand { fn lba(&self) -> u64; fn sector_count(&self) -> usize; - fn buffer(&self) -> Option<(PhysicalAddress, usize)>; + fn buffer(&self) -> Option<(BusAddress, usize)>; unsafe fn into_response(self) -> Self::Response; fn prd_count(&self) -> usize { @@ -64,44 +62,44 @@ register_structs! { } pub struct AtaIdentify { - buffer: PageBox>, + buffer: DmaBuffer>, } pub struct AtaReadDmaEx { lba: u64, sector_count: usize, - buffer_base: PhysicalAddress, + buffer_base: BusAddress, buffer_size: usize, } impl AtaIdentify { - pub fn create() -> Result { - PageBox::new_uninit() + pub fn create(dma: &dyn DmaAllocator) -> Result { + DmaBuffer::new_uninit(dma) .map(Self::with_data) .map_err(AhciError::MemoryError) } - pub fn with_data(buffer: PageBox>) -> Self { + pub fn with_data(buffer: DmaBuffer>) -> Self { Self { buffer } } } impl AtaReadDmaEx { - pub fn new(lba: u64, sector_count: usize, buffer: &PageSlice>) -> Self { + pub fn new(lba: u64, sector_count: usize, buffer: &mut DmaBuffer<[MaybeUninit]>) -> Self { assert_eq!(buffer.len() % SECTOR_SIZE, 0); assert_ne!(buffer.len(), 0); Self { lba, sector_count, - buffer_base: unsafe { buffer.as_physical_address() }, + buffer_base: buffer.bus_address(), buffer_size: buffer.len(), } } } impl AtaCommand for AtaIdentify { - type Response = PageBox; + type Response = DmaBuffer; const COMMAND_ID: AtaCommandId = AtaCommandId::Identify; @@ -113,14 +111,14 @@ impl AtaCommand for AtaIdentify { 0 } - fn buffer(&self) -> Option<(PhysicalAddress, usize)> { - let base = unsafe { self.buffer.as_physical_address() }; + fn buffer(&self) -> Option<(BusAddress, usize)> { + let base = self.buffer.bus_address(); let size = size_of::(); Some((base, size)) } unsafe fn into_response(self) -> Self::Response { - self.buffer.assume_init() + DmaBuffer::assume_init(self.buffer) } } @@ -137,7 +135,7 @@ impl AtaCommand for AtaReadDmaEx { self.sector_count } - fn buffer(&self) -> Option<(PhysicalAddress, usize)> { + fn buffer(&self) -> Option<(BusAddress, usize)> { Some((self.buffer_base, self.buffer_size)) } diff --git a/kernel/driver/block/ahci/src/data.rs b/kernel/driver/block/ahci/src/data.rs index e309ef9b..1d015de1 100644 --- a/kernel/driver/block/ahci/src/data.rs +++ b/kernel/driver/block/ahci/src/data.rs @@ -2,7 +2,7 @@ use core::mem::size_of; use alloc::string::String; use bytemuck::{Pod, Zeroable}; -use libk_mm::address::PhysicalAddress; +use libk::dma::BusAddress; use libk_util::{ConstAssert, IsTrue}; use static_assertions::const_assert_eq; @@ -174,7 +174,7 @@ impl CommandTable { } impl CommandListEntry { - pub fn new(command_table_entry: PhysicalAddress, prd_count: usize) -> Result { + pub fn new(command_table_entry: BusAddress, prd_count: usize) -> Result { if prd_count > 0xFFFF { todo!() } @@ -183,7 +183,7 @@ impl CommandListEntry { attr: (size_of::() / size_of::()) as _, prdtl: prd_count as _, prdbc: 0, - ctba: command_table_entry.into(), + ctba: command_table_entry.into_u64(), _0: [0; 4], }) } @@ -201,18 +201,14 @@ unsafe impl Zeroable for CommandTable { } impl PhysicalRegionDescriptor { - pub fn new( - address: PhysicalAddress, - byte_count: usize, - is_last: bool, - ) -> Result { + pub fn new(address: BusAddress, byte_count: usize, is_last: bool) -> Result { if byte_count > MAX_PRD_SIZE { return Err(AhciError::RegionTooLarge); } let dbc_mask = (is_last as u32) << 31; Ok(Self { - buffer_address: address.into(), + buffer_address: address.into_u64(), _0: 0, dbc: ((byte_count as u32 - 1) << 1) | 1 | dbc_mask, }) diff --git a/kernel/driver/block/ahci/src/lib.rs b/kernel/driver/block/ahci/src/lib.rs index a34fbf92..362fa78b 100644 --- a/kernel/driver/block/ahci/src/lib.rs +++ b/kernel/driver/block/ahci/src/lib.rs @@ -9,11 +9,12 @@ use bytemuck::Zeroable; use data::ReceivedFis; use device_api::{ device::{Device, DeviceInitContext}, + dma::DmaAllocator, interrupt::{InterruptAffinity, InterruptHandler, IrqVector}, }; use error::AhciError; -use libk::{device::manager::probe_partitions, fs::devfs, task::runtime}; -use libk_mm::{address::AsPhysicalAddress, device::DeviceMemoryIo, PageBox}; +use libk::{device::manager::probe_partitions, dma::DmaBuffer, fs::devfs, task::runtime}; +use libk_mm::device::DeviceMemoryIo; use libk_util::{sync::IrqSafeSpinlock, OneTimeInit}; use port::AhciPort; use regs::{PortRegs, Regs}; @@ -40,8 +41,9 @@ const MAX_DRIVES: usize = (b'z' - b'a') as usize; pub struct AhciController { regs: IrqSafeSpinlock>, + dma: Arc, ports: OneTimeInit>>, - received_fis_buffers: OneTimeInit<[Option>; 16]>, + received_fis_buffers: OneTimeInit<[Option>; 16]>, version: Version, max_port_count: usize, @@ -81,8 +83,9 @@ impl AhciController { let regs = self.regs.lock(); let port = ®s.PORTS[i]; - let buffer = PageBox::new(ReceivedFis::zeroed()).map_err(AhciError::MemoryError)?; - port.set_received_fis_address_64(unsafe { buffer.as_physical_address() }); + let buffer = DmaBuffer::new(&*self.dma, ReceivedFis::zeroed()) + .map_err(AhciError::MemoryError)?; + port.set_received_fis_address_64(buffer.bus_address()); *fis_buffer_slot = Some(buffer); } @@ -181,9 +184,7 @@ impl InterruptHandler for AhciController { } impl Device for AhciController { - unsafe fn init(self: Arc, cx: DeviceInitContext) -> Result<(), Error> { - // TODO use DmaAllocator instead of PageBox - let _ = cx; + unsafe fn init(self: Arc, _cx: DeviceInitContext) -> Result<(), Error> { // Do the init in background runtime::spawn(self.late_init())?; Ok(()) @@ -241,7 +242,7 @@ pci_driver! { "ahci" } - fn probe(&self, info: &PciDeviceInfo) -> Result, Error> { + fn probe(&self, info: &PciDeviceInfo, dma: &Arc) -> Result, Error> { let bar5 = info.config_space.bar(5).ok_or(Error::InvalidOperation)?; let bar5 = bar5.as_memory().ok_or(Error::InvalidOperation)?; @@ -270,6 +271,7 @@ pci_driver! { let ahci = Arc::new(AhciController { regs: IrqSafeSpinlock::new(regs), + dma: dma.clone(), ports: OneTimeInit::new(), received_fis_buffers: OneTimeInit::new(), version, diff --git a/kernel/driver/block/ahci/src/port.rs b/kernel/driver/block/ahci/src/port.rs index 026b4020..8a918ff9 100644 --- a/kernel/driver/block/ahci/src/port.rs +++ b/kernel/driver/block/ahci/src/port.rs @@ -8,14 +8,11 @@ use core::{ use alloc::{boxed::Box, string::String, sync::Arc}; use async_trait::async_trait; use bytemuck::Zeroable; -use device_api::device::Device; +use device_api::{device::Device, dma::DmaAllocator}; use futures_util::task::AtomicWaker; -use libk::{device::block::BlockDevice, error::Error}; +use libk::{device::block::BlockDevice, dma::DmaBuffer, error::Error}; use libk_mm::{ - address::{AsPhysicalAddress, PhysicalAddress}, - device::DeviceMemoryIo, - table::MapAttributes, - PageBox, PageProvider, PageSlice, + address::PhysicalAddress, device::DeviceMemoryIo, table::MapAttributes, PageProvider, PageSlice, }; use libk_util::{sync::IrqSafeSpinlock, waker::QueueWaker, OneTimeInit}; use tock_registers::interfaces::{Readable, Writeable}; @@ -37,8 +34,8 @@ struct PortInner { regs: DeviceMemoryIo<'static, PortRegs>, #[allow(unused)] - received_fis: PageBox, - command_list: PageBox<[CommandListEntry]>, + received_fis: DmaBuffer, + command_list: DmaBuffer<[CommandListEntry]>, } pub struct PortInfo { @@ -90,18 +87,16 @@ impl Drop for SubmittedCommand<'_> { impl PortInner { fn submit_command( &mut self, + dma: &dyn DmaAllocator, index: usize, command: &C, ) -> Result<(), AhciError> { let list_entry = &mut self.command_list[index]; let mut table_entry = - PageBox::new(CommandTable::zeroed()).map_err(AhciError::MemoryError)?; + DmaBuffer::new(dma, CommandTable::zeroed()).map_err(AhciError::MemoryError)?; table_entry.setup_command(command)?; - *list_entry = CommandListEntry::new( - unsafe { table_entry.as_physical_address() }, - command.prd_count(), - )?; + *list_entry = CommandListEntry::new(table_entry.bus_address(), command.prd_count())?; // Sync before send // XXX do this properly @@ -137,12 +132,14 @@ impl AhciPort { return Err(AhciError::DeviceError); } - let received_fis = PageBox::new(ReceivedFis::zeroed()).map_err(AhciError::MemoryError)?; - let command_list = PageBox::new_slice(CommandListEntry::zeroed(), COMMAND_LIST_LENGTH) - .map_err(AhciError::MemoryError)?; + let received_fis = + DmaBuffer::new(&*ahci.dma, ReceivedFis::zeroed()).map_err(AhciError::MemoryError)?; + let command_list = + DmaBuffer::new_slice(&*ahci.dma, CommandListEntry::zeroed(), COMMAND_LIST_LENGTH) + .map_err(AhciError::MemoryError)?; - regs.set_received_fis_address_64(unsafe { received_fis.as_physical_address() }); - regs.set_command_list_address_64(unsafe { command_list.as_physical_address() }); + regs.set_received_fis_address_64(received_fis.bus_address()); + regs.set_command_list_address_64(command_list.bus_address()); regs.IE.write( IE::DPE::SET @@ -182,7 +179,9 @@ impl AhciPort { } pub async fn init_inner(&self) -> Result<(), AhciError> { - let identify = self.perform_command(AtaIdentify::create()?).await?; + let identify = self + .perform_command(AtaIdentify::create(&*self.ahci.dma)?) + .await?; let model = identify.model_number.to_string(); let serial = identify.serial_number.to_string(); @@ -237,7 +236,11 @@ impl AhciPort { return Err(AhciError::RegionTooLarge); } let index = self.allocate_command().await; - if let Err(error) = self.inner.lock().submit_command(index, command) { + if let Err(error) = self + .inner + .lock() + .submit_command(&*self.ahci.dma, index, command) + { self.free_command(index); return Err(error); } @@ -302,6 +305,7 @@ impl AhciPort { #[async_trait] impl BlockDevice for AhciPort { + // TODO read directly into cache async fn read_aligned( &self, position: u64, @@ -314,13 +318,15 @@ impl BlockDevice for AhciPort { return Err(Error::InvalidOperation); } + let mut dma_buffer = DmaBuffer::new_uninit_slice(&*self.ahci.dma, buffer.len())?; let lba = position / SECTOR_SIZE as u64; - let command = AtaReadDmaEx::new(lba, buffer.len() / SECTOR_SIZE, buffer); - self.submit(&command) - .await? - .wait_for_completion() - .await - .map_err(AhciError::into) + + let command = AtaReadDmaEx::new(lba, buffer.len() / SECTOR_SIZE, &mut dma_buffer); + self.submit(&command).await?.wait_for_completion().await?; + + buffer.copy_from_slice(&dma_buffer[..]); + + Ok(()) } async fn write_aligned(&self, _position: u64, _buffer: &PageSlice) -> Result<(), Error> { diff --git a/kernel/driver/block/ahci/src/regs.rs b/kernel/driver/block/ahci/src/regs.rs index a754eb69..f7ff0f25 100644 --- a/kernel/driver/block/ahci/src/regs.rs +++ b/kernel/driver/block/ahci/src/regs.rs @@ -1,4 +1,4 @@ -use libk_mm::address::PhysicalAddress; +use libk::dma::BusAddress; use tock_registers::{ interfaces::{ReadWriteable, Readable, Writeable}, register_bitfields, register_structs, @@ -141,14 +141,14 @@ impl PortRegs { Ok(()) } - pub fn set_received_fis_address_64(&self, address: PhysicalAddress) { - let address: u64 = address.into(); + pub fn set_received_fis_address_64(&self, address: BusAddress) { + let address: u64 = address.into_u64(); self.FB.set(address as u32); self.FBU.set((address >> 32) as u32); } - pub fn set_command_list_address_64(&self, address: PhysicalAddress) { - let address: u64 = address.into(); + pub fn set_command_list_address_64(&self, address: BusAddress) { + let address: u64 = address.into_u64(); self.CLB.set(address as u32); self.CLBU.set((address >> 32) as u32); } diff --git a/kernel/driver/block/nvme/src/command.rs b/kernel/driver/block/nvme/src/command.rs index cb853dc7..a40a4026 100644 --- a/kernel/driver/block/nvme/src/command.rs +++ b/kernel/driver/block/nvme/src/command.rs @@ -2,7 +2,7 @@ use core::fmt::{self, Write}; -use libk_mm::address::PhysicalAddress; +use libk::dma::BusAddress; use tock_registers::{interfaces::Readable, register_structs, registers::ReadOnly, UIntLike}; use crate::queue::PhysicalRegionPage; @@ -74,7 +74,7 @@ pub struct CreateIoCompletionQueue { pub id: u32, pub size: usize, pub vector: u32, - pub data: PhysicalAddress, + pub data: BusAddress, } #[derive(Clone, Copy, Debug)] @@ -82,7 +82,7 @@ pub struct CreateIoSubmissionQueue { pub id: u32, pub cq_id: u32, pub size: usize, - pub data: PhysicalAddress, + pub data: BusAddress, } // Replies diff --git a/kernel/driver/block/nvme/src/drive.rs b/kernel/driver/block/nvme/src/drive.rs index 97b661ab..4399af3f 100644 --- a/kernel/driver/block/nvme/src/drive.rs +++ b/kernel/driver/block/nvme/src/drive.rs @@ -3,7 +3,7 @@ use core::mem::MaybeUninit; use alloc::{boxed::Box, sync::Arc}; use async_trait::async_trait; use device_api::device::Device; -use libk::{device::block::BlockDevice, error::Error}; +use libk::{device::block::BlockDevice, dma::DmaBuffer, error::Error}; use libk_mm::{ address::{AsPhysicalAddress, PhysicalAddress}, table::MapAttributes, @@ -30,7 +30,9 @@ impl NvmeNamespace { max_transfer_size: usize, ) -> Result, NvmeError> { let admin_q = controller.admin_q.get(); - let identify = admin_q.request(IdentifyNamespaceRequest { nsid }).await?; + let identify = admin_q + .request(&*controller.dma, IdentifyNamespaceRequest { nsid }) + .await?; let current_lba_format_idx = identify.current_lba_fmt_idx(); let current_lba_format = identify.lba_fmt(current_lba_format_idx).unwrap(); @@ -76,6 +78,7 @@ impl Device for NvmeNamespace { #[async_trait] impl BlockDevice for NvmeNamespace { + // TODO read directly to cache async fn read_aligned( &self, position: u64, @@ -84,23 +87,27 @@ impl BlockDevice for NvmeNamespace { debug_assert_eq!(position % self.block_size() as u64, 0); let lba = position / self.block_size() as u64; debug_assert_eq!(buffer.len() % self.block_size(), 0); - let buffer_address = unsafe { buffer.as_physical_address() }; - debug_assert_eq!(buffer_address.into_u64() % self.block_size() as u64, 0); let lba_count = buffer.len() / self.block_size(); + let mut dma_buffer = DmaBuffer::new_uninit_slice(&*self.controller.dma, buffer.len())?; + let result = self .controller .perform_io( self.nsid, lba, lba_count, - buffer_address, + dma_buffer.bus_address(), buffer.len(), IoDirection::Read, ) .await; - log::info!(target: "io", "read #{lba}, {lba_count} blocks -> {result:?} @ {buffer_address:#x}"); + log::info!(target: "io", "read #{lba}, {lba_count} blocks -> {result:?} @ {dma_buffer:p}"); + + if result.is_ok() { + buffer.copy_from_slice(&dma_buffer[..]); + } result.map_err(NvmeError::into) } @@ -109,8 +116,8 @@ impl BlockDevice for NvmeNamespace { debug_assert_eq!(position % self.block_size() as u64, 0); let lba = position / self.block_size() as u64; debug_assert_eq!(buffer.len() % self.block_size(), 0); - let buffer_address = unsafe { buffer.as_physical_address() }; - debug_assert_eq!(buffer_address.into_u64() % self.block_size() as u64, 0); + // let buffer_address = unsafe { buffer.as_physical_address() }; + // debug_assert_eq!(buffer_address.into_u64() % self.block_size() as u64, 0); let lba_count = buffer.len() / self.block_size(); // TODO ArchitectureImpl::flush_data_cache() @@ -119,19 +126,20 @@ impl BlockDevice for NvmeNamespace { core::arch::asm!("wbinvd"); } + let dma_buffer = DmaBuffer::from_slice(&*self.controller.dma, &buffer[..])?; let result = self .controller .perform_io( self.nsid, lba, lba_count, - buffer_address, + dma_buffer.bus_address(), buffer.len(), IoDirection::Write, ) .await; - log::info!(target: "io", "write -> #{lba}, {lba_count} blocks -> {result:?} @ {buffer_address:#x}"); + log::info!(target: "io", "write -> #{lba}, {lba_count} blocks -> {result:?} @ {dma_buffer:p}"); result.map_err(NvmeError::into) } diff --git a/kernel/driver/block/nvme/src/lib.rs b/kernel/driver/block/nvme/src/lib.rs index 54e893a1..8dbaae4b 100644 --- a/kernel/driver/block/nvme/src/lib.rs +++ b/kernel/driver/block/nvme/src/lib.rs @@ -16,11 +16,13 @@ use alloc::{collections::BTreeMap, format, sync::Arc, vec::Vec}; use command::{IdentifyActiveNamespaceIdListRequest, IdentifyControllerRequest}; use device_api::{ device::{Device, DeviceInitContext}, + dma::DmaAllocator, interrupt::{InterruptAffinity, InterruptHandler, IrqVector}, }; use drive::NvmeNamespace; use libk::{ device::manager::probe_partitions, + dma::BusAddress, fs::devfs, task::{cpu_count, cpu_index, runtime}, }; @@ -137,6 +139,7 @@ pub struct NvmeController { controller_id: OneTimeInit, pci: PciDeviceInfo, + dma: Arc, doorbell_shift: usize, min_page_size: usize, @@ -183,15 +186,22 @@ impl NvmeController { let id = i as u32; let (sq_doorbell, cq_doorbell) = unsafe { self.doorbell_pair(i) }; - let queue = QueuePair::new(id, i, Self::IO_QUEUE_SIZE, sq_doorbell, cq_doorbell) - .map_err(NvmeError::MemoryError)?; + let queue = QueuePair::new( + &*self.dma, + id, + i, + Self::IO_QUEUE_SIZE, + sq_doorbell, + cq_doorbell, + ) + .map_err(NvmeError::MemoryError)?; admin_q .request_no_data(CreateIoCompletionQueue { id, vector: id, size: Self::IO_QUEUE_SIZE, - data: queue.cq_physical_pointer(), + data: queue.cq_bus_pointer(), }) .await?; @@ -200,7 +210,7 @@ impl NvmeController { id, cq_id: id, size: Self::IO_QUEUE_SIZE, - data: queue.sq_physical_pointer(), + data: queue.sq_bus_pointer(), }) .await?; @@ -233,7 +243,9 @@ impl NvmeController { let admin_q = self.admin_q.get(); // Identify the controller - let identify = admin_q.request(IdentifyControllerRequest).await?; + let identify = admin_q + .request(&*self.dma, IdentifyControllerRequest) + .await?; let max_transfer_size = if identify.mdts == 0 { // Pick some sane default value @@ -257,7 +269,10 @@ impl NvmeController { let admin_q = self.admin_q.get(); let namespaces = admin_q - .request(IdentifyActiveNamespaceIdListRequest { start_id: 0 }) + .request( + &*self.dma, + IdentifyActiveNamespaceIdListRequest { start_id: 0 }, + ) .await?; let count = namespaces.entries.iter().position(|&x| x == 0).unwrap(); @@ -282,11 +297,11 @@ impl NvmeController { nsid: u32, lba: u64, lba_count: usize, - buffer_address: PhysicalAddress, + buffer_address: BusAddress, transfer_size: usize, direction: IoDirection, ) -> Result<(), NvmeError> { - let prp_list = PrpList::from_buffer(buffer_address, transfer_size)?; + let prp_list = PrpList::from_buffer(&*self.dma, buffer_address, transfer_size)?; let _guard = IrqGuard::acquire(); let cpu_index = cpu_index(); @@ -345,10 +360,7 @@ impl InterruptHandler for NvmeController { } impl Device for NvmeController { - unsafe fn init(self: Arc, cx: DeviceInitContext) -> Result<(), Error> { - // TODO use DmaAllocator instead of PageBox - let _ = cx; - + unsafe fn init(self: Arc, _cx: DeviceInitContext) -> Result<(), Error> { let regs = self.regs.lock(); let timeout = Duration::from_millis(regs.CAP.read(CAP::TO) * 500); @@ -370,6 +382,7 @@ impl Device for NvmeController { let admin_cq_doorbell = unsafe { regs.doorbell_ptr(self.doorbell_shift, true, 0) }; log::debug!("sq_doorbell for adminq = {:p}", admin_sq_doorbell); let admin_q = QueuePair::new( + &*self.dma, 0, 0, Self::ADMIN_QUEUE_SIZE, @@ -382,8 +395,8 @@ impl Device for NvmeController { AQA::ASQS.val(Self::ADMIN_QUEUE_SIZE as u32 - 1) + AQA::ACQS.val(Self::ADMIN_QUEUE_SIZE as u32 - 1), ); - regs.ASQ.set(admin_q.sq_physical_pointer().into()); - regs.ACQ.set(admin_q.cq_physical_pointer().into()); + regs.ASQ.set(admin_q.sq_bus_pointer().into_u64()); + regs.ACQ.set(admin_q.cq_bus_pointer().into_u64()); // Configure the controller const IOSQES: u32 = size_of::().ilog2(); @@ -467,7 +480,7 @@ pci_driver! { "nvme" } - fn probe(&self, info: &PciDeviceInfo) -> Result, Error> { + fn probe(&self, info: &PciDeviceInfo, dma: &Arc) -> Result, Error> { let bar0 = info .config_space .bar(0) @@ -503,6 +516,7 @@ pci_driver! { controller_id: OneTimeInit::new(), pci: info.clone(), + dma: dma.clone(), io_queue_count: AtomicUsize::new(1), doorbell_shift, diff --git a/kernel/driver/block/nvme/src/queue.rs b/kernel/driver/block/nvme/src/queue.rs index 8e12480b..b963ba29 100644 --- a/kernel/driver/block/nvme/src/queue.rs +++ b/kernel/driver/block/nvme/src/queue.rs @@ -2,10 +2,9 @@ use core::{future::poll_fn, mem::size_of, ptr::null_mut, task::Poll}; use alloc::collections::{BTreeMap, BTreeSet}; use bytemuck::{Pod, Zeroable}; -use libk_mm::{ - address::{AsPhysicalAddress, PhysicalAddress}, - PageBox, -}; +use device_api::dma::DmaAllocator; +use libk::dma::{BusAddress, DmaBuffer}; +use libk_mm::address::AsPhysicalAddress; use libk_util::{sync::IrqSafeSpinlock, waker::QueueWaker}; use static_assertions::const_assert; use yggdrasil_abi::error::Error; @@ -58,7 +57,7 @@ pub struct CompletionQueueEntry { } pub struct Queue { - data: PageBox<[T]>, + data: DmaBuffer<[T]>, mask: usize, head: usize, tail: usize, @@ -82,8 +81,8 @@ pub struct QueuePair { #[allow(unused)] vector: usize, - sq_base: PhysicalAddress, - cq_base: PhysicalAddress, + sq_base: BusAddress, + cq_base: BusAddress, pub completion_notify: QueueWaker, @@ -94,7 +93,7 @@ pub struct PrpList { prp1: PhysicalRegionPage, prp2: PhysicalRegionPage, #[allow(unused)] - list: Option>, + list: Option>, } impl PrpList { @@ -106,7 +105,11 @@ impl PrpList { } } - pub fn from_buffer(base: PhysicalAddress, size: usize) -> Result { + pub fn from_buffer( + dma: &dyn DmaAllocator, + base: BusAddress, + size: usize, + ) -> Result { // TODO hardcoded page size if base.into_u64() % 0x1000 != 0 { todo!(); @@ -126,12 +129,13 @@ impl PrpList { }), _ => { let count = (size + 0xFFF) / 0x1000; - let list = PageBox::new_slice_with(|i| base.add((i + 1) * 0x1000), count - 1) - .map_err(NvmeError::MemoryError)?; + let list = + DmaBuffer::new_slice_with(dma, |i| base.add((i + 1) * 0x1000), count - 1) + .map_err(NvmeError::MemoryError)?; Ok(Self { prp1: PhysicalRegionPage::with_addr(base), - prp2: PhysicalRegionPage::with_addr(unsafe { list.as_physical_address() }), + prp2: PhysicalRegionPage::with_addr(list.bus_address()), list: Some(list), }) } @@ -146,7 +150,7 @@ impl PhysicalRegionPage { Self(0) } - pub const fn with_addr(address: PhysicalAddress) -> Self { + pub const fn with_addr(address: BusAddress) -> Self { Self(address.into_u64()) } } @@ -197,7 +201,7 @@ impl CompletionQueueEntry { impl Queue { pub fn new( - data: PageBox<[T]>, + data: DmaBuffer<[T]>, head_doorbell: *mut u32, tail_doorbell: *mut u32, phase: bool, @@ -277,17 +281,18 @@ impl Queue { impl QueuePair { pub fn new( + dma: &dyn DmaAllocator, id: u32, vector: usize, capacity: usize, sq_doorbell: *mut u32, cq_doorbell: *mut u32, ) -> Result { - let sq_data = PageBox::new_slice(SubmissionQueueEntry::zeroed(), capacity)?; - let cq_data = PageBox::new_slice(CompletionQueueEntry::zeroed(), capacity)?; + let sq_data = DmaBuffer::new_slice(dma, SubmissionQueueEntry::zeroed(), capacity)?; + let cq_data = DmaBuffer::new_slice(dma, CompletionQueueEntry::zeroed(), capacity)?; - let sq_base = unsafe { sq_data.as_physical_address() }; - let cq_base = unsafe { cq_data.as_physical_address() }; + let sq_base = sq_data.bus_address(); + let cq_base = cq_data.bus_address(); log::debug!("Allocated queue pair: sq={:p}, cq={:p}", sq_data, cq_data); @@ -313,12 +318,12 @@ impl QueuePair { } #[inline] - pub fn sq_physical_pointer(&self) -> PhysicalAddress { + pub fn sq_bus_pointer(&self) -> BusAddress { self.sq_base } #[inline] - pub fn cq_physical_pointer(&self) -> PhysicalAddress { + pub fn cq_bus_pointer(&self) -> BusAddress { self.cq_base } @@ -385,16 +390,17 @@ impl QueuePair { pub async fn request<'r, R: Request>( &'r self, + dma: &dyn DmaAllocator, req: R, - ) -> Result, NvmeError> + ) -> Result, NvmeError> where R::Response: 'r, { - let response = PageBox::new_uninit().map_err(NvmeError::MemoryError)?; - let list = PrpList::from_buffer(unsafe { response.as_physical_address() }, size_of::())?; + let response = DmaBuffer::new_uninit(dma).map_err(NvmeError::MemoryError)?; + let list = PrpList::from_buffer(dma, response.bus_address(), size_of::())?; let command_id = self.submit(req, &list, true)?; let result = self.wait_for_completion(command_id, response).await?; - Ok(unsafe { result.assume_init() }) + Ok(unsafe { DmaBuffer::assume_init(result) }) } pub fn process_completions(&self) -> usize { diff --git a/kernel/driver/block/scsi/src/lib.rs b/kernel/driver/block/scsi/src/lib.rs index a7d8ad66..68bb97a8 100644 --- a/kernel/driver/block/scsi/src/lib.rs +++ b/kernel/driver/block/scsi/src/lib.rs @@ -35,6 +35,7 @@ pub struct ScsiDevice { transport: IrqSafeSpinlock, lba_count: u64, lba_size: usize, + max_lba_per_request: usize, index: OneTimeInit, names: IrqSafeRwLock>, } @@ -76,16 +77,20 @@ impl ScsiDevice { // transport.perform_command(0, ScsiInquiry).await?; let capacity_info = transport.perform_command(0, ScsiReadCapacity).await?; + let max_lba_per_request = + transport.max_bytes_per_request() / capacity_info.block_size as usize; log::info!( - "scsi: lba_size={}, lba_count={}", + "scsi: lba_size={}, lba_count={}, max_lba_per_request={}", capacity_info.block_size, - capacity_info.block_count + capacity_info.block_count, + max_lba_per_request ); Ok(Arc::new(Self { transport: IrqSafeSpinlock::new(transport), lba_count: capacity_info.block_count.into(), lba_size: capacity_info.block_size as usize, + max_lba_per_request, index: OneTimeInit::new(), names: IrqSafeRwLock::new(Vec::new()), })) @@ -100,34 +105,44 @@ impl ScsiDevice { #[async_trait] impl BlockDevice for ScsiDevice { + // TODO avoid copies by reading directly into the cache? async fn read_aligned( &self, position: u64, buffer: &mut PageSlice>, ) -> Result<(), Error> { if buffer.len() % self.lba_size != 0 { + log::warn!("scsi: buffer is not multiple of LBA size"); return Err(Error::InvalidArgument); } + let lba_start = position / self.lba_size as u64; let lba_count = buffer.len() / self.lba_size; + let lba_end = lba_start + lba_count as u64; + if lba_start.saturating_add(lba_count as u64) >= self.lba_count { + log::warn!("scsi: read beyond medium end"); return Err(Error::InvalidArgument); } - log::info!("scsi: read lba={lba_start}, count={lba_count}"); - let mut transport = self.transport.lock(); - for i in 0..lba_count { - let lba = lba_start + i as u64; - let offset = self.lba_size * i; - let slice = unsafe { - MaybeUninit::slice_assume_init_mut(&mut buffer[offset..offset + self.lba_size]) - }; - let len = transport.read(0, lba, slice).await?; - if len != self.lba_size { - log::warn!("scsi: truncated read received at lba {lba}"); - return Err(Error::InvalidOperation); + let mut transport = self.transport.lock(); + + let mut offset = 0; + for i in (0..lba_count).step_by(self.max_lba_per_request) { + let lba = lba_start + i as u64; + let end = (lba + self.max_lba_per_request as u64).min(lba_end); + let count = (end - lba) as usize; + let amount = count * self.lba_size; + + let slice = + unsafe { MaybeUninit::slice_assume_init_mut(&mut buffer[offset..offset + amount]) }; + let len = transport.read(0, lba, count as _, slice).await?; + if len != amount { + return Err(Error::InvalidArgument); } + offset += amount; } + Ok(()) } @@ -145,7 +160,7 @@ impl BlockDevice for ScsiDevice { } fn max_blocks_per_request(&self) -> usize { - 65536 / self.lba_size + self.max_lba_per_request } } diff --git a/kernel/driver/block/scsi/src/transport.rs b/kernel/driver/block/scsi/src/transport.rs index 887961bc..52655643 100644 --- a/kernel/driver/block/scsi/src/transport.rs +++ b/kernel/driver/block/scsi/src/transport.rs @@ -13,6 +13,8 @@ pub trait ScsiTransport: Send + Sync { request_data: &[u8], response_buffer: &mut [u8], ) -> Result; + + fn max_bytes_per_request(&self) -> usize; } pub struct ScsiTransportWrapper { @@ -26,11 +28,18 @@ impl ScsiTransportWrapper { } } - pub async fn read(&mut self, lun: u8, lba: u64, buffer: &mut [u8]) -> Result { - if lba >= u32::MAX as u64 || buffer.len() > u16::MAX as usize { + pub async fn read( + &mut self, + lun: u8, + lba: u64, + lba_count: u16, + buffer: &mut [u8], + ) -> Result { + if lba >= u32::MAX as u64 { return Err(Error::InvalidArgument); } let lba_bytes = (lba as u32).to_be_bytes(); + let lba_count = (lba_count as u16).to_be_bytes(); // Issue a READ (10) command let request_buffer = [ 0x28, @@ -40,8 +49,8 @@ impl ScsiTransportWrapper { lba_bytes[2], lba_bytes[3], 0x00, - 0x00, - 0x01, + lba_count[0], + lba_count[1], 0x00, ]; @@ -69,4 +78,8 @@ impl ScsiTransportWrapper { R::parse_response(&response_buffer[..response_len]) } + + pub fn max_bytes_per_request(&self) -> usize { + self.inner.max_bytes_per_request() + } } diff --git a/kernel/driver/bus/pci/src/driver.rs b/kernel/driver/bus/pci/src/driver.rs index 5c60e794..6c48fbc9 100644 --- a/kernel/driver/bus/pci/src/driver.rs +++ b/kernel/driver/bus/pci/src/driver.rs @@ -1,5 +1,5 @@ use alloc::{sync::Arc, vec::Vec}; -use device_api::device::Device; +use device_api::{device::Device, dma::DmaAllocator}; use libk::error::Error; use libk_util::sync::spin_rwlock::IrqSafeRwLock; @@ -17,7 +17,11 @@ pub struct PciDriverMatch { } pub trait PciDriver: Sync { - fn probe(&self, info: &PciDeviceInfo) -> Result, Error>; + fn probe( + &self, + info: &PciDeviceInfo, + dma: &Arc, + ) -> Result, Error>; fn driver_name(&self) -> &str; } diff --git a/kernel/driver/bus/pci/src/lib.rs b/kernel/driver/bus/pci/src/lib.rs index 6a1da2ed..adb69213 100644 --- a/kernel/driver/bus/pci/src/lib.rs +++ b/kernel/driver/bus/pci/src/lib.rs @@ -13,7 +13,7 @@ use alloc::{format, sync::Arc, vec::Vec}; use bitflags::bitflags; use device::{PciBusDevice, PciDeviceInfo}; -use device_api::device::DeviceInitContext; +use device_api::{device::DeviceInitContext, dma::DmaAllocator}; use interrupt::{PciInterruptMap, PciMsiMap}; use libk::{ dma::DummyDmaAllocator, @@ -617,11 +617,13 @@ fn setup_bus_device(device: &mut PciBusDevice) -> Result<(), Error> { if let Some(driver) = driver::lookup_driver(&device.info) { log::info!("{} -> {}", device.info.address, driver.driver_name()); - let instance = driver.probe(&device.info)?; + let dma: Arc = Arc::new(DummyDmaAllocator); let cx = DeviceInitContext { - dma_allocator: Arc::new(DummyDmaAllocator), + dma_allocator: dma.clone(), }; + let instance = driver.probe(&device.info, &dma)?; + unsafe { instance.clone().init(cx) }?; device.device.replace(instance); diff --git a/kernel/driver/bus/usb/src/class_driver/hid_keyboard.rs b/kernel/driver/bus/usb/src/class_driver/hid_keyboard.rs index 1076399d..92ca0d9e 100644 --- a/kernel/driver/bus/usb/src/class_driver/hid_keyboard.rs +++ b/kernel/driver/bus/usb/src/class_driver/hid_keyboard.rs @@ -2,7 +2,6 @@ use core::mem::MaybeUninit; use alloc::{boxed::Box, sync::Arc}; use async_trait::async_trait; -use libk_mm::PageBox; use yggdrasil_abi::io::{KeyboardKey, KeyboardKeyEvent}; use crate::{device::UsbDeviceAccess, error::UsbError, info::UsbDeviceClass}; @@ -136,14 +135,14 @@ impl UsbDriver for UsbHidKeyboardDriver { .open_interrupt_in_pipe(1, config.endpoints[0].max_packet_size as u16) .await?; - let mut buffer = PageBox::new_slice(0, 8).map_err(UsbError::MemoryError)?; + let mut buffer = [0; 8]; let mut state = KeyboardState::new(); let mut events = [MaybeUninit::uninit(); 16]; loop { let mut event_count = 0; - let len = pipe.read(buffer.as_slice_mut()).await?; + let len = pipe.read(&mut buffer).await?; if len < 8 { continue; } diff --git a/kernel/driver/bus/usb/src/class_driver/mass_storage.rs b/kernel/driver/bus/usb/src/class_driver/mass_storage.rs index 68aeafa7..3596d186 100644 --- a/kernel/driver/bus/usb/src/class_driver/mass_storage.rs +++ b/kernel/driver/bus/usb/src/class_driver/mass_storage.rs @@ -1,8 +1,9 @@ +use core::mem::MaybeUninit; + use alloc::{boxed::Box, sync::Arc}; use async_trait::async_trait; use bytemuck::{Pod, Zeroable}; -use libk::error::Error; -use libk_mm::PageBox; +use libk::{dma::DmaBuffer, error::Error}; use ygg_driver_scsi::{transport::ScsiTransport, ScsiDevice}; use crate::{ @@ -49,7 +50,7 @@ struct Bbb { device: Arc, in_pipe: UsbBulkInPipeAccess, out_pipe: UsbBulkOutPipeAccess, - buffer: PageBox<[u8]>, + buffer: DmaBuffer<[MaybeUninit]>, last_tag: u32, } @@ -61,7 +62,7 @@ impl Bbb { in_pipe: UsbBulkInPipeAccess, out_pipe: UsbBulkOutPipeAccess, ) -> Result { - let buffer = PageBox::new_slice(0, 4096).map_err(UsbError::MemoryError)?; + let buffer = in_pipe.allocate_dma_buffer(32768)?; Ok(Self { device, in_pipe, @@ -82,36 +83,35 @@ impl Bbb { ) -> Result { self.last_tag = self.last_tag.wrapping_add(1); - self.buffer[..size_of::()].fill(0); - - let cbw = bytemuck::from_bytes_mut::(&mut self.buffer[..size_of::()]); + let flags = if !host_to_dev { 1 << 7 } else { 0 }; let tag = self.last_tag; + let mut cbw_bytes = [0; 32]; + let cbw = bytemuck::from_bytes_mut::(&mut cbw_bytes); cbw.signature = 0x43425355; - cbw.tag = tag; cbw.transfer_length = response_len as u32; - if !host_to_dev { - cbw.flags = 1 << 7; - } + cbw.flags = flags; + cbw.tag = tag; cbw.lun = lun; cbw.cb_length = command.len() as u8; cbw.cb_data[..command.len()].copy_from_slice(command); self.out_pipe - .write(self.buffer.as_slice().subslice(..31)) + .write(&cbw_bytes[..31]) .await - .inspect_err(|error| log::error!("msc: out pipe error: {error:?}"))?; + .inspect_err(|error| log::error!("msc: CBW send error: {error:?}"))?; Ok(tag) } async fn read_csw(&mut self, tag: u32) -> Result<(), Error> { + let mut csw_bytes = [0; 16]; self.in_pipe - .read(self.buffer.as_slice_mut().subslice_mut(..13)) + .read_exact(&mut csw_bytes[..13]) .await - .inspect_err(|error| log::error!("msc: in pipe error: {error:?}"))?; + .inspect_err(|error| log::error!("msc: CSW receive error: {error:?}"))?; + let csw = bytemuck::from_bytes::(&csw_bytes); - let csw = bytemuck::from_bytes::(&self.buffer[..size_of::()]); if csw.signature != 0x53425355 { log::warn!("msc: invalid csw signature"); return Err(Error::InvalidArgument); @@ -129,19 +129,24 @@ impl Bbb { } async fn read_response_data(&mut self, buffer: &mut [u8]) -> Result { - // TODO limit by max_packet_size + if buffer.is_empty() { + return Ok(0); + } let len = self .in_pipe - .read(self.buffer.as_slice_mut().subslice_mut(..buffer.len())) + .read_dma(&mut self.buffer, buffer.len()) .await - .inspect_err(|error| log::error!("msc: in pipe error: {error:?}"))?; - buffer[..len].copy_from_slice(&self.buffer[..len]); + .inspect_err(|error| log::error!("msc: DMA read error: {error:?}"))?; + let dma_slice = unsafe { MaybeUninit::slice_assume_init_ref(&self.buffer[..len]) }; + buffer[..len].copy_from_slice(dma_slice); + Ok(len) } } #[async_trait] impl ScsiTransport for Bbb { + // TODO DMA support for SCSI async fn perform_request_raw( &mut self, lun: u8, @@ -149,20 +154,21 @@ impl ScsiTransport for Bbb { response_buffer: &mut [u8], ) -> Result { if request_data.len() > 16 || response_buffer.len() > self.buffer.len() { - return Err(Error::InvalidArgument); + todo!() + // return Err(Error::InvalidArgument); } let tag = self .send_cbw(lun, false, request_data, response_buffer.len()) .await?; - let response_len = if response_buffer.is_empty() { - 0 - } else { - self.read_response_data(response_buffer).await? - }; + let response_len = self.read_response_data(response_buffer).await?; self.read_csw(tag).await?; Ok(response_len) } + + fn max_bytes_per_request(&self) -> usize { + self.buffer.len() + } } impl UsbDeviceDetachHandler for DetachHandler { diff --git a/kernel/driver/bus/usb/src/device.rs b/kernel/driver/bus/usb/src/device.rs index 40e358fc..ab61f518 100644 --- a/kernel/driver/bus/usb/src/device.rs +++ b/kernel/driver/bus/usb/src/device.rs @@ -2,7 +2,6 @@ use core::{fmt, ops::Deref}; use alloc::{boxed::Box, sync::Arc, vec::Vec}; use async_trait::async_trait; -use libk_mm::PageBox; use libk_util::sync::spin_rwlock::{IrqSafeRwLock, IrqSafeRwLockReadGuard}; use crate::{ @@ -87,7 +86,6 @@ impl UsbDeviceAccess { /// * Device has been assigned a bus address pub async fn setup(raw: Arc) -> Result { let control = raw.control_pipe(); - let mut string_buffer = PageBox::new_uninit().map_err(UsbError::MemoryError)?; let device_desc = control.query_device_descriptor().await?; @@ -102,12 +100,8 @@ impl UsbDeviceAccess { ) })?; - let manufacturer = control - .query_string(device_desc.manufacturer_str, &mut string_buffer) - .await?; - let product = control - .query_string(device_desc.product_str, &mut string_buffer) - .await?; + let manufacturer = control.query_string(device_desc.manufacturer_str).await?; + let product = control.query_string(device_desc.product_str).await?; let info = UsbDeviceInfo { manufacturer, @@ -208,12 +202,11 @@ impl UsbDeviceAccess { if index >= self.info.num_configurations { return Err(UsbError::InvalidConfiguration); } - let mut string_buffer = PageBox::new_uninit().map_err(UsbError::MemoryError)?; let control_pipe = self.control_pipe(); let query = control_pipe.query_configuration_descriptor(index).await?; let configuration_name = control_pipe - .query_string(query.configuration().config_str, &mut string_buffer) + .query_string(query.configuration().config_str) .await?; let mut endpoints = Vec::new(); @@ -230,9 +223,7 @@ impl UsbDeviceAccess { }); } ConfigurationDescriptorEntry::Interface(iface) => { - let name = control_pipe - .query_string(iface.interface_str, &mut string_buffer) - .await?; + let name = control_pipe.query_string(iface.interface_str).await?; interfaces.push(UsbInterfaceInfo { name, number: iface.interface_number, diff --git a/kernel/driver/bus/usb/src/error.rs b/kernel/driver/bus/usb/src/error.rs index 173224dd..48c06995 100644 --- a/kernel/driver/bus/usb/src/error.rs +++ b/kernel/driver/bus/usb/src/error.rs @@ -43,7 +43,10 @@ impl From for Error { fn from(value: UsbError) -> Self { match value { UsbError::MemoryError(e) => e, - _ => Error::InvalidOperation, + e => { + log::warn!("usb: {e:?}"); + Error::InvalidOperation + } } } } diff --git a/kernel/driver/bus/usb/src/lib.rs b/kernel/driver/bus/usb/src/lib.rs index c63922d1..d70c12d1 100644 --- a/kernel/driver/bus/usb/src/lib.rs +++ b/kernel/driver/bus/usb/src/lib.rs @@ -1,6 +1,13 @@ #![no_std] #![allow(clippy::new_without_default)] -#![feature(iter_array_chunks, maybe_uninit_slice)] +#![feature( + iter_array_chunks, + maybe_uninit_slice, + maybe_uninit_as_bytes, + maybe_uninit_uninit_array, + maybe_uninit_write_slice, + maybe_uninit_fill +)] extern crate alloc; diff --git a/kernel/driver/bus/usb/src/pipe/control.rs b/kernel/driver/bus/usb/src/pipe/control.rs index a02950ba..31b3fbb6 100644 --- a/kernel/driver/bus/usb/src/pipe/control.rs +++ b/kernel/driver/bus/usb/src/pipe/control.rs @@ -1,5 +1,4 @@ use core::{ - cmp::Ordering, mem::{size_of, MaybeUninit}, ops::Deref, }; @@ -7,10 +6,9 @@ use core::{ use alloc::{boxed::Box, string::String}; use async_trait::async_trait; use bytemuck::{Pod, Zeroable}; -use libk_mm::{PageBox, PageSlice}; +use libk_mm::PageBox; use crate::{ - communication::UsbDirection, descriptor::{ UsbConfigurationDescriptor, UsbDeviceDescriptor, UsbDeviceQualifier, UsbEndpointDescriptor, UsbInterfaceDescriptor, UsbOtherSpeedConfiguration, @@ -86,10 +84,16 @@ fn decode_usb_string(bytes: &[u8]) -> Result { #[async_trait] pub trait UsbControlPipe: Send + Sync { - async fn control_transfer( + async fn control_transfer(&self, setup: ControlTransferSetup) -> Result<(), UsbError>; + async fn control_transfer_in( &self, setup: ControlTransferSetup, - data: Option<(&mut PageSlice>, UsbDirection)>, + buffer: &mut [MaybeUninit], + ) -> Result; + async fn control_transfer_out( + &self, + setup: ControlTransferSetup, + buffer: &[u8], ) -> Result; } @@ -173,28 +177,10 @@ impl ConfigurationDescriptorQuery { } impl UsbControlPipeAccess { - pub async fn query_device_descriptor2(&self) -> Result, UsbError> { - let mut output = PageBox::new_uninit().map_err(UsbError::MemoryError)?; + pub async fn query_device_descriptor(&self) -> Result { + let mut buffer = MaybeUninit::uninit(); - self.control_transfer( - ControlTransferSetup { - bm_request_type: 0b10000000, - b_request: 0xFF, - // b_request: 0x06, - w_value: 0x100, - w_index: 0, - w_length: size_of::() as _, - }, - Some((PageBox::as_bytes_mut(&mut output), UsbDirection::In)), - ) - .await?; - - Ok(unsafe { output.assume_init() }) - } - pub async fn query_device_descriptor(&self) -> Result, UsbError> { - let mut output = PageBox::new_uninit().map_err(UsbError::MemoryError)?; - - self.control_transfer( + self.control_transfer_in( ControlTransferSetup { bm_request_type: 0b10000000, b_request: 0x06, @@ -202,19 +188,19 @@ impl UsbControlPipeAccess { w_index: 0, w_length: size_of::() as _, }, - Some((PageBox::as_bytes_mut(&mut output), UsbDirection::In)), + buffer.as_bytes_mut(), ) .await?; - Ok(unsafe { output.assume_init() }) + Ok(unsafe { buffer.assume_init() }) } async fn fill_configuation_descriptor( &self, index: u8, - buffer: &mut PageBox<[MaybeUninit]>, + buffer: &mut [MaybeUninit], ) -> Result<(), UsbError> { - self.control_transfer( + self.control_transfer_in( ControlTransferSetup { bm_request_type: 0b10000000, b_request: 0x06, @@ -222,30 +208,28 @@ impl UsbControlPipeAccess { w_index: 0, w_length: buffer.len().try_into().unwrap(), }, - Some((buffer.as_slice_mut(), UsbDirection::In)), + buffer, ) .await?; Ok(()) } - pub async fn query_string( - &self, - index: u8, - buffer: &mut PageBox>, - ) -> Result { - self.control_transfer( - ControlTransferSetup { - bm_request_type: 0b10000000, - b_request: 0x06, - w_value: 0x300 | (index as u16), - w_index: 0, - w_length: 4096, - }, - Some((PageBox::as_bytes_mut(buffer), UsbDirection::In)), - ) - .await?; - let data = unsafe { buffer.assume_init_ref() }; + pub async fn query_string(&self, index: u8) -> Result { + let mut buffer = MaybeUninit::uninit_array::<256>(); + let len = self + .control_transfer_in( + ControlTransferSetup { + bm_request_type: 0b10000000, + b_request: 0x06, + w_value: 0x300 | (index as u16), + w_index: 0, + w_length: 4096, + }, + &mut buffer[..], + ) + .await?; + let data = unsafe { MaybeUninit::slice_assume_init_ref(&buffer[..len]) }; let len = data[0] as usize; decode_usb_string(&data[2..len]) @@ -255,35 +239,20 @@ impl UsbControlPipeAccess { &self, index: u8, ) -> Result { - // First, query the real length of the descriptor - let mut buffer = PageBox::new_uninit_slice(size_of::()) - .map_err(UsbError::MemoryError)?; - self.fill_configuation_descriptor(index, &mut buffer) + // 4K should be enough for a configuration descriptor + let mut buffer = PageBox::new_uninit_slice(4096).map_err(UsbError::MemoryError)?; + + self.fill_configuation_descriptor(index, &mut buffer[..]) .await?; - let buffer = unsafe { PageBox::assume_init_slice(buffer) }; - let desc: &UsbConfigurationDescriptor = bytemuck::from_bytes(&buffer); - let total_len = desc.total_length as usize; - - // Return if everything's ready at this point - match total_len.cmp(&size_of::()) { - Ordering::Less => todo!(), - Ordering::Equal => return Ok(ConfigurationDescriptorQuery { buffer }), - _ => (), - } - - // Otherwise, query the rest of the data - let mut buffer = PageBox::new_uninit_slice(total_len).map_err(UsbError::MemoryError)?; - self.fill_configuation_descriptor(index, &mut buffer) - .await?; let buffer = unsafe { PageBox::assume_init_slice(buffer) }; let desc: &UsbConfigurationDescriptor = bytemuck::from_bytes(&buffer[..size_of::()]); let total_len = desc.total_length as usize; - if total_len != buffer.len() { - todo!(); + if total_len > 4096 { + unimplemented!("4KiB wasn't enough"); } Ok(ConfigurationDescriptorQuery { buffer }) @@ -294,18 +263,14 @@ impl UsbControlPipeAccess { w_value: u16, w_index: u16, ) -> Result<(), UsbError> { - self.control_transfer( - ControlTransferSetup { - bm_request_type: D::BM_REQUEST_TYPE, - b_request: D::B_REQUEST, - w_value, - w_index, - w_length: 0, - }, - None, - ) - .await?; - Ok(()) + self.control_transfer(ControlTransferSetup { + bm_request_type: D::BM_REQUEST_TYPE, + b_request: D::B_REQUEST, + w_value, + w_index, + w_length: 0, + }) + .await } pub async fn class_specific_request( @@ -313,18 +278,14 @@ impl UsbControlPipeAccess { w_value: u16, w_index: u16, ) -> Result<(), UsbError> { - self.control_transfer( - ControlTransferSetup { - bm_request_type: D::BM_REQUEST_TYPE, - b_request: D::B_REQUEST, - w_value, - w_index, - w_length: 0, - }, - None, - ) - .await?; - Ok(()) + self.control_transfer(ControlTransferSetup { + bm_request_type: D::BM_REQUEST_TYPE, + b_request: D::B_REQUEST, + w_value, + w_index, + w_length: 0, + }) + .await } pub async fn set_configuration(&self, value: u16) -> Result<(), UsbError> { diff --git a/kernel/driver/bus/usb/src/pipe/mod.rs b/kernel/driver/bus/usb/src/pipe/mod.rs index 1915e0ba..a14759ea 100644 --- a/kernel/driver/bus/usb/src/pipe/mod.rs +++ b/kernel/driver/bus/usb/src/pipe/mod.rs @@ -1,2 +1,12 @@ +use core::mem::MaybeUninit; + +use libk::dma::DmaBuffer; + +use crate::error::UsbError; + pub mod control; pub mod normal; + +pub trait UsbGenericPipe: Send + Sync { + fn allocate_dma_buffer(&self, len: usize) -> Result]>, UsbError>; +} diff --git a/kernel/driver/bus/usb/src/pipe/normal.rs b/kernel/driver/bus/usb/src/pipe/normal.rs index 29abcadd..b5496d7e 100644 --- a/kernel/driver/bus/usb/src/pipe/normal.rs +++ b/kernel/driver/bus/usb/src/pipe/normal.rs @@ -1,15 +1,30 @@ -use core::ops::Deref; +use core::{mem::MaybeUninit, ops::Deref}; use alloc::boxed::Box; use async_trait::async_trait; -use libk_mm::PageSlice; +use libk::dma::DmaBuffer; use crate::error::{TransferError, UsbError}; +use super::UsbGenericPipe; + #[async_trait] -pub trait UsbNormalPipeIn: Send + Sync { - async fn read(&self, buffer: &mut PageSlice) -> Result; - async fn read_exact(&self, buffer: &mut PageSlice) -> Result<(), UsbError> { +pub trait UsbNormalPipeIn: UsbGenericPipe { + async fn read_dma( + &self, + buffer: &mut DmaBuffer<[MaybeUninit]>, + limit: usize, + ) -> Result; + + async fn read(&self, buffer: &mut [u8]) -> Result { + let mut dma_buffer = self.allocate_dma_buffer(buffer.len())?; + let len = self.read_dma(&mut dma_buffer, buffer.len()).await?; + let dma_slice = unsafe { MaybeUninit::slice_assume_init_ref(&dma_buffer[..len]) }; + buffer[..len].copy_from_slice(dma_slice); + Ok(len) + } + + async fn read_exact(&self, buffer: &mut [u8]) -> Result<(), UsbError> { match self.read(buffer).await { Ok(len) if len == buffer.len() => Ok(()), Ok(len) => Err(UsbError::TransferFailed(TransferError::ShortPacket(len))), @@ -19,8 +34,16 @@ pub trait UsbNormalPipeIn: Send + Sync { } #[async_trait] -pub trait UsbNormalPipeOut: Send + Sync { - async fn write(&self, buffer: &PageSlice) -> Result; +pub trait UsbNormalPipeOut: UsbGenericPipe { + async fn write_dma(&self, buffer: &DmaBuffer<[u8]>) -> Result; + + async fn write(&self, buffer: &[u8]) -> Result { + let mut dma_buffer = self.allocate_dma_buffer(buffer.len())?; + MaybeUninit::copy_from_slice(&mut dma_buffer, buffer); + let dma_buffer = unsafe { DmaBuffer::assume_init_slice(dma_buffer) }; + + self.write_dma(&dma_buffer).await + } } pub struct UsbBulkInPipeAccess(pub Box); diff --git a/kernel/driver/net/core/src/ethernet.rs b/kernel/driver/net/core/src/ethernet.rs index 4938ca26..8f2b49c7 100644 --- a/kernel/driver/net/core/src/ethernet.rs +++ b/kernel/driver/net/core/src/ethernet.rs @@ -2,7 +2,7 @@ use core::{fmt, mem::size_of}; use alloc::sync::Arc; use bytemuck::Pod; -use libk_mm::PageBox; +use libk::dma::DmaBuffer; use yggdrasil_abi::{ error::Error, net::{ @@ -24,7 +24,7 @@ pub struct L2Packet { pub l2_offset: usize, pub l3_offset: usize, - pub data: Arc>, + pub data: Arc>, } /// Defines an Ethernet link speed diff --git a/kernel/driver/net/core/src/interface.rs b/kernel/driver/net/core/src/interface.rs index 0d9ed3b6..2200083e 100644 --- a/kernel/driver/net/core/src/interface.rs +++ b/kernel/driver/net/core/src/interface.rs @@ -1,12 +1,12 @@ use core::{ - mem::size_of, + mem::{size_of, MaybeUninit}, net::IpAddr, sync::atomic::{AtomicU32, AtomicUsize, Ordering}, }; use alloc::{boxed::Box, collections::BTreeMap, format, sync::Arc}; +use libk::dma::DmaBuffer; // TODO: link state management? -use libk_mm::PageBox; use libk_util::{ sync::spin_rwlock::{IrqSafeRwLock, IrqSafeRwLockReadGuard}, OneTimeInit, @@ -19,7 +19,9 @@ use yggdrasil_abi::{ use crate::l3::{arp::ArpTable, Route}; pub trait NetworkDevice: Sync + Send { - fn transmit(&self, packet: PageBox<[u8]>) -> Result<(), Error>; + fn allocate_transmit_buffer(&self, len: usize) -> Result]>, Error>; + fn transmit_buffer(&self, buffer: DmaBuffer<[u8]>) -> Result<(), Error>; + fn packet_prefix_size(&self) -> usize; fn read_hardware_address(&self) -> MacAddress; @@ -103,12 +105,15 @@ impl NetworkInterface { let l2_offset = self.device.packet_prefix_size(); let l2_data_offset = l2_offset + size_of::(); - let mut packet = PageBox::new_slice(0, l2_data_offset + l2_data.len())?; + let packet = self + .device + .allocate_transmit_buffer(l2_data_offset + l2_data.len())?; + let mut packet = unsafe { DmaBuffer::assume_init_slice(packet) }; packet[l2_offset..l2_data_offset].copy_from_slice(bytemuck::bytes_of(l2_frame)); - packet[l2_data_offset..].copy_from_slice(l2_data); + packet[l2_data_offset..l2_data_offset + l2_data.len()].copy_from_slice(l2_data); - self.device.transmit(packet) + self.device.transmit_buffer(packet) } } diff --git a/kernel/driver/net/core/src/l3/mod.rs b/kernel/driver/net/core/src/l3/mod.rs index 6229c4d9..0b53bcb7 100644 --- a/kernel/driver/net/core/src/l3/mod.rs +++ b/kernel/driver/net/core/src/l3/mod.rs @@ -6,7 +6,7 @@ use core::{ use alloc::{sync::Arc, vec::Vec}; use bytemuck::{Pod, Zeroable}; -use libk_mm::PageBox; +use libk::dma::DmaBuffer; use libk_util::sync::spin_rwlock::{ IrqSafeRwLock, IrqSafeRwLockReadGuard, IrqSafeRwLockWriteGuard, }; @@ -40,7 +40,7 @@ pub struct L3Packet { pub l4_offset: usize, pub data_length: usize, - pub data: Arc>, + pub data: Arc>, } pub trait IpFrame: Pod { @@ -221,7 +221,7 @@ pub async fn send_l4_ip_resolved(packet: &L4ResolvedPacket<'_, '_>) -> Result<() let l3_frame = packet.make_l3_frame()?; let mut builder = PacketBuilder::new( - packet.interface.device.packet_prefix_size(), + packet.interface.device.as_ref(), size_of::() + size_of::() + packet.total_l4_len(), )?; builder.push(&EthernetFrame { @@ -235,7 +235,7 @@ pub async fn send_l4_ip_resolved(packet: &L4ResolvedPacket<'_, '_>) -> Result<() builder.push_bytes(packet.l4_data)?; let (sent_packet, _len) = builder.finish(); - packet.interface.device.transmit(sent_packet) + packet.interface.device.transmit_buffer(sent_packet) } pub async fn send_l4_ip(packet: &L4UnresolvedPacket<'_>) -> Result<(), Error> { diff --git a/kernel/driver/net/core/src/lib.rs b/kernel/driver/net/core/src/lib.rs index fcd56ccd..8ac0c7ce 100644 --- a/kernel/driver/net/core/src/lib.rs +++ b/kernel/driver/net/core/src/lib.rs @@ -9,9 +9,9 @@ use core::mem::size_of; use alloc::sync::Arc; use bytemuck::Pod; use ethernet::L2Packet; +use interface::NetworkDevice; use l3::L3Packet; -use libk::task::runtime; -use libk_mm::PageBox; +use libk::{dma::DmaBuffer, task::runtime}; use libk_util::queue::UnboundedMpmcQueue; use yggdrasil_abi::{error::Error, net::protocols::EthernetFrame}; @@ -28,20 +28,22 @@ pub use interface::register_interface; pub struct Packet { // TODO info about "received" interface - buffer: PageBox<[u8]>, + buffer: DmaBuffer<[u8]>, offset: usize, iface: u32, } pub struct PacketBuilder { - data: PageBox<[u8]>, + data: DmaBuffer<[u8]>, pos: usize, len: usize, } impl PacketBuilder { - pub fn new(l2_offset: usize, l2_size: usize) -> Result { - let data = PageBox::new_slice(0, l2_offset + l2_size)?; + pub fn new(nic: &dyn NetworkDevice, l2_size: usize) -> Result { + let l2_offset = nic.packet_prefix_size(); + let data = nic.allocate_transmit_buffer(l2_offset + l2_size)?; + let data = unsafe { DmaBuffer::assume_init_slice(data) }; Ok(Self { data, pos: l2_offset, @@ -63,14 +65,14 @@ impl PacketBuilder { Ok(()) } - pub fn finish(self) -> (PageBox<[u8]>, usize) { + pub fn finish(self) -> (DmaBuffer<[u8]>, usize) { (self.data, self.len) } } impl Packet { #[inline] - pub fn new(buffer: PageBox<[u8]>, offset: usize, iface: u32) -> Self { + pub fn new(buffer: DmaBuffer<[u8]>, offset: usize, iface: u32) -> Self { Self { buffer, offset, diff --git a/kernel/driver/net/core/src/socket/raw.rs b/kernel/driver/net/core/src/socket/raw.rs index 965cdf4d..7c0af767 100644 --- a/kernel/driver/net/core/src/socket/raw.rs +++ b/kernel/driver/net/core/src/socket/raw.rs @@ -8,11 +8,11 @@ use core::{ use alloc::{boxed::Box, collections::btree_map::BTreeMap, sync::Arc, vec::Vec}; use async_trait::async_trait; use libk::{ + dma::DmaBuffer, error::Error, task::runtime::maybe_timeout, vfs::{FileReadiness, Socket}, }; -use libk_mm::PageBox; use libk_util::{queue::BoundedMpmcQueue, sync::spin_rwlock::IrqSafeRwLock}; use yggdrasil_abi::{ net::{ @@ -60,7 +60,6 @@ impl RawSocket { if let Some(ids) = bound_sockets.get(&packet.interface_id) { for id in ids { let socket = raw_sockets.get(id).unwrap(); - log::info!("Packet -> {id}"); socket.bound_packet_received(packet.clone()); } } @@ -172,9 +171,12 @@ impl Socket for RawSocket { if message.payload.len() > 4096 - l2_offset { return Err(Error::InvalidArgument); } - let mut packet = PageBox::new_slice(0, l2_offset + message.payload.len())?; + let packet = interface + .device + .allocate_transmit_buffer(l2_offset + message.payload.len())?; + let mut packet = unsafe { DmaBuffer::assume_init_slice(packet) }; packet[l2_offset..l2_offset + message.payload.len()].copy_from_slice(message.payload); - interface.device.transmit(packet)?; + interface.device.transmit_buffer(packet)?; Ok(message.payload.len()) } async fn send_message( diff --git a/kernel/driver/net/loopback/Cargo.toml b/kernel/driver/net/loopback/Cargo.toml index 29dd3d5e..6f7dc4bc 100644 --- a/kernel/driver/net/loopback/Cargo.toml +++ b/kernel/driver/net/loopback/Cargo.toml @@ -5,8 +5,10 @@ edition = "2021" [dependencies] yggdrasil-abi.workspace = true +device-api.workspace = true libk-util.workspace = true libk-mm.workspace = true +libk.workspace = true ygg_driver_net_core = { path = "../../net/core" } diff --git a/kernel/driver/net/loopback/src/lib.rs b/kernel/driver/net/loopback/src/lib.rs index b39b6843..a97654f4 100644 --- a/kernel/driver/net/loopback/src/lib.rs +++ b/kernel/driver/net/loopback/src/lib.rs @@ -2,10 +2,14 @@ extern crate alloc; -use core::net::{IpAddr, Ipv4Addr}; +use core::{ + mem::MaybeUninit, + net::{IpAddr, Ipv4Addr}, +}; use alloc::sync::Arc; -use libk_mm::PageBox; +use device_api::dma::DmaAllocator; +use libk::dma::{DmaBuffer, DummyDmaAllocator}; use libk_util::OneTimeInit; use ygg_driver_net_core::{ interface::{NetworkDevice, NetworkInterfaceType}, @@ -13,11 +17,18 @@ use ygg_driver_net_core::{ }; use yggdrasil_abi::{error::Error, net::MacAddress}; -struct LoopbackDevice; +struct LoopbackDevice { + allocator: Arc, +} impl NetworkDevice for LoopbackDevice { - fn transmit(&self, packet: PageBox<[u8]>) -> Result<(), Error> { - ygg_driver_net_core::receive_packet(Packet::new(packet, 0, *LOOPBACK_ID.get())) + fn allocate_transmit_buffer(&self, len: usize) -> Result]>, Error> { + DmaBuffer::new_uninit_slice(&*self.allocator, len) + } + + fn transmit_buffer(&self, buffer: DmaBuffer<[u8]>) -> Result<(), Error> { + let packet = Packet::new(buffer, 0, *LOOPBACK_ID.get()); + ygg_driver_net_core::receive_packet(packet) } fn packet_prefix_size(&self) -> usize { @@ -32,7 +43,9 @@ impl NetworkDevice for LoopbackDevice { static LOOPBACK_ID: OneTimeInit = OneTimeInit::new(); pub fn init() { - let loopback = Arc::new(LoopbackDevice); + let loopback = Arc::new(LoopbackDevice { + allocator: Arc::new(DummyDmaAllocator), + }); let interface = ygg_driver_net_core::register_interface(NetworkInterfaceType::Loopback, loopback); diff --git a/kernel/driver/net/rtl81xx/src/lib.rs b/kernel/driver/net/rtl81xx/src/lib.rs index d9467f6d..3138cfe8 100644 --- a/kernel/driver/net/rtl81xx/src/lib.rs +++ b/kernel/driver/net/rtl81xx/src/lib.rs @@ -1,7 +1,7 @@ #![no_std] use alloc::sync::Arc; -use device_api::device::Device; +use device_api::{device::Device, dma::DmaAllocator}; use libk::error::Error; use rtl8139::Rtl8139; use rtl8168::Rtl8168; @@ -23,7 +23,7 @@ pci_driver! { "rtl8139" } - fn probe(&self, info: &PciDeviceInfo) -> Result, Error> { + fn probe(&self, info: &PciDeviceInfo, dma: &Arc) -> Result, Error> { info.init_interrupts(PreferredInterruptMode::Msi(false))?; // Enable MMIO + interrupts + bus mastering @@ -38,7 +38,7 @@ pci_driver! { .and_then(PciBaseAddress::as_memory) .ok_or(Error::InvalidArgument)?; - let device = Rtl8139::new(base, info.clone())?; + let device = Rtl8139::new(dma.clone(), base, info.clone())?; Ok(Arc::new(device)) } } @@ -51,7 +51,7 @@ pci_driver! { "rtl816x" } - fn probe(&self, info: &PciDeviceInfo) -> Result, Error> { + fn probe(&self, info: &PciDeviceInfo, dma: &Arc) -> Result, Error> { let base = info .config_space .bar(2) @@ -65,7 +65,7 @@ pci_driver! { // Enable MMIO + interrupts + bus mastering info.set_command(true, true, false, true); - let device = Rtl8168::new(base, info.clone())?; + let device = Rtl8168::new(dma.clone(), base, info.clone())?; Ok(Arc::new(device)) } } diff --git a/kernel/driver/net/rtl81xx/src/rtl8139.rs b/kernel/driver/net/rtl81xx/src/rtl8139.rs index 3cf04238..270e9c2b 100644 --- a/kernel/driver/net/rtl81xx/src/rtl8139.rs +++ b/kernel/driver/net/rtl81xx/src/rtl8139.rs @@ -3,14 +3,11 @@ use core::mem::MaybeUninit; use alloc::sync::Arc; use device_api::{ device::{Device, DeviceInitContext}, + dma::DmaAllocator, interrupt::{InterruptHandler, IrqVector}, }; -use libk::error::Error; -use libk_mm::{ - address::{AsPhysicalAddress, PhysicalAddress}, - device::DeviceMemoryIo, - PageBox, -}; +use libk::{dma::DmaBuffer, error::Error}; +use libk_mm::{address::PhysicalAddress, device::DeviceMemoryIo}; use libk_util::{queue::BoundedQueue, sync::IrqSafeSpinlock, OneTimeInit}; use tock_registers::{ interfaces::{ReadWriteable, Readable, Writeable}, @@ -207,17 +204,17 @@ register_structs! { } struct Rx { - buffer: PageBox<[MaybeUninit]>, + buffer: DmaBuffer<[MaybeUninit]>, rd: usize, } // TODO place a secondary Tx queue here, to send the queued packets when more slots become // available struct Tx { - buffers: [Option>; 4], + buffers: [Option>; 4], wr: usize, rd: usize, - queue: BoundedQueue>, + queue: BoundedQueue>, } pub struct Rtl8139 { @@ -228,13 +225,13 @@ pub struct Rtl8139 { nic: OneTimeInit, rx: OneTimeInit>, tx: IrqSafeSpinlock, + + dma: Arc, } impl Tx { - pub fn tx_now(&mut self, regs: &Regs, packet: PageBox<[u8]>) -> Result<(), Error> { - let packet_address = unsafe { packet.as_physical_address() } - .try_into_u32() - .map_err(|_| Error::InvalidArgument)?; + pub fn tx_now(&mut self, regs: &Regs, packet: DmaBuffer<[u8]>) -> Result<(), Error> { + let packet_address = packet.bus_address().try_into_u32()?; let packet_len = packet.len(); if packet_len > 1500 { return Err(Error::InvalidArgument); @@ -259,7 +256,11 @@ impl Rtl8139 { const RX_BUFFER_LEN: usize = 8192; const RX_BUFFER_OVERFLOW: usize = 4096; - pub fn new(base: PhysicalAddress, info: PciDeviceInfo) -> Result { + pub fn new( + dma: Arc, + base: PhysicalAddress, + info: PciDeviceInfo, + ) -> Result { let regs = unsafe { DeviceMemoryIo::::map(base, Default::default()) }?; let mac0 = regs.IDRn[0].get().to_le_bytes(); let mac1 = regs.IDRn[1].get().to_le_bytes(); @@ -278,6 +279,8 @@ impl Rtl8139 { rd: 0, queue: BoundedQueue::new(64), }), + + dma, }) } } @@ -318,9 +321,10 @@ impl InterruptHandler for Rtl8139 { let rx_len = u16::from_le_bytes([rx_len_0, rx_len_1]) as usize; if rx_len >= 16 { - if let Ok(mut packet_buf) = PageBox::new_uninit_slice(rx_len) { + if let Ok(mut packet_buf) = DmaBuffer::new_uninit_slice(&*self.dma, rx_len) { packet_buf.copy_from_slice(&rx.buffer[rx_pos + 4..rx_pos + rx_len + 4]); - let packet_buf = unsafe { packet_buf.assume_init_slice() }; + let packet_buf = unsafe { DmaBuffer::assume_init_slice(packet_buf) }; + // let packet_buf = unsafe { packet_buf.assume_init_slice() }; let packet = Packet::new(packet_buf, 0, nic); ygg_driver_net_core::receive_packet(packet).ok(); } @@ -342,18 +346,16 @@ impl InterruptHandler for Rtl8139 { } impl Device for Rtl8139 { - unsafe fn init(self: Arc, cx: DeviceInitContext) -> Result<(), Error> { - // TODO use DmaAllocator instead of PageBox - let _ = cx; - + unsafe fn init(self: Arc, _cx: DeviceInitContext) -> Result<(), Error> { log::info!("Initialize rtl8139 driver"); log::info!("MAC: {}", self.mac); // Setup the initial Rx buffer - let rx_buffer = PageBox::new_uninit_slice(Self::RX_BUFFER_LEN + Self::RX_BUFFER_OVERFLOW)?; - let rx_buffer_address = unsafe { rx_buffer.as_physical_address() } - .try_into_u32() - .map_err(|_| Error::InvalidArgument)?; + let rx_buffer = DmaBuffer::new_uninit_slice( + &*self.dma, + Self::RX_BUFFER_LEN + Self::RX_BUFFER_OVERFLOW, + )?; + let rx_buffer_address = rx_buffer.bus_address().try_into_u32()?; self.pci.map_interrupt(Default::default(), self.clone())?; @@ -402,7 +404,11 @@ impl Device for Rtl8139 { } impl NetworkDevice for Rtl8139 { - fn transmit(&self, packet: PageBox<[u8]>) -> Result<(), Error> { + fn allocate_transmit_buffer(&self, len: usize) -> Result]>, Error> { + DmaBuffer::new_uninit_slice(&*self.dma, len) + } + + fn transmit_buffer(&self, packet: DmaBuffer<[u8]>) -> Result<(), Error> { let mut tx = self.tx.lock(); // Buffer still in Tx, cannot send diff --git a/kernel/driver/net/rtl81xx/src/rtl8168.rs b/kernel/driver/net/rtl81xx/src/rtl8168.rs index 6d5ca0ef..59a3a88f 100644 --- a/kernel/driver/net/rtl81xx/src/rtl8168.rs +++ b/kernel/driver/net/rtl81xx/src/rtl8168.rs @@ -6,14 +6,14 @@ use core::{ use alloc::{sync::Arc, vec::Vec}; use device_api::{ device::{Device, DeviceInitContext}, + dma::DmaAllocator, interrupt::{InterruptHandler, IrqVector}, }; -use libk::error::Error; -use libk_mm::{ - address::{AsPhysicalAddress, PhysicalAddress}, - device::DeviceMemoryIo, - PageBox, +use libk::{ + dma::{BusAddress, DmaBuffer}, + error::Error, }; +use libk_mm::{address::PhysicalAddress, device::DeviceMemoryIo, L3_PAGE_SIZE}; use libk_util::{sync::IrqSafeSpinlock, OneTimeInit}; use tock_registers::{ interfaces::{ReadWriteable, Readable, Writeable}, @@ -218,18 +218,18 @@ enum Revision { struct Descriptor { cmd: u32, vlan_cmd: u32, - address: PhysicalAddress, + address: BusAddress, } struct RxRing { - entries: PageBox<[Descriptor]>, - buffers: Vec]>>, + entries: DmaBuffer<[Descriptor]>, + buffers: Vec]>>, rd: usize, } struct TxRing { - entries: PageBox<[Descriptor]>, - buffers: Vec>>, + entries: DmaBuffer<[Descriptor]>, + buffers: Vec>>, wr: usize, rd: usize, } @@ -242,20 +242,18 @@ pub struct Rtl8168 { nic: OneTimeInit, rx: OneTimeInit>, tx: OneTimeInit>, + + dma: Arc, } impl RxRing { - pub fn with_capacity(capacity: usize) -> Result { + pub fn with_capacity(dma: &dyn DmaAllocator, capacity: usize) -> Result { let buffers = (0..capacity) - .map(|_| PageBox::new_uninit_slice(4096)) - .collect::, _>>()?; - let entries = PageBox::new_slice_with( - |i| { - Descriptor::new_rx( - unsafe { buffers[i].as_physical_address() }, - i == capacity - 1, - ) - }, + .map(|_| DmaBuffer::new_uninit_slice(dma, L3_PAGE_SIZE)) + .collect::, Error>>()?; + let entries = DmaBuffer::new_slice_with( + dma, + |i| Descriptor::new_rx(buffers[i].bus_address(), i == capacity - 1), capacity, )?; Ok(Self { @@ -265,17 +263,21 @@ impl RxRing { }) } - pub fn consume)>(&mut self, handler: F) -> Result { + pub fn consume)>( + &mut self, + dma: &dyn DmaAllocator, + handler: F, + ) -> Result { let mut count = 0; loop { let index = self.rd % self.entries.len(); let entry = &self.entries[index]; if entry.is_host_owned() { - let new_buffer = PageBox::new_uninit_slice(4096)?; - let new_buffer_address = unsafe { new_buffer.as_physical_address() }; + let new_buffer = DmaBuffer::new_uninit_slice(dma, 4096)?; + let new_buffer_address = new_buffer.bus_address(); let buffer = mem::replace(&mut self.buffers[index], new_buffer); - let buffer = unsafe { buffer.assume_init_slice() }; + let buffer = unsafe { DmaBuffer::assume_init_slice(buffer) }; handler(buffer); self.entries[index].setup_rx(new_buffer_address); @@ -289,15 +291,15 @@ impl RxRing { Ok(count) } - pub fn base_address(&self) -> PhysicalAddress { - unsafe { self.entries.as_physical_address() } + pub fn base_address(&self) -> BusAddress { + self.entries.bus_address() } } impl TxRing { - pub fn with_capacity(capacity: usize) -> Result { + pub fn with_capacity(dma: &dyn DmaAllocator, capacity: usize) -> Result { let entries = - PageBox::new_slice_with(|i| Descriptor::empty_tx(i == capacity - 1), capacity)?; + DmaBuffer::new_slice_with(dma, |i| Descriptor::empty_tx(i == capacity - 1), capacity)?; let buffers = (0..capacity).map(|_| None).collect(); Ok(Self { @@ -312,8 +314,8 @@ impl TxRing { self.wr.wrapping_add(1) % self.entries.len() == self.rd % self.entries.len() } - pub fn push(&mut self, packet: PageBox<[u8]>) -> Result<(), Error> { - let packet_base = unsafe { packet.as_physical_address() }; + pub fn push(&mut self, packet: DmaBuffer<[u8]>) -> Result<(), Error> { + let packet_base = packet.bus_address(); let packet_size = packet.len(); // TODO packet size checks @@ -348,8 +350,8 @@ impl TxRing { Ok(count) } - pub fn base_address(&self) -> PhysicalAddress { - unsafe { self.entries.as_physical_address() } + pub fn base_address(&self) -> BusAddress { + self.entries.bus_address() } } @@ -363,7 +365,7 @@ impl Descriptor { // First segment of a packet const CMD_FS: u32 = 1 << 29; - pub fn new_rx(buffer: PhysicalAddress, last: bool) -> Self { + pub fn new_rx(buffer: BusAddress, last: bool) -> Self { let mut cmd = Self::CMD_OWN | 0x1000; if last { cmd |= Self::CMD_END; @@ -380,11 +382,11 @@ impl Descriptor { Self { cmd, vlan_cmd: 0, - address: PhysicalAddress::ZERO, + address: BusAddress::ZERO, } } - pub fn setup_rx(&mut self, buffer: PhysicalAddress) { + pub fn setup_rx(&mut self, buffer: BusAddress) { let cmd = self.cmd; self.address = buffer; self.vlan_cmd = 0; @@ -394,7 +396,7 @@ impl Descriptor { } } - pub fn setup_tx(&mut self, buffer: PhysicalAddress, size: usize) { + pub fn setup_tx(&mut self, buffer: BusAddress, size: usize) { let mut cmd = self.cmd; cmd |= Self::CMD_OWN | Self::CMD_FS | Self::CMD_LS | (size as u32); self.address = buffer; @@ -527,7 +529,11 @@ impl Regs { } impl Rtl8168 { - pub fn new(base: PhysicalAddress, info: PciDeviceInfo) -> Result { + pub fn new( + dma: Arc, + base: PhysicalAddress, + info: PciDeviceInfo, + ) -> Result { let regs = unsafe { DeviceMemoryIo::::map(base, Default::default()) }?; let mac0 = regs.IDRn[0].get().to_le_bytes(); let mac1 = regs.IDRn[1].get().to_le_bytes(); @@ -540,6 +546,8 @@ impl Rtl8168 { nic: OneTimeInit::new(), rx: OneTimeInit::new(), tx: OneTimeInit::new(), + + dma, }) } } @@ -564,7 +572,7 @@ impl InterruptHandler for Rtl8168 { let mut rx = self.rx.get().lock(); let nic = *self.nic.get(); let count = rx - .consume(|buffer| { + .consume(&*self.dma, |buffer| { // TODO add packet len hint to packets let packet = Packet::new(buffer, 0, nic); ygg_driver_net_core::receive_packet(packet).ok(); @@ -592,14 +600,12 @@ impl InterruptHandler for Rtl8168 { } impl Device for Rtl8168 { - unsafe fn init(self: Arc, cx: DeviceInitContext) -> Result<(), Error> { - // TODO use DmaAllocator instead of PageBox - let _ = cx; + unsafe fn init(self: Arc, _cx: DeviceInitContext) -> Result<(), Error> { log::info!("Initialize rtl8168"); log::info!("MAC: {}", self.mac); - let rx_ring = RxRing::with_capacity(256)?; - let tx_ring = TxRing::with_capacity(256)?; + let rx_ring = RxRing::with_capacity(&*self.dma, 256)?; + let tx_ring = TxRing::with_capacity(&*self.dma, 256)?; let rx_ring_base = rx_ring.base_address().into_u64(); let tx_ring_base = tx_ring.base_address().into_u64(); @@ -688,11 +694,15 @@ impl Device for Rtl8168 { } impl NetworkDevice for Rtl8168 { - fn transmit(&self, packet: PageBox<[u8]>) -> Result<(), Error> { + fn allocate_transmit_buffer(&self, len: usize) -> Result]>, Error> { + DmaBuffer::new_uninit_slice(&*self.dma, len) + } + + fn transmit_buffer(&self, buffer: DmaBuffer<[u8]>) -> Result<(), Error> { let mut tx = self.tx.get().lock(); let regs = self.regs.lock(); - tx.push(packet)?; + tx.push(buffer)?; regs.TPPOLL.write(TPPOLL::NPQ::SET); Ok(()) diff --git a/kernel/driver/usb/xhci/src/context.rs b/kernel/driver/usb/xhci/src/context.rs index 5c1ba48d..5c78eeb7 100644 --- a/kernel/driver/usb/xhci/src/context.rs +++ b/kernel/driver/usb/xhci/src/context.rs @@ -1,9 +1,7 @@ use core::ops::{Deref, DerefMut}; -use libk_mm::{ - address::{AsPhysicalAddress, PhysicalAddress}, - PageBox, -}; +use device_api::dma::DmaAllocator; +use libk::dma::{BusAddress, DmaBuffer}; use libk_util::sync::spin_rwlock::IrqSafeRwLock; use xhci_lib::context::{self, DeviceHandler, InputHandler}; use ygg_driver_usb::error::UsbError; @@ -11,22 +9,22 @@ use ygg_driver_usb::error::UsbError; use crate::regs::{ContextSize, PortNumber}; pub enum XhciDeviceContext { - Context32(IrqSafeRwLock>), - Context64(IrqSafeRwLock>), + Context32(IrqSafeRwLock>), + Context64(IrqSafeRwLock>), } pub enum XhciInputContext { - Context32(PageBox), - Context64(PageBox), + Context32(DmaBuffer), + Context64(DmaBuffer), } impl XhciDeviceContext { - pub fn new(size: ContextSize) -> Result { + pub fn new(dma: &dyn DmaAllocator, size: ContextSize) -> Result { match size { - ContextSize::Context32 => PageBox::new(context::Device::new_32byte()) + ContextSize::Context32 => DmaBuffer::new(dma, context::Device::new_32byte()) .map(IrqSafeRwLock::new) .map(Self::Context32), - ContextSize::Context64 => PageBox::new(context::Device::new_64byte()) + ContextSize::Context64 => DmaBuffer::new(dma, context::Device::new_64byte()) .map(IrqSafeRwLock::new) .map(Self::Context64), } @@ -40,35 +38,38 @@ impl XhciDeviceContext { } } - pub fn physical_address(&self) -> PhysicalAddress { + pub fn bus_address(&self) -> BusAddress { match self { - Self::Context32(cx) => unsafe { cx.read().as_physical_address() }, - Self::Context64(cx) => unsafe { cx.read().as_physical_address() }, + Self::Context32(cx) => cx.read().bus_address(), + Self::Context64(cx) => cx.read().bus_address(), } } + // pub fn physical_address(&self) -> PhysicalAddress { + // } } impl XhciInputContext { - pub fn new(size: ContextSize) -> Result { + pub fn new(dma: &dyn DmaAllocator, size: ContextSize) -> Result { match size { ContextSize::Context32 => { - PageBox::new(context::Input::new_32byte()).map(Self::Context32) + DmaBuffer::new(dma, context::Input::new_32byte()).map(Self::Context32) } ContextSize::Context64 => { - PageBox::new(context::Input::new_64byte()).map(Self::Context64) + DmaBuffer::new(dma, context::Input::new_64byte()).map(Self::Context64) } } .map_err(UsbError::MemoryError) } pub fn new_address_device( + dma: &dyn DmaAllocator, size: ContextSize, root_hub_port_number: PortNumber, max_packet_size: usize, speed: u8, - dequeue_pointer: PhysicalAddress, + dequeue_pointer: BusAddress, ) -> Result { - let mut cx = Self::new(size)?; + let mut cx = Self::new(dma, size)?; { let control = cx.control_mut(); @@ -100,10 +101,16 @@ impl XhciInputContext { Ok(cx) } - pub fn physical_address(&self) -> PhysicalAddress { + // pub fn physical_address(&self) -> PhysicalAddress { + // match self { + // Self::Context32(cx) => unsafe { cx.as_physical_address() }, + // Self::Context64(cx) => unsafe { cx.as_physical_address() }, + // } + // } + pub fn bus_address(&self) -> BusAddress { match self { - Self::Context32(cx) => unsafe { cx.as_physical_address() }, - Self::Context64(cx) => unsafe { cx.as_physical_address() }, + Self::Context32(cx) => cx.bus_address(), + Self::Context64(cx) => cx.bus_address(), } } } diff --git a/kernel/driver/usb/xhci/src/controller.rs b/kernel/driver/usb/xhci/src/controller.rs index 4838d384..f82ec3b5 100644 --- a/kernel/driver/usb/xhci/src/controller.rs +++ b/kernel/driver/usb/xhci/src/controller.rs @@ -1,15 +1,19 @@ -use core::sync::atomic::{AtomicU8, Ordering}; +use core::{ + mem::MaybeUninit, + sync::atomic::{AtomicU8, Ordering}, +}; use alloc::{boxed::Box, collections::btree_map::BTreeMap, sync::Arc, vec::Vec}; use async_trait::async_trait; use device_api::{ device::{Device, DeviceInitContext}, + dma::DmaAllocator, interrupt::{InterruptHandler, IrqVector}, }; -use libk::{error::Error, task::runtime}; -use libk_mm::{ - address::{AsPhysicalAddress, PhysicalAddress}, - PageBox, +use libk::{ + dma::{BusAddress, DmaBuffer}, + error::Error, + task::runtime, }; use libk_util::{ sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock}, @@ -50,8 +54,8 @@ use crate::{ #[allow(unused)] struct ScratchpadArray { - buffers: Vec>, - array: PageBox<[PhysicalAddress]>, + buffers: Vec]>>, + array: DmaBuffer<[BusAddress]>, } struct RootHubPort { @@ -63,8 +67,9 @@ pub struct Xhci { pub(crate) regs: Regs, #[allow(unused)] pci: PciDeviceInfo, + pub(crate) dma: Arc, - dcbaa: IrqSafeRwLock>, + dcbaa: IrqSafeRwLock>, #[allow(unused)] scratchpads: Option, pub(crate) command_ring: CommandRing, @@ -80,31 +85,39 @@ pub struct Xhci { } impl ScratchpadArray { - pub fn new(capacity: usize, element_size: usize) -> Result { + pub fn new( + dma: &dyn DmaAllocator, + capacity: usize, + element_size: usize, + ) -> Result { let buffers = (0..capacity) - .map(|_| PageBox::new_slice(0, element_size)) + .map(|_| DmaBuffer::new_zeroed_slice(dma, element_size)) .collect::, _>>() .map_err(UsbError::MemoryError)?; - let array = - PageBox::new_slice_with(|i| unsafe { buffers[i].as_physical_address() }, capacity) - .map_err(UsbError::MemoryError)?; + let array = DmaBuffer::new_slice_with(dma, |i| buffers[i].bus_address(), capacity) + .map_err(UsbError::MemoryError)?; Ok(Self { buffers, array }) } } impl Xhci { - pub fn new(pci: PciDeviceInfo, regs: Regs) -> Result { - let mut dcbaa = PageBox::new_slice(PhysicalAddress::ZERO, regs.slot_count + 1) + pub fn new( + dma: Arc, + pci: PciDeviceInfo, + regs: Regs, + ) -> Result { + let mut dcbaa = DmaBuffer::new_slice(&*dma, BusAddress::ZERO, regs.slot_count + 1) .map_err(UsbError::MemoryError)?; - let command_ring = CommandRing::new(128)?; - let event_ring = EventRing::new(128)?; - let erst = EventRingSegmentTable::for_event_rings(&[&event_ring])?; + let command_ring = CommandRing::new(&*dma, 128)?; + let event_ring = EventRing::new(&*dma, 128)?; + let erst = EventRingSegmentTable::for_event_rings(&*dma, &[&event_ring])?; // Setup scratch buffers // TODO: Linux seems to just ignore the PAGESIZE, it's (1 << 0) everywhere let scratchpads = if regs.scratch_count != 0 { - let array = ScratchpadArray::new(regs.scratch_count, 0x1000)?; - dcbaa[0] = unsafe { array.array.as_physical_address() }; + let array = ScratchpadArray::new(&*dma, regs.scratch_count, 0x1000)?; + dcbaa[0] = array.array.bus_address(); + // dcbaa[0] = unsafe { array.array.as_physical_address() }; Some(array) } else { None @@ -141,6 +154,7 @@ impl Xhci { Ok(Self { regs, pci, + dma, dcbaa: IrqSafeRwLock::new(dcbaa), scratchpads, @@ -157,7 +171,7 @@ impl Xhci { }) } - fn notify_endpoint(&self, slot_id: u8, endpoint_id: u8, address: PhysicalAddress, status: u32) { + fn notify_endpoint(&self, slot_id: u8, endpoint_id: u8, address: BusAddress, status: u32) { if let Some(endpoint) = self.endpoints.read().get(&(slot_id, endpoint_id)) { endpoint.notify(address, status); } else { @@ -178,12 +192,12 @@ impl Xhci { slot_type: u8, speed: UsbSpeed, ) -> Result<(u8, Arc, Arc), UsbError> { - let device_context = XhciDeviceContext::new(self.regs.context_size)?; + let device_context = XhciDeviceContext::new(&*self.dma, self.regs.context_size)?; let slot_id = self.command_ring.enable_slot(&**self, slot_type).await?; let (control_pipe, control_ring) = ControlPipe::new(self.clone(), slot_id, 1, 128)?; let control_pipe = UsbControlPipeAccess(Box::new(control_pipe)); - self.dcbaa.write()[slot_id as usize] = device_context.physical_address(); + self.dcbaa.write()[slot_id as usize] = device_context.bus_address(); self.endpoints .write() .insert((slot_id, 1), control_ring.clone()); @@ -270,14 +284,16 @@ impl Xhci { .await?; let input_cx = XhciInputContext::new_address_device( + &*self.dma, self.regs.context_size, number, max_packet_size, speed, - control_ring.base(), + control_ring.bus_address(), )?; + self.command_ring - .address_device(&**self, slot_id, input_cx.physical_address(), false) + .address_device(&**self, slot_id, &input_cx, false) .await .inspect_err(|error| { log::error!("Port {number} Address Device TRB (BSR=0) failed: {error:?}") @@ -426,7 +442,7 @@ impl CommandExecutor for Xhci { &self, slot_id: u8, endpoint_id: u8, - dequeue_pointer: PhysicalAddress, + dequeue_pointer: BusAddress, dequeue_cycle: bool, ) -> Result<(), UsbError> { log::warn!("xhci: reset stalled endpoint {slot_id}:{endpoint_id}"); @@ -443,16 +459,13 @@ impl CommandExecutor for Xhci { } impl Device for Xhci { - unsafe fn init(self: Arc, cx: DeviceInitContext) -> Result<(), Error> { - // TODO use DmaAllocator instead of PageBox - let _ = cx; - + unsafe fn init(self: Arc, _cx: DeviceInitContext) -> Result<(), Error> { self.regs.hc_reset(10000000)?; log::info!("xHC reset complete"); // Configure the HC - let dcbaap = unsafe { self.dcbaa.read().as_physical_address() }; - let cr_base = self.command_ring.base(); + let dcbaap = self.dcbaa.read().bus_address(); + let cr_base = self.command_ring.bus_base(); let op = self.regs.operational.write(); let rt = self.regs.runtime.write(); diff --git a/kernel/driver/usb/xhci/src/device.rs b/kernel/driver/usb/xhci/src/device.rs index b7f281ad..7d635641 100644 --- a/kernel/driver/usb/xhci/src/device.rs +++ b/kernel/driver/usb/xhci/src/device.rs @@ -124,10 +124,12 @@ impl XhciBusDevice { (UsbEndpointType::Interrupt, UsbDirection::Out) => context::EndpointType::InterruptOut, (UsbEndpointType::Bulk, UsbDirection::In) => context::EndpointType::BulkIn, (UsbEndpointType::Bulk, UsbDirection::Out) => context::EndpointType::BulkOut, - _ => todo!(), + (UsbEndpointType::Isochronous, UsbDirection::In) => context::EndpointType::IsochIn, + (UsbEndpointType::Isochronous, UsbDirection::Out) => context::EndpointType::IsochOut, + (UsbEndpointType::Control, _) => context::EndpointType::Control, }; - let mut input_cx = XhciInputContext::new(self.xhci.regs.context_size)?; + let mut input_cx = XhciInputContext::new(&*self.xhci.dma, self.xhci.regs.context_size)?; { let control = input_cx.control_mut(); @@ -150,17 +152,13 @@ impl XhciBusDevice { // TODO Pick a better value here endpoint.set_interval(3); endpoint.set_max_packet_size(max_packet_size); - endpoint.set_tr_dequeue_pointer(ring.base().into_u64()); + endpoint.set_tr_dequeue_pointer(ring.bus_address().into_u64()); endpoint.set_dequeue_cycle_state(); } self.xhci .command_ring - .configure_endpoint( - self.xhci.as_ref(), - self.slot_id, - input_cx.physical_address(), - ) + .configure_endpoint(self.xhci.as_ref(), self.slot_id, &input_cx) .await?; self.endpoints.write().insert(dci, ring.clone()); diff --git a/kernel/driver/usb/xhci/src/lib.rs b/kernel/driver/usb/xhci/src/lib.rs index a82ca20e..6bbb0c78 100644 --- a/kernel/driver/usb/xhci/src/lib.rs +++ b/kernel/driver/usb/xhci/src/lib.rs @@ -1,12 +1,12 @@ #![no_std] #![allow(clippy::new_without_default)] -#![feature(iter_array_chunks, let_chains)] +#![feature(iter_array_chunks, let_chains, maybe_uninit_slice)] extern crate alloc; use alloc::sync::Arc; use controller::Xhci; -use device_api::{device::Device, interrupt::InterruptAffinity}; +use device_api::{device::Device, dma::DmaAllocator, interrupt::InterruptAffinity}; use regs::Regs; use ygg_driver_pci::{ capability::{DevicePowerState, PowerManagementCapability}, @@ -32,7 +32,7 @@ pci_driver! { "xhci" } - fn probe(&self, info: &PciDeviceInfo) -> Result, Error> { + fn probe(&self, info: &PciDeviceInfo, dma: &Arc) -> Result, Error> { // TODO Chip Hardware Reset let bar0 = info .config_space @@ -60,7 +60,7 @@ pci_driver! { info.init_interrupts(PreferredInterruptMode::Msi(true))?; let regs = Regs::map(bar0)?; - let xhci = Arc::new(Xhci::new(info.clone(), regs)?); + let xhci = Arc::new(Xhci::new(dma.clone(), info.clone(), regs)?); info.map_interrupt(InterruptAffinity::Any, xhci.clone())?; diff --git a/kernel/driver/usb/xhci/src/pipe.rs b/kernel/driver/usb/xhci/src/pipe.rs index 5b87b5b2..b91ab053 100644 --- a/kernel/driver/usb/xhci/src/pipe.rs +++ b/kernel/driver/usb/xhci/src/pipe.rs @@ -2,17 +2,20 @@ use core::mem::MaybeUninit; use alloc::{boxed::Box, sync::Arc}; use async_trait::async_trait; -use libk_mm::{address::AsPhysicalAddress, PageSlice}; +use libk::dma::DmaBuffer; use ygg_driver_usb::{ - communication::UsbDirection, error::{TransferError, UsbError}, pipe::{ control::{ControlTransferSetup, UsbControlPipe}, normal::{UsbNormalPipeIn, UsbNormalPipeOut}, + UsbGenericPipe, }, }; -use crate::{controller::Xhci, ring::transfer::TransferRing}; +use crate::{ + controller::Xhci, + ring::transfer::{ControlDataStage, TransferRing}, +}; pub struct ControlPipe { xhci: Arc, @@ -36,7 +39,12 @@ impl ControlPipe { endpoint_id: u8, capacity: usize, ) -> Result<(Self, Arc), UsbError> { - let ring = Arc::new(TransferRing::new(slot_id, endpoint_id, capacity)?); + let ring = Arc::new(TransferRing::new( + &*xhci.dma, + slot_id, + endpoint_id, + capacity, + )?); Ok(( Self { xhci, @@ -54,7 +62,12 @@ impl NormalInPipe { endpoint_id: u8, capacity: usize, ) -> Result<(Self, Arc), UsbError> { - let ring = Arc::new(TransferRing::new(slot_id, endpoint_id, capacity)?); + let ring = Arc::new(TransferRing::new( + &*xhci.dma, + slot_id, + endpoint_id, + capacity, + )?); Ok(( Self { xhci, @@ -72,7 +85,12 @@ impl NormalOutPipe { endpoint_id: u8, capacity: usize, ) -> Result<(Self, Arc), UsbError> { - let ring = Arc::new(TransferRing::new(slot_id, endpoint_id, capacity)?); + let ring = Arc::new(TransferRing::new( + &*xhci.dma, + slot_id, + endpoint_id, + capacity, + )?); Ok(( Self { xhci, @@ -85,45 +103,92 @@ impl NormalOutPipe { #[async_trait] impl UsbControlPipe for ControlPipe { - async fn control_transfer( + async fn control_transfer(&self, setup: ControlTransferSetup) -> Result<(), UsbError> { + self.ring + .control_transfer(self.xhci.as_ref(), setup, ControlDataStage::None) + .await?; + Ok(()) + } + + async fn control_transfer_in( &self, setup: ControlTransferSetup, - data: Option<(&mut PageSlice>, UsbDirection)>, + buffer: &mut [MaybeUninit], ) -> Result { - let data_len = data.as_ref().map_or(0, |(data, _)| data.len()); + let data_len = buffer.len(); + let mut dma_buffer = DmaBuffer::new_uninit_slice(&*self.xhci.dma, data_len) + .map_err(UsbError::MemoryError)?; + let result = self .ring - .control_transfer(self.xhci.as_ref(), setup, data) + .control_transfer( + self.xhci.as_ref(), + setup, + ControlDataStage::In(&mut dma_buffer), + ) .await; - allow_short_packet(data_len, result) + let result = allow_short_packet(data_len, result); + + match result { + Ok(len) => { + buffer[..len].copy_from_slice(&dma_buffer[..len]); + Ok(len) + } + Err(error) => Err(error), + } + } + + async fn control_transfer_out( + &self, + setup: ControlTransferSetup, + buffer: &[u8], + ) -> Result { + let mut dma_buffer = + DmaBuffer::from_slice(&*self.xhci.dma, buffer).map_err(UsbError::MemoryError)?; + + self.ring + .control_transfer( + self.xhci.as_ref(), + setup, + ControlDataStage::Out(&mut dma_buffer), + ) + .await + } +} + +impl UsbGenericPipe for NormalInPipe { + fn allocate_dma_buffer(&self, len: usize) -> Result]>, UsbError> { + DmaBuffer::new_uninit_slice(&*self.xhci.dma, len).map_err(UsbError::MemoryError) } } #[async_trait] impl UsbNormalPipeIn for NormalInPipe { - async fn read(&self, buffer: &mut PageSlice) -> Result { - let data_len = buffer.len(); + async fn read_dma( + &self, + buffer: &mut DmaBuffer<[MaybeUninit]>, + limit: usize, + ) -> Result { + let len = limit.min(buffer.len()); let result = self .ring - .normal_transfer( - self.xhci.as_ref(), - unsafe { buffer.as_physical_address() }, - buffer.len(), - ) + .normal_transfer(self.xhci.as_ref(), buffer.bus_address(), len) .await; - allow_short_packet(data_len, result) + allow_short_packet(len, result) + } +} + +impl UsbGenericPipe for NormalOutPipe { + fn allocate_dma_buffer(&self, len: usize) -> Result]>, UsbError> { + DmaBuffer::new_uninit_slice(&*self.xhci.dma, len).map_err(UsbError::MemoryError) } } #[async_trait] impl UsbNormalPipeOut for NormalOutPipe { - async fn write(&self, buffer: &PageSlice) -> Result { + async fn write_dma(&self, buffer: &DmaBuffer<[u8]>) -> Result { self.ring - .normal_transfer( - self.xhci.as_ref(), - unsafe { buffer.as_physical_address() }, - buffer.len(), - ) + .normal_transfer(self.xhci.as_ref(), buffer.bus_address(), buffer.len()) .await } } diff --git a/kernel/driver/usb/xhci/src/regs/operational.rs b/kernel/driver/usb/xhci/src/regs/operational.rs index 22e9b7a5..9c3e5814 100644 --- a/kernel/driver/usb/xhci/src/regs/operational.rs +++ b/kernel/driver/usb/xhci/src/regs/operational.rs @@ -1,5 +1,4 @@ -use libk::error::Error; -use libk_mm::address::PhysicalAddress; +use libk::{dma::BusAddress, error::Error}; use tock_registers::{ fields::FieldValue, interfaces::{Readable, Writeable}, @@ -116,12 +115,12 @@ impl OperationalRegs { } } - pub fn set_dcbaap(&self, value: PhysicalAddress) { + pub fn set_dcbaap(&self, value: BusAddress) { let value = value.into_u64(); self.DCBAAP.set(value); } - pub fn set_crcr(&self, value: PhysicalAddress, dcs: bool) { + pub fn set_crcr(&self, value: BusAddress, dcs: bool) { let mut value = value.into_u64(); if dcs { value |= 1; diff --git a/kernel/driver/usb/xhci/src/regs/runtime.rs b/kernel/driver/usb/xhci/src/regs/runtime.rs index 7ef2fd07..92b7811a 100644 --- a/kernel/driver/usb/xhci/src/regs/runtime.rs +++ b/kernel/driver/usb/xhci/src/regs/runtime.rs @@ -1,11 +1,11 @@ -use libk_mm::address::PhysicalAddress; +use libk::dma::BusAddress; use tock_registers::{ interfaces::{Readable, Writeable}, register_bitfields, register_structs, registers::ReadWrite, }; -use crate::ring::{EventRing, EventRingSegmentTable}; +use crate::ring::{EventRing, EventRingSegmentTable, GenericRing}; register_bitfields! { u32, @@ -53,8 +53,8 @@ impl RuntimeRegs { ) { let interrupter = &self.IRn[index]; - let erdp = event_ring.dequeue_pointer().into_u64(); - let erstba = erst.physical_address().into_u64(); + let erdp = event_ring.bus_base().into_u64(); + let erstba = erst.bus_address().into_u64(); interrupter.ERSTSZ.set(erst.capacity() as u32); interrupter.ERSTBA.set(erstba); @@ -62,7 +62,7 @@ impl RuntimeRegs { interrupter.IMAN.write(IMAN::IE::SET); } - pub fn set_interrupter_dequeue_pointer(&self, index: usize, erdp: PhysicalAddress) { + pub fn set_interrupter_dequeue_pointer(&self, index: usize, erdp: BusAddress) { let _ = self.IRn[index].ERDP.get(); self.IRn[index].ERDP.set(erdp.into_u64() | (1 << 3)); } diff --git a/kernel/driver/usb/xhci/src/ring/command.rs b/kernel/driver/usb/xhci/src/ring/command.rs index 3732380e..e70d8465 100644 --- a/kernel/driver/usb/xhci/src/ring/command.rs +++ b/kernel/driver/usb/xhci/src/ring/command.rs @@ -7,10 +7,8 @@ use core::{ use alloc::collections::BTreeMap; use bytemuck::{Pod, Zeroable}; -use libk_mm::{ - address::{AsPhysicalAddress, PhysicalAddress}, - PageBox, -}; +use device_api::dma::DmaAllocator; +use libk::dma::{BusAddress, DmaBuffer}; use libk_util::{ sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock}, waker::QueueWaker, @@ -18,10 +16,12 @@ use libk_util::{ use ygg_driver_usb::error::UsbError; use yggdrasil_abi::define_bitfields; +use crate::context::XhciInputContext; + use super::{CommandExecutor, GenericRing, LinkTrb}; struct CommandRingInner { - trbs: PageBox<[MaybeUninit]>, + trbs: DmaBuffer<[MaybeUninit]>, enqueue_index: usize, #[allow(unused)] dequeue_index: usize, @@ -31,23 +31,18 @@ struct CommandRingInner { pub struct CommandRing { inner: IrqSafeSpinlock, // TODO maybe use Vec of "slots"? - completions: IrqSafeRwLock>, + completions: IrqSafeRwLock>, completion_notify: QueueWaker, - capacity: usize, } impl GenericRing for CommandRing { - fn base(&self) -> PhysicalAddress { - unsafe { self.inner.lock().trbs.as_physical_address() } - } - - fn capacity(&self) -> usize { - self.capacity + fn bus_base(&self) -> BusAddress { + self.inner.lock().trbs.bus_address() } } impl CommandRingInner { - fn enqueue(&mut self, trb: C) -> PhysicalAddress { + fn enqueue(&mut self, trb: C) -> BusAddress { let mut raw: RawCommandTrb = bytemuck::cast(trb); raw.flags.set_ty(C::TRB_TYPE as _); @@ -55,8 +50,12 @@ impl CommandRingInner { self.trbs[self.enqueue_index].write(raw); - let address = unsafe { self.trbs.as_physical_address() } + let address = self + .trbs + .bus_address() .add(self.enqueue_index * size_of::()); + // let address = unsafe { self.trbs.as_physical_address() } + // .add(self.enqueue_index * size_of::()); // Move to the next TRB slot self.enqueue_index += 1; @@ -72,7 +71,7 @@ impl CommandRingInner { } fn enqueue_link(&mut self) { - let base = unsafe { self.trbs.as_physical_address() }; + let base = self.trbs.bus_address(); let link = LinkTrb::new(base, self.cycle_bit); self.trbs[self.enqueue_index].write(bytemuck::cast(link)); @@ -88,8 +87,8 @@ impl CommandRingInner { } impl CommandRing { - pub fn new(capacity: usize) -> Result { - let trbs = PageBox::new_zeroed_slice(capacity).map_err(UsbError::MemoryError)?; + pub fn new(dma: &dyn DmaAllocator, capacity: usize) -> Result { + let trbs = DmaBuffer::new_zeroed_slice(dma, capacity).map_err(UsbError::MemoryError)?; Ok(Self { inner: IrqSafeSpinlock::new(CommandRingInner { @@ -100,11 +99,10 @@ impl CommandRing { }), completions: IrqSafeRwLock::new(BTreeMap::new()), completion_notify: QueueWaker::new(), - capacity, }) } - pub fn enqueue(&self, trb: C) -> PhysicalAddress { + pub fn enqueue(&self, trb: C) -> BusAddress { let mut inner = self.inner.lock(); inner.enqueue(trb) } @@ -113,12 +111,12 @@ impl CommandRing { &self, executor: &E, slot_id: u8, - cx_physical_address: PhysicalAddress, + input_context: &XhciInputContext, bsr: bool, ) -> Result<(), UsbError> { self.submit_and_wait( executor, - AddressDeviceCommandTrb::new(cx_physical_address, slot_id, bsr), + AddressDeviceCommandTrb::new(input_context.bus_address(), slot_id, bsr), ) .await?; Ok(()) @@ -128,11 +126,11 @@ impl CommandRing { &self, executor: &E, slot_id: u8, - cx_physical_address: PhysicalAddress, + input_context: &XhciInputContext, ) -> Result<(), UsbError> { self.submit_and_wait( executor, - ConfigureEndpointCommandTrb::new(cx_physical_address, slot_id), + ConfigureEndpointCommandTrb::new(input_context.bus_address(), slot_id), ) .await?; Ok(()) @@ -158,7 +156,7 @@ impl CommandRing { executor: &E, slot_id: u8, endpoint_id: u8, - dequeue_pointer: PhysicalAddress, + dequeue_pointer: BusAddress, dequeue_cycle: bool, ) -> Result<(), UsbError> { self.submit_and_wait( @@ -219,11 +217,11 @@ impl CommandRing { .await } - pub fn get_completion(&self, address: PhysicalAddress) -> Option { + pub fn get_completion(&self, address: BusAddress) -> Option { self.completions.write().remove(&address) } - pub fn notify(&self, address: PhysicalAddress, reply: CommandReply) { + pub fn notify(&self, address: BusAddress, reply: CommandReply) { self.inner.lock().advance(); self.completions.write().insert(address, reply); self.completion_notify.wake_all(); @@ -296,7 +294,7 @@ pub struct DisableSlotCommandTrb { #[derive(Clone, Copy, Debug, Pod, Zeroable)] #[repr(C, align(16))] pub struct AddressDeviceCommandTrb { - pub input_context_address: PhysicalAddress, + pub input_context_address: BusAddress, _0: u32, pub flags: AddressDeviceCommandFlags, } @@ -304,7 +302,7 @@ pub struct AddressDeviceCommandTrb { #[derive(Clone, Copy, Debug, Pod, Zeroable)] #[repr(C, align(16))] pub struct ConfigureEndpointCommandTrb { - pub input_context_address: PhysicalAddress, + pub input_context_address: BusAddress, _0: u32, pub flags: ConfigureEndpointCommandFlags, } @@ -361,7 +359,7 @@ impl DisableSlotCommandTrb { } impl AddressDeviceCommandTrb { - pub fn new(input_context_address: PhysicalAddress, slot_id: u8, bsr: bool) -> Self { + pub fn new(input_context_address: BusAddress, slot_id: u8, bsr: bool) -> Self { Self { input_context_address, _0: 0, @@ -371,7 +369,7 @@ impl AddressDeviceCommandTrb { } impl ConfigureEndpointCommandTrb { - pub fn new(input_context_address: PhysicalAddress, slot_id: u8) -> Self { + pub fn new(input_context_address: BusAddress, slot_id: u8) -> Self { Self { input_context_address, _0: 0, @@ -397,7 +395,7 @@ impl SetTrDequeuePointerCommandTrb { pub fn new( slot_id: u8, endpoint_id: u8, - dequeue_pointer: PhysicalAddress, + dequeue_pointer: BusAddress, dequeue_cycle: bool, ) -> Self { let mut value = dequeue_pointer.into_u64(); diff --git a/kernel/driver/usb/xhci/src/ring/event.rs b/kernel/driver/usb/xhci/src/ring/event.rs index a5a2f4a8..b3d3fc51 100644 --- a/kernel/driver/usb/xhci/src/ring/event.rs +++ b/kernel/driver/usb/xhci/src/ring/event.rs @@ -1,10 +1,8 @@ use core::mem::{size_of, MaybeUninit}; use bytemuck::{Pod, Zeroable}; -use libk_mm::{ - address::{AsPhysicalAddress, PhysicalAddress}, - PageBox, -}; +use device_api::dma::DmaAllocator; +use libk::dma::{BusAddress, DmaBuffer}; use libk_util::sync::IrqSafeSpinlock; use ygg_driver_usb::error::UsbError; use yggdrasil_abi::define_bitfields; @@ -14,11 +12,11 @@ use super::{command::CommandReply, GenericRing}; pub enum Event { PortChange(usize), CommandCompletion { - address: PhysicalAddress, + address: BusAddress, reply: CommandReply, }, Transfer { - address: PhysicalAddress, + address: BusAddress, slot_id: u8, endpoint_id: u8, status: u32, @@ -27,7 +25,7 @@ pub enum Event { #[repr(C, align(16))] pub struct EventRingSegment { - address: PhysicalAddress, + address: BusAddress, // Number of TRBs supported by the ring segment. Valid values are 16 to 4096 size: u16, _0: u16, @@ -35,11 +33,11 @@ pub struct EventRingSegment { } pub struct EventRingSegmentTable { - entries: PageBox<[EventRingSegment]>, + entries: DmaBuffer<[EventRingSegment]>, } struct EventRingInner { - trbs: PageBox<[MaybeUninit]>, + trbs: DmaBuffer<[MaybeUninit]>, dequeue_index: usize, cycle_bit: bool, } @@ -50,20 +48,24 @@ pub struct EventRing { } impl EventRingSegmentTable { - pub fn for_event_rings(rings: &[&EventRing]) -> Result { - let entries = PageBox::from_iter_exact(rings.iter().map(|ring| EventRingSegment { - address: ring.base(), - size: ring.capacity().try_into().unwrap(), - _0: 0, - _1: 0, - })) + pub fn for_event_rings(dma: &dyn DmaAllocator, rings: &[&EventRing]) -> Result { + let entries = DmaBuffer::new_slice_with( + dma, + |i| EventRingSegment { + address: rings[i].bus_base(), + size: rings[i].capacity.try_into().unwrap(), + _0: 0, + _1: 0, + }, + rings.len(), + ) .map_err(UsbError::MemoryError)?; Ok(Self { entries }) } - pub fn physical_address(&self) -> PhysicalAddress { - unsafe { self.entries.as_physical_address() } + pub fn bus_address(&self) -> BusAddress { + self.entries.bus_address() } pub fn capacity(&self) -> usize { @@ -72,12 +74,8 @@ impl EventRingSegmentTable { } impl GenericRing for EventRing { - fn base(&self) -> PhysicalAddress { - unsafe { self.inner.lock().trbs.as_physical_address() } - } - - fn capacity(&self) -> usize { - self.capacity + fn bus_base(&self) -> BusAddress { + self.inner.lock().trbs.bus_address() } } @@ -103,8 +101,8 @@ impl EventRingInner { } impl EventRing { - pub fn new(capacity: usize) -> Result { - let trbs = PageBox::new_zeroed_slice(capacity).map_err(UsbError::MemoryError)?; + pub fn new(dma: &dyn DmaAllocator, capacity: usize) -> Result { + let trbs = DmaBuffer::new_zeroed_slice(dma, capacity).map_err(UsbError::MemoryError)?; Ok(Self { inner: IrqSafeSpinlock::new(EventRingInner { @@ -120,9 +118,11 @@ impl EventRing { self.inner.lock().try_dequeue() } - pub fn dequeue_pointer(&self) -> PhysicalAddress { + pub fn dequeue_pointer(&self) -> BusAddress { let i = self.inner.lock(); - unsafe { i.trbs.as_physical_address() }.add(i.dequeue_index * size_of::()) + i.trbs + .bus_address() + .add(i.dequeue_index * size_of::()) } } @@ -171,7 +171,7 @@ define_bitfields! { #[derive(Clone, Copy, Debug, Pod, Zeroable)] #[repr(C, align(16))] pub struct TransferEventTrb { - pub address: PhysicalAddress, + pub address: BusAddress, pub status: TransferEventStatus, pub flags: TransferEventFlags, } @@ -179,7 +179,7 @@ pub struct TransferEventTrb { #[derive(Clone, Copy, Debug, Pod, Zeroable)] #[repr(C, align(16))] pub struct CommandCompletionEventTrb { - pub address: PhysicalAddress, + pub address: BusAddress, pub status: CommandCompletionEventStatus, pub flags: CommandCompletionEventFlags, } diff --git a/kernel/driver/usb/xhci/src/ring/mod.rs b/kernel/driver/usb/xhci/src/ring/mod.rs index ca4c04a1..fa5190a2 100644 --- a/kernel/driver/usb/xhci/src/ring/mod.rs +++ b/kernel/driver/usb/xhci/src/ring/mod.rs @@ -1,11 +1,7 @@ -//use bytemuck::{Pod, Zeroable}; -//use libk_mm::address::PhysicalAddress; -//use yggdrasil_abi::define_bitfields; - use alloc::boxed::Box; use async_trait::async_trait; use bytemuck::{Pod, Zeroable}; -use libk_mm::address::PhysicalAddress; +use libk::dma::BusAddress; use ygg_driver_usb::error::UsbError; use yggdrasil_abi::define_bitfields; @@ -23,14 +19,13 @@ pub trait CommandExecutor { &self, slot_id: u8, endpoint_id: u8, - dequeue_pointer: PhysicalAddress, + dequeue_pointer: BusAddress, dequeue_cycle: bool, ) -> Result<(), UsbError>; } pub trait GenericRing { - fn capacity(&self) -> usize; - fn base(&self) -> PhysicalAddress; + fn bus_base(&self) -> BusAddress; } define_bitfields! { @@ -45,13 +40,13 @@ define_bitfields! { #[derive(Clone, Copy, Debug, Pod, Zeroable)] #[repr(C, align(16))] pub struct LinkTrb { - pub address: PhysicalAddress, + pub address: BusAddress, _0: u32, pub flags: LinkTrbFlags, } impl LinkTrb { - pub fn new(address: PhysicalAddress, cycle_bit: bool) -> Self { + pub fn new(address: BusAddress, cycle_bit: bool) -> Self { Self { address, _0: 0, diff --git a/kernel/driver/usb/xhci/src/ring/transfer.rs b/kernel/driver/usb/xhci/src/ring/transfer.rs index af505f9b..cd062188 100644 --- a/kernel/driver/usb/xhci/src/ring/transfer.rs +++ b/kernel/driver/usb/xhci/src/ring/transfer.rs @@ -10,11 +10,9 @@ use alloc::{ vec::Vec, }; use bytemuck::{Pod, Zeroable}; +use device_api::dma::DmaAllocator; use futures_util::task::AtomicWaker; -use libk_mm::{ - address::{AsPhysicalAddress, PhysicalAddress}, - PageBox, PageSlice, -}; +use libk::dma::{BusAddress, DmaBuffer}; use libk_util::{ queue::BoundedQueue, sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock, IrqSafeSpinlockGuard}, @@ -29,7 +27,7 @@ use yggdrasil_abi::define_bitfields; use super::{CommandExecutor, LinkTrb}; struct TransferRingInner { - trbs: PageBox<[MaybeUninit]>, + trbs: DmaBuffer<[MaybeUninit]>, enqueue_index: usize, dequeue_index: usize, cycle_bit: bool, @@ -37,7 +35,7 @@ struct TransferRingInner { pub struct TransferRing { inner: IrqSafeSpinlock, - base: PhysicalAddress, + bus_base: BusAddress, capacity: usize, slot_id: u8, @@ -50,7 +48,7 @@ pub struct TransferRing { pub struct TransactionBuilder<'a> { inner: IrqSafeSpinlockGuard<'a, TransferRingInner>, ring: &'a Arc, - pending: Vec, + pending: Vec, } pub struct Transaction { @@ -66,15 +64,26 @@ pub enum TransactionEvent { Shutdown, } +pub enum ControlDataStage<'a> { + None, + In(&'a mut DmaBuffer<[MaybeUninit]>), + Out(&'a DmaBuffer<[u8]>), +} + impl TransferRing { - pub fn new(slot_id: u8, endpoint_id: u8, capacity: usize) -> Result { - let inner = TransferRingInner::new(capacity)?; - let base = unsafe { inner.trbs.as_physical_address() }; + pub fn new( + dma: &dyn DmaAllocator, + slot_id: u8, + endpoint_id: u8, + capacity: usize, + ) -> Result { + let inner = TransferRingInner::new(dma, capacity)?; + let bus_base = inner.trbs.bus_address(); let transactions = (0..capacity).map(|_| None).collect(); Ok(Self { inner: IrqSafeSpinlock::new(inner), - base, + bus_base, capacity, slot_id, @@ -105,7 +114,7 @@ impl TransferRing { ) { if let Err(TransferError::Stall) = result { let dequeue = self - .base + .bus_base .add(transaction.next_dequeue * size_of::()); if let Err(rerror) = executor .reset_endpoint( @@ -130,7 +139,7 @@ impl TransferRing { pub async fn normal_transfer( self: &Arc, executor: &E, - buffer: PhysicalAddress, + buffer: BusAddress, length: usize, ) -> Result { if length == 0 { @@ -154,12 +163,12 @@ impl TransferRing { self: &Arc, executor: &E, setup: ControlTransferSetup, - buffer: Option<(&mut PageSlice>, UsbDirection)>, + data: ControlDataStage<'_>, ) -> Result { let mut builder = self.transaction_builder()?; - let data_len = buffer.as_ref().map_or(0, |(buffer, _)| buffer.len()); - let (setup, data, status) = builder.enqueue_control(setup, buffer)?; + let data_len = data.len(); + let (setup, data, status) = builder.enqueue_control(setup, data)?; let transaction = builder.submit(executor); @@ -188,18 +197,19 @@ impl TransferRing { } } - pub fn notify(&self, address: PhysicalAddress, status: u32) { + pub fn notify(&self, address: BusAddress, status: u32) { if status == 0 { return; } - if address < self.base || address - self.base >= size_of::() * self.capacity + if address < self.bus_base + || address - self.bus_base >= size_of::() * self.capacity { log::warn!("xhci: event outside of trb array: {address:#x}"); return; } - let index = (address - self.base) / size_of::(); + let index = (address - self.bus_base) / size_of::(); if let Some(tx) = self.transactions.write()[index] .take() .and_then(|tx| tx.upgrade()) @@ -210,8 +220,8 @@ impl TransferRing { } } - pub fn base(&self) -> PhysicalAddress { - self.base + pub fn bus_address(&self) -> BusAddress { + self.bus_base } } @@ -221,14 +231,10 @@ impl TransactionBuilder<'_> { pub fn enqueue(&mut self, trb: C, ioc: bool) -> Result { let address = self.inner.enqueue(trb, ioc)?; self.pending.push(address); - Ok((address - self.ring.base) / size_of::()) + Ok((address - self.ring.bus_base) / size_of::()) } - pub fn enqueue_normal( - &mut self, - buffer: PhysicalAddress, - length: usize, - ) -> Result { + pub fn enqueue_normal(&mut self, buffer: BusAddress, length: usize) -> Result { let trb_count = length.div_ceil(Self::TRB_SIZE_LIMIT); if self.inner.free_capacity() <= trb_count || trb_count == 0 { return Err(UsbError::DeviceBusy); @@ -253,11 +259,11 @@ impl TransactionBuilder<'_> { pub fn enqueue_control( &mut self, setup: ControlTransferSetup, - buffer: Option<(&mut PageSlice>, UsbDirection)>, + buffer: ControlDataStage, ) -> Result<(usize, Option, usize), UsbError> { // Check ring capacity first // TODO larger DATA stages - let trb_count = 2 + if buffer.is_some() { 1 } else { 0 }; + let trb_count = 2 + if buffer.len() != 0 { 1 } else { 0 }; if self.inner.free_capacity() <= trb_count { return Err(UsbError::DeviceBusy); } @@ -266,21 +272,33 @@ impl TransactionBuilder<'_> { let setup_stage = self .enqueue(ControlTransferSetupTrb::new(setup), true) .unwrap(); - let data_stage = if let Some((buffer, direction)) = buffer { - Some( - self.enqueue( + + let data_stage = match buffer { + ControlDataStage::None => None, + ControlDataStage::In(buffer) => { + let index = self.enqueue( ControlTransferDataTrb::new( - unsafe { buffer.as_physical_address() }, + buffer.bus_address(), buffer.len(), - direction, + UsbDirection::In, ), true, - ) - .unwrap(), - ) - } else { - None + )?; + Some(index) + } + ControlDataStage::Out(buffer) => { + let index = self.enqueue( + ControlTransferDataTrb::new( + buffer.bus_address(), + buffer.len(), + UsbDirection::Out, + ), + true, + )?; + Some(index) + } }; + let status_stage = self .enqueue(ControlTransferStatusTrb::new(UsbDirection::In), true) .unwrap(); @@ -298,7 +316,7 @@ impl TransactionBuilder<'_> { let mut transactions = self.ring.transactions.write(); for &pending in self.pending.iter() { - let index = (pending - self.ring.base) / size_of::(); + let index = (pending - self.ring.bus_base) / size_of::(); transactions[index] = Some(Arc::downgrade(&transaction)); } @@ -313,8 +331,9 @@ impl TransactionBuilder<'_> { } impl TransferRingInner { - fn new(capacity: usize) -> Result { - let trbs = PageBox::new_zeroed_slice(capacity).map_err(UsbError::MemoryError)?; + fn new(dma: &dyn DmaAllocator, capacity: usize) -> Result { + let trbs = DmaBuffer::new_zeroed_slice(dma, capacity).map_err(UsbError::MemoryError)?; + Ok(Self { trbs, enqueue_index: 0, @@ -323,7 +342,7 @@ impl TransferRingInner { }) } - fn enqueue(&mut self, trb: C, ioc: bool) -> Result { + fn enqueue(&mut self, trb: C, ioc: bool) -> Result { if (self.enqueue_index + 1) % (self.trbs.len() - 1) == self.dequeue_index { log::warn!("xhci: transfer ring full"); return Err(UsbError::DeviceBusy); @@ -337,7 +356,9 @@ impl TransferRingInner { self.trbs[self.enqueue_index].write(raw); - let address = unsafe { self.trbs.as_physical_address() } + let address = self + .trbs + .bus_address() .add(self.enqueue_index * size_of::()); self.enqueue_index += 1; @@ -352,7 +373,7 @@ impl TransferRingInner { } fn enqueue_link(&mut self) { - let base = unsafe { self.trbs.as_physical_address() }; + let base = self.trbs.bus_address(); let link = LinkTrb::new(base, self.cycle_bit); self.trbs[self.enqueue_index].write(bytemuck::cast(link)); @@ -363,6 +384,16 @@ impl TransferRingInner { } } +impl ControlDataStage<'_> { + pub fn len(&self) -> usize { + match self { + Self::None => 0, + Self::In(buf) => buf.len(), + Self::Out(buf) => buf.len(), + } + } +} + impl Transaction { pub fn notify(&self, trb_index: usize, status: u32) { self.event_queue @@ -505,7 +536,7 @@ define_bitfields! { #[derive(Clone, Copy, Debug, Pod, Zeroable)] #[repr(C, align(16))] pub struct NormalTransferTrb { - pub buffer: PhysicalAddress, + pub buffer: BusAddress, pub flags: NormalTransferFlags, } @@ -519,7 +550,7 @@ pub struct ControlTransferSetupTrb { #[derive(Clone, Copy, Debug, Pod, Zeroable)] #[repr(C, align(16))] pub struct ControlTransferDataTrb { - pub buffer: PhysicalAddress, + pub buffer: BusAddress, pub flags: ControlTransferDataFlags, } @@ -542,7 +573,7 @@ pub trait TransferTrb: Pod { } impl NormalTransferTrb { - pub fn new(buffer: PhysicalAddress, length: usize) -> Self { + pub fn new(buffer: BusAddress, length: usize) -> Self { Self { buffer, flags: NormalTransferFlags::new(length.try_into().unwrap()), @@ -566,7 +597,7 @@ impl ControlTransferSetupTrb { } impl ControlTransferDataTrb { - pub fn new(buffer: PhysicalAddress, length: usize, direction: UsbDirection) -> Self { + pub fn new(buffer: BusAddress, length: usize, direction: UsbDirection) -> Self { Self { buffer, flags: ControlTransferDataFlags::new( diff --git a/kernel/driver/virtio/core/Cargo.toml b/kernel/driver/virtio/core/Cargo.toml index 333d3a7f..2d9a14a9 100644 --- a/kernel/driver/virtio/core/Cargo.toml +++ b/kernel/driver/virtio/core/Cargo.toml @@ -7,6 +7,7 @@ edition = "2021" yggdrasil-abi.workspace = true libk-util.workspace = true libk-mm.workspace = true +libk.workspace = true device-api = { workspace = true, features = ["derive"] } ygg_driver_pci = { path = "../../bus/pci", optional = true } diff --git a/kernel/driver/virtio/core/src/queue.rs b/kernel/driver/virtio/core/src/queue.rs index c5beb1c6..9d3e5c90 100644 --- a/kernel/driver/virtio/core/src/queue.rs +++ b/kernel/driver/virtio/core/src/queue.rs @@ -8,14 +8,15 @@ use core::{ sync::atomic::{fence, Ordering}, }; -use libk_mm::{address::AsPhysicalAddress, PageBox}; +use device_api::dma::DmaAllocator; +use libk::dma::{BusAddress, DmaBuffer}; use crate::{error::Error, transport::Transport}; #[derive(Debug)] #[repr(C)] struct Descriptor { - address: u64, + address: BusAddress, len: u32, flags: u16, next: u16, @@ -29,7 +30,7 @@ struct Descriptor { // used_event: u16 // } struct AvailableRing { - data: PageBox<[MaybeUninit]>, + data: DmaBuffer<[MaybeUninit]>, } // Layout: @@ -41,13 +42,12 @@ struct AvailableRing { // _pad: u16 // } struct UsedRing { - data: PageBox<[MaybeUninit]>, - + data: DmaBuffer<[MaybeUninit]>, used_count: usize, } pub struct VirtQueue { - descriptor_table: PageBox<[MaybeUninit]>, + descriptor_table: DmaBuffer<[MaybeUninit]>, available: AvailableRing, used: UsedRing, @@ -63,8 +63,12 @@ pub struct VirtQueue { } impl AvailableRing { - pub fn with_capacity(no_irq: bool, capacity: usize) -> Result { - let mut data = PageBox::new_zeroed_slice(capacity + 3)?; + pub fn with_capacity( + dma: &dyn DmaAllocator, + no_irq: bool, + capacity: usize, + ) -> Result { + let mut data = DmaBuffer::new_zeroed_slice(dma, capacity + 3)?; if no_irq { data[0].write(1); @@ -85,8 +89,8 @@ impl AvailableRing { } impl UsedRing { - pub fn with_capacity(capacity: usize) -> Result { - let mut data = PageBox::new_zeroed_slice(capacity * 2 + 2)?; + pub fn with_capacity(dma: &dyn DmaAllocator, capacity: usize) -> Result { + let mut data = DmaBuffer::new_zeroed_slice(dma, capacity * 2 + 2)?; data[0].write(0); @@ -110,6 +114,7 @@ impl UsedRing { impl VirtQueue { pub fn with_capacity( transport: &mut T, + dma: &dyn DmaAllocator, index: u16, capacity: usize, msix_vector: Option, @@ -127,16 +132,16 @@ impl VirtQueue { return Err(Error::QueueTooLarge); } - let descriptor_table = PageBox::new_zeroed_slice(capacity)?; - let available = AvailableRing::with_capacity(no_avail_irq, capacity)?; - let used = UsedRing::with_capacity(capacity)?; + let descriptor_table = DmaBuffer::new_zeroed_slice(dma, capacity)?; + let available = AvailableRing::with_capacity(dma, no_avail_irq, capacity)?; + let used = UsedRing::with_capacity(dma, capacity)?; transport.set_queue( index, capacity as u16, - unsafe { descriptor_table.as_physical_address() }, - unsafe { available.data.as_physical_address() }, - unsafe { used.data.as_physical_address() }, + descriptor_table.bus_address(), + available.data.bus_address(), + used.data.bus_address(), msix_vector, ); @@ -163,6 +168,7 @@ impl VirtQueue { pub fn with_max_capacity( transport: &mut T, + dma: &dyn DmaAllocator, index: u16, capacity: usize, msix_vector: Option, @@ -171,16 +177,16 @@ impl VirtQueue { let max_capacity = transport.max_queue_size(index); let capacity = capacity.min(max_capacity as usize); - Self::with_capacity(transport, index, capacity, msix_vector, no_avail_irq) + Self::with_capacity(transport, dma, index, capacity, msix_vector, no_avail_irq) } /// # Safety /// - /// Invariants: PageBox remains valid and allocated until it is properly dequeued. + /// Invariants: DmaBuffer remains valid and allocated until it is properly dequeued. pub unsafe fn add<'a, 'b>( &mut self, - input: &'a [&'b mut PageBox<[u8]>], - output: &'a [&'b PageBox<[u8]>], + input: &'a [&'b mut DmaBuffer<[MaybeUninit]>], + output: &'a [&'b DmaBuffer<[u8]>], ) -> Result { if input.is_empty() && output.is_empty() { return Err(Error::EmptyTransaction); @@ -209,8 +215,8 @@ impl VirtQueue { unsafe fn add_direct<'a, 'b>( &mut self, - input: &'a [&'b mut PageBox<[u8]>], - output: &'a [&'b PageBox<[u8]>], + input: &'a [&'b mut DmaBuffer<[MaybeUninit]>], + output: &'a [&'b DmaBuffer<[u8]>], ) -> u16 { let head = self.free_head; let mut last = self.free_head; @@ -221,7 +227,7 @@ impl VirtQueue { let next = (self.free_head + 1) % self.capacity as u16; desc.write(Descriptor { - address: item.as_physical_address().into(), + address: item.bus_address(), len: item.len().try_into().unwrap(), // TODO flags: (1 << 0), @@ -238,7 +244,7 @@ impl VirtQueue { let next = (self.free_head + 1) % self.capacity as u16; desc.write(Descriptor { - address: item.as_physical_address().into(), + address: item.bus_address(), len: item.len().try_into().unwrap(), // TODO MAGIC flags: (1 << 0) | (1 << 1), @@ -265,8 +271,8 @@ impl VirtQueue { pub fn add_notify_wait_pop<'a, 'b, T: Transport>( &mut self, - input: &'a [&'b mut PageBox<[u8]>], - output: &'a [&'b PageBox<[u8]>], + input: &'a [&'b mut DmaBuffer<[MaybeUninit]>], + output: &'a [&'b DmaBuffer<[u8]>], transport: &mut T, ) -> Result { let token = unsafe { self.add(input, output) }?; @@ -333,7 +339,7 @@ impl VirtQueue { assert_ne!(current.len, 0); let next_head = (current.flags & (1 << 0) != 0).then_some(current.next); - current.address = 0; + current.address = BusAddress::ZERO; current.flags = 0; current.next = 0; current.len = 0; diff --git a/kernel/driver/virtio/core/src/transport/mod.rs b/kernel/driver/virtio/core/src/transport/mod.rs index 55d79efc..89ade893 100644 --- a/kernel/driver/virtio/core/src/transport/mod.rs +++ b/kernel/driver/virtio/core/src/transport/mod.rs @@ -1,6 +1,7 @@ use core::mem::size_of; -use libk_mm::{address::PhysicalAddress, device::DeviceMemoryIo}; +use libk::dma::BusAddress; +use libk_mm::device::DeviceMemoryIo; use tock_registers::{ interfaces::{Readable, Writeable}, registers::WriteOnly, @@ -56,17 +57,17 @@ pub trait Transport: Send { &mut self, queue: u16, capacity: u16, - descriptor_table_phys: PhysicalAddress, - available_ring_phys: PhysicalAddress, - used_ring_phys: PhysicalAddress, + descriptor_table_phys: BusAddress, + available_ring_phys: BusAddress, + used_ring_phys: BusAddress, msix_vector: Option, ) { let cfg = self.common_cfg(); cfg.queue_select.set(queue); cfg.queue_size.set(capacity); - cfg.queue_desc.set(descriptor_table_phys.into()); - cfg.queue_driver.set(available_ring_phys.into()); - cfg.queue_device.set(used_ring_phys.into()); + cfg.queue_desc.set(descriptor_table_phys.into_u64()); + cfg.queue_driver.set(available_ring_phys.into_u64()); + cfg.queue_device.set(used_ring_phys.into_u64()); if self.supports_msix() { cfg.queue_msix_vector.set(msix_vector.unwrap_or(0xFFFF)); } else { diff --git a/kernel/driver/virtio/gpu/src/command.rs b/kernel/driver/virtio/gpu/src/command.rs index 86ca898c..1f831aea 100644 --- a/kernel/driver/virtio/gpu/src/command.rs +++ b/kernel/driver/virtio/gpu/src/command.rs @@ -1,6 +1,12 @@ +use core::mem::MaybeUninit; + use bytemuck::{Pod, Zeroable}; -use libk::{device::display::PixelFormat, error::Error}; -use libk_mm::{address::PhysicalAddress, PageBox}; +use device_api::dma::DmaAllocator; +use libk::{ + device::display::PixelFormat, + dma::{BusAddress, DmaBuffer}, + error::Error, +}; use libk_util::sync::IrqSafeSpinlockGuard; use ygg_driver_virtio_core::{queue::VirtQueue, transport::Transport}; @@ -102,10 +108,12 @@ impl<'a, T: Transport> ControlLock<'a, T> { fn send_recv<'r, Req: Pod>( &mut self, + dma: &dyn DmaAllocator, req: &Req, - buffer: &'r mut PageBox<[u8]>, + buffer: &'r mut DmaBuffer<[MaybeUninit]>, ) -> Result<(&'r ControlHeader, &'r [u8]), Error> { - let mut request = PageBox::new_slice(0u8, size_of::())?; + let request = DmaBuffer::new_uninit_slice(dma, size_of::())?; + let mut request = unsafe { DmaBuffer::assume_init_slice(request) }; request.copy_from_slice(bytemuck::bytes_of(req)); let len = self @@ -121,18 +129,20 @@ impl<'a, T: Transport> ControlLock<'a, T> { return Err(Error::InvalidArgument); } - let header = bytemuck::from_bytes(&buffer[..size_of::()]); - let data = &buffer[size_of::()..len]; + let payload = unsafe { MaybeUninit::slice_assume_init_ref(&buffer[..len]) }; + let header = bytemuck::from_bytes(&payload[..size_of::()]); + let data = &payload[size_of::()..len]; Ok((header, data)) } fn send_recv_no_data( &mut self, + dma: &dyn DmaAllocator, req: &Req, - buffer: &mut PageBox<[u8]>, + buffer: &mut DmaBuffer<[MaybeUninit]>, ) -> Result<(), Error> { - let (response, _) = self.send_recv(req, buffer)?; + let (response, _) = self.send_recv(dma, req, buffer)?; if response.ty == 0x1100 { Ok(()) } else { @@ -142,8 +152,9 @@ impl<'a, T: Transport> ControlLock<'a, T> { pub fn query_scanouts<'r>( &mut self, + dma: &dyn DmaAllocator, max_scanouts: usize, - buffer: &'r mut PageBox<[u8]>, + buffer: &'r mut DmaBuffer<[MaybeUninit]>, ) -> Result<&'r [ScanoutInfo], Error> { let request = ControlHeader { ty: 0x0100, @@ -154,7 +165,7 @@ impl<'a, T: Transport> ControlLock<'a, T> { _0: [0; 3], }; - let (response, data) = self.send_recv(&request, buffer)?; + let (response, data) = self.send_recv(dma, &request, buffer)?; if response.ty != 0x1101 { log::warn!("virtio-gpu: query_scanouts returned {:#x}", response.ty); return Err(Error::InvalidArgument); @@ -169,7 +180,8 @@ impl<'a, T: Transport> ControlLock<'a, T> { pub fn create_resource_2d( &mut self, - buffer: &mut PageBox<[u8]>, + dma: &dyn DmaAllocator, + buffer: &mut DmaBuffer<[MaybeUninit]>, width: u32, height: u32, pixel_format: PixelFormat, @@ -189,16 +201,17 @@ impl<'a, T: Transport> ControlLock<'a, T> { format, }; - self.send_recv_no_data(&request, buffer)?; + self.send_recv_no_data(dma, &request, buffer)?; Ok(1) } pub fn attach_backing( &mut self, - buffer: &mut PageBox<[u8]>, + dma: &dyn DmaAllocator, + buffer: &mut DmaBuffer<[MaybeUninit]>, resource_id: u32, - base: PhysicalAddress, + base: BusAddress, length: u32, ) -> Result<(), Error> { let request = ResourceAttachBacking { @@ -215,12 +228,13 @@ impl<'a, T: Transport> ControlLock<'a, T> { }, }; - self.send_recv_no_data(&request, buffer) + self.send_recv_no_data(dma, &request, buffer) } pub fn set_scanout( &mut self, - buffer: &mut PageBox<[u8]>, + dma: &dyn DmaAllocator, + buffer: &mut DmaBuffer<[MaybeUninit]>, scanout_id: u32, resource_id: u32, width: u32, @@ -241,12 +255,13 @@ impl<'a, T: Transport> ControlLock<'a, T> { resource_id, }; - self.send_recv_no_data(&request, buffer) + self.send_recv_no_data(dma, &request, buffer) } pub fn transfer_to_host_2d( &mut self, - buffer: &mut PageBox<[u8]>, + dma: &dyn DmaAllocator, + buffer: &mut DmaBuffer<[MaybeUninit]>, resource_id: u32, r: Rect, ) -> Result<(), Error> { @@ -261,12 +276,13 @@ impl<'a, T: Transport> ControlLock<'a, T> { _0: 0, }; - self.send_recv_no_data(&request, buffer) + self.send_recv_no_data(dma, &request, buffer) } pub fn resource_flush( &mut self, - buffer: &mut PageBox<[u8]>, + dma: &dyn DmaAllocator, + buffer: &mut DmaBuffer<[MaybeUninit]>, resource_id: u32, r: Rect, ) -> Result<(), Error> { @@ -280,6 +296,6 @@ impl<'a, T: Transport> ControlLock<'a, T> { _0: 0, }; - self.send_recv_no_data(&request, buffer) + self.send_recv_no_data(dma, &request, buffer) } } diff --git a/kernel/driver/virtio/gpu/src/lib.rs b/kernel/driver/virtio/gpu/src/lib.rs index 39eb93c3..0eebd54d 100644 --- a/kernel/driver/virtio/gpu/src/lib.rs +++ b/kernel/driver/virtio/gpu/src/lib.rs @@ -1,3 +1,4 @@ +#![feature(maybe_uninit_slice)] #![no_std] extern crate alloc; @@ -6,18 +7,23 @@ use core::mem::MaybeUninit; use alloc::{sync::Arc, vec::Vec}; use command::{ControlLock, ScanoutInfo}; -use device_api::device::{Device, DeviceInitContext}; -use libk::device::{ - display::{ - DisplayDevice, DisplayMode, DisplayOwner, DriverFlags, FramebufferInfo, PixelFormat, +use device_api::{ + device::{Device, DeviceInitContext}, + dma::DmaAllocator, +}; +use libk::{ + device::{ + display::{ + DisplayDevice, DisplayMode, DisplayOwner, DriverFlags, FramebufferInfo, PixelFormat, + }, + manager::DEVICE_REGISTRY, }, - manager::DEVICE_REGISTRY, + dma::DmaBuffer, }; use libk_mm::{ - address::{PhysicalAddress, Virtualize}, - phys, + address::{AsPhysicalAddress, PhysicalAddress}, table::MapAttributes, - PageBox, PageProvider, L3_PAGE_SIZE, + PageProvider, L3_PAGE_SIZE, }; use libk_util::{ sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock}, @@ -42,9 +48,7 @@ struct Framebuffer { resource_id: u32, double: bool, - base: PhysicalAddress, - page_count: usize, - kernel_base: usize, + dma_buffer: DmaBuffer<[MaybeUninit]>, stride: usize, size: usize, } @@ -54,7 +58,7 @@ struct Config { framebuffer: Option, owner: DisplayOwner, - response: PageBox<[u8]>, + response: DmaBuffer<[MaybeUninit]>, } pub struct VirtioGpu { @@ -65,11 +69,17 @@ pub struct VirtioGpu { queues: OneTimeInit, config: IrqSafeRwLock, + dma: Arc, + num_scanouts: usize, } impl VirtioGpu { - pub fn new(transport: T, info: Option) -> Result { + pub fn new( + dma: Arc, + transport: T, + info: Option, + ) -> Result { // Read num-scanouts from device config let Some(device_cfg) = transport.device_cfg() else { log::error!("virtio-gpu must have device-specific configuration section"); @@ -93,10 +103,12 @@ impl VirtioGpu { config: IrqSafeRwLock::new(Config { scanouts: Vec::new(), framebuffer: None, - response: PageBox::new_slice(0, 4096)?, + response: DmaBuffer::new_uninit_slice(&*dma, L3_PAGE_SIZE)?, owner: DisplayOwner::None, }), + dma, + num_scanouts: num_scanouts as usize, }) } @@ -141,7 +153,7 @@ impl VirtioGpu { // TODO cursorq let mut transport = self.transport.lock(); - let control = VirtQueue::with_max_capacity(&mut *transport, 0, 128, None, true) + let control = VirtQueue::with_max_capacity(&mut *transport, &*self.dma, 0, 128, None, true) .map_err(|_| Error::InvalidArgument)?; self.queues.init(Queues { @@ -163,7 +175,8 @@ impl VirtioGpu { let mut control = self.control(); let mut config = self.config.write(); - let scanouts = control.query_scanouts(self.num_scanouts, &mut config.response)?; + let scanouts = + control.query_scanouts(&*self.dma, self.num_scanouts, &mut config.response)?; for (i, scanout) in scanouts.iter().enumerate() { log::info!( "virtio-gpu: [{i}] {}x{} + {},{}", @@ -199,21 +212,32 @@ impl VirtioGpu { let stride = w as usize * size_of::(); let size = stride * h as usize; - let page_count = size.div_ceil(L3_PAGE_SIZE); - let base = phys::alloc_pages_contiguous(page_count)?; - let kernel_base = base.virtualize(); + let dma_buffer = DmaBuffer::new_uninit_slice(&*self.dma, size)?; let mut control = self.control(); - let resource_id = - control.create_resource_2d(&mut config.response, w, h, PixelFormat::R8G8B8A8)?; + let resource_id = control.create_resource_2d( + &*self.dma, + &mut config.response, + w, + h, + PixelFormat::R8G8B8A8, + )?; control.attach_backing( + &*self.dma, &mut config.response, resource_id, - base, + dma_buffer.bus_address(), size.try_into().unwrap(), )?; - control.set_scanout(&mut config.response, index as u32, resource_id, w, h)?; + control.set_scanout( + &*self.dma, + &mut config.response, + index as u32, + resource_id, + w, + h, + )?; config.framebuffer = Some(Framebuffer { scanout_index: index, @@ -221,10 +245,9 @@ impl VirtioGpu { resource_id, size, - page_count, stride, - base, - kernel_base, + + dma_buffer, }); Ok(()) @@ -244,8 +267,8 @@ impl VirtioGpu { } else { let resource_id = framebuffer.resource_id; - control.transfer_to_host_2d(&mut config.response, resource_id, r)?; - control.resource_flush(&mut config.response, resource_id, r)?; + control.transfer_to_host_2d(&*self.dma, &mut config.response, resource_id, r)?; + control.resource_flush(&*self.dma, &mut config.response, resource_id, r)?; Ok(()) } @@ -253,9 +276,7 @@ impl VirtioGpu { } impl Device for VirtioGpu { - unsafe fn init(self: Arc, cx: DeviceInitContext) -> Result<(), Error> { - // TODO use DmaAllocator instead of PageBox - let _ = cx; + unsafe fn init(self: Arc, _cx: DeviceInitContext) -> Result<(), Error> { let status = self.begin_init()?; self.setup_queues()?; self.finish_init(status); @@ -280,7 +301,7 @@ impl PageProvider for VirtioGpu { // TODO check that the page is mapped by framebuffer owner let config = self.config.read(); let framebuffer = config.framebuffer.as_ref().ok_or(Error::DoesNotExist)?; - if offset as usize + L3_PAGE_SIZE > framebuffer.page_count * L3_PAGE_SIZE { + if offset as usize + L3_PAGE_SIZE > framebuffer.dma_buffer.page_count() * L3_PAGE_SIZE { log::warn!( "virtio-gpu: offset {:#x} outside of framebuffer bounds {:#x}", offset, @@ -288,7 +309,7 @@ impl PageProvider for VirtioGpu { ); return Err(Error::InvalidMemoryOperation); } - let phys = framebuffer.base.add(offset as usize); + let phys = unsafe { framebuffer.dma_buffer.as_physical_address() }.add(offset as usize); Ok(phys) } @@ -360,8 +381,8 @@ impl DisplayDevice for VirtioGpu { } output[0].write(FramebufferInfo { - base: framebuffer.base, - kernel_base: Some(framebuffer.kernel_base), + base: unsafe { framebuffer.dma_buffer.as_physical_address() }, + kernel_base: Some(framebuffer.dma_buffer.as_ptr().addr()), stride: framebuffer.stride, size: framebuffer.size, }); @@ -398,7 +419,7 @@ pci_driver! { "virtio-gpu" } - fn probe(&self, info: &PciDeviceInfo) -> Result, Error> { + fn probe(&self, info: &PciDeviceInfo, dma: &Arc) -> Result, Error> { let space = &info.config_space; let transport = PciTransport::from_config_space(space) @@ -406,9 +427,8 @@ pci_driver! { log::error!("Couldn't set up PCI virtio transport: {error:?}"); }) .map_err(|_| Error::InvalidArgument)?; - let device = VirtioGpu::new(transport, Some(info.clone()))?; + let device = VirtioGpu::new(dma.clone(), transport, Some(info.clone()))?; let device = Arc::new(device); - // let device = Box::leak(Box::new(device)); Ok(device) } diff --git a/kernel/driver/virtio/net/Cargo.toml b/kernel/driver/virtio/net/Cargo.toml index c2b78b57..82b51644 100644 --- a/kernel/driver/virtio/net/Cargo.toml +++ b/kernel/driver/virtio/net/Cargo.toml @@ -7,6 +7,7 @@ edition = "2021" yggdrasil-abi.workspace = true libk-util.workspace = true libk-mm.workspace = true +libk.workspace = true device-api = { workspace = true, features = ["derive"] } ygg_driver_virtio_core = { path = "../core" } diff --git a/kernel/driver/virtio/net/src/lib.rs b/kernel/driver/virtio/net/src/lib.rs index 377e2600..7a5615bc 100644 --- a/kernel/driver/virtio/net/src/lib.rs +++ b/kernel/driver/virtio/net/src/lib.rs @@ -3,15 +3,16 @@ extern crate alloc; -use core::mem::size_of; +use core::mem::{size_of, MaybeUninit}; use alloc::{collections::BTreeMap, sync::Arc}; use bytemuck::{Pod, Zeroable}; use device_api::{ device::{Device, DeviceInitContext}, + dma::DmaAllocator, interrupt::{InterruptAffinity, InterruptHandler, IrqVector}, }; -use libk_mm::PageBox; +use libk::dma::DmaBuffer; use libk_util::{ sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock, IrqSafeSpinlockGuard}, OneTimeInit, @@ -43,7 +44,8 @@ pub struct VirtioNet { mac: IrqSafeRwLock, - pending_packets: IrqSafeRwLock>>, + pending_packets: IrqSafeRwLock]>>>, + dma: Arc, pci_device_info: Option, } @@ -71,7 +73,11 @@ impl Queues { impl VirtioNet { const PACKET_SIZE: usize = 4096; - pub fn new(transport: T, pci_device_info: Option) -> Self { + pub fn new( + dma: Arc, + transport: T, + pci_device_info: Option, + ) -> Self { // Read MAC from device config let device_cfg = transport .device_cfg() @@ -90,6 +96,7 @@ impl VirtioNet { pending_packets: IrqSafeRwLock::new(BTreeMap::new()), pci_device_info, + dma, } } @@ -99,7 +106,7 @@ impl VirtioNet { let mut packets = self.pending_packets.write(); for _ in 0..buffers { - let mut packet = PageBox::new_slice(0, Self::PACKET_SIZE).unwrap(); + let mut packet = DmaBuffer::new_uninit_slice(&*self.dma, Self::PACKET_SIZE).unwrap(); let token = unsafe { queue.add(&[&mut packet], &[]).unwrap() }; packets.insert(token, packet); } @@ -117,11 +124,12 @@ impl VirtioNet { let mut pending_packets = self.pending_packets.write(); let packet = pending_packets.remove(&token).unwrap(); - let mut buffer = PageBox::new_slice(0, Self::PACKET_SIZE).unwrap(); + let mut buffer = DmaBuffer::new_uninit_slice(&*self.dma, Self::PACKET_SIZE).unwrap(); let token = unsafe { queue.add(&[&mut buffer], &[]).unwrap() }; pending_packets.insert(token, buffer); + let packet = unsafe { DmaBuffer::assume_init_slice(packet) }; let packet = Packet::new(packet, size_of::(), interface_id); ygg_driver_net_core::receive_packet(packet).unwrap(); count += 1 @@ -191,10 +199,17 @@ impl VirtioNet { let mut transport = self.transport.lock(); // Setup the virtqs - let rx = VirtQueue::with_max_capacity(&mut *transport, 0, 128, receive_vector, false) + let rx = VirtQueue::with_max_capacity( + &mut *transport, + &*self.dma, + 0, + 128, + receive_vector, + false, + ) + .map_err(cvt_error)?; + let tx = VirtQueue::with_max_capacity(&mut *transport, &*self.dma, 1, 128, None, true) .map_err(cvt_error)?; - let tx = - VirtQueue::with_max_capacity(&mut *transport, 1, 128, None, true).map_err(cvt_error)?; self.queues.init(Queues { receive: IrqSafeSpinlock::new(rx), @@ -206,7 +221,11 @@ impl VirtioNet { } impl NetworkDevice for VirtioNet { - fn transmit(&self, mut packet: PageBox<[u8]>) -> Result<(), Error> { + fn allocate_transmit_buffer(&self, len: usize) -> Result]>, Error> { + DmaBuffer::new_uninit_slice(&*self.dma, len) + } + + fn transmit_buffer(&self, mut packet: DmaBuffer<[u8]>) -> Result<(), Error> { let queues = self.queues.get(); let mut tx = queues.transmit.lock(); let mut transport = self.transport.lock(); @@ -214,7 +233,6 @@ impl NetworkDevice for VirtioNet { let _len = tx .add_notify_wait_pop(&[], &[&packet], &mut *transport) .unwrap(); - Ok(()) } @@ -253,9 +271,7 @@ impl Device for VirtioNet { "VirtIO Network Device" } - unsafe fn init(self: Arc, cx: DeviceInitContext) -> Result<(), Error> { - // TODO use DmaAllocator instead of PageBox - let _ = cx; + unsafe fn init(self: Arc, _cx: DeviceInitContext) -> Result<(), Error> { let status = self.begin_init()?; // TODO multiqueue @@ -291,11 +307,11 @@ pci_driver! { "virtio-net" } - fn probe(&self, info: &PciDeviceInfo) -> Result, Error> { + fn probe(&self, info: &PciDeviceInfo, dma: &Arc) -> Result, Error> { let space = &info.config_space; let transport = PciTransport::from_config_space(space).unwrap(); - let device = VirtioNet::new(transport, Some(info.clone())); + let device = VirtioNet::new(dma.clone(), transport, Some(info.clone())); let device = Arc::new(device); diff --git a/kernel/lib/device-api/src/dma.rs b/kernel/lib/device-api/src/dma.rs index e3990350..fb220fe4 100644 --- a/kernel/lib/device-api/src/dma.rs +++ b/kernel/lib/device-api/src/dma.rs @@ -1 +1,15 @@ -pub trait DmaAllocator {} +use core::{alloc::Layout, ptr::NonNull}; + +use yggdrasil_abi::error::Error; + +#[derive(Debug)] +pub struct DmaAllocation { + pub host_virtual: NonNull<()>, + pub host_physical: u64, + pub bus_address: u64, + pub page_count: usize, +} + +pub trait DmaAllocator: Send + Sync { + fn allocate(&self, layout: Layout) -> Result; +} diff --git a/kernel/libk/src/device/manager.rs b/kernel/libk/src/device/manager.rs index ed05fb33..215a939a 100644 --- a/kernel/libk/src/device/manager.rs +++ b/kernel/libk/src/device/manager.rs @@ -4,7 +4,6 @@ use core::{ }; use alloc::{collections::BTreeMap, format, sync::Arc, vec::Vec}; -use device_api::device::Device; use libk_util::{sync::spin_rwlock::IrqSafeRwLock, OneTimeInit}; use yggdrasil_abi::{error::Error, io::FileMode}; @@ -47,26 +46,16 @@ pub struct SerialTerminalRegistry { registry: GenericRegistry>, } -struct PendingDevice { - device: Arc, - irq_only: bool, - failed: bool, -} - pub struct DeviceRegistry { pub display: DisplayDeviceRegistry, pub terminal: TerminalRegistry, pub serial_terminal: SerialTerminalRegistry, - - pending_initialization: IrqSafeRwLock>, } pub static DEVICE_REGISTRY: DeviceRegistry = DeviceRegistry { display: DisplayDeviceRegistry::new(), terminal: TerminalRegistry::new(), serial_terminal: SerialTerminalRegistry::new(), - - pending_initialization: IrqSafeRwLock::new(Vec::new()), }; impl TerminalRegistry { @@ -154,13 +143,13 @@ impl Deref for DisplayWrapper { } impl DeviceRegistry { - pub fn add_pending_initialization(&self, device: Arc, irq_only: bool) { - self.pending_initialization.write().push(PendingDevice { - device, - irq_only, - failed: false, - }); - } + // pub fn add_pending_initialization(&self, device: Arc, irq_only: bool) { + // self.pending_initialization.write().push(PendingDevice { + // device, + // irq_only, + // failed: false, + // }); + // } // pub fn run_initialization(&self) { // let mut devices = self.pending_initialization.write(); diff --git a/kernel/libk/src/dma.rs b/kernel/libk/src/dma.rs index cc1ad8fd..4614230d 100644 --- a/kernel/libk/src/dma.rs +++ b/kernel/libk/src/dma.rs @@ -1,5 +1,264 @@ -use device_api::dma::DmaAllocator; +use core::{ + alloc::Layout, + fmt, + mem::{self, MaybeUninit}, + ops::{Deref, DerefMut, Sub}, + ptr::{self, NonNull}, +}; + +use bytemuck::{Pod, Zeroable}; +use device_api::dma::{DmaAllocation, DmaAllocator}; +use libk_mm::{ + address::{AsPhysicalAddress, PhysicalAddress, Virtualize}, + phys, L3_PAGE_SIZE, +}; +use yggdrasil_abi::error::Error; pub struct DummyDmaAllocator; -impl DmaAllocator for DummyDmaAllocator {} +impl DmaAllocator for DummyDmaAllocator { + fn allocate(&self, layout: Layout) -> Result { + if layout.align() > L3_PAGE_SIZE { + return Err(Error::InvalidMemoryOperation); + } + let page_count = layout.size().div_ceil(L3_PAGE_SIZE); + let host_physical = phys::alloc_pages_contiguous(page_count)?; + let host_virtual = unsafe { + NonNull::new_unchecked(ptr::with_exposed_provenance_mut(host_physical.virtualize())) + }; + let bus_address = host_physical.into_u64(); + + Ok(DmaAllocation { + host_physical: host_physical.into_u64(), + host_virtual, + bus_address, + page_count, + }) + } +} + +pub struct DmaBuffer { + host_pointer: NonNull, + host_physical: PhysicalAddress, + bus_address: u64, + page_count: usize, +} + +#[derive(Clone, Copy, PartialEq, Eq, Debug, PartialOrd, Ord, Pod, Zeroable)] +#[repr(transparent)] +pub struct BusAddress(u64); + +impl DmaBuffer { + pub fn new(allocator: &dyn DmaAllocator, value: T) -> Result, Error> { + let mut uninit = DmaBuffer::new_uninit(allocator)?; + uninit.write(value); + Ok(unsafe { DmaBuffer::assume_init(uninit) }) + } + + pub fn new_slice( + allocator: &dyn DmaAllocator, + value: T, + size: usize, + ) -> Result, Error> + where + T: Copy, + { + let mut uninit = DmaBuffer::new_uninit_slice(allocator, size)?; + for i in 0..size { + uninit[i].write(value); + } + Ok(unsafe { DmaBuffer::assume_init_slice(uninit) }) + } + + pub fn from_slice(allocator: &dyn DmaAllocator, source: &[T]) -> Result, Error> + where + T: Copy, + { + let mut uninit = DmaBuffer::new_uninit_slice(allocator, source.len())?; + MaybeUninit::copy_from_slice(&mut uninit[..], source); + Ok(unsafe { DmaBuffer::assume_init_slice(uninit) }) + } + + pub fn new_slice_with T>( + allocator: &dyn DmaAllocator, + init: F, + size: usize, + ) -> Result, Error> { + let mut uninit = DmaBuffer::new_uninit_slice(allocator, size)?; + for i in 0..size { + uninit[i].write(init(i)); + } + Ok(unsafe { DmaBuffer::assume_init_slice(uninit) }) + } + + pub fn new_uninit(allocator: &dyn DmaAllocator) -> Result>, Error> { + let layout = Layout::new::(); + let allocation = allocator.allocate(layout)?; + let host_pointer = allocation.host_virtual.cast(); + Ok(DmaBuffer { + host_pointer, + host_physical: PhysicalAddress::from_u64(allocation.host_physical), + bus_address: allocation.bus_address, + page_count: allocation.page_count, + }) + } + + pub fn new_uninit_slice( + allocator: &dyn DmaAllocator, + size: usize, + ) -> Result]>, Error> { + let layout = Layout::array::(size).map_err(|_| Error::InvalidMemoryOperation)?; + let allocation = allocator.allocate(layout)?; + let host_pointer = NonNull::slice_from_raw_parts(allocation.host_virtual.cast(), size); + Ok(DmaBuffer { + host_pointer, + host_physical: PhysicalAddress::from_u64(allocation.host_physical), + bus_address: allocation.bus_address, + page_count: allocation.page_count, + }) + } + + pub fn new_zeroed_slice( + allocator: &dyn DmaAllocator, + size: usize, + ) -> Result]>, Error> { + let layout = Layout::array::(size).map_err(|_| Error::InvalidMemoryOperation)?; + let allocation = allocator.allocate(layout)?; + unsafe { + let mut slice = NonNull::<[u8]>::slice_from_raw_parts( + allocation.host_virtual.cast(), + layout.size(), + ); + + slice.as_mut().fill(0); + } + let host_pointer = NonNull::slice_from_raw_parts(allocation.host_virtual.cast(), size); + Ok(DmaBuffer { + host_pointer, + host_physical: PhysicalAddress::from_u64(allocation.host_physical), + bus_address: allocation.bus_address, + page_count: allocation.page_count, + }) + } + + pub unsafe fn assume_init(buffer: DmaBuffer>) -> DmaBuffer { + let host_pointer = buffer.host_pointer; + let host_physical = buffer.host_physical; + let bus_address = buffer.bus_address; + let page_count = buffer.page_count; + + mem::forget(buffer); + + let host_pointer = host_pointer.cast(); + + DmaBuffer { + host_pointer, + host_physical, + bus_address, + page_count, + } + } + + pub unsafe fn assume_init_slice(buffer: DmaBuffer<[MaybeUninit]>) -> DmaBuffer<[T]> { + let host_pointer = buffer.host_pointer; + let host_physical = buffer.host_physical; + let bus_address = buffer.bus_address; + let page_count = buffer.page_count; + + mem::forget(buffer); + + let len = host_pointer.len(); + let host_pointer = NonNull::slice_from_raw_parts(host_pointer.cast::(), len); + + DmaBuffer { + host_pointer, + host_physical, + bus_address, + page_count, + } + } +} + +impl DmaBuffer { + #[inline] + pub fn page_count(&self) -> usize { + self.page_count + } + + #[inline] + pub fn bus_address(&self) -> BusAddress { + BusAddress(self.bus_address) + } +} + +unsafe impl Send for DmaBuffer {} +unsafe impl Sync for DmaBuffer {} + +impl Drop for DmaBuffer { + fn drop(&mut self) { + log::trace!("Drop DmaBuffer @ {:#x}", self.host_physical); + unsafe { + ptr::drop_in_place(self.host_pointer.as_ptr()); + for i in 0..self.page_count { + phys::free_page(self.host_physical.add(i * L3_PAGE_SIZE)); + } + } + } +} + +impl AsPhysicalAddress for DmaBuffer { + #[inline] + unsafe fn as_physical_address(&self) -> PhysicalAddress { + self.host_physical + } +} + +impl Deref for DmaBuffer { + type Target = T; + + fn deref(&self) -> &Self::Target { + unsafe { self.host_pointer.as_ref() } + } +} + +impl DerefMut for DmaBuffer { + fn deref_mut(&mut self) -> &mut Self::Target { + unsafe { self.host_pointer.as_mut() } + } +} + +impl fmt::Pointer for DmaBuffer { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "", self.bus_address) + } +} + +impl BusAddress { + pub const ZERO: Self = Self(0); + + pub const fn into_u64(self) -> u64 { + self.0 + } + + pub fn try_into_u32(self) -> Result { + self.0.try_into().map_err(|_| Error::InvalidMemoryOperation) + } + + pub const fn add(self, offset: usize) -> Self { + Self(self.0 + offset as u64) + } +} + +impl Sub for BusAddress { + type Output = usize; + + fn sub(self, rhs: BusAddress) -> Self::Output { + (self.0 - rhs.0).try_into().unwrap() + } +} + +impl fmt::LowerHex for BusAddress { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::LowerHex::fmt(&self.0, f) + } +} diff --git a/kernel/libk/src/lib.rs b/kernel/libk/src/lib.rs index 28d014ed..08145977 100644 --- a/kernel/libk/src/lib.rs +++ b/kernel/libk/src/lib.rs @@ -6,6 +6,7 @@ new_range_api, associated_type_defaults, maybe_uninit_slice, + maybe_uninit_write_slice, step_trait, const_trait_impl, slice_ptr_get, diff --git a/kernel/src/arch/x86/mod.rs b/kernel/src/arch/x86/mod.rs index 7eb80675..b677f7bc 100644 --- a/kernel/src/arch/x86/mod.rs +++ b/kernel/src/arch/x86/mod.rs @@ -4,10 +4,14 @@ use abi::error::Error; use alloc::{sync::Arc, vec::Vec}; -use device_api::{device::Device, interrupt::Irq}; +use device_api::{ + device::{Device, DeviceInitContext}, + interrupt::Irq, +}; use kernel_arch_x86::ISA_IRQ_OFFSET; use libk::{ config, debug, + dma::DummyDmaAllocator, fs::{devfs, sysfs}, task::runtime, }; @@ -50,6 +54,12 @@ pub enum SelectedClockSource { Fallback(Arc), } +pub fn dummy_init_context() -> DeviceInitContext { + DeviceInitContext { + dma_allocator: Arc::new(DummyDmaAllocator), + } +} + // Initialize the bare minimum required to: // * Allocate/manage interrupts // * Print debug output @@ -63,7 +73,12 @@ pub fn init_platform_early(cmdline: &str) -> Result // Initialize async executor queue runtime::init_task_queue(); - let com1_3 = ComPort::setup(0x3F8, 0x3E8, Irq::External(ISA_IRQ_OFFSET + 4))?; + let com1_3 = ComPort::setup( + dummy_init_context(), + 0x3F8, + 0x3E8, + Irq::External(ISA_IRQ_OFFSET + 4), + )?; let i8259 = I8259::setup().expect("Could not initialize i8259 PIC"); #[cfg(any(target_arch = "x86", rust_analyzer))] @@ -71,7 +86,7 @@ pub fn init_platform_early(cmdline: &str) -> Result use libk::device::register_external_interrupt_controller; // No other interrupt handling options - unsafe { i8259.clone().init() }?; + unsafe { i8259.clone().init(dummy_init_context()) }?; register_external_interrupt_controller(i8259.clone()); } diff --git a/kernel/src/arch/x86/peripherals/hpet.rs b/kernel/src/arch/x86/peripherals/hpet.rs index 76900407..902f51fb 100644 --- a/kernel/src/arch/x86/peripherals/hpet.rs +++ b/kernel/src/arch/x86/peripherals/hpet.rs @@ -4,7 +4,7 @@ use abi::error::Error; use acpi::HpetInfo; use alloc::{collections::btree_map::BTreeMap, format, string::String, sync::Arc}; use device_api::{ - device::Device, + device::{Device, DeviceInitContext}, interrupt::{InterruptHandler, Irq, IrqOptions, IrqTrigger, IrqVector}, }; use libk::{device::external_interrupt_controller, task::runtime, time}; @@ -16,6 +16,8 @@ use tock_registers::{ registers::{ReadOnly, ReadWrite}, }; +use crate::arch::x86::dummy_init_context; + register_bitfields! { u64, GENERAL_CAPABILITIES [ @@ -122,7 +124,7 @@ impl InterruptHandler for HpetTimer { } impl Device for HpetTimer { - unsafe fn init(self: Arc) -> Result<(), Error> { + unsafe fn init(self: Arc, _cx: DeviceInitContext) -> Result<(), Error> { if self.period > u32::MAX as u64 && !self.bits64 { log::error!( "HPET period is >32bit and the HPET itself does not support 64bit counters" @@ -182,7 +184,7 @@ impl Device for HpetTimer { } impl Device for Hpet { - unsafe fn init(self: Arc) -> Result<(), Error> { + unsafe fn init(self: Arc, _cx: DeviceInitContext) -> Result<(), Error> { let regs = self.regs.write(); // Reset the device until at least one timer is created @@ -244,9 +246,10 @@ impl Hpet { } pub fn setup_from_acpi(acpi: &HpetInfo) -> Result, Error> { + let cx = dummy_init_context(); let base = PhysicalAddress::from_usize(acpi.base_address); let hpet = Arc::new(Self::new(base)?); - unsafe { hpet.clone().init() }?; + unsafe { hpet.clone().init(cx) }?; Ok(hpet) } @@ -279,7 +282,8 @@ impl Hpet { let timer = if let Ok(timer) = timer { timers.insert(index, timer.clone()); - match unsafe { timer.clone().init() } { + let cx = dummy_init_context(); + match unsafe { timer.clone().init(cx) } { Ok(()) => Ok(timer), Err(error) => Err(error), } diff --git a/kernel/src/arch/x86/peripherals/i8259.rs b/kernel/src/arch/x86/peripherals/i8259.rs index 555a938b..c537432b 100644 --- a/kernel/src/arch/x86/peripherals/i8259.rs +++ b/kernel/src/arch/x86/peripherals/i8259.rs @@ -3,7 +3,7 @@ use abi::error::Error; use alloc::sync::Arc; use device_api::{ - device::Device, + device::{Device, DeviceInitContext}, interrupt::{ ExternalInterruptController, FixedInterruptTable, InterruptHandler, InterruptTable, Irq, IrqOptions, IrqVector, @@ -38,7 +38,7 @@ pub struct I8259 { } impl Device for I8259 { - unsafe fn init(self: Arc) -> Result<(), Error> { + unsafe fn init(self: Arc, _cx: DeviceInitContext) -> Result<(), Error> { self.enable(); Ok(()) } diff --git a/kernel/src/arch/x86/peripherals/ps2/mod.rs b/kernel/src/arch/x86/peripherals/ps2/mod.rs index bcd5f4ca..6d532e3f 100644 --- a/kernel/src/arch/x86/peripherals/ps2/mod.rs +++ b/kernel/src/arch/x86/peripherals/ps2/mod.rs @@ -5,7 +5,7 @@ use abi::{ }; use alloc::sync::Arc; use device_api::{ - device::Device, + device::{Device, DeviceInitContext}, interrupt::{InterruptHandler, Irq, IrqVector}, }; use kernel_arch_x86::{ @@ -113,7 +113,7 @@ impl Device for PS2Controller { "PS/2 Controller" } - unsafe fn init(self: Arc) -> Result<(), Error> { + unsafe fn init(self: Arc, _cx: DeviceInitContext) -> Result<(), Error> { Ok(()) } diff --git a/kernel/src/arch/x86/peripherals/serial.rs b/kernel/src/arch/x86/peripherals/serial.rs index f9de7166..b73131f4 100644 --- a/kernel/src/arch/x86/peripherals/serial.rs +++ b/kernel/src/arch/x86/peripherals/serial.rs @@ -2,7 +2,7 @@ use abi::{error::Error, io::TerminalOptions}; use alloc::sync::Arc; use device_api::{ - device::Device, + device::{Device, DeviceInitContext}, interrupt::{InterruptHandler, Irq, IrqVector}, }; use kernel_arch_x86::intrinsics::{IoPort, IoPortAccess}; @@ -113,7 +113,7 @@ impl Device for Port { "COM port" } - unsafe fn init(self: Arc) -> Result<(), Error> { + unsafe fn init(self: Arc, _cx: DeviceInitContext) -> Result<(), Error> { DEVICE_REGISTRY .serial_terminal .register(self.terminal.clone(), Some(self.clone())) @@ -168,9 +168,14 @@ impl ComPort { }) } - pub fn setup(port_a: u16, port_b: u16, irq: Irq) -> Result, Error> { + pub fn setup( + cx: DeviceInitContext, + port_a: u16, + port_b: u16, + irq: Irq, + ) -> Result, Error> { let this = Arc::new(Self::new(port_a, port_b, irq)?); - unsafe { this.port_a().clone().init() }?; + unsafe { this.port_a().clone().init(cx) }?; Ok(this) } diff --git a/kernel/src/arch/x86_64/apic/ioapic.rs b/kernel/src/arch/x86_64/apic/ioapic.rs index 926ed91d..f54acf4c 100644 --- a/kernel/src/arch/x86_64/apic/ioapic.rs +++ b/kernel/src/arch/x86_64/apic/ioapic.rs @@ -3,7 +3,7 @@ use ::acpi::platform::interrupt::{Apic as AcpiApic, Polarity, TriggerMode}; use abi::error::Error; use alloc::sync::Arc; use device_api::{ - device::Device, + device::{Device, DeviceInitContext}, interrupt::{ ExternalInterruptController, FixedInterruptTable, InterruptHandler, InterruptTable, Irq, IrqLevel, IrqOptions, IrqTrigger, IrqVector, @@ -157,7 +157,7 @@ impl Device for IoApic { "I/O APIC" } - unsafe fn init(self: Arc) -> Result<(), Error> { + unsafe fn init(self: Arc, _cx: DeviceInitContext) -> Result<(), Error> { unreachable!() } } diff --git a/kernel/src/device/interrupt/riscv_plic.rs b/kernel/src/device/interrupt/riscv_plic.rs index 81dfe7f9..4e84b552 100644 --- a/kernel/src/device/interrupt/riscv_plic.rs +++ b/kernel/src/device/interrupt/riscv_plic.rs @@ -4,7 +4,7 @@ use core::sync::atomic::Ordering; use abi::{error::Error, primitive_enum}; use alloc::{sync::Arc, vec::Vec}; use device_api::{ - device::Device, + device::{Device, DeviceInitContext}, interrupt::{ ExternalInterruptController, FixedInterruptTable, FullIrq, InterruptHandler, InterruptTable, Irq, IrqOptions, IrqVector, @@ -209,7 +209,7 @@ impl ExternalInterruptController for Plic { } impl Device for Plic { - unsafe fn init(self: Arc) -> Result<(), Error> { + unsafe fn init(self: Arc, _cx: DeviceInitContext) -> Result<(), Error> { log::info!("Initialize RISC-V PLIC"); let common = DeviceMemoryIo::::map(self.base, Default::default())?; diff --git a/kernel/src/device/serial/ns16550a.rs b/kernel/src/device/serial/ns16550a.rs index c78cf549..aa3d9042 100644 --- a/kernel/src/device/serial/ns16550a.rs +++ b/kernel/src/device/serial/ns16550a.rs @@ -2,7 +2,7 @@ use abi::{error::Error, io::TerminalOptions}; use alloc::sync::Arc; use device_api::{ - device::Device, + device::{Device, DeviceInitContext}, interrupt::{FullIrq, InterruptHandler, IrqVector}, }; use device_tree::driver::{device_tree_driver, Node, ProbeContext}; @@ -108,7 +108,7 @@ impl Io { } impl Device for Ns16550a { - unsafe fn init(self: Arc) -> Result<(), Error> { + unsafe fn init(self: Arc, _cx: DeviceInitContext) -> Result<(), Error> { log::info!("Init ns16550a @ {:#x}", self.base); let mut io = Io { regs: DeviceMemoryIo::map(self.base, Default::default())?, diff --git a/kernel/src/device/serial/snps_dw_apb_uart.rs b/kernel/src/device/serial/snps_dw_apb_uart.rs index 37e176d5..8eb0693a 100644 --- a/kernel/src/device/serial/snps_dw_apb_uart.rs +++ b/kernel/src/device/serial/snps_dw_apb_uart.rs @@ -2,7 +2,7 @@ use abi::{error::Error, io::TerminalOptions}; use alloc::sync::Arc; use device_api::{ - device::Device, + device::{Device, DeviceInitContext}, interrupt::{FullIrq, InterruptHandler, IrqVector}, }; use device_tree::driver::{device_tree_driver, Node, ProbeContext}; @@ -148,7 +148,7 @@ impl InterruptHandler for DwUart { } impl Device for DwUart { - unsafe fn init(self: Arc) -> Result<(), Error> { + unsafe fn init(self: Arc, _cx: DeviceInitContext) -> Result<(), Error> { let regs = DeviceMemoryIo::map(self.base, Default::default())?; let mut io = Io { regs }; io.init();