From 87c7614fd85705ac8085db80bad4e598e45936f3 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Mon, 3 Feb 2025 09:33:02 +0200 Subject: [PATCH] xhci: rework xhci driver, now works on real hw --- Cargo.lock | 35 +- kernel/driver/block/nvme/src/lib.rs | 2 + kernel/driver/block/scsi/src/lib.rs | 11 +- kernel/driver/bus/pci/src/capability.rs | 16 + kernel/driver/bus/pci/src/macros.rs | 35 + .../bus/usb/src/class_driver/hid_keyboard.rs | 62 +- .../bus/usb/src/class_driver/mass_storage.rs | 86 +- kernel/driver/bus/usb/src/class_driver/mod.rs | 16 +- kernel/driver/bus/usb/src/communication.rs | 138 --- kernel/driver/bus/usb/src/descriptor.rs | 2 +- kernel/driver/bus/usb/src/device.rs | 83 +- kernel/driver/bus/usb/src/error.rs | 13 +- kernel/driver/bus/usb/src/info.rs | 28 +- kernel/driver/bus/usb/src/lib.rs | 2 +- kernel/driver/bus/usb/src/pipe/bulk.rs | 55 -- kernel/driver/bus/usb/src/pipe/control.rs | 158 ++-- kernel/driver/bus/usb/src/pipe/interrupt.rs | 32 - kernel/driver/bus/usb/src/pipe/mod.rs | 9 +- kernel/driver/bus/usb/src/pipe/normal.rs | 62 ++ kernel/driver/usb/xhci/Cargo.toml | 11 +- kernel/driver/usb/xhci/src/context.rs | 17 +- kernel/driver/usb/xhci/src/controller.rs | 757 +++++++-------- kernel/driver/usb/xhci/src/device.rs | 321 +++---- kernel/driver/usb/xhci/src/lib.rs | 35 +- kernel/driver/usb/xhci/src/pipe.rs | 227 ++--- kernel/driver/usb/xhci/src/regs.rs | 335 ------- kernel/driver/usb/xhci/src/regs/capability.rs | 66 ++ kernel/driver/usb/xhci/src/regs/extended.rs | 122 +++ kernel/driver/usb/xhci/src/regs/mod.rs | 184 ++++ .../driver/usb/xhci/src/regs/operational.rs | 140 +++ kernel/driver/usb/xhci/src/regs/runtime.rs | 69 ++ kernel/driver/usb/xhci/src/ring/command.rs | 106 +++ kernel/driver/usb/xhci/src/ring/mod.rs | 29 +- kernel/driver/usb/xhci/src/ring/transfer.rs | 869 ++++++++---------- kernel/driver/usb/xhci/src/util.rs | 24 +- kernel/libk/libk-mm/src/device.rs | 5 +- kernel/libk/src/debug.rs | 4 +- kernel/src/arch/x86/mod.rs | 16 +- kernel/src/panic.rs | 25 +- 39 files changed, 2112 insertions(+), 2095 deletions(-) create mode 100644 kernel/driver/bus/pci/src/macros.rs delete mode 100644 kernel/driver/bus/usb/src/pipe/bulk.rs delete mode 100644 kernel/driver/bus/usb/src/pipe/interrupt.rs create mode 100644 kernel/driver/bus/usb/src/pipe/normal.rs delete mode 100644 kernel/driver/usb/xhci/src/regs.rs create mode 100644 kernel/driver/usb/xhci/src/regs/capability.rs create mode 100644 kernel/driver/usb/xhci/src/regs/extended.rs create mode 100644 kernel/driver/usb/xhci/src/regs/mod.rs create mode 100644 kernel/driver/usb/xhci/src/regs/operational.rs create mode 100644 kernel/driver/usb/xhci/src/regs/runtime.rs diff --git a/Cargo.lock b/Cargo.lock index e1e6ed52..8d546fa9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,7 +8,7 @@ version = "10.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a21cd0131c25c438e19cd6a774adf7e3f64f7f4d723022882facc2dee0f8bc9" dependencies = [ - "tock-registers 0.9.0", + "tock-registers", ] [[package]] @@ -1125,7 +1125,7 @@ dependencies = [ "libk-mm-interface", "memtables", "static_assertions", - "tock-registers 0.9.0", + "tock-registers", "yggdrasil-abi", ] @@ -1149,7 +1149,7 @@ dependencies = [ "libk-mm-interface", "log", "static_assertions", - "tock-registers 0.9.0", + "tock-registers", "yggdrasil-abi", ] @@ -1173,7 +1173,7 @@ dependencies = [ "log", "memtables", "static_assertions", - "tock-registers 0.9.0", + "tock-registers", "yggdrasil-abi", ] @@ -1186,7 +1186,7 @@ dependencies = [ "kernel-arch-interface", "log", "static_assertions", - "tock-registers 0.9.0", + "tock-registers", ] [[package]] @@ -1201,7 +1201,7 @@ dependencies = [ "log", "memtables", "static_assertions", - "tock-registers 0.9.0", + "tock-registers", "yggdrasil-abi", ] @@ -2105,12 +2105,6 @@ dependencies = [ "zerovec", ] -[[package]] -name = "tock-registers" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "696941a0aee7e276a165a978b37918fd5d22c55c3d6bda197813070ca9c0f21c" - [[package]] name = "tock-registers" version = "0.9.0" @@ -2648,7 +2642,7 @@ dependencies = [ "log", "memoffset 0.9.1", "static_assertions", - "tock-registers 0.9.0", + "tock-registers", "ygg_driver_pci", "yggdrasil-abi", ] @@ -2702,7 +2696,7 @@ dependencies = [ "libk-mm", "libk-util", "log", - "tock-registers 0.9.0", + "tock-registers", "ygg_driver_net_core", "ygg_driver_pci", "yggdrasil-abi", @@ -2722,7 +2716,7 @@ dependencies = [ "libk-util", "log", "static_assertions", - "tock-registers 0.9.0", + "tock-registers", "ygg_driver_pci", "yggdrasil-abi", ] @@ -2739,7 +2733,7 @@ dependencies = [ "libk-mm", "libk-util", "log", - "tock-registers 0.9.0", + "tock-registers", "ygg_driver_acpi", "yggdrasil-abi", ] @@ -2778,6 +2772,7 @@ dependencies = [ name = "ygg_driver_usb_xhci" version = "0.1.0" dependencies = [ + "async-trait", "atomic_enum", "bytemuck", "device-api", @@ -2786,7 +2781,7 @@ dependencies = [ "libk-mm", "libk-util", "log", - "tock-registers 0.8.1", + "tock-registers", "xhci", "ygg_driver_pci", "ygg_driver_usb", @@ -2802,7 +2797,7 @@ dependencies = [ "libk-mm", "libk-util", "log", - "tock-registers 0.9.0", + "tock-registers", "ygg_driver_pci", "yggdrasil-abi", ] @@ -2832,7 +2827,7 @@ dependencies = [ "libk-mm", "libk-util", "log", - "tock-registers 0.9.0", + "tock-registers", "ygg_driver_net_core", "ygg_driver_pci", "ygg_driver_virtio_core", @@ -2890,7 +2885,7 @@ dependencies = [ "memtables", "prettyplease", "static_assertions", - "tock-registers 0.9.0", + "tock-registers", "vmalloc", "yboot-proto", "ygg_driver_acpi", diff --git a/kernel/driver/block/nvme/src/lib.rs b/kernel/driver/block/nvme/src/lib.rs index b7f9e141..0bcc1d9d 100644 --- a/kernel/driver/block/nvme/src/lib.rs +++ b/kernel/driver/block/nvme/src/lib.rs @@ -1,6 +1,8 @@ #![feature(const_trait_impl, let_chains, if_let_guard, maybe_uninit_slice)] #![allow(missing_docs)] #![no_std] +// TODO +#![allow(unused)] extern crate alloc; diff --git a/kernel/driver/block/scsi/src/lib.rs b/kernel/driver/block/scsi/src/lib.rs index 6e46a048..a7d8ad66 100644 --- a/kernel/driver/block/scsi/src/lib.rs +++ b/kernel/driver/block/scsi/src/lib.rs @@ -8,7 +8,7 @@ use alloc::{ boxed::Box, collections::btree_map::BTreeMap, format, string::String, sync::Arc, vec::Vec, }; use async_trait::async_trait; -use command::{ScsiInquiry, ScsiReadCapacity, ScsiRequestSense, ScsiTestUnitReady}; +use command::{ScsiReadCapacity, ScsiRequestSense, ScsiTestUnitReady}; use device_api::device::Device; use libk::{ device::{block::BlockDevice, manager::probe_partitions}, @@ -44,8 +44,6 @@ impl ScsiDevice { pub async fn setup(transport: T) -> Result, Error> { let mut transport = ScsiTransportWrapper::new(transport); - transport.perform_command(0, ScsiInquiry).await?; - let mut attempts = 5; let mut timeout = 100; while attempts > 0 { @@ -63,6 +61,8 @@ impl ScsiDevice { break; } + log::warn!("scsi: unit not ready [{attempts}/5]"); + runtime::sleep(Duration::from_millis(timeout)).await; timeout *= 2; attempts -= 1; @@ -72,6 +72,9 @@ impl ScsiDevice { return Err(Error::DoesNotExist); } + // TODO INQUIRY fails for real USB flash drives + // transport.perform_command(0, ScsiInquiry).await?; + let capacity_info = transport.perform_command(0, ScsiReadCapacity).await?; log::info!( "scsi: lba_size={}, lba_count={}", @@ -142,7 +145,7 @@ impl BlockDevice for ScsiDevice { } fn max_blocks_per_request(&self) -> usize { - 8 + 65536 / self.lba_size } } diff --git a/kernel/driver/bus/pci/src/capability.rs b/kernel/driver/bus/pci/src/capability.rs index 4f7f4e44..6af5d5a3 100644 --- a/kernel/driver/bus/pci/src/capability.rs +++ b/kernel/driver/bus/pci/src/capability.rs @@ -310,6 +310,22 @@ impl<'s, S: PciConfigurationSpace + ?Sized + 's> PowerManagementData<'s, S> { } } + pub fn set_pme_en(&self, state: bool) { + let pmcsr = self.space.read_u16(self.offset + 4); + let new = if state { + pmcsr | (1 << 8) + } else { + pmcsr & !(1 << 8) + }; + if pmcsr == new { + return; + } + + log::info!("Set PMCSR.PME_En = {state}"); + + self.space.write_u16(self.offset + 4, new); + } + pub fn get_device_power_state(&self) -> DevicePowerState { let pmcsr = self.space.read_u16(self.offset + 4); match pmcsr & 0x3 { diff --git a/kernel/driver/bus/pci/src/macros.rs b/kernel/driver/bus/pci/src/macros.rs new file mode 100644 index 00000000..6b8786f6 --- /dev/null +++ b/kernel/driver/bus/pci/src/macros.rs @@ -0,0 +1,35 @@ +pub macro pci_driver_match { + (class ($class:literal:$subclass:literal:$prog_if:literal)) => { + $crate::driver::PciMatch::Class($class, Some($subclass), Some($prog_if)) + }, + (class ($class:literal:$subclass:literal)) => { + $crate::driver::PciMatch::Class($class, Some($subclass), None) + }, + (class $class:literal) => { + $crate::driver::PciMatch::Class($class, None, None) + }, + (device ($vendor:literal:$device:literal)) => { + $crate::driver::PciMatch::Vendor($vendor, $device) + } +} + +pub macro pci_driver( + matches: [$($kind:ident $match:tt),+], + driver: $driver:tt +) { + #[link_section = ".init_array"] + #[used] + static __REGISTER_FN: extern "C" fn() = __register_fn; + + extern "C" fn __register_fn() { + struct Driver; + impl $crate::driver::PciDriver for Driver $driver + static DRIVER: Driver = Driver; + + log::info!("register pci driver: {:?}", $crate::driver::PciDriver::driver_name(&Driver)); + $( + let pmatch = $crate::macros::pci_driver_match!($kind $match); + $crate::driver::register_match(pmatch, &DRIVER); + )+ + } +} diff --git a/kernel/driver/bus/usb/src/class_driver/hid_keyboard.rs b/kernel/driver/bus/usb/src/class_driver/hid_keyboard.rs index 2dbe33d1..1a03981c 100644 --- a/kernel/driver/bus/usb/src/class_driver/hid_keyboard.rs +++ b/kernel/driver/bus/usb/src/class_driver/hid_keyboard.rs @@ -1,8 +1,7 @@ - use core::mem::MaybeUninit; -use alloc::sync::Arc; -use futures_util::{future::BoxFuture, FutureExt}; +use alloc::{boxed::Box, sync::Arc}; +use async_trait::async_trait; use libk_mm::PageBox; use yggdrasil_abi::io::{KeyboardKey, KeyboardKeyEvent}; @@ -126,41 +125,41 @@ impl KeyboardState { } } +#[async_trait] impl UsbDriver for UsbHidKeyboardDriver { - fn run( - self: Arc, - device: Arc, - ) -> BoxFuture<'static, Result<(), UsbError>> { - async move { - // TODO not sure whether to use boot protocol (easy) or GetReport - let config = device.select_configuration(|_| true).await?.unwrap(); - assert_eq!(config.endpoints.len(), 1); + async fn run(self: Arc, device: Arc) -> Result<(), UsbError> { + // TODO not sure whether to use boot protocol (easy) or GetReport + let config = device.select_configuration(|_| true).await?.unwrap(); - let pipe = device.open_interrupt_in_pipe(1).await?; + log::info!("Setup HID keyboard"); + let pipe = device + .open_interrupt_in_pipe(1, config.endpoints[0].max_packet_size as u16) + .await?; - let mut buffer = PageBox::new_slice(0, 8).map_err(UsbError::MemoryError)?; - let mut state = KeyboardState::new(); - let mut events = [MaybeUninit::uninit(); 16]; + let mut buffer = PageBox::new_slice(0, 8).map_err(UsbError::MemoryError)?; + let mut state = KeyboardState::new(); + let mut events = [MaybeUninit::uninit(); 16]; - loop { - let mut event_count = 0; + loop { + let mut event_count = 0; - let data = pipe.read(&mut buffer).await?; + let len = pipe.read(buffer.as_slice_mut()).await?; + if len < 8 { + continue; + } - event_count += state.retain_modifiers(data[0], &mut events); - event_count += state.press_modifiers(data[0], &mut events[event_count..]); - event_count += state.retain(&data[2..], &mut events[event_count..]); - event_count += state.press(&data[2..], &mut events[event_count..]); + event_count += state.retain_modifiers(buffer[0], &mut events); + event_count += state.press_modifiers(buffer[0], &mut events[event_count..]); + event_count += state.retain(&buffer[2..], &mut events[event_count..]); + event_count += state.press(&buffer[2..], &mut events[event_count..]); - let events = unsafe { MaybeUninit::slice_assume_init_ref(&events[..event_count]) }; + let events = unsafe { MaybeUninit::slice_assume_init_ref(&events[..event_count]) }; - for &event in events { - log::trace!("Generic Keyboard: {:?}", event); - ygg_driver_input::send_event(event); - } + for &event in events { + log::info!("Generic Keyboard: {:?}", event); + ygg_driver_input::send_event(event); } } - .boxed() } fn name(&self) -> &'static str { @@ -168,6 +167,11 @@ impl UsbDriver for UsbHidKeyboardDriver { } fn probe(&self, class: &UsbClassInfo, _device: &UsbDeviceAccess) -> bool { - class.class == UsbDeviceClass::Hid && class.subclass == 0x01 + log::info!( + "class = {:?}, subclass = {:02x}", + class.class, + class.subclass + ); + class.class == UsbDeviceClass::Hid && (class.subclass == 0x00 || class.subclass == 0x01) } } diff --git a/kernel/driver/bus/usb/src/class_driver/mass_storage.rs b/kernel/driver/bus/usb/src/class_driver/mass_storage.rs index 814db8e0..68aeafa7 100644 --- a/kernel/driver/bus/usb/src/class_driver/mass_storage.rs +++ b/kernel/driver/bus/usb/src/class_driver/mass_storage.rs @@ -1,18 +1,18 @@ use alloc::{boxed::Box, sync::Arc}; use async_trait::async_trait; use bytemuck::{Pod, Zeroable}; -use futures_util::{future::BoxFuture, FutureExt}; use libk::error::Error; use libk_mm::PageBox; use ygg_driver_scsi::{transport::ScsiTransport, ScsiDevice}; use crate::{ + communication::UsbDirection, device::{UsbDeviceAccess, UsbDeviceDetachHandler}, error::UsbError, - info::UsbDeviceClass, + info::{UsbDeviceClass, UsbEndpointType}, pipe::{ - bulk::{UsbBulkInPipeAccess, UsbBulkOutPipeAccess}, control::UsbClassSpecificRequest, + normal::{UsbBulkInPipeAccess, UsbBulkOutPipeAccess}, }, }; @@ -99,7 +99,8 @@ impl Bbb { self.out_pipe .write(self.buffer.as_slice().subslice(..31)) - .await?; + .await + .inspect_err(|error| log::error!("msc: out pipe error: {error:?}"))?; Ok(tag) } @@ -107,7 +108,8 @@ impl Bbb { async fn read_csw(&mut self, tag: u32) -> Result<(), Error> { self.in_pipe .read(self.buffer.as_slice_mut().subslice_mut(..13)) - .await?; + .await + .inspect_err(|error| log::error!("msc: in pipe error: {error:?}"))?; let csw = bytemuck::from_bytes::(&self.buffer[..size_of::()]); if csw.signature != 0x53425355 { @@ -128,12 +130,13 @@ impl Bbb { async fn read_response_data(&mut self, buffer: &mut [u8]) -> Result { // TODO limit by max_packet_size - let bytes = self + let len = self .in_pipe .read(self.buffer.as_slice_mut().subslice_mut(..buffer.len())) - .await?; - buffer[..bytes.len()].copy_from_slice(bytes); - Ok(bytes.len()) + .await + .inspect_err(|error| log::error!("msc: in pipe error: {error:?}"))?; + buffer[..len].copy_from_slice(&self.buffer[..len]); + Ok(len) } } @@ -178,42 +181,47 @@ impl UsbClassSpecificRequest for BulkOnlyMassStorageReset { const B_REQUEST: u8 = 0b11111111; } +#[async_trait] impl UsbDriver for UsbMassStorageDriverBulkOnly { - fn run( - self: Arc, - device: Arc, - ) -> BoxFuture<'static, Result<(), UsbError>> { - async move { - // TODO filter to only accept BBB config - let config = device.select_configuration(|_| true).await?.unwrap(); - // Bulk-in, bulk-out - assert_eq!(config.endpoints.len(), 2); - // TODO those indices may be different - let control_pipe = device.control_pipe(); - let in_pipe = device.open_bulk_in_pipe(1).await?; - let out_pipe = device.open_bulk_out_pipe(2).await?; + async fn run(self: Arc, device: Arc) -> Result<(), UsbError> { + // TODO filter to only accept BBB config + let config = device.select_configuration(|_| true).await?.unwrap(); + // Bulk-in, bulk-out + assert_eq!(config.endpoints.len(), 2); + // TODO those indices may be different + let control_pipe = device.control_pipe(); + let (in_index, in_info) = config + .find_endpoint(|ep| ep.is(UsbEndpointType::Bulk, UsbDirection::In)) + .ok_or(UsbError::InvalidConfiguration)?; + let (out_index, out_info) = config + .find_endpoint(|ep| ep.is(UsbEndpointType::Bulk, UsbDirection::Out)) + .ok_or(UsbError::InvalidConfiguration)?; + let in_pipe = device + .open_bulk_in_pipe(in_index, in_info.max_packet_size as u16) + .await?; + let out_pipe = device + .open_bulk_out_pipe(out_index, out_info.max_packet_size as u16) + .await?; - // Perform a Bulk-Only Mass Storage Reset - // TODO interface id? - control_pipe - .class_specific_request::(0, 0) - .await?; + // Perform a Bulk-Only Mass Storage Reset + // TODO interface id? + control_pipe + .class_specific_request::(0, 0) + .await?; - // TODO get max LUN + // TODO get max LUN - let bbb = Bbb::new(device.clone(), in_pipe, out_pipe)?; - let scsi = ScsiDevice::setup(bbb) - .await - .inspect_err(|error| log::error!("msc: scsi error {error:?}")) - .map_err(|_| UsbError::DriverError)?; - let detach = DetachHandler(scsi.clone()); - device.set_detach_handler(Arc::new(detach)); + let bbb = Bbb::new(device.clone(), in_pipe, out_pipe)?; + let scsi = ScsiDevice::setup(bbb) + .await + .inspect_err(|error| log::error!("msc: scsi error {error:?}")) + .map_err(|_| UsbError::DriverError)?; + let detach = DetachHandler(scsi.clone()); + device.set_detach_handler(Arc::new(detach)); - ygg_driver_scsi::attach(scsi).ok(); + ygg_driver_scsi::attach(scsi).ok(); - Ok(()) - } - .boxed() + Ok(()) } fn name(&self) -> &'static str { diff --git a/kernel/driver/bus/usb/src/class_driver/mod.rs b/kernel/driver/bus/usb/src/class_driver/mod.rs index 8c7a23a2..6979aa45 100644 --- a/kernel/driver/bus/usb/src/class_driver/mod.rs +++ b/kernel/driver/bus/usb/src/class_driver/mod.rs @@ -1,5 +1,5 @@ -use alloc::{sync::Arc, vec::Vec}; -use futures_util::future::BoxFuture; +use alloc::{boxed::Box, sync::Arc, vec::Vec}; +use async_trait::async_trait; use libk::task::runtime; use libk_util::sync::spin_rwlock::IrqSafeRwLock; @@ -21,24 +21,22 @@ pub struct UsbClassInfo { pub interface_protocol_number: u8, } +#[async_trait] pub trait UsbDriver: Send + Sync { - fn name(&self) -> &'static str; - fn run( - self: Arc, - device: Arc, - ) -> BoxFuture<'static, Result<(), UsbError>>; + async fn run(self: Arc, device: Arc) -> Result<(), UsbError>; + fn name(&self) -> &'static str; fn probe(&self, class: &UsbClassInfo, device: &UsbDeviceAccess) -> bool; } async fn extract_class_info(device: &UsbDeviceAccess) -> Result, UsbError> { - if device.num_configurations != 1 { + if device.info.num_configurations != 1 { return Ok(None); } let device_info = &device.info; let config_info = device.query_configuration_info(0).await?; - if config_info.interfaces.len() == 1 { + if config_info.interfaces.len() >= 1 { let if_info = &config_info.interfaces[0]; let class = if device_info.device_class == UsbDeviceClass::FromInterface { diff --git a/kernel/driver/bus/usb/src/communication.rs b/kernel/driver/bus/usb/src/communication.rs index c390a136..b429b846 100644 --- a/kernel/driver/bus/usb/src/communication.rs +++ b/kernel/driver/bus/usb/src/communication.rs @@ -1,149 +1,11 @@ -use core::{ - future::poll_fn, - sync::atomic::{AtomicU32, Ordering}, - task::{Context, Poll}, -}; - -use alloc::{sync::Arc, vec::Vec}; -use futures_util::task::AtomicWaker; -use libk_mm::address::PhysicalAddress; - -use crate::error::{TransferError, UsbError}; - #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub enum UsbDirection { Out, In, } -#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] -#[repr(transparent)] -pub struct UsbTransferToken(pub u64); - -#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] -#[repr(transparent)] -pub struct UsbTransferResult(pub u32); - -pub struct UsbTransferStatus { - pub result: AtomicU32, - pub notify: AtomicWaker, -} - -pub struct UsbControlTransfer { - pub id: UsbTransferToken, - pub length: usize, - pub direction: UsbDirection, - pub elements: Vec, - pub status: Arc, -} - -pub struct UsbInterruptTransfer { - pub address: PhysicalAddress, - pub length: usize, - pub direction: UsbDirection, - pub status: Arc, -} - -pub struct UsbBulkTransfer { - pub address: PhysicalAddress, - pub length: usize, - pub direction: UsbDirection, - pub status: Arc, -} - impl UsbDirection { pub const fn is_device_to_host(self) -> bool { matches!(self, UsbDirection::In) } } - -// TODO this is xHCI-specific -impl UsbTransferResult { - pub fn is_aborted(&self) -> bool { - self.0 == u32::MAX - } - - pub fn is_success(&self) -> bool { - (self.0 >> 24) & 0xFF == 1 - } - - pub fn sub_length(&self) -> usize { - (self.0 & 0xFFFFFF) as _ - } - - pub fn error_code(&self) -> TransferError { - match self.0 >> 24 { - 0 => TransferError::InvalidTransfer, - 2 => TransferError::BufferError, - 4 => TransferError::UsbTransactionError, - 6 => TransferError::Stall, - 13 => TransferError::ShortPacket(self.0 & 0xFFFFFF), - _ => TransferError::Other, - } - } -} - -impl UsbControlTransfer { - pub async fn wait(&self) -> Result { - let sub_length = self.status.wait().await?; - Ok(self.length.saturating_sub(sub_length)) - } -} - -impl UsbInterruptTransfer { - pub async fn wait(&self) -> Result { - let sub_length = self.status.wait().await?; - Ok(self.length.saturating_sub(sub_length)) - } -} - -impl UsbBulkTransfer { - pub async fn wait(&self) -> Result { - let sub_length = self.status.wait().await?; - Ok(self.length.saturating_sub(sub_length)) - } -} - -impl UsbTransferStatus { - pub fn new() -> Self { - Self { - result: AtomicU32::new(0), - notify: AtomicWaker::new(), - } - } - - pub(crate) async fn wait(&self) -> Result { - poll_fn(|cx| { - self.poll(cx).map(|v| { - if v.is_success() { - Ok(v.sub_length()) - } else if v.is_aborted() { - Err(UsbError::DeviceDisconnected) - } else { - Err(UsbError::TransferFailed(v.error_code())) - } - }) - }) - .await - } - - pub fn signal(&self, status: u32) { - self.result.store(status, Ordering::Release); - self.notify.wake(); - } - - pub fn abort(&self) { - self.result.store(u32::MAX, Ordering::Release); - self.notify.wake(); - } - - pub fn poll(&self, cx: &mut Context<'_>) -> Poll { - self.notify.register(cx.waker()); - let value = self.result.load(Ordering::Acquire); - if value != 0 { - Poll::Ready(UsbTransferResult(value)) - } else { - Poll::Pending - } - } -} diff --git a/kernel/driver/bus/usb/src/descriptor.rs b/kernel/driver/bus/usb/src/descriptor.rs index c221a36c..5b388a95 100644 --- a/kernel/driver/bus/usb/src/descriptor.rs +++ b/kernel/driver/bus/usb/src/descriptor.rs @@ -1,10 +1,10 @@ use bytemuck::{Pod, Zeroable}; use crate::{ + communication::UsbDirection, device::UsbSpeed, error::UsbError, info::{UsbDeviceClass, UsbDeviceProtocol, UsbEndpointType, UsbVersion}, - UsbDirection, }; #[derive(Clone, Copy, Debug, Default, Pod, Zeroable)] diff --git a/kernel/driver/bus/usb/src/device.rs b/kernel/driver/bus/usb/src/device.rs index 0404da0a..40e358fc 100644 --- a/kernel/driver/bus/usb/src/device.rs +++ b/kernel/driver/bus/usb/src/device.rs @@ -1,17 +1,22 @@ use core::{fmt, ops::Deref}; use alloc::{boxed::Box, sync::Arc, vec::Vec}; -use futures_util::future::BoxFuture; +use async_trait::async_trait; use libk_mm::PageBox; use libk_util::sync::spin_rwlock::{IrqSafeRwLock, IrqSafeRwLockReadGuard}; use crate::{ error::UsbError, - info::{UsbConfigurationInfo, UsbDeviceInfo, UsbEndpointInfo, UsbInterfaceInfo, UsbVersion}, + info::{ + UsbConfigurationInfo, UsbDeviceInfo, UsbEndpointInfo, UsbEndpointType, UsbInterfaceInfo, + UsbVersion, + }, pipe::{ - bulk::{UsbBulkInPipeAccess, UsbBulkOutPipeAccess}, control::{ConfigurationDescriptorEntry, UsbControlPipeAccess}, - interrupt::UsbInterruptInPipeAccess, + normal::{ + UsbBulkInPipeAccess, UsbBulkOutPipeAccess, UsbInterruptInPipeAccess, UsbNormalPipeIn, + UsbNormalPipeOut, + }, }, UsbHostController, }; @@ -25,9 +30,8 @@ pub struct UsbBusAddress { } pub struct UsbDeviceAccess { - pub device: Box, + pub device: Arc, pub info: UsbDeviceInfo, - pub num_configurations: u8, pub current_configuration: IrqSafeRwLock>, } @@ -43,23 +47,25 @@ pub trait UsbDeviceDetachHandler: Send + Sync { fn handle_device_detach(&self); } +#[async_trait] #[allow(unused)] pub trait UsbDevice: Send + Sync { // Endpoint "0" fn control_pipe(&self) -> &UsbControlPipeAccess; - fn open_interrupt_in_pipe( + async fn open_normal_in_pipe( &self, number: u8, - ) -> BoxFuture> { - unimplemented!() - } - fn open_bulk_in_pipe(&self, number: u8) -> BoxFuture> { - unimplemented!() - } - fn open_bulk_out_pipe(&self, number: u8) -> BoxFuture> { - unimplemented!() - } + max_packet_size: u16, + ty: UsbEndpointType, + ) -> Result, UsbError>; + + async fn open_normal_out_pipe( + &self, + number: u8, + max_packet_size: u16, + ty: UsbEndpointType, + ) -> Result, UsbError>; fn port_number(&self) -> u8; fn bus_address(&self) -> UsbBusAddress; @@ -79,7 +85,7 @@ impl UsbDeviceAccess { /// * Device is not yet configured /// * Control pipe for the device has been properly set up /// * Device has been assigned a bus address - pub async fn setup(raw: Box) -> Result { + pub async fn setup(raw: Arc) -> Result { let control = raw.control_pipe(); let mut string_buffer = PageBox::new_uninit().map_err(UsbError::MemoryError)?; @@ -116,17 +122,54 @@ impl UsbDeviceAccess { device_protocol: device_desc.protocol(), device_protocol_number: device_desc.device_protocol, + num_configurations: device_desc.num_configurations, + max_packet_size: device_desc.max_packet_size(usb_version, raw.speed())?, }; Ok(Self { device: raw, info, - num_configurations: device_desc.num_configurations, current_configuration: IrqSafeRwLock::new(None), }) } + pub async fn open_interrupt_in_pipe( + &self, + number: u8, + max_packet_size: u16, + ) -> Result { + let pipe = self + .device + .open_normal_in_pipe(number, max_packet_size, UsbEndpointType::Interrupt) + .await?; + Ok(UsbInterruptInPipeAccess(pipe)) + } + + pub async fn open_bulk_in_pipe( + &self, + number: u8, + max_packet_size: u16, + ) -> Result { + let pipe = self + .device + .open_normal_in_pipe(number, max_packet_size, UsbEndpointType::Bulk) + .await?; + Ok(UsbBulkInPipeAccess(pipe)) + } + + pub async fn open_bulk_out_pipe( + &self, + number: u8, + max_packet_size: u16, + ) -> Result { + let pipe = self + .device + .open_normal_out_pipe(number, max_packet_size, UsbEndpointType::Bulk) + .await?; + Ok(UsbBulkOutPipeAccess(pipe)) + } + pub fn read_current_configuration( &self, ) -> IrqSafeRwLockReadGuard> { @@ -140,7 +183,7 @@ impl UsbDeviceAccess { let mut current_config = self.current_configuration.write(); let control_pipe = self.control_pipe(); - for i in 0..self.num_configurations { + for i in 0..self.info.num_configurations { let info = self.query_configuration_info(i).await?; if predicate(&info) { @@ -162,7 +205,7 @@ impl UsbDeviceAccess { &self, index: u8, ) -> Result { - if index >= self.num_configurations { + if index >= self.info.num_configurations { return Err(UsbError::InvalidConfiguration); } let mut string_buffer = PageBox::new_uninit().map_err(UsbError::MemoryError)?; diff --git a/kernel/driver/bus/usb/src/error.rs b/kernel/driver/bus/usb/src/error.rs index d17dcc6e..173224dd 100644 --- a/kernel/driver/bus/usb/src/error.rs +++ b/kernel/driver/bus/usb/src/error.rs @@ -1,13 +1,13 @@ use yggdrasil_abi::error::Error; -#[derive(Debug)] +#[derive(Debug, Clone, Copy)] pub enum TransferError { InvalidTransfer, - ShortPacket(u32), + ShortPacket(usize), BufferError, UsbTransactionError, Stall, - Other, + Other(u8), } #[derive(Debug)] @@ -25,6 +25,7 @@ pub enum UsbError { InvalidConfiguration, InvalidDescriptorField, // Runtime errors + TimedOut, DeviceBusy, DeviceDisconnected, TransferFailed(TransferError), @@ -32,6 +33,12 @@ pub enum UsbError { DriverError, } +impl From for UsbError { + fn from(value: TransferError) -> Self { + Self::TransferFailed(value) + } +} + impl From for Error { fn from(value: UsbError) -> Self { match value { diff --git a/kernel/driver/bus/usb/src/info.rs b/kernel/driver/bus/usb/src/info.rs index c2ea9fc2..4366f851 100644 --- a/kernel/driver/bus/usb/src/info.rs +++ b/kernel/driver/bus/usb/src/info.rs @@ -3,9 +3,9 @@ use core::fmt; use alloc::{string::String, vec::Vec}; use yggdrasil_abi::primitive_enum; -use crate::UsbDirection; +use crate::communication::UsbDirection; -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone, Copy, PartialEq)] pub enum UsbEndpointType { Control, Isochronous, @@ -33,6 +33,7 @@ pub enum UsbUsageType { pub enum UsbVersion { Usb11, Usb20, + Usb21, Usb30, Usb31, Usb32, @@ -98,6 +99,8 @@ pub struct UsbDeviceInfo { /// Max packet size for endpoint zero pub max_packet_size: usize, + + pub num_configurations: u8, } impl UsbVersion { @@ -108,7 +111,8 @@ impl UsbVersion { pub fn from_bcd_usb(value: u16) -> Option { match value { 0x110 => Some(UsbVersion::Usb11), - 0x200 => Some(UsbVersion::Usb20), + 0x200..=0x20F => Some(UsbVersion::Usb20), + 0x210..=0x21F => Some(UsbVersion::Usb21), 0x300 => Some(UsbVersion::Usb30), 0x310 => Some(UsbVersion::Usb31), 0x320 => Some(UsbVersion::Usb32), @@ -122,6 +126,7 @@ impl fmt::Display for UsbVersion { let string = match self { Self::Usb11 => "USB1.1", Self::Usb20 => "USB2.0", + Self::Usb21 => "USB2.1", Self::Usb30 => "USB3.0", Self::Usb31 => "USB3.1", Self::Usb32 => "USB3.2", @@ -129,3 +134,20 @@ impl fmt::Display for UsbVersion { f.write_str(string) } } + +impl UsbEndpointInfo { + pub fn is(&self, ty: UsbEndpointType, dir: UsbDirection) -> bool { + self.ty == ty && self.direction == dir + } +} + +impl UsbConfigurationInfo { + pub fn find_endpoint bool>( + &self, + predicate: P, + ) -> Option<(u8, &UsbEndpointInfo)> { + let index = self.endpoints.iter().position(predicate)?; + let info = &self.endpoints[index]; + Some((index as u8 + 1, info)) + } +} diff --git a/kernel/driver/bus/usb/src/lib.rs b/kernel/driver/bus/usb/src/lib.rs index 33fdb293..c63922d1 100644 --- a/kernel/driver/bus/usb/src/lib.rs +++ b/kernel/driver/bus/usb/src/lib.rs @@ -15,7 +15,7 @@ pub mod util; pub mod class_driver; -pub use communication::{UsbControlTransfer, UsbDirection, UsbTransferStatus, UsbTransferToken}; +// pub use communication::{UsbControlTransfer, UsbDirection, UsbTransferStatus, UsbTransferToken}; pub trait UsbEndpoint: Sync {} diff --git a/kernel/driver/bus/usb/src/pipe/bulk.rs b/kernel/driver/bus/usb/src/pipe/bulk.rs deleted file mode 100644 index f7919c72..00000000 --- a/kernel/driver/bus/usb/src/pipe/bulk.rs +++ /dev/null @@ -1,55 +0,0 @@ -use core::ops::Deref; - -use alloc::boxed::Box; -use libk_mm::PageSlice; - -use crate::{communication::UsbBulkTransfer, error::UsbError}; - -use super::UsbGenericPipe; - -pub trait UsbBulkInPipe: UsbGenericPipe + Send + Sync { - fn start_read(&self, buffer: &mut PageSlice) -> Result; - fn complete_transfer(&self, transfer: UsbBulkTransfer); -} - -pub trait UsbBulkOutPipe: UsbGenericPipe + Send + Sync { - fn start_write(&self, buffer: &PageSlice) -> Result; - fn complete_transfer(&self, transfer: UsbBulkTransfer); -} - -pub struct UsbBulkInPipeAccess(pub Box); -pub struct UsbBulkOutPipeAccess(pub Box); - -impl UsbBulkInPipeAccess { - pub async fn read<'a>(&self, buffer: &'a mut PageSlice) -> Result<&'a [u8], UsbError> { - let transfer = self.start_read(buffer)?; - let len = transfer.wait().await?; - self.complete_transfer(transfer); - Ok(&buffer[..len]) - } -} - -impl Deref for UsbBulkInPipeAccess { - type Target = dyn UsbBulkInPipe; - - fn deref(&self) -> &Self::Target { - &*self.0 - } -} - -impl UsbBulkOutPipeAccess { - pub async fn write<'a>(&self, buffer: &'a PageSlice) -> Result<(), UsbError> { - let transfer = self.start_write(buffer)?; - transfer.wait().await?; - self.complete_transfer(transfer); - Ok(()) - } -} - -impl Deref for UsbBulkOutPipeAccess { - type Target = dyn UsbBulkOutPipe; - - fn deref(&self) -> &Self::Target { - &*self.0 - } -} diff --git a/kernel/driver/bus/usb/src/pipe/control.rs b/kernel/driver/bus/usb/src/pipe/control.rs index ed58c48a..a02950ba 100644 --- a/kernel/driver/bus/usb/src/pipe/control.rs +++ b/kernel/driver/bus/usb/src/pipe/control.rs @@ -5,23 +5,19 @@ use core::{ }; use alloc::{boxed::Box, string::String}; +use async_trait::async_trait; use bytemuck::{Pod, Zeroable}; -use libk_mm::{ - address::{AsPhysicalAddress, PhysicalAddress}, - PageBox, -}; +use libk_mm::{PageBox, PageSlice}; use crate::{ + communication::UsbDirection, descriptor::{ UsbConfigurationDescriptor, UsbDeviceDescriptor, UsbDeviceQualifier, UsbEndpointDescriptor, UsbInterfaceDescriptor, UsbOtherSpeedConfiguration, }, error::UsbError, - UsbControlTransfer, UsbDirection, }; -use super::UsbGenericPipe; - #[derive(Debug)] pub struct ControlTransferSetup { pub bm_request_type: u8, @@ -88,28 +84,17 @@ fn decode_usb_string(bytes: &[u8]) -> Result { // Pipe impl -pub trait UsbControlPipe: UsbGenericPipe + Send + Sync { - fn start_transfer( +#[async_trait] +pub trait UsbControlPipe: Send + Sync { + async fn control_transfer( &self, setup: ControlTransferSetup, - data: Option<(PhysicalAddress, usize, UsbDirection)>, - ) -> Result; - - fn complete_transfer(&self, transfer: UsbControlTransfer); + data: Option<(&mut PageSlice>, UsbDirection)>, + ) -> Result; } pub struct UsbControlPipeAccess(pub Box); -fn input_buffer( - data: &mut PageBox>, -) -> (PhysicalAddress, usize, UsbDirection) { - ( - unsafe { data.as_physical_address() }, - size_of::(), - UsbDirection::In, - ) -} - #[derive(Debug)] pub enum ConfigurationDescriptorEntry<'a> { Configuration(&'a UsbConfigurationDescriptor), @@ -188,15 +173,40 @@ impl ConfigurationDescriptorQuery { } impl UsbControlPipeAccess { - pub async fn perform_value_control( - &self, - setup: ControlTransferSetup, - buffer: Option<(PhysicalAddress, usize, UsbDirection)>, - ) -> Result<(), UsbError> { - let transfer = self.start_transfer(setup, buffer)?; - transfer.status.wait().await?; - self.complete_transfer(transfer); - Ok(()) + pub async fn query_device_descriptor2(&self) -> Result, UsbError> { + let mut output = PageBox::new_uninit().map_err(UsbError::MemoryError)?; + + self.control_transfer( + ControlTransferSetup { + bm_request_type: 0b10000000, + b_request: 0xFF, + // b_request: 0x06, + w_value: 0x100, + w_index: 0, + w_length: size_of::() as _, + }, + Some((PageBox::as_bytes_mut(&mut output), UsbDirection::In)), + ) + .await?; + + Ok(unsafe { output.assume_init() }) + } + pub async fn query_device_descriptor(&self) -> Result, UsbError> { + let mut output = PageBox::new_uninit().map_err(UsbError::MemoryError)?; + + self.control_transfer( + ControlTransferSetup { + bm_request_type: 0b10000000, + b_request: 0x06, + w_value: 0x100, + w_index: 0, + w_length: size_of::() as _, + }, + Some((PageBox::as_bytes_mut(&mut output), UsbDirection::In)), + ) + .await?; + + Ok(unsafe { output.assume_init() }) } async fn fill_configuation_descriptor( @@ -204,7 +214,7 @@ impl UsbControlPipeAccess { index: u8, buffer: &mut PageBox<[MaybeUninit]>, ) -> Result<(), UsbError> { - self.perform_value_control( + self.control_transfer( ControlTransferSetup { bm_request_type: 0b10000000, b_request: 0x06, @@ -212,13 +222,33 @@ impl UsbControlPipeAccess { w_index: 0, w_length: buffer.len().try_into().unwrap(), }, - Some(( - unsafe { buffer.as_physical_address() }, - buffer.len(), - UsbDirection::In, - )), + Some((buffer.as_slice_mut(), UsbDirection::In)), ) - .await + .await?; + Ok(()) + } + + pub async fn query_string( + &self, + index: u8, + buffer: &mut PageBox>, + ) -> Result { + self.control_transfer( + ControlTransferSetup { + bm_request_type: 0b10000000, + b_request: 0x06, + w_value: 0x300 | (index as u16), + w_index: 0, + w_length: 4096, + }, + Some((PageBox::as_bytes_mut(buffer), UsbDirection::In)), + ) + .await?; + let data = unsafe { buffer.assume_init_ref() }; + + let len = data[0] as usize; + + decode_usb_string(&data[2..len]) } pub async fn query_configuration_descriptor( @@ -259,52 +289,12 @@ impl UsbControlPipeAccess { Ok(ConfigurationDescriptorQuery { buffer }) } - pub async fn query_device_descriptor(&self) -> Result, UsbError> { - let mut output = PageBox::new_uninit().map_err(UsbError::MemoryError)?; - self.perform_value_control( - ControlTransferSetup { - bm_request_type: 0b10000000, - b_request: 0x06, - w_value: 0x100, - w_index: 0, - w_length: size_of::() as _, - }, - Some(input_buffer(&mut output)), - ) - .await?; - - Ok(unsafe { output.assume_init() }) - } - - pub async fn query_string( - &self, - index: u8, - buffer: &mut PageBox>, - ) -> Result { - self.perform_value_control( - ControlTransferSetup { - bm_request_type: 0b10000000, - b_request: 0x06, - w_value: 0x300 | (index as u16), - w_index: 0, - w_length: 4096, - }, - Some(input_buffer(buffer)), - ) - .await?; - let data = unsafe { buffer.assume_init_ref() }; - - let len = data[0] as usize; - - decode_usb_string(&data[2..len]) - } - pub async fn perform_action( &self, w_value: u16, w_index: u16, ) -> Result<(), UsbError> { - self.perform_value_control( + self.control_transfer( ControlTransferSetup { bm_request_type: D::BM_REQUEST_TYPE, b_request: D::B_REQUEST, @@ -314,7 +304,8 @@ impl UsbControlPipeAccess { }, None, ) - .await + .await?; + Ok(()) } pub async fn class_specific_request( @@ -322,7 +313,7 @@ impl UsbControlPipeAccess { w_value: u16, w_index: u16, ) -> Result<(), UsbError> { - self.perform_value_control( + self.control_transfer( ControlTransferSetup { bm_request_type: D::BM_REQUEST_TYPE, b_request: D::B_REQUEST, @@ -332,7 +323,8 @@ impl UsbControlPipeAccess { }, None, ) - .await + .await?; + Ok(()) } pub async fn set_configuration(&self, value: u16) -> Result<(), UsbError> { diff --git a/kernel/driver/bus/usb/src/pipe/interrupt.rs b/kernel/driver/bus/usb/src/pipe/interrupt.rs deleted file mode 100644 index a474db1a..00000000 --- a/kernel/driver/bus/usb/src/pipe/interrupt.rs +++ /dev/null @@ -1,32 +0,0 @@ -use core::ops::Deref; - -use alloc::boxed::Box; -use libk_mm::PageBox; - -use crate::{communication::UsbInterruptTransfer, error::UsbError}; - -use super::UsbGenericPipe; - -pub trait UsbInterruptInPipe: UsbGenericPipe + Send + Sync { - fn start_read(&self, buffer: &mut PageBox<[u8]>) -> Result; - fn complete_transfer(&self, transfer: UsbInterruptTransfer); -} - -pub struct UsbInterruptInPipeAccess(pub Box); - -impl UsbInterruptInPipeAccess { - pub async fn read<'a>(&self, buffer: &'a mut PageBox<[u8]>) -> Result<&'a [u8], UsbError> { - let transfer = self.start_read(buffer)?; - let len = transfer.wait().await?; - self.complete_transfer(transfer); - Ok(&buffer[..len]) - } -} - -impl Deref for UsbInterruptInPipeAccess { - type Target = dyn UsbInterruptInPipe; - - fn deref(&self) -> &Self::Target { - &*self.0 - } -} diff --git a/kernel/driver/bus/usb/src/pipe/mod.rs b/kernel/driver/bus/usb/src/pipe/mod.rs index b09f2bd8..1915e0ba 100644 --- a/kernel/driver/bus/usb/src/pipe/mod.rs +++ b/kernel/driver/bus/usb/src/pipe/mod.rs @@ -1,9 +1,2 @@ -pub mod bulk; pub mod control; -pub mod interrupt; - -pub trait UsbGenericPipe {} - -pub enum UsbPipe { - Control(control::UsbControlPipeAccess), -} +pub mod normal; diff --git a/kernel/driver/bus/usb/src/pipe/normal.rs b/kernel/driver/bus/usb/src/pipe/normal.rs new file mode 100644 index 00000000..29abcadd --- /dev/null +++ b/kernel/driver/bus/usb/src/pipe/normal.rs @@ -0,0 +1,62 @@ +use core::ops::Deref; + +use alloc::boxed::Box; +use async_trait::async_trait; +use libk_mm::PageSlice; + +use crate::error::{TransferError, UsbError}; + +#[async_trait] +pub trait UsbNormalPipeIn: Send + Sync { + async fn read(&self, buffer: &mut PageSlice) -> Result; + async fn read_exact(&self, buffer: &mut PageSlice) -> Result<(), UsbError> { + match self.read(buffer).await { + Ok(len) if len == buffer.len() => Ok(()), + Ok(len) => Err(UsbError::TransferFailed(TransferError::ShortPacket(len))), + Err(error) => Err(error), + } + } +} + +#[async_trait] +pub trait UsbNormalPipeOut: Send + Sync { + async fn write(&self, buffer: &PageSlice) -> Result; +} + +pub struct UsbBulkInPipeAccess(pub Box); +pub struct UsbBulkOutPipeAccess(pub Box); + +pub struct UsbInterruptInPipeAccess(pub Box); +pub struct UsbInterruptOutPipeAccess(pub Box); + +impl Deref for UsbBulkInPipeAccess { + type Target = dyn UsbNormalPipeIn; + + fn deref(&self) -> &Self::Target { + self.0.deref() + } +} + +impl Deref for UsbBulkOutPipeAccess { + type Target = dyn UsbNormalPipeOut; + + fn deref(&self) -> &Self::Target { + self.0.deref() + } +} + +impl Deref for UsbInterruptInPipeAccess { + type Target = dyn UsbNormalPipeIn; + + fn deref(&self) -> &Self::Target { + self.0.deref() + } +} + +impl Deref for UsbInterruptOutPipeAccess { + type Target = dyn UsbNormalPipeOut; + + fn deref(&self) -> &Self::Target { + self.0.deref() + } +} diff --git a/kernel/driver/usb/xhci/Cargo.toml b/kernel/driver/usb/xhci/Cargo.toml index 9dd43ebb..fe688727 100644 --- a/kernel/driver/usb/xhci/Cargo.toml +++ b/kernel/driver/usb/xhci/Cargo.toml @@ -16,8 +16,9 @@ libk = { path = "../../../libk" } xhci_lib = { git = "https://github.com/rust-osdev/xhci.git", package = "xhci" } -atomic_enum = "0.3.0" -log = "0.4.22" -tock-registers = "0.8.1" -bytemuck = { version = "1.16.1", features = ["derive"] } -futures-util = { version = "0.3.30", default-features = false, features = ["alloc", "async-await"] } +async-trait.workspace = true +log.workspace = true +atomic_enum.workspace = true +tock-registers.workspace = true +bytemuck.workspace = true +futures-util.workspace = true diff --git a/kernel/driver/usb/xhci/src/context.rs b/kernel/driver/usb/xhci/src/context.rs index e212a2de..5c1ba48d 100644 --- a/kernel/driver/usb/xhci/src/context.rs +++ b/kernel/driver/usb/xhci/src/context.rs @@ -8,10 +8,7 @@ use libk_util::sync::spin_rwlock::IrqSafeRwLock; use xhci_lib::context::{self, DeviceHandler, InputHandler}; use ygg_driver_usb::error::UsbError; -use crate::{ - controller::{ContextSize, PortNumber}, - regs::PortSpeed, -}; +use crate::regs::{ContextSize, PortNumber}; pub enum XhciDeviceContext { Context32(IrqSafeRwLock>), @@ -66,10 +63,9 @@ impl XhciInputContext { pub fn new_address_device( size: ContextSize, - bus_address: u8, root_hub_port_number: PortNumber, - speed: PortSpeed, - max_packet_size: Option, // if not set, a default one for the given speed is used + max_packet_size: usize, + speed: u8, dequeue_pointer: PhysicalAddress, ) -> Result { let mut cx = Self::new(size)?; @@ -86,9 +82,8 @@ impl XhciInputContext { slot.set_context_entries(1); slot.set_interrupter_target(0); - slot.set_usb_device_address(bus_address); slot.set_root_hub_port_number(root_hub_port_number.into()); - slot.set_speed(speed.into()); + slot.set_speed(speed); } { @@ -97,9 +92,7 @@ impl XhciInputContext { ep0.set_endpoint_type(context::EndpointType::Control); ep0.set_error_count(3); // Use the provided max_packet_size, or the default one for the given speed - ep0.set_max_packet_size( - max_packet_size.unwrap_or(speed.default_max_packet_size()) as u16 - ); + ep0.set_max_packet_size(max_packet_size as u16); ep0.set_tr_dequeue_pointer(dequeue_pointer.into_u64()); ep0.set_dequeue_cycle_state(); } diff --git a/kernel/driver/usb/xhci/src/controller.rs b/kernel/driver/usb/xhci/src/controller.rs index cd2209e9..efd7af01 100644 --- a/kernel/driver/usb/xhci/src/controller.rs +++ b/kernel/driver/usb/xhci/src/controller.rs @@ -1,13 +1,9 @@ -use core::{ - fmt, - mem::MaybeUninit, - num::NonZeroU8, - sync::atomic::{AtomicUsize, Ordering}, -}; +use core::sync::atomic::{AtomicU8, Ordering}; -use alloc::{boxed::Box, collections::BTreeMap, sync::Arc, vec, vec::Vec}; +use alloc::{boxed::Box, collections::btree_map::BTreeMap, sync::Arc, vec::Vec}; +use async_trait::async_trait; use device_api::{device::Device, interrupt::InterruptHandler}; -use libk::task::runtime; +use libk::{error::Error, task::runtime}; use libk_mm::{ address::{AsPhysicalAddress, PhysicalAddress}, PageBox, @@ -16,492 +12,384 @@ use libk_util::{ sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock}, OneTimeInit, }; -use xhci_lib::ExtendedCapability; +use tock_registers::{ + interfaces::{ReadWriteable, Readable, Writeable}, + LocalRegisterCopy, +}; +use ygg_driver_pci::device::PciDeviceInfo; use ygg_driver_usb::{ bus::UsbBusManager, - device::{UsbBusAddress, UsbDeviceAccess}, + device::{UsbBusAddress, UsbDeviceAccess, UsbSpeed}, error::UsbError, info::UsbVersion, pipe::control::UsbControlPipeAccess, - util::UsbAddressAllocator, UsbHostController, }; -use yggdrasil_abi::error::Error; use crate::{ context::{XhciDeviceContext, XhciInputContext}, device::XhciBusDevice, pipe::ControlPipe, - regs::{Mapper, PortSpeed, Regs}, + regs::{ + self, + extended::ExtendedCapability, + operational::{PortRegs, CONFIG, PORTSC, USBCMD, USBSTS}, + PortNumber, Regs, + }, ring::{ - CommandExecutor, CommandRing, ControlTransferRing, Event, EventRing, EventRingSegmentTable, - GenericTransferRing, + command::CommandRing, + event::{Event, EventRing, EventRingSegmentTable}, + transfer::TransferRing, + CommandExecutor, GenericRing, }, util::EventBitmap, }; -#[derive(Debug, PartialEq, Eq, Clone, Copy)] -pub enum ContextSize { - Context32, - Context64, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] -pub struct PortNumber(NonZeroU8); - -impl PortNumber { - pub fn new(port: u8) -> Option { - NonZeroU8::new(port).map(Self) - } - - pub fn from_port_index(index: usize) -> Option { - let as_u8 = (index + 1).try_into().ok()?; - Some(Self(unsafe { NonZeroU8::new_unchecked(as_u8) })) - } - - pub fn port_index(&self) -> usize { - u8::from(self.0) as usize - 1 - } -} - #[allow(unused)] -struct Scratchpads { - buffers: Vec]>>, +struct ScratchpadArray { + buffers: Vec>, array: PageBox<[PhysicalAddress]>, } -#[allow(unused)] struct RootHubPort { version: UsbVersion, slot_type: u8, - max_hub_depth: Option, } pub struct Xhci { - regs: Regs, - - bus_address: OneTimeInit, - address_allocator: UsbAddressAllocator, - - port_count: usize, - pub(crate) context_size: ContextSize, + pub(crate) regs: Regs, + #[allow(unused)] + pci: PciDeviceInfo, dcbaa: IrqSafeRwLock>, - endpoints: IrqSafeRwLock>>, - event_ring: EventRing, - pub(crate) command_ring: CommandRing, #[allow(unused)] - scratchpads: Option, - - root_hub_map: BTreeMap, + scratchpads: Option, + pub(crate) command_ring: CommandRing, + event_ring: EventRing, + erst: EventRingSegmentTable, + root_hub_ports: Vec>, + pub(crate) endpoints: IrqSafeRwLock>>, + pub(crate) slots: Vec>>>, + pub(crate) port_slot_map: Vec, + bus_index: OneTimeInit, port_event_map: EventBitmap, } -impl Scratchpads { - pub fn new(count: usize) -> Result, UsbError> { - if count == 0 { - return Ok(None); - } - - let buffers = (0..count) - .map(|_| PageBox::new_uninit_slice(4096)) +impl ScratchpadArray { + pub fn new(capacity: usize, element_size: usize) -> Result { + let buffers = (0..capacity) + .map(|_| PageBox::new_slice(0, element_size)) .collect::, _>>() .map_err(UsbError::MemoryError)?; - let array = PageBox::new_slice_with(|i| unsafe { buffers[i].as_physical_address() }, count) - .map_err(UsbError::MemoryError)?; - - Ok(Some(Self { buffers, array })) + let array = + PageBox::new_slice_with(|i| unsafe { buffers[i].as_physical_address() }, capacity) + .map_err(UsbError::MemoryError)?; + Ok(Self { buffers, array }) } } impl Xhci { - // Extract all info about the xHC, but don't do any init besides performing a BIOS->OS handoff - pub fn new( - regs: xhci_lib::Registers, - mut extended: xhci_lib::extended_capabilities::List, - ) -> Result { - let event_ring = EventRing::new(128)?; + pub fn new(pci: PciDeviceInfo, regs: Regs) -> Result { + let mut dcbaa = PageBox::new_slice(PhysicalAddress::ZERO, regs.slot_count + 1) + .map_err(UsbError::MemoryError)?; let command_ring = CommandRing::new(128)?; + let event_ring = EventRing::new(128)?; + let erst = EventRingSegmentTable::for_event_rings(&[&event_ring])?; - let regs = Regs::from(regs); - - let scratchpad_count = regs.scratchpad_count(); - let port_count = regs.port_count(); - let slot_count = regs.max_slot_count(); - let context_size = regs.context_size(); - let context_size = match context_size { - 32 => ContextSize::Context32, - 64 => ContextSize::Context64, - _ => { - log::error!("Unhandled context size: {context_size}"); - return Err(UsbError::InvalidConfiguration); - } + // Setup scratch buffers + // TODO: Linux seems to just ignore the PAGESIZE, it's (1 << 0) everywhere + let scratchpads = if regs.scratch_count != 0 { + let array = ScratchpadArray::new(regs.scratch_count, 0x1000)?; + dcbaa[0] = unsafe { array.array.as_physical_address() }; + Some(array) + } else { + None }; - let scratchpads = Scratchpads::new(scratchpad_count)?; - let mut dcbaa = PageBox::new_slice(PhysicalAddress::ZERO, slot_count + 1) - .map_err(UsbError::MemoryError)?; - - if let Some(scratchpads) = scratchpads.as_ref() { - dcbaa[0] = unsafe { scratchpads.array.as_physical_address() }; - } - - let mut root_hub_map = BTreeMap::new(); - for cap in extended.into_iter() { - let Ok(cap) = cap else { - continue; - }; + let mut root_hub_ports: Vec> = (0..regs.port_count).map(|_| None).collect(); + for cap in regs.extended_capabilities.iter() { match cap { - ExtendedCapability::UsbLegacySupport(mut legsup) => { - let mut handoff = false; - legsup.usblegsup.update_volatile(|legsup| { - if legsup.hc_bios_owned_semaphore() { - handoff = true; - legsup.set_hc_os_owned_semaphore(); - } - }); - - if !handoff { - continue; - } - log::info!("xhci: BIOS->OS handoff started"); - - let mut timeout = 10000000; - while timeout > 0 { - let status = legsup.usblegsup.read_volatile(); - if !status.hc_bios_owned_semaphore() && status.hc_os_owned_semaphore() { - break; - } - - core::hint::spin_loop(); - timeout -= 1; - } - if timeout == 0 { - log::error!("xhci: BIOS->OS handoff failed"); - return Err(UsbError::DeviceBusy); - } - log::info!("xhci: BIOS->OS handoff finished"); - } - ExtendedCapability::XhciSupportedProtocol(proto) => { - let header = proto.header.read_volatile(); - - if header.name_string() != 0x20425355 { - log::warn!("Skip unknown xHCI supported protocol capability"); - continue; - } - let version_major = header.major_revision(); - let version_minor = header.minor_revision(); - let slot_type = header.protocol_slot_type(); - let max_hub_depth = header.hub_depth(); - let version = ((version_major as u16) << 8) | (version_minor as u16); - let Some(version) = UsbVersion::from_bcd_usb(version) else { - log::warn!("Skip unknown xHCI supported protocol revision: {version_major:x}.{version_minor:x}"); + ExtendedCapability::ProtocolSupport(support) => { + let Some(version) = support.usb_revision() else { continue; }; - let port_range = header.compatible_port_offset() - ..header.compatible_port_offset() + header.compatible_port_count(); - log::info!("Ports {port_range:?}: USB {version_major:x}.{version_minor:x}, slot type {slot_type}, max hub depth {max_hub_depth}"); - - for port in port_range { - let Some(number) = PortNumber::new(port) else { - continue; - }; - root_hub_map.insert( - number, - RootHubPort { - version, - slot_type, - max_hub_depth: (max_hub_depth != 0) - .then_some(max_hub_depth as usize), - }, - ); + for port in support.port_range() { + log::info!("* Port {port}: {version}"); + root_hub_ports[port as usize - 1] = Some(RootHubPort { + version, + slot_type: support.slot_type(), + }); } } - _ => (), + ExtendedCapability::LegacySupport(legsup) => { + legsup.write().perform_bios_handoff(10000000)?; + } } } + let port_slot_map = (0..regs.port_count).map(|_| AtomicU8::new(0)).collect(); + let slots = (0..regs.slot_count) + .map(|_| IrqSafeRwLock::new(None)) + .collect(); + Ok(Self { regs, - - bus_address: OneTimeInit::new(), - address_allocator: UsbAddressAllocator::new(), - - port_count, - context_size, - - event_ring, - command_ring, + pci, dcbaa: IrqSafeRwLock::new(dcbaa), - endpoints: IrqSafeRwLock::new(BTreeMap::new()), scratchpads, - root_hub_map, + command_ring, + event_ring, + erst, + root_hub_ports, + bus_index: OneTimeInit::new(), + endpoints: IrqSafeRwLock::new(BTreeMap::new()), + slots, + port_slot_map, port_event_map: EventBitmap::new(), }) } - pub fn register_device_context(&self, slot_id: u8, context: PhysicalAddress) { - self.dcbaa.write()[slot_id as usize] = context; + fn notify_endpoint(&self, slot_id: u8, endpoint_id: u8, address: PhysicalAddress, status: u32) { + if let Some(endpoint) = self.endpoints.read().get(&(slot_id, endpoint_id)) { + endpoint.notify(address, status); + } else { + log::warn!("Endpoint {slot_id}:{endpoint_id} does not exist"); + } } - pub fn register_endpoint( - &self, - slot_id: u8, - endpoint_id: u8, - ring: Arc, - ) { - self.endpoints.write().insert((slot_id, endpoint_id), ring); - } - - pub fn shutdown_endpoint(&self, slot_id: u8, endpoint_id: u8) { - if let Some(endpoint) = self.endpoints.write().remove(&(slot_id, endpoint_id)) { + fn kill_endpoint(&self, slot_id: u8, endpoint_id: u8) { + let endpoint = self.endpoints.write().remove(&(slot_id, endpoint_id)); + if let Some(endpoint) = endpoint { endpoint.shutdown(); - } else { - log::warn!( - "Endpoint {}:{} does not exist, maybe already shut down?", - slot_id, - endpoint_id - ); } } - pub fn notify_transfer( - &self, - slot_id: u8, - endpoint_id: u8, - address: PhysicalAddress, - status: u32, - ) { - if let Some(ep) = self.endpoints.read().get(&(slot_id, endpoint_id)) { - ep.notify(address, status); - } else { - log::warn!("No endpoint slot={slot_id}, ep={endpoint_id}"); - } - } + async fn allocate_device_slot( + self: &Arc, + port_id: PortNumber, + slot_type: u8, + speed: UsbSpeed, + ) -> Result<(u8, Arc, Arc), UsbError> { + let device_context = XhciDeviceContext::new(self.regs.context_size)?; + let slot_id = self.command_ring.enable_slot(&**self, slot_type).await?; + let (control_pipe, control_ring) = ControlPipe::new(self.clone(), slot_id, 1, 128)?; + let control_pipe = UsbControlPipeAccess(Box::new(control_pipe)); - async fn reset_port(&self, port: PortNumber) -> Result<(), UsbError> { - let index = port.port_index(); + self.dcbaa.write()[slot_id as usize] = device_context.physical_address(); + self.endpoints + .write() + .insert((slot_id, 1), control_ring.clone()); - // Set port reset and wait for it to clear - self.regs.ports.update(index, |regs| { - regs.portsc.set_port_reset(); + let slot = Arc::new(XhciBusDevice { + port_id, + slot_id, + device_context, + control_pipe, + speed, + detach_handler: IrqSafeSpinlock::new(None), + endpoints: IrqSafeRwLock::new(BTreeMap::from_iter([(1, control_ring.clone())])), + xhci: self.clone(), + bus_address: OneTimeInit::new(), }); - // TODO timeout - loop { - self.port_event_map.wait_specific(index).await; + *self.slots[slot_id as usize].write() = Some(slot.clone()); + self.port_slot_map[port_id.index()].store(slot_id, Ordering::Release); - let mut status = None; - self.regs.ports.update(index, |regs| { - // Port became disconnected - if !regs.portsc.port_enabled_disabled() || !regs.portsc.current_connect_status() { - log::warn!("xhci: port {port} disconnected during reset"); - status = Some(Err(UsbError::DeviceDisconnected)); - return; - } - - if !regs.portsc.port_reset() { - regs.portsc.clear_port_reset_change(); - status = Some(Ok(())); - } - }); - - if let Some(status) = status { - return status; - } - } + Ok((slot_id, slot, control_ring)) } - async fn setup_connected_port(self: Arc, port: PortNumber) -> Result<(), UsbError> { - // TODO cleanup after a failed device init: - // * Deallocate the bus address - // * Issue a Disable Slot TRB - // * Remove Device Context from DCBAA - // * Deregister the device's Default Control Endpoint + // TODO proper timeout + async fn reset_port(&self, regs: &PortRegs) -> Result<(), UsbError> { + // Reset the port + regs.modify_portsc_preserving(PORTSC::PR::SET); - let index = port.port_index(); - let root_hub_port = self - .root_hub_map - .get(&port) + let mut timeout = 10000000; + while timeout > 0 { + let status = regs.PORTSC.extract(); + + if status.matches_all(PORTSC::PR::CLEAR) { + break; + } + + timeout -= 1; + } + + // Clear PRC + regs.modify_portsc_preserving(PORTSC::PRC::SET); + Ok(()) + } + + // TODO clean up resources if init fails + async fn setup_port( + self: &Arc, + regs: &PortRegs, + number: PortNumber, + ) -> Result<(), UsbError> { + let root_hub_port = self.root_hub_ports[number.index()] + .as_ref() .ok_or(UsbError::PortInitFailed)?; - log::info!( - "xhci: setup {} device at port {port}", - root_hub_port.version - ); + let need_reset = !root_hub_port.version.is_version_3(); - if root_hub_port.version == UsbVersion::Usb20 { - // Port needs a reset first - log::info!("xhci: reset port {port}"); - self.reset_port(port).await?; + if need_reset { + self.reset_port(regs).await?; } - let speed = PortSpeed::try_from(self.regs.ports.read(index).portsc.port_speed()) - .map_err(|_| UsbError::PortInitFailed)?; + let status = regs.PORTSC.extract(); - log::info!("xhci: port {port} effective speed {speed:?}"); + let speed = status.read(PORTSC::PS) as u8; + let pls = status.read(PORTSC::PLS); - // Allocate a device slot - let slot_id = self - .command_ring - .enable_slot(&*self, root_hub_port.slot_type) - .await?; - - // Allocate some address for the device - let bus_address = self.address_allocator.allocate()?; - - // Allocate a default endpoint ring - let control_ring = Arc::new(ControlTransferRing::new(slot_id, 1, 128)?); - - // Register control endpoint - self.register_endpoint(slot_id, 1, control_ring.clone()); - - let control_pipe = UsbControlPipeAccess(Box::new(ControlPipe::new( - self.clone(), - slot_id, - control_ring.clone(), - ))); - - // Setup Device Context - let device_cx = XhciDeviceContext::new(self.context_size)?; - self.register_device_context(slot_id, device_cx.physical_address()); - - // Issue an Address Device TRB with BSR=1 first - let input_cx = XhciInputContext::new_address_device( - self.context_size, - bus_address, - port, - speed, - None, - control_ring.dequeue_pointer(), - )?; - self.command_ring - .address_device(&*self, slot_id, input_cx.physical_address(), true) - .await?; - - let state = device_cx.map(|cx| cx.slot().slot_state()); - log::info!("xhci: port {port} slot {slot_id} state after BSR=1: {state:?}"); - - // After an Address Device with BSR=1, retrieving a Device Descriptor is possible - // This is needed for Full-speed devices, where the max_packet_size is 8 at first, but - // is determined by the Device Descriptor when it's available - let device_descriptor = control_pipe.query_device_descriptor().await?; - let max_packet_size = - device_descriptor.max_packet_size(root_hub_port.version, speed.into())?; - - log::info!("xhci: port {port}:"); - log::info!(" * max_packet_size = {max_packet_size}"); - let max_packet_size = Some(max_packet_size); - - // Reset the control endpoint for a proper dequeue pointer and a proper DCS - control_ring.reset(); - - // Issue an Address Device TRB with BSR=0 - let input_cx = XhciInputContext::new_address_device( - self.context_size, - bus_address, - port, - speed, - max_packet_size, - control_ring.dequeue_pointer(), - )?; - self.command_ring - .address_device(&*self, slot_id, input_cx.physical_address(), false) - .await - .inspect_err(|error| { - log::error!("xhci: port {port} Address Device (BSR=0) error: {error:?}") - })?; - - let state = device_cx.map(|cx| cx.slot().slot_state()); - log::info!("xhci: port {port} slot {slot_id} state after BSR=0: {state:?}"); - - log::info!("xhci: port {port} device addressed ({bus_address})"); - - // Hand off the device to the general USB stack - - let bus_address = UsbBusAddress { - bus: *self.bus_address.get(), - device: bus_address, + let (usb_speed, max_packet_size) = match speed { + 1 => (UsbSpeed::Full, 8), + 2 => (UsbSpeed::Low, 8), + 3 => (UsbSpeed::High, 64), + 4 => (UsbSpeed::Super, 512), + _ => { + log::error!("Port {number} invalid speed value {speed}"); + return Err(UsbError::DeviceDisconnected); + } }; - let device = Box::new(XhciBusDevice { - xhci: self.clone(), - slot_id, - port_id: port.into(), - bus_address, - speed, - control_pipe, - device_context: device_cx, - rings: IrqSafeRwLock::new(vec![control_ring]), - detach_handler: IrqSafeSpinlock::new(None), - }); - let device = UsbDeviceAccess::setup(device).await?; + // PLS != 0 after reset + if pls != 0 { + log::error!("Port {number} pls!=0 after reset"); + return Err(UsbError::DeviceDisconnected); + } - UsbBusManager::register_device(Arc::new(device)); + let (slot_id, slot, control_ring) = self + .allocate_device_slot(number, root_hub_port.slot_type, usb_speed) + .await?; + + let input_cx = XhciInputContext::new_address_device( + self.regs.context_size, + number, + max_packet_size, + speed, + control_ring.base(), + )?; + self.command_ring + .address_device(&**self, slot_id, input_cx.physical_address(), false) + .await + .inspect_err(|error| { + log::error!("Port {number} Address Device TRB (BSR=0) failed: {error:?}") + })?; + + // TODO update max_packet_size for default control endpoint after fetching the device + // descriptor + + let bus_address = slot + .device_context + .map(|device| device.slot().usb_device_address()); + slot.bus_address.init(UsbBusAddress { + bus: *self.bus_index.get(), + device: bus_address, + }); + + let device = UsbDeviceAccess::setup(slot).await?; + UsbBusManager::register_device(device.into()); Ok(()) } - async fn port_manager_task(self: Arc) -> Result<(), UsbError> { - // Inject events for the root hub ports - for (&port, _) in self.root_hub_map.iter() { - self.port_event_map.signal(port.port_index()); + async fn handle_disconnect(&self, number: PortNumber) -> Result<(), UsbError> { + let slot_id = self.port_slot_map[number.index()].swap(0, Ordering::Acquire); + if slot_id == 0 { + return Ok(()); + } + log::info!("Port {number} disconnected"); + + let slot = self.slots[slot_id as usize].write().take(); + if let Some(slot) = slot { + UsbBusManager::detach_device(*slot.bus_address.get()); + + // Clean up xHC resources + for (&endpoint_id, endpoint) in slot.endpoints.read().iter() { + endpoint.shutdown(); + self.kill_endpoint(slot_id, endpoint_id); + } + } else { + log::warn!("No device in slot {slot_id}"); } - let mut show_disconnect_mask = 0; - loop { - let events = self.port_event_map.wait_any(self.port_count).await; + // TODO stop endpoints on the xHC side? + self.command_ring.disable_slot(self, slot_id).await?; - for port_index in events { - let port = PortNumber::from_port_index(port_index).unwrap(); + Ok(()) + } - let mut connected = None; - - self.regs.ports.update(port_index, |regs| { - if regs.portsc.port_enabled_disabled_change() { - regs.portsc.clear_port_enabled_disabled_change(); - } - if regs.portsc.connect_status_change() { - regs.portsc.clear_connect_status_change(); - connected = Some(regs.portsc.current_connect_status()); - } - }); - - let Some(state) = connected else { + async fn port_handler_task(self: Arc) -> Result<(), Error> { + // Inject notify for ports that are already connected + { + let ports = self.regs.ports.write(); + for (index, port) in self.root_hub_ports.iter().enumerate() { + if port.is_none() { continue; - }; + } - if state { - if show_disconnect_mask & (1 << port_index) == 0 { - log::info!("xhci: port {port} connected"); - if let Err(error) = self.clone().setup_connected_port(port).await { - show_disconnect_mask &= !(1 << port_index); - log::error!("xhci: port {port} setup failed: {error:?}"); - } else { - show_disconnect_mask |= 1 << port_index; + if ports[index].PORTSC.matches_all(PORTSC::CCS::SET) { + self.port_event_map.signal(index); + } + } + } + + loop { + let events = self.port_event_map.wait_any(self.regs.port_count).await; + + for index in events { + let number = PortNumber::from_index(index); + let ports = self.regs.ports.write(); + let regs = &ports[index]; + + let status = regs.PORTSC.extract(); + let neutral = regs::portsc_to_neutral(status); + let mut clear = LocalRegisterCopy::::new(0); + + // Clear affected change bits + let connected = if status.matches_all(PORTSC::CSC::SET) { + clear.modify(PORTSC::CSC::SET); + Some(status.matches_all(PORTSC::CCS::SET)) + } else { + None + }; + if status.matches_all(PORTSC::PEC::SET) { + clear.modify(PORTSC::PEC::SET); + } + if status.matches_all(PORTSC::PLC::SET) { + clear.modify(PORTSC::PLC::SET); + } + + regs.PORTSC.set(neutral.get() | clear.get()); + + if let Some(connected) = connected { + if connected { + log::info!("Port {number} connected"); + if let Err(error) = self.setup_port(regs, number).await { + log::error!("Port {number} setup error: {error:?}"); } } - } else { - if show_disconnect_mask & (1 << port_index) != 0 { - log::warn!("xhci: port {port} disconnected"); - show_disconnect_mask &= !(1 << port_index); + + if !connected { + if let Err(error) = self.handle_disconnect(number).await { + log::error!("Port {number} disconnect error: {error:?}"); + } } } } } } - fn handle_event(self: Arc) { + fn handle_event(&self) { while let Some(event) = self.event_ring.try_dequeue() { match event { - Event::PortChange(port) => { - if port > 0 { - self.port_event_map.signal(port - 1); + Event::PortChange(number) => { + if number > 0 { + self.port_event_map.signal(number - 1); } } Event::CommandCompletion { address, reply } => { @@ -513,79 +401,98 @@ impl Xhci { endpoint_id, status, } => { - self.notify_transfer(slot_id, endpoint_id, address, status); + self.notify_endpoint(slot_id, endpoint_id, address, status); } } } self.regs - .set_interrupter_0_dequeue_pointer(self.event_ring.dequeue_pointer()); + .runtime + .write() + .set_interrupter_dequeue_pointer(0, self.event_ring.dequeue_pointer()); } } -impl UsbHostController for Xhci {} - +#[async_trait] impl CommandExecutor for Xhci { fn ring_doorbell(&self, index: usize, target: u8) { - self.regs.ring_doorbell(index, target); + self.regs.doorbells[index].store(target as u32, Ordering::Release); + } + + async fn reset_endpoint( + &self, + slot_id: u8, + endpoint_id: u8, + dequeue_pointer: PhysicalAddress, + dequeue_cycle: bool, + ) -> Result<(), UsbError> { + log::warn!("xhci: reset stalled endpoint {slot_id}:{endpoint_id}"); + + self.command_ring + .reset_endpoint(&*self, slot_id, endpoint_id, false) + .await?; + self.command_ring + .set_tr_dequeue_pointer(&*self, slot_id, endpoint_id, dequeue_pointer, dequeue_cycle) + .await?; + + Ok(()) } } impl Device for Xhci { unsafe fn init(self: Arc) -> Result<(), Error> { - static XHCI_COUNT: AtomicUsize = AtomicUsize::new(0); - log::info!("Init USB xHCI"); + self.regs.hc_reset(10000000)?; + log::info!("xHC reset complete"); - if XHCI_COUNT.fetch_add(1, Ordering::Release) != 0 { - log::warn!("Skip second xhci init"); - return Ok(()); - } + // Configure the HC + let dcbaap = unsafe { self.dcbaa.read().as_physical_address() }; + let cr_base = self.command_ring.base(); - self.regs.reset(); - self.regs.set_max_slot_count(); + let op = self.regs.operational.write(); + let rt = self.regs.runtime.write(); - let erst = EventRingSegmentTable::for_event_rings(&[&self.event_ring])?; - let dcbaa = self.dcbaa.read(); + log::info!("xhci: configure HC"); + op.CONFIG + .modify(CONFIG::MaxSlotsEn.val(self.regs.slot_count as u32)); + op.set_dcbaap(dcbaap); + op.set_crcr(cr_base, true); - self.regs - .configure(&dcbaa, &self.command_ring, &self.event_ring, &erst); + log::info!("xhci: configure interrupter"); + rt.configure_interrupter(0, &self.event_ring, &self.erst); + + // Enable interrupts and start the HC + log::info!("xhci: start HC"); + op.USBCMD.modify(USBCMD::INTE::SET + USBCMD::HSEE::SET); + op.USBCMD.modify(USBCMD::RS::SET); + op.wait_usbsts_bit(USBSTS::CNR::CLEAR, 100000000)?; let bus = UsbBusManager::register_bus(self.clone()); - self.bus_address.init(bus); + self.bus_index.init(bus); - // Start the port manager task - runtime::spawn(self.clone().port_manager_task())?; + runtime::spawn(self.clone().port_handler_task()).ok(); Ok(()) } fn display_name(&self) -> &str { - "USB xHCI" + "xHCI" } } impl InterruptHandler for Xhci { fn handle_irq(self: Arc, _vector: Option) -> bool { - if let Some(status) = self.regs.handle_interrupt() { - if status.event_interrupt() { - self.handle_event(); - } + let status = self.regs.handle_interrupt(0); - true - } else { - false + if status.matches_all(USBSTS::HSE::SET) { + log::error!("xhci: Host System Error occurred"); } + + if status.matches_all(USBSTS::EINT::SET) { + self.handle_event(); + } + + true } } -impl fmt::Display for PortNumber { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Display::fmt(&self.0, f) - } -} - -impl From for u8 { - fn from(value: PortNumber) -> Self { - value.0.into() - } -} +impl UsbHostController for Xhci {} diff --git a/kernel/driver/usb/xhci/src/device.rs b/kernel/driver/usb/xhci/src/device.rs index 702b1753..b7f281ad 100644 --- a/kernel/driver/usb/xhci/src/device.rs +++ b/kernel/driver/usb/xhci/src/device.rs @@ -1,47 +1,177 @@ -use alloc::{boxed::Box, sync::Arc, vec::Vec}; -use futures_util::{future::BoxFuture, FutureExt}; -use libk_util::sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock}; +use alloc::{boxed::Box, collections::btree_map::BTreeMap, sync::Arc}; +use async_trait::async_trait; +use libk_util::{ + sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock}, + OneTimeInit, +}; use xhci_lib::context; use ygg_driver_usb::{ + communication::UsbDirection, device::{UsbBusAddress, UsbDevice, UsbDeviceDetachHandler, UsbSpeed}, error::UsbError, info::UsbEndpointType, pipe::{ - bulk::{UsbBulkInPipeAccess, UsbBulkOutPipeAccess}, control::UsbControlPipeAccess, - interrupt::UsbInterruptInPipeAccess, + normal::{UsbNormalPipeIn, UsbNormalPipeOut}, }, - UsbDirection, UsbHostController, + UsbHostController, }; use crate::{ context::{XhciDeviceContext, XhciInputContext}, - pipe::{BulkInPipe, BulkOutPipe, InterruptInPipe}, - regs::PortSpeed, - ring::{ - transfer::{BulkInTransferRing, BulkOutTransferRing, InterruptInTransferRing}, - GenericTransferRing, - }, + pipe::{NormalInPipe, NormalOutPipe}, + regs::PortNumber, + ring::transfer::TransferRing, Xhci, }; pub struct XhciBusDevice { - pub(crate) port_id: u8, + pub(crate) port_id: PortNumber, pub(crate) slot_id: u8, - pub(crate) bus_address: UsbBusAddress, - - pub(crate) speed: PortSpeed, + pub(crate) speed: UsbSpeed, + pub(crate) bus_address: OneTimeInit, pub(crate) xhci: Arc, - #[allow(unused)] pub(crate) device_context: XhciDeviceContext, - pub(crate) rings: IrqSafeRwLock>>, + pub(crate) endpoints: IrqSafeRwLock>>, pub(crate) control_pipe: UsbControlPipeAccess, pub(crate) detach_handler: IrqSafeSpinlock>>, } +#[async_trait] +impl UsbDevice for XhciBusDevice { + fn port_number(&self) -> u8 { + self.port_id.into() + } + + fn bus_address(&self) -> UsbBusAddress { + *self.bus_address.get() + } + + fn control_pipe(&self) -> &UsbControlPipeAccess { + &self.control_pipe + } + + fn handle_detach(&self) { + if let Some(handler) = self.detach_handler.lock().as_ref() { + handler.handle_device_detach(); + } + } + + fn set_detach_handler(&self, handler: Arc) { + *self.detach_handler.lock() = Some(handler); + } + + fn controller_ref(&self) -> &dyn UsbHostController { + self.xhci.as_ref() + } + + fn debug(&self) {} + + fn speed(&self) -> UsbSpeed { + self.speed + } + + async fn open_normal_in_pipe( + &self, + number: u8, + max_packet_size: u16, + ty: UsbEndpointType, + ) -> Result, UsbError> { + let dci = Self::dci(ty, UsbDirection::In, number) as u8; + let (pipe, ring) = NormalInPipe::new(self.xhci.clone(), self.slot_id, dci, 128)?; + self.add_endpoint(number, ring.clone(), ty, UsbDirection::In, max_packet_size) + .await?; + Ok(Box::new(pipe)) + } + + async fn open_normal_out_pipe( + &self, + number: u8, + max_packet_size: u16, + ty: UsbEndpointType, + ) -> Result, UsbError> { + let dci = Self::dci(ty, UsbDirection::Out, number) as u8; + let (pipe, ring) = NormalOutPipe::new(self.xhci.clone(), self.slot_id, dci, 128)?; + self.add_endpoint(number, ring.clone(), ty, UsbDirection::Out, max_packet_size) + .await?; + Ok(Box::new(pipe)) + } +} + impl XhciBusDevice { + pub async fn add_endpoint( + &self, + endpoint_number: u8, + ring: Arc, + ty: UsbEndpointType, + direction: UsbDirection, + max_packet_size: u16, + ) -> Result<(), UsbError> { + let dci = Self::dci(ty, direction, endpoint_number) as u8; + + log::debug!( + "Setup endpoint dci #{}: {} {:?} {:?}", + dci, + endpoint_number, + ty, + direction + ); + + let ep_type = match (ty, direction) { + (UsbEndpointType::Interrupt, UsbDirection::In) => context::EndpointType::InterruptIn, + (UsbEndpointType::Interrupt, UsbDirection::Out) => context::EndpointType::InterruptOut, + (UsbEndpointType::Bulk, UsbDirection::In) => context::EndpointType::BulkIn, + (UsbEndpointType::Bulk, UsbDirection::Out) => context::EndpointType::BulkOut, + _ => todo!(), + }; + + let mut input_cx = XhciInputContext::new(self.xhci.regs.context_size)?; + + { + let control = input_cx.control_mut(); + + control.set_add_context_flag(0); + control.set_add_context_flag(dci as usize); + } + + { + let slot = input_cx.device_mut().slot_mut(); + + slot.set_context_entries(dci + 1); + } + + { + let endpoint = input_cx.device_mut().endpoint_mut(dci as usize); + + endpoint.set_endpoint_type(ep_type); + endpoint.set_error_count(3); + // TODO Pick a better value here + endpoint.set_interval(3); + endpoint.set_max_packet_size(max_packet_size); + endpoint.set_tr_dequeue_pointer(ring.base().into_u64()); + endpoint.set_dequeue_cycle_state(); + } + + self.xhci + .command_ring + .configure_endpoint( + self.xhci.as_ref(), + self.slot_id, + input_cx.physical_address(), + ) + .await?; + + self.endpoints.write().insert(dci, ring.clone()); + self.xhci + .endpoints + .write() + .insert((self.slot_id, dci), ring); + + Ok(()) + } + fn dci(ty: UsbEndpointType, dir: UsbDirection, number: u8) -> usize { match ty { UsbEndpointType::Control => (number as usize) * 2 + 1, @@ -55,159 +185,4 @@ impl XhciBusDevice { } } } - - async fn setup_endpoint_inner( - &self, - ring: Arc, - dci: u8, - ty: UsbEndpointType, - direction: UsbDirection, - ) -> Result<(), UsbError> { - log::debug!("Setup endpoint dci #{}: {:?} {:?}", dci, ty, direction); - - let ep_type = match (ty, direction) { - (UsbEndpointType::Interrupt, UsbDirection::In) => context::EndpointType::InterruptIn, - (UsbEndpointType::Interrupt, UsbDirection::Out) => context::EndpointType::InterruptOut, - (UsbEndpointType::Bulk, UsbDirection::In) => context::EndpointType::BulkIn, - (UsbEndpointType::Bulk, UsbDirection::Out) => context::EndpointType::BulkOut, - _ => todo!(), - }; - - let mut input_cx = XhciInputContext::new(self.xhci.context_size)?; - - { - let control = input_cx.control_mut(); - - control.set_add_context_flag(0); - control.set_add_context_flag(dci as _); - } - - { - let slot = input_cx.device_mut().slot_mut(); - - slot.set_context_entries(31); - slot.set_interrupter_target(0); - slot.set_usb_device_address(self.bus_address.device); - slot.set_root_hub_port_number(self.port_id); - slot.set_speed(self.speed.into()); - } - - { - let ep_cx = input_cx.device_mut().endpoint_mut(dci as _); - - ep_cx.set_endpoint_type(ep_type); - ep_cx.set_error_count(3); - // TODO - ep_cx.set_max_packet_size(8); - ep_cx.set_tr_dequeue_pointer(ring.dequeue_pointer().into_u64()); - ep_cx.set_dequeue_cycle_state(); - } - - self.xhci - .command_ring - .configure_endpoint( - self.xhci.as_ref(), - self.slot_id, - input_cx.physical_address(), - ) - .await?; - - self.rings.write().push(ring.clone()); - self.xhci.register_endpoint(self.slot_id, dci, ring); - - Ok(()) - } -} - -impl UsbDevice for XhciBusDevice { - fn control_pipe(&self) -> &UsbControlPipeAccess { - &self.control_pipe - } - - fn port_number(&self) -> u8 { - self.port_id - } - - fn bus_address(&self) -> UsbBusAddress { - self.bus_address - } - - fn controller_ref(&self) -> &dyn UsbHostController { - self.xhci.as_ref() - } - - fn open_interrupt_in_pipe( - &self, - number: u8, - ) -> BoxFuture> { - async move { - let dci = Self::dci(UsbEndpointType::Interrupt, UsbDirection::In, number) as u8; - let ring = Arc::new(InterruptInTransferRing::new(self.slot_id, dci as _, 128)?); - - self.setup_endpoint_inner( - ring.clone(), - dci, - UsbEndpointType::Interrupt, - UsbDirection::In, - ) - .await?; - - let pipe = InterruptInPipe::new(self.xhci.clone(), self.slot_id, number, dci, ring); - - Ok(UsbInterruptInPipeAccess(Box::new(pipe))) - } - .boxed() - } - - fn open_bulk_in_pipe(&self, number: u8) -> BoxFuture> { - async move { - let dci = Self::dci(UsbEndpointType::Bulk, UsbDirection::In, number) as u8; - let ring = Arc::new(BulkInTransferRing::new(self.slot_id, dci as _, 128)?); - - self.setup_endpoint_inner(ring.clone(), dci, UsbEndpointType::Bulk, UsbDirection::In) - .await?; - - let pipe = BulkInPipe::new(self.xhci.clone(), self.slot_id, number, dci, ring); - - Ok(UsbBulkInPipeAccess(Box::new(pipe))) - } - .boxed() - } - - fn open_bulk_out_pipe(&self, number: u8) -> BoxFuture> { - async move { - let dci = Self::dci(UsbEndpointType::Bulk, UsbDirection::Out, number) as u8; - let ring = Arc::new(BulkOutTransferRing::new(self.slot_id, dci as _, 128)?); - - self.setup_endpoint_inner(ring.clone(), dci, UsbEndpointType::Bulk, UsbDirection::Out) - .await?; - - let pipe = BulkOutPipe::new(self.xhci.clone(), self.slot_id, number, dci, ring); - - Ok(UsbBulkOutPipeAccess(Box::new(pipe))) - } - .boxed() - } - - fn set_detach_handler(&self, handler: Arc) { - *self.detach_handler.lock() = Some(handler); - } - - fn handle_detach(&self) { - log::info!("Device detach handler"); - for ring in self.rings.write().drain(..) { - self.xhci - .shutdown_endpoint(ring.slot_id(), ring.endpoint_id()); - } - - if let Some(handler) = self.detach_handler.lock().take() { - handler.handle_device_detach(); - } - } - - fn speed(&self) -> UsbSpeed { - self.speed.into() - } - - fn debug(&self) {} } diff --git a/kernel/driver/usb/xhci/src/lib.rs b/kernel/driver/usb/xhci/src/lib.rs index c3beda25..e04e6621 100644 --- a/kernel/driver/usb/xhci/src/lib.rs +++ b/kernel/driver/usb/xhci/src/lib.rs @@ -1,14 +1,17 @@ #![no_std] #![allow(clippy::new_without_default)] -#![feature(iter_array_chunks)] +#![feature(iter_array_chunks, let_chains)] extern crate alloc; use alloc::sync::Arc; +use controller::Xhci; use device_api::{device::Device, interrupt::InterruptAffinity}; -use regs::Mapper; -use xhci_lib::extended_capabilities; +use regs::Regs; +// use regs::Mapper; +// use xhci_lib::extended_capabilities; use ygg_driver_pci::{ + capability::{DevicePowerState, PowerManagementCapability}, device::{PciDeviceInfo, PreferredInterruptMode}, PciCommandRegister, PciConfigurationSpace, }; @@ -21,7 +24,6 @@ mod pipe; mod regs; mod ring; mod util; -pub use controller::Xhci; pub fn probe(info: &PciDeviceInfo) -> Result, Error> { // TODO Chip Hardware Reset @@ -32,22 +34,27 @@ pub fn probe(info: &PciDeviceInfo) -> Result, Error> { .as_memory() .expect("xHCI's BAR0 is not memory-type"); + if let Some(power) = info.config_space.capability::() { + log::info!("xHC has power management capability"); + let power_state = power.get_device_power_state(); + + if power_state != DevicePowerState::D0 { + power.set_device_power_state(DevicePowerState::D0); + } + // Enable PME# signal generation + power.set_pme_en(true); + } + let mut cmd = PciCommandRegister::from_bits_retain(info.config_space.command()); cmd &= !(PciCommandRegister::DISABLE_INTERRUPTS | PciCommandRegister::ENABLE_IO); cmd |= PciCommandRegister::ENABLE_MEMORY | PciCommandRegister::BUS_MASTER; info.config_space.set_command(cmd.bits()); - let base = bar0.try_into_usize().unwrap(); - let mapper = Mapper::new(); - let regs = unsafe { xhci_lib::Registers::new(base, mapper.clone()) }; - let extended = unsafe { - extended_capabilities::List::new(base, regs.capability.hccparams1.read_volatile(), mapper) - } - .ok_or(Error::InvalidArgument) - .inspect_err(|_| log::error!("Cannot proceed with xhci init: no extended capabilities"))?; - let xhci = Arc::new(Xhci::new(regs, extended)?); - info.init_interrupts(PreferredInterruptMode::Msi(true))?; + + let regs = Regs::map(bar0)?; + let xhci = Arc::new(Xhci::new(info.clone(), regs)?); + info.map_interrupt(InterruptAffinity::Any, xhci.clone())?; Ok(xhci) diff --git a/kernel/driver/usb/xhci/src/pipe.rs b/kernel/driver/usb/xhci/src/pipe.rs index 43d5f833..5b87b5b2 100644 --- a/kernel/driver/usb/xhci/src/pipe.rs +++ b/kernel/driver/usb/xhci/src/pipe.rs @@ -1,170 +1,139 @@ -use alloc::sync::Arc; -use libk_mm::{address::PhysicalAddress, PageBox, PageSlice}; +use core::mem::MaybeUninit; + +use alloc::{boxed::Box, sync::Arc}; +use async_trait::async_trait; +use libk_mm::{address::AsPhysicalAddress, PageSlice}; use ygg_driver_usb::{ - communication::{UsbBulkTransfer, UsbInterruptTransfer}, - error::UsbError, + communication::UsbDirection, + error::{TransferError, UsbError}, pipe::{ - bulk::{UsbBulkInPipe, UsbBulkOutPipe}, control::{ControlTransferSetup, UsbControlPipe}, - interrupt::UsbInterruptInPipe, - UsbGenericPipe, + normal::{UsbNormalPipeIn, UsbNormalPipeOut}, }, - UsbControlTransfer, UsbDirection, }; -use crate::{ - ring::{ - transfer::{BulkInTransferRing, BulkOutTransferRing, InterruptInTransferRing}, - ControlTransferRing, - }, - Xhci, -}; +use crate::{controller::Xhci, ring::transfer::TransferRing}; pub struct ControlPipe { xhci: Arc, - ring: Arc, + pub(crate) ring: Arc, } -#[allow(unused)] -pub struct InterruptInPipe { +pub struct NormalInPipe { xhci: Arc, - - slot_id: u8, - endpoint_id: u8, - dci: u8, - - ring: Arc, + pub(crate) ring: Arc, } -#[allow(unused)] -pub struct BulkInPipe { +pub struct NormalOutPipe { xhci: Arc, - - slot_id: u8, - endpoint_id: u8, - dci: u8, - - ring: Arc, -} -#[allow(unused)] -pub struct BulkOutPipe { - xhci: Arc, - - slot_id: u8, - endpoint_id: u8, - dci: u8, - - ring: Arc, -} - -impl UsbGenericPipe for ControlPipe {} - -impl UsbControlPipe for ControlPipe { - fn start_transfer( - &self, - setup: ControlTransferSetup, - data: Option<(PhysicalAddress, usize, UsbDirection)>, - ) -> Result { - self.ring.start_transfer(self.xhci.as_ref(), setup, data) - } - - fn complete_transfer(&self, transfer: UsbControlTransfer) { - self.ring.complete_transfer(transfer) - } + pub(crate) ring: Arc, } impl ControlPipe { - pub fn new(xhci: Arc, _slot_id: u8, ring: Arc) -> Self { - Self { xhci, ring } - } -} - -impl UsbGenericPipe for InterruptInPipe {} - -impl UsbInterruptInPipe for InterruptInPipe { - fn start_read(&self, buffer: &mut PageBox<[u8]>) -> Result { - self.ring.start_transfer(self.xhci.as_ref(), buffer) - } - - fn complete_transfer(&self, transfer: UsbInterruptTransfer) { - self.ring.complete_transfer(transfer) - } -} - -impl InterruptInPipe { pub fn new( xhci: Arc, slot_id: u8, endpoint_id: u8, - dci: u8, - ring: Arc, - ) -> Self { - Self { - xhci, - slot_id, - endpoint_id, - dci, + capacity: usize, + ) -> Result<(Self, Arc), UsbError> { + let ring = Arc::new(TransferRing::new(slot_id, endpoint_id, capacity)?); + Ok(( + Self { + xhci, + ring: ring.clone(), + }, ring, - } + )) } } -impl UsbGenericPipe for BulkInPipe {} - -impl UsbBulkInPipe for BulkInPipe { - fn start_read(&self, buffer: &mut PageSlice) -> Result { - self.ring.start_transfer(self.xhci.as_ref(), buffer) - } - - fn complete_transfer(&self, transfer: UsbBulkTransfer) { - self.ring.complete_transfer(transfer) - } -} - -impl BulkInPipe { +impl NormalInPipe { pub fn new( xhci: Arc, slot_id: u8, endpoint_id: u8, - dci: u8, - ring: Arc, - ) -> Self { - Self { - xhci, - slot_id, - endpoint_id, - dci, + capacity: usize, + ) -> Result<(Self, Arc), UsbError> { + let ring = Arc::new(TransferRing::new(slot_id, endpoint_id, capacity)?); + Ok(( + Self { + xhci, + ring: ring.clone(), + }, ring, - } + )) } } -impl UsbGenericPipe for BulkOutPipe {} - -impl UsbBulkOutPipe for BulkOutPipe { - fn start_write(&self, buffer: &PageSlice) -> Result { - self.ring.start_transfer(self.xhci.as_ref(), buffer) - } - - fn complete_transfer(&self, transfer: UsbBulkTransfer) { - self.ring.complete_transfer(transfer) - } -} - -impl BulkOutPipe { +impl NormalOutPipe { pub fn new( xhci: Arc, slot_id: u8, endpoint_id: u8, - dci: u8, - ring: Arc, - ) -> Self { - Self { - xhci, - slot_id, - endpoint_id, - dci, + capacity: usize, + ) -> Result<(Self, Arc), UsbError> { + let ring = Arc::new(TransferRing::new(slot_id, endpoint_id, capacity)?); + Ok(( + Self { + xhci, + ring: ring.clone(), + }, ring, - } + )) + } +} + +#[async_trait] +impl UsbControlPipe for ControlPipe { + async fn control_transfer( + &self, + setup: ControlTransferSetup, + data: Option<(&mut PageSlice>, UsbDirection)>, + ) -> Result { + let data_len = data.as_ref().map_or(0, |(data, _)| data.len()); + let result = self + .ring + .control_transfer(self.xhci.as_ref(), setup, data) + .await; + allow_short_packet(data_len, result) + } +} + +#[async_trait] +impl UsbNormalPipeIn for NormalInPipe { + async fn read(&self, buffer: &mut PageSlice) -> Result { + let data_len = buffer.len(); + let result = self + .ring + .normal_transfer( + self.xhci.as_ref(), + unsafe { buffer.as_physical_address() }, + buffer.len(), + ) + .await; + allow_short_packet(data_len, result) + } +} + +#[async_trait] +impl UsbNormalPipeOut for NormalOutPipe { + async fn write(&self, buffer: &PageSlice) -> Result { + self.ring + .normal_transfer( + self.xhci.as_ref(), + unsafe { buffer.as_physical_address() }, + buffer.len(), + ) + .await + } +} + +fn allow_short_packet(data_len: usize, result: Result) -> Result { + match result { + // Short packets are okay for control transfers + Err(UsbError::TransferFailed(TransferError::ShortPacket(residual))) => { + Ok(data_len.saturating_sub(residual)) + } + result => result, } } diff --git a/kernel/driver/usb/xhci/src/regs.rs b/kernel/driver/usb/xhci/src/regs.rs deleted file mode 100644 index b8eb075b..00000000 --- a/kernel/driver/usb/xhci/src/regs.rs +++ /dev/null @@ -1,335 +0,0 @@ -use core::{cell::UnsafeCell, num::NonZeroUsize}; - -use alloc::{sync::Arc, vec::Vec}; -use libk_mm::{ - address::{AsPhysicalAddress, PhysicalAddress}, - device::RawDeviceMemoryMapping, - PageBox, -}; -use libk_util::sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock}; -use xhci_lib::{ - accessor::{array, marker}, - registers::{ - operational::UsbStatusRegister, Capability, Doorbell, InterrupterRegisterSet, Operational, - PortRegisterSet, - }, -}; -use ygg_driver_usb::device::UsbSpeed; -use yggdrasil_abi::primitive_enum; - -use crate::ring::{CommandRing, EventRing, EventRingSegmentTable, GenericRing}; - -struct MapperInner { - mappings: Vec<(RawDeviceMemoryMapping, usize)>, -} - -#[derive(Clone)] -pub struct Mapper { - mapper: Arc>, -} - -pub struct LockedArray { - array: UnsafeCell>, - locks: Vec>, -} - -unsafe impl Sync for LockedArray {} -unsafe impl Send for LockedArray {} - -pub struct Regs { - operational: IrqSafeRwLock>, - interrupters: IrqSafeRwLock>, - capability: Capability, - doorbells: LockedArray, - pub ports: LockedArray, -} - -impl LockedArray { - #[inline] - #[allow(clippy::mut_from_ref)] - unsafe fn get_mut(&self) -> &mut array::Generic { - &mut *self.array.get() - } - - pub fn update(&self, index: usize, f: U) { - let _guard = self.locks[index].write(); - unsafe { self.get_mut() }.update_volatile_at(index, f); - } - - pub fn read(&self, index: usize) -> T { - let _guard = self.locks[index].read(); - unsafe { self.get_mut() }.read_volatile_at(index) - } -} - -impl From> for LockedArray { - fn from(value: array::Generic) -> Self { - let locks = Vec::from_iter((0..value.len()).map(|_| IrqSafeRwLock::new(()))); - - Self { - array: UnsafeCell::new(value), - locks, - } - } -} - -impl From> for Regs { - fn from(value: xhci_lib::Registers) -> Self { - Self { - operational: IrqSafeRwLock::new(value.operational), - capability: value.capability, - interrupters: IrqSafeRwLock::new(value.interrupter_register_set), - doorbells: LockedArray::from(value.doorbell), - ports: LockedArray::from(value.port_register_set), - } - } -} - -impl Regs { - pub fn reset(&self) { - let mut o = self.operational.write(); - - // TODO Get ownership from firmware - - // Stop the controller - o.usbcmd.update_volatile(|u| { - u.clear_run_stop(); - }); - - while !o.usbsts.read_volatile().hc_halted() { - core::hint::spin_loop(); - } - - // Reset the controller - o.usbcmd.update_volatile(|u| { - u.set_host_controller_reset(); - }); - while o.usbcmd.read_volatile().host_controller_reset() - || o.usbsts.read_volatile().controller_not_ready() - { - core::hint::spin_loop(); - } - } - - pub fn max_slot_count(&self) -> usize { - self.capability - .hcsparams1 - .read_volatile() - .number_of_device_slots() as _ - } - - pub fn set_max_slot_count(&self) -> usize { - let device_slot_count = self.max_slot_count(); - let mut o = self.operational.write(); - // Set max slots enabled - o.config.update_volatile(|u| { - u.set_max_device_slots_enabled(device_slot_count as _); - }); - - device_slot_count as _ - } - - pub fn context_size(&self) -> usize { - match self.capability.hccparams1.read_volatile().context_size() { - true => 64, - false => 32, - } - } - - pub fn port_count(&self) -> usize { - self.capability.hcsparams1.read_volatile().number_of_ports() as _ - } - - pub fn scratchpad_count(&self) -> usize { - self.capability - .hcsparams2 - .read_volatile() - .max_scratchpad_buffers() as usize - } - - pub fn configure( - &self, - dcbaa: &PageBox<[PhysicalAddress]>, - cmd_ring: &CommandRing, - evt_ring: &EventRing, - erst: &EventRingSegmentTable, - ) { - let mut o = self.operational.write(); - let mut i = self.interrupters.write(); - - o.dcbaap.update_volatile(|u| unsafe { - u.set(dcbaa.as_physical_address().into()); - }); - o.crcr.update_volatile(|u| { - u.set_command_ring_pointer(cmd_ring.base().into()); - u.set_ring_cycle_state(); - }); - - let mut intr0 = i.interrupter_mut(0); - intr0.erstsz.update_volatile(|u| { - u.set(erst.capacity().try_into().unwrap()); - }); - intr0.erdp.update_volatile(|u| { - log::debug!("::: Dequeue Pointer: {:#x}", evt_ring.dequeue_pointer()); - u.set_event_ring_dequeue_pointer(evt_ring.dequeue_pointer().into()); - }); - intr0.erstba.update_volatile(|u| { - u.set(erst.physical_address().into()); - }); - // intr0.imod.update_volatile(|u| { - // u.set_interrupt_moderation_interval(0) - // .set_interrupt_moderation_counter(0); - // }); - intr0.iman.update_volatile(|u| { - u.set_interrupt_enable(); - }); - - o.usbcmd.update_volatile(|u| { - u.set_interrupter_enable().set_run_stop(); - }); - - // Wait for the controller to come out of "not ready" state - while o.usbsts.read_volatile().controller_not_ready() { - core::hint::spin_loop(); - } - } - - pub fn handle_interrupt(&self) -> Option { - let mut o = self.operational.write(); - let mut i = self.interrupters.write(); - - let status = o.usbsts.read_volatile(); - - if !status.event_interrupt() { - return None; - } - - o.usbsts.write_volatile(status); - - if status.host_system_error() { - return Some(status); - } - - // Acknowledge interrupts - let mut intr0 = i.interrupter_mut(0); - intr0.iman.update_volatile(|u| { - u.clear_interrupt_pending(); - }); - - Some(status) - } - - pub fn set_interrupter_0_dequeue_pointer(&self, pointer: PhysicalAddress) { - let mut i = self.interrupters.write(); - - i.interrupter_mut(0).erdp.update_volatile(|u| { - u.set_event_ring_dequeue_pointer(pointer.into()); - u.clear_event_handler_busy(); - }); - } - - pub fn ring_doorbell(&self, index: usize, target: u8) { - self.doorbells.update(index, |u| { - u.set_doorbell_target(target); - }); - } -} - -impl Mapper { - pub fn new() -> Self { - Self { - mapper: Arc::new(IrqSafeSpinlock::new(MapperInner { - mappings: Vec::new(), - })), - } - } -} - -impl xhci_lib::accessor::Mapper for Mapper { - // FIXME really slow, but at least reduces the number of unneeded mappings - unsafe fn map(&mut self, phys_start: usize, bytes: usize) -> NonZeroUsize { - let mut mapper = self.mapper.lock(); - for (mapping, refcount) in mapper.mappings.iter_mut() { - if phys_start as u64 >= mapping.physical_base - && ((phys_start + bytes) as u64) - <= mapping.physical_base + (mapping.page_size * mapping.page_count) as u64 - { - *refcount += 1; - return NonZeroUsize::new_unchecked( - mapping.base_address + phys_start - mapping.physical_base as usize, - ); - } - } - let mapping = RawDeviceMemoryMapping::map(phys_start as u64, bytes, Default::default()) - .inspect_err(|error| { - log::error!( - "Cannot map xHC MMIO region {:#x?}: {error:?}", - phys_start..phys_start + bytes - ) - }) - .expect("Could not map an USB xHCI region"); - let address = mapping.address; - log::info!("xhci: map {:#x} -> {:#x}", mapping.base_address, address); - mapper.mappings.push((mapping, 1)); - NonZeroUsize::new_unchecked(address) - } - - fn unmap(&mut self, _virt_start: usize, _bytes: usize) { - // let mut mapper = self.mapper.lock(); - // let index = mapper.mappings.iter().position(|(mapping, _)| { - // virt_start >= mapping.base_address - // && virt_start + bytes - // <= mapping.base_address + mapping.page_count * mapping.page_size - // }); - - // if let Some(index) = index { - // let (entry, refcount) = &mut mapper.mappings[index]; - // *refcount -= 1; - // if *refcount == 0 { - // log::info!("xhci: unmap {:#x}", entry.base_address); - // mapper.mappings.remove(index); - // } - // } - } -} - -// Register value definitions - -primitive_enum! { - pub enum PortSpeed: u8 { - Full = 1, - Low = 2, - High = 3, - SuperGen1x1 = 4, - SuperGen2x1 = 5, - SuperGen1x2 = 6, - SuperGen2x2 = 7, - } -} - -impl PortSpeed { - pub fn default_max_packet_size(&self) -> usize { - match self { - Self::Low => 8, - Self::High => 64, - Self::SuperGen1x1 | Self::SuperGen1x2 | Self::SuperGen2x1 | Self::SuperGen2x2 => 512, - - // See Section 4.3., point 7. of the initialization list - Self::Full => 8, - } - } -} - -impl From for UsbSpeed { - fn from(value: PortSpeed) -> Self { - match value { - PortSpeed::Low => UsbSpeed::Low, - PortSpeed::Full => UsbSpeed::Full, - PortSpeed::High => UsbSpeed::High, - PortSpeed::SuperGen1x1 - | PortSpeed::SuperGen1x2 - | PortSpeed::SuperGen2x1 - | PortSpeed::SuperGen2x2 => UsbSpeed::Super, - } - } -} diff --git a/kernel/driver/usb/xhci/src/regs/capability.rs b/kernel/driver/usb/xhci/src/regs/capability.rs new file mode 100644 index 00000000..0e596482 --- /dev/null +++ b/kernel/driver/usb/xhci/src/regs/capability.rs @@ -0,0 +1,66 @@ +use tock_registers::{register_bitfields, register_structs, registers::ReadOnly}; + +register_bitfields! { + u32, + pub HCSPARAMS1 [ + MaxSlots OFFSET(0) NUMBITS(8) [], + MaxIntrs OFFSET(8) NUMBITS(11) [], + MaxPorts OFFSET(24) NUMBITS(8) [], + ], + pub HCSPARAMS2 [ + IST OFFSET(0) NUMBITS(4) [], + ERSTMax OFFSET(4) NUMBITS(4) [], + MaxScratchpadBufsHi OFFSET(21) NUMBITS(5) [], + ScratchpadRestore OFFSET(26) NUMBITS(1) [], + MaxScratchpadBufsLo OFFSET(27) NUMBITS(5) [], + ], + pub HCSPARAMS3 [ + U1ExitLatency OFFSET(0) NUMBITS(8) [], + U2ExitLatency OFFSET(16) NUMBITS(16) [], + ], + pub HCCPARAMS1 [ + AC64 OFFSET(0) NUMBITS(1) [], + BNC OFFSET(1) NUMBITS(1) [], + CSZ OFFSET(2) NUMBITS(1) [], + PPC OFFSET(3) NUMBITS(1) [], + PIND OFFSET(4) NUMBITS(1) [], + LHRC OFFSET(5) NUMBITS(1) [], + LTC OFFSET(6) NUMBITS(1) [], + NSS OFFSET(7) NUMBITS(1) [], + PAE OFFSET(8) NUMBITS(1) [], + SPC OFFSET(9) NUMBITS(1) [], + SEC OFFSET(10) NUMBITS(1) [], + CFC OFFSET(11) NUMBITS(1) [], + MaxPSASize OFFSET(12) NUMBITS(4) [], + XECP OFFSET(16) NUMBITS(16) [], + ], + pub HCCPARAMS2 [ + U3C OFFSET(0) NUMBITS(1) [], + CMC OFFSET(1) NUMBITS(1) [], + FSC OFFSET(2) NUMBITS(1) [], + CTC OFFSET(3) NUMBITS(1) [], + LEC OFFSET(4) NUMBITS(1) [], + CIC OFFSET(5) NUMBITS(1) [], + ETC OFFSET(6) NUMBITS(1) [], + ETC_TSC OFFSET(7) NUMBITS(1) [], + GSC OFFSET(8) NUMBITS(1) [], + VTC OFFSET(9) NUMBITS(1) [], + ], +} + +register_structs! { + #[allow(non_snake_case)] + pub CapabilityRegs { + (0x00 => pub CAPLENGTH: ReadOnly), + (0x01 => _0), + (0x02 => pub HCIVERSION: ReadOnly), + (0x04 => pub HCSPARAMS1: ReadOnly), + (0x08 => pub HCSPARAMS2: ReadOnly), + (0x0C => pub HCSPARAMS3: ReadOnly), + (0x10 => pub HCCPARAMS1: ReadOnly), + (0x14 => pub DBOFF: ReadOnly), + (0x18 => pub RTSOFF: ReadOnly), + (0x1C => pub HCCPARAMS2: ReadOnly), + (0x20 => @END), + } +} diff --git a/kernel/driver/usb/xhci/src/regs/extended.rs b/kernel/driver/usb/xhci/src/regs/extended.rs new file mode 100644 index 00000000..0bb65b6c --- /dev/null +++ b/kernel/driver/usb/xhci/src/regs/extended.rs @@ -0,0 +1,122 @@ +use core::{ + ops::Range, + sync::atomic::{AtomicU32, Ordering}, +}; + +use alloc::vec::Vec; +use libk::error::Error; +use libk_mm::{address::PhysicalAddress, device::DeviceMemoryIo}; +use libk_util::sync::spin_rwlock::IrqSafeRwLock; +use ygg_driver_usb::{error::UsbError, info::UsbVersion}; + +pub struct ProtocolSupport { + words: [u32; 4], +} + +pub struct LegacySupport(DeviceMemoryIo<'static, [AtomicU32; 4]>); + +pub enum ExtendedCapability { + LegacySupport(IrqSafeRwLock), + ProtocolSupport(ProtocolSupport), +} + +impl ExtendedCapability { + pub fn parse_list( + base: PhysicalAddress, + extended_capability_offset: usize, + ) -> Result, Error> { + let mut extended_capabilities = Vec::new(); + + if extended_capability_offset != 0 { + let extended_capability_base = base.add(extended_capability_offset); + let mut current = extended_capability_base; + + loop { + let header = + unsafe { DeviceMemoryIo::<[AtomicU32; 4]>::map(current, Default::default()) }?; + let word0 = header[0].load(Ordering::Acquire); + let id = word0 as u8; + let next_pointer = ((word0 >> 8) & 0xFF) as usize; + + match id { + 0x01 => extended_capabilities.push(ExtendedCapability::LegacySupport( + IrqSafeRwLock::new(LegacySupport(header)), + )), + 0x02 => { + let words = [ + word0, + header[1].load(Ordering::Acquire), + header[2].load(Ordering::Acquire), + header[3].load(Ordering::Acquire), + ]; + extended_capabilities.push(ExtendedCapability::ProtocolSupport( + ProtocolSupport { words }, + )) + } + _ => (), + } + + if next_pointer == 0 { + break; + } else { + current = current.add(next_pointer << 2); + } + } + } + + Ok(extended_capabilities) + } +} + +impl ProtocolSupport { + pub fn usb_revision(&self) -> Option { + UsbVersion::from_bcd_usb((self.words[0] >> 16) as u16) + } + + pub fn slot_type(&self) -> u8 { + (self.words[3] & 0x1F) as u8 + } + + pub fn port_range(&self) -> Range { + let port_offset = self.words[2] as u8; + let port_count = (self.words[2] >> 8) as u8; + + port_offset.max(1)..port_offset.saturating_add(port_count) + } +} + +impl LegacySupport { + fn is_bios_owned(&self) -> bool { + self.0[0].load(Ordering::Acquire) & (1 << 16) != 0 + } + + fn set_os_owned(&mut self) { + self.0[0].fetch_or(1 << 24, Ordering::Release); + } + + pub fn perform_bios_handoff(&mut self, mut timeout_cycles: u64) -> Result<(), UsbError> { + if !self.is_bios_owned() { + return Ok(()); + } + + // Clear SMI events + self.0[1].fetch_and( + !((1 << 0) | (1 << 4) | (1 << 13) | (1 << 14) | (1 << 15)), + Ordering::Release, + ); + + // Set OS owned semaphore + self.set_os_owned(); + while timeout_cycles > 0 && self.is_bios_owned() { + core::hint::spin_loop(); + timeout_cycles -= 1; + } + + if timeout_cycles > 0 { + Ok(()) + } else { + log::error!("xhci: BIOS handoff timed out"); + Err(UsbError::DeviceBusy) + } + } +} diff --git a/kernel/driver/usb/xhci/src/regs/mod.rs b/kernel/driver/usb/xhci/src/regs/mod.rs new file mode 100644 index 00000000..20de0684 --- /dev/null +++ b/kernel/driver/usb/xhci/src/regs/mod.rs @@ -0,0 +1,184 @@ +use core::{fmt, sync::atomic::AtomicU32}; + +use alloc::vec::Vec; +use capability::{CapabilityRegs, HCCPARAMS1, HCSPARAMS1, HCSPARAMS2}; +use extended::ExtendedCapability; +use libk::error::Error; +use libk_mm::{address::PhysicalAddress, device::DeviceMemoryIo}; +use libk_util::sync::spin_rwlock::IrqSafeRwLock; +use operational::{OperationalRegs, PortRegs, PORTSC, USBCMD, USBSTS}; +use runtime::{RuntimeRegs, IMAN}; +use tock_registers::{ + interfaces::{ReadWriteable, Readable, Writeable}, + LocalRegisterCopy, +}; + +pub mod capability; +pub mod extended; +pub mod operational; +pub mod runtime; + +pub struct Regs { + #[allow(unused)] + pub(crate) capability: DeviceMemoryIo<'static, CapabilityRegs>, + pub(crate) operational: IrqSafeRwLock>, + pub(crate) runtime: IrqSafeRwLock>, + pub(crate) doorbells: DeviceMemoryIo<'static, [AtomicU32]>, + pub(crate) extended_capabilities: Vec, + // TODO per-port locks + pub(crate) ports: IrqSafeRwLock>, + pub(crate) port_count: usize, + pub(crate) slot_count: usize, + pub(crate) scratch_count: usize, + pub(crate) context_size: ContextSize, +} + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum ContextSize { + Context32, + Context64, +} + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +#[repr(transparent)] +pub struct PortNumber(u8); + +impl Regs { + const PORT_REGS_OFFSET: usize = 0x400; + + pub fn map(base: PhysicalAddress) -> Result { + let capability = + unsafe { DeviceMemoryIo::::map(base, Default::default()) }?; + + let operational_offset = capability.CAPLENGTH.get() as usize; + let doorbell_offset = capability.DBOFF.get() & !0x3; + let runtime_offset = capability.RTSOFF.get() & !0x1F; + let extended_capability_offset = + (capability.HCCPARAMS1.read(HCCPARAMS1::XECP) as usize) << 2; + + let port_count = capability.HCSPARAMS1.read(HCSPARAMS1::MaxPorts) as usize; + let slot_count = capability.HCSPARAMS1.read(HCSPARAMS1::MaxSlots) as usize; + + let operational_base = base.add(operational_offset); + let doorbell_base = base.add(doorbell_offset as usize); + let runtime_base = base.add(runtime_offset as usize); + + let scratch_lo = capability.HCSPARAMS2.read(HCSPARAMS2::MaxScratchpadBufsLo); + let scratch_hi = capability.HCSPARAMS2.read(HCSPARAMS2::MaxScratchpadBufsHi); + let scratch_count = ((scratch_hi << 5) | scratch_lo) as usize; + + let context_size = if capability.HCCPARAMS1.matches_all(HCCPARAMS1::CSZ::SET) { + ContextSize::Context64 + } else { + ContextSize::Context32 + }; + + let operational = unsafe { + DeviceMemoryIo::::map(operational_base, Default::default()) + }?; + let runtime = + unsafe { DeviceMemoryIo::::map(runtime_base, Default::default()) }?; + + let ports = unsafe { + DeviceMemoryIo::map_slice( + operational_base.add(Self::PORT_REGS_OFFSET), + port_count, + Default::default(), + ) + }?; + let doorbells = + unsafe { DeviceMemoryIo::map_slice(doorbell_base, 256, Default::default()) }?; + let extended_capabilities = + ExtendedCapability::parse_list(base, extended_capability_offset)?; + + Ok(Self { + capability, + operational: IrqSafeRwLock::new(operational), + runtime: IrqSafeRwLock::new(runtime), + doorbells, + ports: IrqSafeRwLock::new(ports), + extended_capabilities, + + port_count, + slot_count, + scratch_count, + context_size, + }) + } + + pub fn hc_reset(&self, timeout_cycles: u64) -> Result<(), Error> { + let op = self.operational.write(); + + // Wait for CNR to get cleared + op.wait_usbsts_bit(USBSTS::CNR::CLEAR, timeout_cycles)?; + + // Clear run/stop and wait for the HC to halt + op.USBCMD.modify(USBCMD::RS::CLEAR); + op.wait_usbsts_bit(USBSTS::HCH::SET, timeout_cycles)?; + + // Reset the HC, wait for CNR to get cleared again + op.USBCMD.modify(USBCMD::HCRST::SET); + op.wait_usbsts_bit(USBSTS::CNR::CLEAR, timeout_cycles)?; + + // Halt the HC again + op.USBCMD.modify(USBCMD::RS::CLEAR); + op.wait_usbsts_bit(USBSTS::HCH::SET, timeout_cycles)?; + + Ok(()) + } + + pub fn handle_interrupt(&self, vector: usize) -> LocalRegisterCopy { + let op = self.operational.write(); + let rt = self.runtime.write(); + + let status = op.USBSTS.extract(); + + // Clear the RW1C bits + op.USBSTS.set(status.get()); + + // Acknowledge interrupts for given vector (vector == interrupter index) + let interrupter = &rt.IRn[vector]; + + interrupter.IMAN.modify(IMAN::IP::SET); + + status + } +} + +pub fn portsc_to_neutral( + portsc: LocalRegisterCopy, +) -> LocalRegisterCopy { + const RO_MASK: u32 = (1 << 0) | (1 << 3) | (0xF << 10) | (1 << 30); + const RWS_MASK: u32 = (0xF << 5) + | (1 << 9) + | (0x3 << 14) + | (1 << 25) + | (1 << 26) + | (1 << 27) + | (1 << 30) + | (1 << 31); + + portsc.bitand(RO_MASK | RWS_MASK) +} + +impl Into for PortNumber { + fn into(self) -> u8 { + self.0 + } +} + +impl PortNumber { + pub fn from_index(index: usize) -> Self { + Self(index as u8 + 1) + } + + pub fn index(&self) -> usize { + self.0 as usize - 1 + } +} + +impl fmt::Display for PortNumber { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(&self.0, f) + } +} diff --git a/kernel/driver/usb/xhci/src/regs/operational.rs b/kernel/driver/usb/xhci/src/regs/operational.rs new file mode 100644 index 00000000..22e9b7a5 --- /dev/null +++ b/kernel/driver/usb/xhci/src/regs/operational.rs @@ -0,0 +1,140 @@ +use libk::error::Error; +use libk_mm::address::PhysicalAddress; +use tock_registers::{ + fields::FieldValue, + interfaces::{Readable, Writeable}, + register_bitfields, register_structs, + registers::{ReadOnly, ReadWrite}, +}; + +register_bitfields! { + u32, + pub USBCMD [ + RS OFFSET(0) NUMBITS(1) [], + HCRST OFFSET(1) NUMBITS(1) [], + INTE OFFSET(2) NUMBITS(1) [], + HSEE OFFSET(3) NUMBITS(1) [], + LHCRST OFFSET(7) NUMBITS(1) [], + CSS OFFSET(8) NUMBITS(1) [], + CRS OFFSET(9) NUMBITS(1) [], + EWE OFFSET(10) NUMBITS(1) [], + EU3S OFFSET(11) NUMBITS(1) [], + CME OFFSET(13) NUMBITS(1) [], + ETE OFFSET(14) NUMBITS(1) [], + TSC_EN OFFSET(15) NUMBITS(1) [], + VTIOE OFFSET(16) NUMBITS(1) [], + ], + pub USBSTS [ + HCH OFFSET(0) NUMBITS(1) [], + HSE OFFSET(2) NUMBITS(1) [], + EINT OFFSET(3) NUMBITS(1) [], + PCD OFFSET(4) NUMBITS(1) [], + SSS OFFSET(8) NUMBITS(1) [], + RSS OFFSET(9) NUMBITS(1) [], + SRE OFFSET(10) NUMBITS(1) [], + CNR OFFSET(11) NUMBITS(1) [], + HCE OFFSET(12) NUMBITS(1) [], + ], + pub CONFIG [ + MaxSlotsEn OFFSET(0) NUMBITS(8) [], + U3E OFFSET(8) NUMBITS(1) [], + CIE OFFSET(9) NUMBITS(1) [], + ], + + // Port registers + pub PORTSC [ + CCS OFFSET(0) NUMBITS(1) [], + PED OFFSET(1) NUMBITS(1) [], + OCA OFFSET(3) NUMBITS(1) [], + PR OFFSET(4) NUMBITS(1) [], + PLS OFFSET(5) NUMBITS(4) [], + PP OFFSET(9) NUMBITS(1) [], + PS OFFSET(10) NUMBITS(4) [], + PIC OFFSET(14) NUMBITS(2) [], + LWS OFFSET(16) NUMBITS(1) [], + CSC OFFSET(17) NUMBITS(1) [], + PEC OFFSET(18) NUMBITS(1) [], + WRC OFFSET(19) NUMBITS(1) [], + OCC OFFSET(20) NUMBITS(1) [], + PRC OFFSET(21) NUMBITS(1) [], + PLC OFFSET(22) NUMBITS(1) [], + CEC OFFSET(23) NUMBITS(1) [], + ], + pub PORTLI [ + LEC OFFSET(0) NUMBITS(16) [], + RLC OFFSET(16) NUMBITS(4) [], + TLC OFFSET(20) NUMBITS(4) [], + ], +} + +register_structs! { + #[allow(non_snake_case)] + pub OperationalRegs { + (0x000 => pub USBCMD: ReadWrite), + (0x004 => pub USBSTS: ReadWrite), + (0x008 => pub PAGESIZE: ReadOnly), + (0x00C => _0), + (0x014 => pub DNCTRL: ReadWrite), + (0x018 => pub CRCR: ReadWrite), + // (0x018 => pub CRCR_LO: ReadWrite), + // (0x01C => pub CRCR_HI: ReadWrite), + (0x020 => _1), + (0x030 => pub DCBAAP: ReadWrite), + // (0x030 => pub DCBAAP_LO: ReadWrite), + // (0x034 => pub DCBAAP_HI: ReadWrite), + (0x038 => pub CONFIG: ReadWrite), + (0x03C => _2), + (0x400 => @END), + } +} + +register_structs! { + #[allow(non_snake_case)] + pub PortRegs { + (0x00 => pub PORTSC: ReadWrite), + (0x04 => pub PORTPMSC: ReadWrite), + (0x08 => pub PORTLI: ReadWrite), + (0x0C => pub PORTHLPMC: ReadWrite), + (0x10 => @END), + } +} + +impl OperationalRegs { + pub fn wait_usbsts_bit( + &self, + bit: FieldValue, + mut timeout: u64, + ) -> Result<(), Error> { + while timeout > 0 && !self.USBSTS.matches_all(bit) { + core::hint::spin_loop(); + timeout -= 1; + } + if timeout > 0 { + Ok(()) + } else { + Err(Error::TimedOut) + } + } + + pub fn set_dcbaap(&self, value: PhysicalAddress) { + let value = value.into_u64(); + self.DCBAAP.set(value); + } + + pub fn set_crcr(&self, value: PhysicalAddress, dcs: bool) { + let mut value = value.into_u64(); + if dcs { + value |= 1; + } + self.CRCR.set(value); + } +} + +impl PortRegs { + pub fn modify_portsc_preserving(&self, value: FieldValue) { + let neutral = super::portsc_to_neutral(self.PORTSC.extract()); + self.PORTSC.set(neutral.get() | value.value); + } +} + +unsafe impl Sync for PortRegs {} diff --git a/kernel/driver/usb/xhci/src/regs/runtime.rs b/kernel/driver/usb/xhci/src/regs/runtime.rs new file mode 100644 index 00000000..7ef2fd07 --- /dev/null +++ b/kernel/driver/usb/xhci/src/regs/runtime.rs @@ -0,0 +1,69 @@ +use libk_mm::address::PhysicalAddress; +use tock_registers::{ + interfaces::{Readable, Writeable}, + register_bitfields, register_structs, + registers::ReadWrite, +}; + +use crate::ring::{EventRing, EventRingSegmentTable}; + +register_bitfields! { + u32, + pub IMAN [ + IP OFFSET(0) NUMBITS(1) [], + IE OFFSET(1) NUMBITS(1) [], + ], + pub IMOD [ + IMODI OFFSET(0) NUMBITS(16) [], + IMODC OFFSET(16) NUMBITS(16) [], + ], +} + +register_structs! { + #[allow(non_snake_case)] + pub InterrupterRegs { + (0x00 => pub IMAN: ReadWrite), + (0x04 => pub IMOD: ReadWrite), + (0x08 => pub ERSTSZ: ReadWrite), + (0x0C => _0), + (0x10 => pub ERSTBA: ReadWrite), + (0x18 => pub ERDP: ReadWrite), + (0x20 => @END), + } +} + +register_structs! { + #[allow(non_snake_case)] + pub RuntimeRegs { + (0x0000 => pub MFINDEX: ReadWrite), + (0x0004 => _0), + // NOTE there're 1024 of them, but I'm not going to use all of those anyway, so use 1023 + // for a nice round number + (0x0020 => pub IRn: [InterrupterRegs; 1023]), + (0x8000 => @END), + } +} + +impl RuntimeRegs { + pub fn configure_interrupter( + &self, + index: usize, + event_ring: &EventRing, + erst: &EventRingSegmentTable, + ) { + let interrupter = &self.IRn[index]; + + let erdp = event_ring.dequeue_pointer().into_u64(); + let erstba = erst.physical_address().into_u64(); + + interrupter.ERSTSZ.set(erst.capacity() as u32); + interrupter.ERSTBA.set(erstba); + interrupter.ERDP.set(erdp); + interrupter.IMAN.write(IMAN::IE::SET); + } + + pub fn set_interrupter_dequeue_pointer(&self, index: usize, erdp: PhysicalAddress) { + let _ = self.IRn[index].ERDP.get(); + self.IRn[index].ERDP.set(erdp.into_u64() | (1 << 3)); + } +} diff --git a/kernel/driver/usb/xhci/src/ring/command.rs b/kernel/driver/usb/xhci/src/ring/command.rs index 9ce1d16f..3732380e 100644 --- a/kernel/driver/usb/xhci/src/ring/command.rs +++ b/kernel/driver/usb/xhci/src/ring/command.rs @@ -138,6 +138,42 @@ impl CommandRing { Ok(()) } + pub async fn reset_endpoint( + &self, + executor: &E, + slot_id: u8, + endpoint_id: u8, + preserve_transaction_state: bool, + ) -> Result<(), UsbError> { + self.submit_and_wait( + executor, + ResetEndpointCommandTrb::new(slot_id, endpoint_id, preserve_transaction_state), + ) + .await?; + Ok(()) + } + + pub async fn set_tr_dequeue_pointer( + &self, + executor: &E, + slot_id: u8, + endpoint_id: u8, + dequeue_pointer: PhysicalAddress, + dequeue_cycle: bool, + ) -> Result<(), UsbError> { + self.submit_and_wait( + executor, + SetTrDequeuePointerCommandTrb::new( + slot_id, + endpoint_id, + dequeue_pointer, + dequeue_cycle, + ), + ) + .await?; + Ok(()) + } + pub async fn enable_slot( &self, executor: &E, @@ -221,6 +257,21 @@ define_bitfields! { } } +define_bitfields! { + pub ResetEndpointCommandFlags: u32 { + (24..32) => slot_id, + (16..21) => endpoint_id, + 9 => tsp, + } +} + +define_bitfields! { + pub SetTrDequeuePointerCommandFlags: u32 { + (24..32) => slot_id, + (16..21) => endpoint_id, + } +} + define_bitfields! { pub RawCommandFlags : u32 { (10..16) => ty + set_ty, @@ -258,6 +309,21 @@ pub struct ConfigureEndpointCommandTrb { pub flags: ConfigureEndpointCommandFlags, } +#[derive(Clone, Copy, Debug, Pod, Zeroable)] +#[repr(C, align(16))] +pub struct ResetEndpointCommandTrb { + _0: [u32; 3], + pub flags: ResetEndpointCommandFlags, +} + +#[derive(Clone, Copy, Debug, Pod, Zeroable)] +#[repr(C, align(16))] +pub struct SetTrDequeuePointerCommandTrb { + pub value: u64, + _0: u32, + pub flags: SetTrDequeuePointerCommandFlags, +} + #[derive(Clone, Copy, Debug, Pod, Zeroable)] #[repr(C, align(16))] pub struct RawCommandTrb { @@ -314,6 +380,38 @@ impl ConfigureEndpointCommandTrb { } } +impl ResetEndpointCommandTrb { + pub fn new(slot_id: u8, endpoint_id: u8, preserve_transaction_state: bool) -> Self { + Self { + _0: [0; 3], + flags: ResetEndpointCommandFlags::new( + slot_id as _, + endpoint_id as _, + preserve_transaction_state, + ), + } + } +} + +impl SetTrDequeuePointerCommandTrb { + pub fn new( + slot_id: u8, + endpoint_id: u8, + dequeue_pointer: PhysicalAddress, + dequeue_cycle: bool, + ) -> Self { + let mut value = dequeue_pointer.into_u64(); + if dequeue_cycle { + value |= 1; + } + Self { + value, + _0: 0, + flags: SetTrDequeuePointerCommandFlags::new(slot_id as _, endpoint_id as _), + } + } +} + impl CommandTrb for EnableSlotCommandTrb { const TRB_TYPE: u8 = 9; } @@ -329,3 +427,11 @@ impl CommandTrb for AddressDeviceCommandTrb { impl CommandTrb for ConfigureEndpointCommandTrb { const TRB_TYPE: u8 = 12; } + +impl CommandTrb for ResetEndpointCommandTrb { + const TRB_TYPE: u8 = 14; +} + +impl CommandTrb for SetTrDequeuePointerCommandTrb { + const TRB_TYPE: u8 = 16; +} diff --git a/kernel/driver/usb/xhci/src/ring/mod.rs b/kernel/driver/usb/xhci/src/ring/mod.rs index 8f589fef..ca4c04a1 100644 --- a/kernel/driver/usb/xhci/src/ring/mod.rs +++ b/kernel/driver/usb/xhci/src/ring/mod.rs @@ -1,17 +1,31 @@ +//use bytemuck::{Pod, Zeroable}; +//use libk_mm::address::PhysicalAddress; +//use yggdrasil_abi::define_bitfields; + +use alloc::boxed::Box; +use async_trait::async_trait; use bytemuck::{Pod, Zeroable}; use libk_mm::address::PhysicalAddress; +use ygg_driver_usb::error::UsbError; use yggdrasil_abi::define_bitfields; pub mod command; pub mod event; pub mod transfer; -pub use command::CommandRing; -pub use event::{Event, EventRing, EventRingSegmentTable}; -pub use transfer::ControlTransferRing; +pub use event::{EventRing, EventRingSegmentTable}; +#[async_trait] pub trait CommandExecutor { fn ring_doorbell(&self, index: usize, target: u8); + + async fn reset_endpoint( + &self, + slot_id: u8, + endpoint_id: u8, + dequeue_pointer: PhysicalAddress, + dequeue_cycle: bool, + ) -> Result<(), UsbError>; } pub trait GenericRing { @@ -19,15 +33,6 @@ pub trait GenericRing { fn base(&self) -> PhysicalAddress; } -pub trait GenericTransferRing: GenericRing + Send + Sync { - fn slot_id(&self) -> u8; - fn endpoint_id(&self) -> u8; - - fn dequeue_pointer(&self) -> PhysicalAddress; - fn notify(&self, address: PhysicalAddress, status: u32); - fn shutdown(&self); -} - define_bitfields! { pub LinkTrbFlags : u32 { (10..16) => ty, diff --git a/kernel/driver/usb/xhci/src/ring/transfer.rs b/kernel/driver/usb/xhci/src/ring/transfer.rs index af539948..af505f9b 100644 --- a/kernel/driver/usb/xhci/src/ring/transfer.rs +++ b/kernel/driver/usb/xhci/src/ring/transfer.rs @@ -1,26 +1,32 @@ use core::{ - mem::{size_of, MaybeUninit}, - sync::atomic::{AtomicBool, AtomicU64, Ordering}, + future::poll_fn, + mem::MaybeUninit, + sync::atomic::{AtomicBool, Ordering}, + task::Poll, }; -use alloc::{collections::BTreeMap, sync::Arc, vec::Vec}; +use alloc::{ + sync::{Arc, Weak}, + vec::Vec, +}; use bytemuck::{Pod, Zeroable}; +use futures_util::task::AtomicWaker; use libk_mm::{ address::{AsPhysicalAddress, PhysicalAddress}, PageBox, PageSlice, }; -use libk_util::sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock, IrqSafeSpinlockGuard}; +use libk_util::{ + queue::BoundedQueue, + sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock, IrqSafeSpinlockGuard}, +}; use ygg_driver_usb::{ - communication::{UsbBulkTransfer, UsbInterruptTransfer}, - error::UsbError, + communication::UsbDirection, + error::{TransferError, UsbError}, pipe::control::ControlTransferSetup, - UsbControlTransfer, UsbDirection, UsbTransferStatus, UsbTransferToken, }; use yggdrasil_abi::define_bitfields; -use crate::ring::LinkTrb; - -use super::{CommandExecutor, GenericRing, GenericTransferRing}; +use super::{CommandExecutor, LinkTrb}; struct TransferRingInner { trbs: PageBox<[MaybeUninit]>, @@ -29,134 +35,320 @@ struct TransferRingInner { cycle_bit: bool, } -// TODO split TransferRing into Normal, Control, etc -pub struct ControlTransferRing { +pub struct TransferRing { inner: IrqSafeSpinlock, + base: PhysicalAddress, capacity: usize, - // TODO this is inefficient and ugly - pending_trbs: IrqSafeRwLock>, - completions: IrqSafeRwLock>>, - slot_id: u8, - ep_id: u8, + endpoint_id: u8, - transfer_id: AtomicU64, + transactions: IrqSafeRwLock>>>, shutdown: AtomicBool, } -pub struct InterruptInTransferRing { - inner: IrqSafeSpinlock, - capacity: usize, - - completions: IrqSafeRwLock>>, - - slot_id: u8, - ep_id: u8, - - shutdown: AtomicBool, +pub struct TransactionBuilder<'a> { + inner: IrqSafeSpinlockGuard<'a, TransferRingInner>, + ring: &'a Arc, + pending: Vec, } -pub struct BulkInTransferRing { - inner: IrqSafeSpinlock, - capacity: usize, - - completions: IrqSafeRwLock>>, - - slot_id: u8, - ep_id: u8, - - shutdown: AtomicBool, -} -pub struct BulkOutTransferRing { - inner: IrqSafeSpinlock, - capacity: usize, - - completions: IrqSafeRwLock>>, - - slot_id: u8, - ep_id: u8, - - shutdown: AtomicBool, +pub struct Transaction { + event_queue: BoundedQueue, + event_notify: AtomicWaker, + next_dequeue: usize, + next_cycle: bool, } -struct TransferBuilder<'a> { - ring: &'a ControlTransferRing, - ring_inner: IrqSafeSpinlockGuard<'a, TransferRingInner>, - - token: UsbTransferToken, - direction: UsbDirection, - addresses: Vec, - status: Arc, +#[derive(Debug)] +pub enum TransactionEvent { + Status(usize, u32), + Shutdown, } -impl<'a> TransferBuilder<'a> { - pub fn new(ring: &'a ControlTransferRing, direction: UsbDirection) -> Self { - let ring_inner = ring.inner.lock(); - let token = UsbTransferToken(ring.transfer_id.fetch_add(1, Ordering::AcqRel)); - let status = Arc::new(UsbTransferStatus::new()); +impl TransferRing { + pub fn new(slot_id: u8, endpoint_id: u8, capacity: usize) -> Result { + let inner = TransferRingInner::new(capacity)?; + let base = unsafe { inner.trbs.as_physical_address() }; + let transactions = (0..capacity).map(|_| None).collect(); - ring.completions.write().insert(token, status.clone()); + Ok(Self { + inner: IrqSafeSpinlock::new(inner), + base, + capacity, - Self { - ring, - ring_inner, + slot_id, + endpoint_id, - token, - direction, - status, - addresses: Vec::new(), + transactions: IrqSafeRwLock::new(transactions), + shutdown: AtomicBool::new(false), + }) + } + + pub fn transaction_builder(self: &Arc) -> Result { + if self.shutdown.load(Ordering::Acquire) { + return Err(UsbError::DeviceDisconnected); + } + + Ok(TransactionBuilder { + inner: self.inner.lock(), + ring: self, + pending: Vec::new(), + }) + } + + async fn handle_stall( + &self, + executor: &E, + result: &Result, + transaction: &Transaction, + ) { + if let Err(TransferError::Stall) = result { + let dequeue = self + .base + .add(transaction.next_dequeue * size_of::()); + if let Err(rerror) = executor + .reset_endpoint( + self.slot_id, + self.endpoint_id, + dequeue, + transaction.next_cycle, + ) + .await + { + log::error!( + "xhci: could not reset endpoint after stall {}:{}: {rerror:?}", + self.slot_id, + self.endpoint_id + ); + + self.shutdown.store(true, Ordering::Release); + } } } - pub fn push_trb(&mut self, trb: C) -> &mut Self { - let address = self.ring_inner.enqueue(trb); - self.addresses.push(address); - self.ring.pending_trbs.write().insert(address, self.token); - self + pub async fn normal_transfer( + self: &Arc, + executor: &E, + buffer: PhysicalAddress, + length: usize, + ) -> Result { + if length == 0 { + return Ok(0); + } + let mut builder = self.transaction_builder()?; + let last_data_trb = builder.enqueue_normal(buffer, length)?; + let transaction = builder.submit(executor); + + let status = transaction.wait_normal(last_data_trb).await; + + builder.inner.dequeue_index = builder.inner.enqueue_index; + self.handle_stall(executor, &status, &transaction).await; + + let residual = status?; + Ok(length.saturating_sub(residual)) } - pub fn start(self, executor: &E, length: usize) -> UsbControlTransfer { - executor.ring_doorbell(self.ring.slot_id as _, self.ring.ep_id); + // Helper functions, shorthands for transaction_builder().....finish() + kick() + pub async fn control_transfer( + self: &Arc, + executor: &E, + setup: ControlTransferSetup, + buffer: Option<(&mut PageSlice>, UsbDirection)>, + ) -> Result { + let mut builder = self.transaction_builder()?; - UsbControlTransfer { - id: self.token, - length, + let data_len = buffer.as_ref().map_or(0, |(buffer, _)| buffer.len()); + let (setup, data, status) = builder.enqueue_control(setup, buffer)?; - direction: self.direction, - elements: self.addresses, - status: self.status, + let transaction = builder.submit(executor); + + // TODO timeout + let status = transaction.wait_control(setup, data, status).await; + + builder.inner.dequeue_index = builder.inner.enqueue_index; + self.handle_stall(executor, &status, &transaction).await; + + let residual = status?; + Ok(data_len.saturating_sub(residual)) + } + + pub fn kick(&self, executor: &E) { + executor.ring_doorbell(self.slot_id as usize, self.endpoint_id); + } + + pub fn shutdown(&self) { + self.shutdown.store(true, Ordering::Release); + // Shutdown transactions + let transactions = self.transactions.read(); + for index in 0..self.capacity { + if let Some(tx) = transactions[index].as_ref().and_then(Weak::upgrade) { + tx.shutdown(); + } } } + + pub fn notify(&self, address: PhysicalAddress, status: u32) { + if status == 0 { + return; + } + + if address < self.base || address - self.base >= size_of::() * self.capacity + { + log::warn!("xhci: event outside of trb array: {address:#x}"); + return; + } + + let index = (address - self.base) / size_of::(); + if let Some(tx) = self.transactions.write()[index] + .take() + .and_then(|tx| tx.upgrade()) + { + tx.notify(index, status); + } else { + log::warn!("xhci: no transaction @ {index} to notify"); + } + } + + pub fn base(&self) -> PhysicalAddress { + self.base + } +} + +impl TransactionBuilder<'_> { + const TRB_SIZE_LIMIT: usize = 65536; + + pub fn enqueue(&mut self, trb: C, ioc: bool) -> Result { + let address = self.inner.enqueue(trb, ioc)?; + self.pending.push(address); + Ok((address - self.ring.base) / size_of::()) + } + + pub fn enqueue_normal( + &mut self, + buffer: PhysicalAddress, + length: usize, + ) -> Result { + let trb_count = length.div_ceil(Self::TRB_SIZE_LIMIT); + if self.inner.free_capacity() <= trb_count || trb_count == 0 { + return Err(UsbError::DeviceBusy); + } + + let mut last_trb = 0; + for i in 0..trb_count { + let offset = i * Self::TRB_SIZE_LIMIT; + let amount = (length - offset).min(Self::TRB_SIZE_LIMIT); + + last_trb = self + .enqueue( + NormalTransferTrb::new(buffer.add(offset), amount), + i == trb_count - 1, + ) + .unwrap(); + } + + Ok(last_trb) + } + + pub fn enqueue_control( + &mut self, + setup: ControlTransferSetup, + buffer: Option<(&mut PageSlice>, UsbDirection)>, + ) -> Result<(usize, Option, usize), UsbError> { + // Check ring capacity first + // TODO larger DATA stages + let trb_count = 2 + if buffer.is_some() { 1 } else { 0 }; + if self.inner.free_capacity() <= trb_count { + return Err(UsbError::DeviceBusy); + } + + // unwrap()s are okay here, capacity checked above + let setup_stage = self + .enqueue(ControlTransferSetupTrb::new(setup), true) + .unwrap(); + let data_stage = if let Some((buffer, direction)) = buffer { + Some( + self.enqueue( + ControlTransferDataTrb::new( + unsafe { buffer.as_physical_address() }, + buffer.len(), + direction, + ), + true, + ) + .unwrap(), + ) + } else { + None + }; + let status_stage = self + .enqueue(ControlTransferStatusTrb::new(UsbDirection::In), true) + .unwrap(); + + Ok((setup_stage, data_stage, status_stage)) + } + + pub fn finish(&mut self) -> Arc { + let transaction = Arc::new(Transaction { + event_queue: BoundedQueue::new(self.pending.len()), + event_notify: AtomicWaker::new(), + next_dequeue: self.inner.enqueue_index, + next_cycle: self.inner.cycle_bit, + }); + + let mut transactions = self.ring.transactions.write(); + for &pending in self.pending.iter() { + let index = (pending - self.ring.base) / size_of::(); + transactions[index] = Some(Arc::downgrade(&transaction)); + } + + transaction + } + + pub fn submit(&mut self, executor: &E) -> Arc { + let transaction = self.finish(); + self.ring.kick(executor); + transaction + } } impl TransferRingInner { - fn enqueue(&mut self, trb: C) -> PhysicalAddress { + fn new(capacity: usize) -> Result { + let trbs = PageBox::new_zeroed_slice(capacity).map_err(UsbError::MemoryError)?; + Ok(Self { + trbs, + enqueue_index: 0, + dequeue_index: 0, + cycle_bit: true, + }) + } + + fn enqueue(&mut self, trb: C, ioc: bool) -> Result { if (self.enqueue_index + 1) % (self.trbs.len() - 1) == self.dequeue_index { - todo!("Ring full"); + log::warn!("xhci: transfer ring full"); + return Err(UsbError::DeviceBusy); } let mut raw: RawTransferTrb = bytemuck::cast(trb); - raw.flags.set_ty(C::TRB_TYPE as _); + raw.flags.set_ty(C::TRB_TYPE as u32); raw.flags.set_cycle(self.cycle_bit); + raw.flags.set_ioc(ioc); self.trbs[self.enqueue_index].write(raw); let address = unsafe { self.trbs.as_physical_address() } .add(self.enqueue_index * size_of::()); - // Move to the next TRB slot self.enqueue_index += 1; if self.enqueue_index >= self.trbs.len() - 1 { self.enqueue_link(); - // Wrap around self.cycle_bit = !self.cycle_bit; self.enqueue_index = 0; } - address + Ok(address) } fn enqueue_link(&mut self) { @@ -166,445 +358,109 @@ impl TransferRingInner { self.trbs[self.enqueue_index].write(bytemuck::cast(link)); } - fn advance(&mut self) { - self.dequeue_index += 1; - - if self.dequeue_index >= self.trbs.len() - 1 { - self.dequeue_index = 0; - } + fn free_capacity(&self) -> usize { + self.enqueue_index + self.trbs.len() - self.dequeue_index } } -impl GenericRing for ControlTransferRing { - fn base(&self) -> PhysicalAddress { - unsafe { self.inner.lock().trbs.as_physical_address() } +impl Transaction { + pub fn notify(&self, trb_index: usize, status: u32) { + self.event_queue + .push(TransactionEvent::Status(trb_index, status)) + .ok(); + self.event_notify.wake(); } - fn capacity(&self) -> usize { - self.capacity - } -} - -impl GenericTransferRing for ControlTransferRing { - fn dequeue_pointer(&self) -> PhysicalAddress { - let inner = self.inner.lock(); - unsafe { inner.trbs.as_physical_address() } - .add(inner.dequeue_index * size_of::()) + pub fn shutdown(&self) { + self.event_queue.push(TransactionEvent::Shutdown).ok(); + self.event_notify.wake(); } - fn notify(&self, address: PhysicalAddress, value: u32) { - if value == 0 { - return; - } + pub async fn wait_normal(&self, last_trb: usize) -> Result { + loop { + let event = self.next_event().await; + let status = event.to_result(); - let completions = self.completions.read(); - if let Some(&token) = self.pending_trbs.read().get(&address) { - let Some(status) = completions.get(&token) else { - log::warn!( - "Notification received for non-existent transfer: {:?}", - token - ); - return; - }; - - status.signal(value); + match event { + TransactionEvent::Status(trb_index, _) => { + if status.is_err() || trb_index == last_trb { + break status; + } + } + TransactionEvent::Shutdown => { + log::error!("xhci: abort transaction, endpoint shutdown"); + return Err(TransferError::UsbTransactionError); + } + } } } - fn shutdown(&self) { - self.shutdown.store(true, Ordering::Release); - let mut completions = self.completions.write(); - while let Some((_, status)) = completions.pop_first() { - status.abort(); + pub async fn wait_trb(&self, trb: usize) -> Result { + let event = self.next_event().await; + match event { + TransactionEvent::Status(trb_index, _) => { + if trb_index != trb { + return Err(TransferError::InvalidTransfer); + } + } + TransactionEvent::Shutdown => { + log::error!("xhci: abort transaction, endpoint shutdown"); + return Err(TransferError::UsbTransactionError); + } } + event.to_result() } - fn slot_id(&self) -> u8 { - self.slot_id - } - - fn endpoint_id(&self) -> u8 { - self.ep_id - } -} - -impl GenericRing for InterruptInTransferRing { - fn base(&self) -> PhysicalAddress { - unsafe { self.inner.lock().trbs.as_physical_address() } - } - - fn capacity(&self) -> usize { - self.capacity - } -} - -impl GenericTransferRing for InterruptInTransferRing { - fn dequeue_pointer(&self) -> PhysicalAddress { - let inner = self.inner.lock(); - unsafe { inner.trbs.as_physical_address() } - .add(inner.dequeue_index * size_of::()) - } - - fn notify(&self, address: PhysicalAddress, value: u32) { - if value == 0 { - return; - } - - let mut completions = self.completions.write(); - if let Some(status) = completions.remove(&address) { - status.signal(value); - } - } - - fn shutdown(&self) { - self.shutdown.store(true, Ordering::Release); - let mut completions = self.completions.write(); - while let Some((_, status)) = completions.pop_first() { - status.abort(); - } - } - - fn slot_id(&self) -> u8 { - self.slot_id - } - - fn endpoint_id(&self) -> u8 { - self.ep_id - } -} - -impl GenericRing for BulkInTransferRing { - fn base(&self) -> PhysicalAddress { - unsafe { self.inner.lock().trbs.as_physical_address() } - } - - fn capacity(&self) -> usize { - self.capacity - } -} - -impl GenericTransferRing for BulkInTransferRing { - fn dequeue_pointer(&self) -> PhysicalAddress { - let inner = self.inner.lock(); - unsafe { inner.trbs.as_physical_address() } - .add(inner.dequeue_index * size_of::()) - } - - fn notify(&self, address: PhysicalAddress, value: u32) { - if value == 0 { - return; - } - - let mut completions = self.completions.write(); - if let Some(status) = completions.remove(&address) { - status.signal(value); - } - } - - fn shutdown(&self) { - self.shutdown.store(true, Ordering::Release); - let mut completions = self.completions.write(); - while let Some((_, status)) = completions.pop_first() { - status.abort(); - } - } - - fn slot_id(&self) -> u8 { - self.slot_id - } - - fn endpoint_id(&self) -> u8 { - self.ep_id - } -} - -impl GenericRing for BulkOutTransferRing { - fn capacity(&self) -> usize { - self.capacity - } - - fn base(&self) -> PhysicalAddress { - unsafe { self.inner.lock().trbs.as_physical_address() } - } -} - -impl GenericTransferRing for BulkOutTransferRing { - fn dequeue_pointer(&self) -> PhysicalAddress { - let inner = self.inner.lock(); - unsafe { inner.trbs.as_physical_address() } - .add(inner.dequeue_index * size_of::()) - } - - fn shutdown(&self) { - self.shutdown.store(true, Ordering::Release); - let mut completions = self.completions.write(); - while let Some((_, status)) = completions.pop_first() { - status.abort(); - } - } - - fn notify(&self, address: PhysicalAddress, value: u32) { - if value == 0 { - return; - } - - let mut completions = self.completions.write(); - if let Some(status) = completions.remove(&address) { - status.signal(value); - } - } - - fn endpoint_id(&self) -> u8 { - self.ep_id - } - - fn slot_id(&self) -> u8 { - self.slot_id - } -} - -impl InterruptInTransferRing { - pub fn new(slot_id: u8, ep_id: u8, capacity: usize) -> Result { - let trbs = PageBox::new_zeroed_slice(capacity).map_err(UsbError::MemoryError)?; - - Ok(Self { - inner: IrqSafeSpinlock::new(TransferRingInner { - trbs, - enqueue_index: 0, - dequeue_index: 0, - cycle_bit: true, - }), - completions: IrqSafeRwLock::new(BTreeMap::new()), - slot_id, - ep_id, - capacity, - shutdown: AtomicBool::new(false), - }) - } - - pub fn start_transfer( + pub async fn wait_control( &self, - executor: &E, - buffer: &mut PageBox<[u8]>, - ) -> Result { - // Don't even try to start the transfer - if self.shutdown.load(Ordering::Acquire) { - return Err(UsbError::DeviceDisconnected); - } - - let status = Arc::new(UsbTransferStatus::new()); - let address = self.inner.lock().enqueue(NormalTransferTrb::new( - unsafe { buffer.as_physical_address() }, - buffer.len(), - true, - )); - self.completions.write().insert(address, status.clone()); - - executor.ring_doorbell(self.slot_id as _, self.ep_id); - - Ok(UsbInterruptTransfer { - length: buffer.len(), - direction: UsbDirection::In, - address, - status, - }) + setup_trb: usize, + last_data_trb: Option, + status_trb: usize, + ) -> Result { + self.wait_trb(setup_trb).await?; + let residual = if let Some(last_data_trb) = last_data_trb { + self.wait_normal(last_data_trb).await? + } else { + 0 + }; + self.wait_trb(status_trb).await?; + Ok(residual) } - pub fn complete_transfer(&self, _transfer: UsbInterruptTransfer) { - // Interrupt transfers consist of one TRB each - // TODO: Can two transfers happen simultaneously? e.g. - // - // [TRBa, TRBb] are queued in the ring, both are executing and - // TRBb finishes first - self.inner.lock().advance(); + pub async fn next_event(&self) -> TransactionEvent { + poll_fn(|cx| { + if let Some(event) = self.event_queue.pop() { + Poll::Ready(event) + } else { + self.event_notify.register(cx.waker()); + Poll::Pending + } + }) + .await } } -impl BulkInTransferRing { - pub fn new(slot_id: u8, ep_id: u8, capacity: usize) -> Result { - let trbs = PageBox::new_zeroed_slice(capacity).map_err(UsbError::MemoryError)?; - - Ok(Self { - inner: IrqSafeSpinlock::new(TransferRingInner { - trbs, - enqueue_index: 0, - dequeue_index: 0, - cycle_bit: true, - }), - completions: IrqSafeRwLock::new(BTreeMap::new()), - slot_id, - ep_id, - capacity, - shutdown: AtomicBool::new(false), - }) - } - - pub fn start_transfer( - &self, - executor: &E, - buffer: &mut PageSlice, - ) -> Result { - // Don't even try to start the transfer - if self.shutdown.load(Ordering::Acquire) { - return Err(UsbError::DeviceDisconnected); +impl TransactionEvent { + pub fn to_result(&self) -> Result { + match self { + &Self::Status(_, status) => match status >> 24 { + 1 => Ok((status as usize) & 0xFFFFFF), + 4 => Err(TransferError::UsbTransactionError), + 6 => Err(TransferError::Stall), + 13 => Err(TransferError::ShortPacket((status as usize) & 0xFFFFFF)), + code => Err(TransferError::Other(code as u8)), + }, + Self::Shutdown => Err(TransferError::UsbTransactionError), } - - let status = Arc::new(UsbTransferStatus::new()); - let address = self.inner.lock().enqueue(NormalTransferTrb::new( - unsafe { buffer.as_physical_address() }, - buffer.len(), - true, - )); - self.completions.write().insert(address, status.clone()); - - executor.ring_doorbell(self.slot_id as _, self.ep_id); - - Ok(UsbBulkTransfer { - length: buffer.len(), - direction: UsbDirection::In, - address, - status, - }) - } - - pub fn complete_transfer(&self, _transfer: UsbBulkTransfer) { - // Interrupt transfers consist of one TRB each - // TODO: Can two transfers happen simultaneously? e.g. - // - // [TRBa, TRBb] are queued in the ring, both are executing and - // TRBb finishes first - self.inner.lock().advance(); } } -impl BulkOutTransferRing { - pub fn new(slot_id: u8, ep_id: u8, capacity: usize) -> Result { - let trbs = PageBox::new_zeroed_slice(capacity).map_err(UsbError::MemoryError)?; - - Ok(Self { - inner: IrqSafeSpinlock::new(TransferRingInner { - trbs, - enqueue_index: 0, - dequeue_index: 0, - cycle_bit: true, - }), - completions: IrqSafeRwLock::new(BTreeMap::new()), - slot_id, - ep_id, - capacity, - shutdown: AtomicBool::new(false), - }) - } - - pub fn start_transfer( - &self, - executor: &E, - buffer: &PageSlice, - ) -> Result { - // Don't even try to start the transfer - if self.shutdown.load(Ordering::Acquire) { - return Err(UsbError::DeviceDisconnected); - } - - let status = Arc::new(UsbTransferStatus::new()); - let address = self.inner.lock().enqueue(NormalTransferTrb::new( - unsafe { buffer.as_physical_address() }, - buffer.len(), - true, - )); - self.completions.write().insert(address, status.clone()); - - executor.ring_doorbell(self.slot_id as _, self.ep_id); - - Ok(UsbBulkTransfer { - direction: UsbDirection::Out, - length: buffer.len(), - address, - status, - }) - } - - pub fn complete_transfer(&self, _transfer: UsbBulkTransfer) { - // Interrupt transfers consist of one TRB each - // TODO: Can two transfers happen simultaneously? e.g. - // - // [TRBa, TRBb] are queued in the ring, both are executing and - // TRBb finishes first - self.inner.lock().advance(); - } -} - -impl ControlTransferRing { - pub fn new(slot_id: u8, ep_id: u8, capacity: usize) -> Result { - let trbs = PageBox::new_zeroed_slice(capacity).map_err(UsbError::MemoryError)?; - - Ok(Self { - inner: IrqSafeSpinlock::new(TransferRingInner { - trbs, - enqueue_index: 0, - dequeue_index: 0, - cycle_bit: true, - }), - completions: IrqSafeRwLock::new(BTreeMap::new()), - pending_trbs: IrqSafeRwLock::new(BTreeMap::new()), - slot_id, - ep_id, - capacity, - - transfer_id: AtomicU64::new(0), - - shutdown: AtomicBool::new(false), - }) - } - - pub fn reset(&self) { - let mut inner = self.inner.lock(); - self.transfer_id.store(0, Ordering::Release); - inner.enqueue_index = 0; - inner.dequeue_index = 0; - inner.cycle_bit = true; - } - - pub fn start_transfer( - &self, - executor: &E, - setup: ControlTransferSetup, - buffer: Option<(PhysicalAddress, usize, UsbDirection)>, - ) -> Result { - // Don't even try to start the transfer - if self.shutdown.load(Ordering::Acquire) { - return Err(UsbError::DeviceDisconnected); - } - - let mut builder = TransferBuilder::new(self, UsbDirection::In); - - builder.push_trb(ControlTransferSetupTrb::new(setup)); - if let Some((address, length, direction)) = buffer { - builder.push_trb(ControlTransferDataTrb::new(address, length, direction)); - } - builder.push_trb(ControlTransferStatusTrb::new(UsbDirection::In, true)); - - let transfer = builder.start(executor, 0); - - Ok(transfer) - } - - pub fn complete_transfer(&self, transfer: UsbControlTransfer) { - let mut pending = self.pending_trbs.write(); - let mut inner = self.inner.lock(); - for trb in transfer.elements { - pending.remove(&trb); - inner.advance(); - } - self.completions.write().remove(&transfer.id); - } -} - -// TRB implementations +// TRB definitions define_bitfields! { pub RawTransferFlags : u32 { (10..16) => ty + set_ty, + 5 => ioc + set_ioc, 0 => cycle + set_cycle } } @@ -612,7 +468,6 @@ define_bitfields! { define_bitfields! { pub NormalTransferFlags: u64 { (0..16) => trb_length, - 37 => interrupt_on_completion, } } @@ -644,7 +499,6 @@ define_bitfields! { define_bitfields! { pub ControlTransferStatusFlags : u32 { 16 => direction, - 5 => interrupt_on_completion } } @@ -688,10 +542,10 @@ pub trait TransferTrb: Pod { } impl NormalTransferTrb { - pub fn new(buffer: PhysicalAddress, length: usize, interrupt_on_completion: bool) -> Self { + pub fn new(buffer: PhysicalAddress, length: usize) -> Self { Self { buffer, - flags: NormalTransferFlags::new(length.try_into().unwrap(), interrupt_on_completion), + flags: NormalTransferFlags::new(length.try_into().unwrap()), } } } @@ -724,13 +578,10 @@ impl ControlTransferDataTrb { } impl ControlTransferStatusTrb { - pub const fn new(direction: UsbDirection, interrupt_on_completion: bool) -> Self { + pub const fn new(direction: UsbDirection) -> Self { Self { _0: [0; 3], - flags: ControlTransferStatusFlags::new( - direction.is_device_to_host(), - interrupt_on_completion, - ), + flags: ControlTransferStatusFlags::new(direction.is_device_to_host()), } } } diff --git a/kernel/driver/usb/xhci/src/util.rs b/kernel/driver/usb/xhci/src/util.rs index 60da8fa2..d6b7efc4 100644 --- a/kernel/driver/usb/xhci/src/util.rs +++ b/kernel/driver/usb/xhci/src/util.rs @@ -46,18 +46,18 @@ impl EventBitmap { self.waker.wake(); } - pub async fn wait_specific(&self, bit: usize) { - poll_fn(|cx| { - let state = self.bitmap.fetch_and(!(1 << bit), Ordering::Acquire); - if state & (1 << bit) != 0 { - Poll::Ready(()) - } else { - self.waker.register(cx.waker()); - Poll::Pending - } - }) - .await - } + // pub async fn wait_specific(&self, bit: usize) { + // poll_fn(|cx| { + // let state = self.bitmap.fetch_and(!(1 << bit), Ordering::Acquire); + // if state & (1 << bit) != 0 { + // Poll::Ready(()) + // } else { + // self.waker.register(cx.waker()); + // Poll::Pending + // } + // }) + // .await + // } pub async fn wait_any(&self, max: usize) -> impl Iterator { poll_fn(|cx| { diff --git a/kernel/libk/libk-mm/src/device.rs b/kernel/libk/libk-mm/src/device.rs index 490a722b..0ca571a6 100644 --- a/kernel/libk/libk-mm/src/device.rs +++ b/kernel/libk/libk-mm/src/device.rs @@ -158,8 +158,9 @@ impl AsPhysicalAddress for DeviceMemoryIo<'_, T> { } } -unsafe impl Send for DeviceMemoryIo<'_, T> {} -impl !Sync for DeviceMemoryIo<'_, T> {} +unsafe impl Send for DeviceMemoryIo<'_, T> {} +unsafe impl Sync for DeviceMemoryIo<'_, T> {} +//impl !Sync for DeviceMemoryIo<'_, T> {} impl<'a, T: Sized> DeviceMemoryIoMut<'a, T> { /// Maps a physical address as device memory to a slice `[T; len]` diff --git a/kernel/libk/src/debug.rs b/kernel/libk/src/debug.rs index f998e5de..21c6d2a9 100644 --- a/kernel/libk/src/debug.rs +++ b/kernel/libk/src/debug.rs @@ -322,7 +322,7 @@ impl PanicLoggerSink<'_> { self.log( &log::Record::builder() .level(log::Level::Error) - .target("raw") + .target(":raw") .args(args) .build(), ) @@ -432,7 +432,7 @@ pub fn disable_early_sinks() { } /// Print a trace message coming from a process -pub fn program_trace(process: &Process, thread: &Thread, message: &str) { +pub fn program_trace(process: &Process, _thread: &Thread, message: &str) { log::debug!( target: ":program", "{} ({}) {message}\n", diff --git a/kernel/src/arch/x86/mod.rs b/kernel/src/arch/x86/mod.rs index 730d15cf..47fc22de 100644 --- a/kernel/src/arch/x86/mod.rs +++ b/kernel/src/arch/x86/mod.rs @@ -62,14 +62,14 @@ pub fn register_pci_drivers() { // ygg_driver_nvme::probe, // ); // XXX: i686 hangs in interrupt handler - #[cfg(any(target_arch = "x86_64", rust_analyzer))] - ygg_driver_pci::register_class_driver( - "AHCI Controller", - 0x01, - Some(0x06), - Some(0x01), - ygg_driver_ahci::probe, - ); + // #[cfg(any(target_arch = "x86_64", rust_analyzer))] + // ygg_driver_pci::register_class_driver( + // "AHCI Controller", + // 0x01, + // Some(0x06), + // Some(0x01), + // ygg_driver_ahci::probe, + // ); ygg_driver_pci::register_class_driver( "USB xHCI", 0x0C, diff --git a/kernel/src/panic.rs b/kernel/src/panic.rs index 65c45bcb..8a9ac073 100644 --- a/kernel/src/panic.rs +++ b/kernel/src/panic.rs @@ -1,8 +1,8 @@ //! Kernel panic handler code use core::sync::atomic::{AtomicBool, AtomicU32, Ordering}; -use device_api::interrupt::{IpiDeliveryTarget, IpiMessage}; -use kernel_arch::{sync::hack_locks, Architecture, ArchitectureImpl}; +// use device_api::interrupt::{IpiDeliveryTarget, IpiMessage}; +use kernel_arch::{Architecture, ArchitectureImpl}; use libk::{ arch::{Cpu, LocalCpu}, debug::{self, PanicLoggerSink}, @@ -65,17 +65,18 @@ pub(crate) fn panic_handler(pi: &core::panic::PanicInfo) -> ! { .compare_exchange(false, true, Ordering::Release, Ordering::Acquire) .is_ok() { - // Let other CPUs know we're screwed - if unsafe { - PLATFORM - .send_ipi(IpiDeliveryTarget::OtherCpus, IpiMessage::Panic) - .unwrap_or(false) - } { - let ap_count = ArchitectureImpl::cpu_count() - 1; - PANIC_HANDLED_FENCE.wait_all(ap_count); - } + // TODO IPI leads to triple fault sometimes in x86-64 + // // Let other CPUs know we're screwed + // if unsafe { + // PLATFORM + // .send_ipi(IpiDeliveryTarget::OtherCpus, IpiMessage::Panic) + // .unwrap_or(false) + // } { + // let ap_count = ArchitectureImpl::cpu_count() - 1; + // PANIC_HANDLED_FENCE.wait_all(ap_count); + // } - unsafe { hack_locks() }; + // unsafe { hack_locks() }; dump_panic_info(&cpu, pi);