diff --git a/Cargo.lock b/Cargo.lock index 1078f310..24a72854 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2772,6 +2772,7 @@ version = "0.1.0" dependencies = [ "async-trait", "bytemuck", + "device-api", "kernel-fs", "libk", "libk-mm", @@ -2788,6 +2789,7 @@ name = "ygg_driver_net_igbe" version = "0.1.0" dependencies = [ "device-api", + "futures-util", "libk", "libk-mm", "libk-util", diff --git a/kernel/driver/bsp/sifive/src/ethernet/mod.rs b/kernel/driver/bsp/sifive/src/ethernet/mod.rs index b56c384d..87a44df1 100644 --- a/kernel/driver/bsp/sifive/src/ethernet/mod.rs +++ b/kernel/driver/bsp/sifive/src/ethernet/mod.rs @@ -26,11 +26,12 @@ use ygg_driver_net_core::{ RxPacket, ephy::PhyAccess, interface::{NetworkDevice, NetworkInterfaceType}, + util::GenericQueue, }; use yggdrasil_abi::net::{MacAddress, link::LinkState}; use crate::ethernet::{ - queue::Queue, + queue::Descriptor, regs::{ControlRegs, Regs}, }; @@ -55,7 +56,7 @@ struct Ethernet { regs: IrqSafeSpinlock>, // Operation - queue: OneTimeInit, + queue: OneTimeInit>, softirq: BitmapEvent, iface_id: OneTimeInit, } @@ -126,7 +127,7 @@ impl Ethernet { if status.matches_all(regs::Interrupt::RCOMP::SET) { queue - .consume_rx(&**self.dma.get(), |packet| { + .consume_rx(&**self.dma.get(), None, |packet, _len| { let packet = RxPacket::new(packet, 0, iface); ygg_driver_net_core::receive_packet(packet).ok(); }) @@ -152,7 +153,7 @@ impl Ethernet { fn start_xmit(&self, frame: DmaBuffer<[u8]>) -> Result<(), Error> { let queue = self.queue.get(); - queue.push_xmit(frame)?; + queue.try_push_xmit(frame)?; let regs = self.regs.lock(); regs.network_control @@ -170,10 +171,17 @@ impl Device for Ethernet { let dma = self.dma.init(cx.dma_allocator); let regs = self.regs.lock(); - let queue = self.queue.init(Queue::with_capacity(&**dma, 256, 64)?); + let queue = self.queue.init(GenericQueue::with_capacity( + &**dma, + 256, + 256, + queue::RX_BUFFER_SIZE, + )?); - let rx_queue_base = queue.rx_queue_base().try_into_u32().unwrap(); - let tx_queue_base = queue.tx_queue_base().try_into_u32().unwrap(); + let rx_queue_base = queue.rx_buffer_base().try_into_u32().unwrap(); + let tx_queue_base = queue.tx_buffer_base().try_into_u32().unwrap(); + + regs.interrupt_disable.set(0xFFFFFFFF); regs.rx_buffer_queue_base.set(rx_queue_base); regs.tx_buffer_queue_base.set(tx_queue_base); @@ -197,7 +205,6 @@ impl Device for Ethernet { // if (GEM_BFEXT(DAW64, gem_readl(...))) // hw_dma_cap |= HW_DMA_CAP_64B - regs.interrupt_disable.set(0xFFFFFFFF); regs.interrupt_enable.write( regs::Interrupt::LINK::SET + regs::Interrupt::RCOMP::SET + regs::Interrupt::TCOMP::SET, ); diff --git a/kernel/driver/bsp/sifive/src/ethernet/queue.rs b/kernel/driver/bsp/sifive/src/ethernet/queue.rs index 6897b849..1244650e 100644 --- a/kernel/driver/bsp/sifive/src/ethernet/queue.rs +++ b/kernel/driver/bsp/sifive/src/ethernet/queue.rs @@ -1,12 +1,5 @@ -use core::mem::{self, MaybeUninit}; - -use alloc::vec::Vec; -use device_api::dma::DmaAllocator; -use libk::{ - dma::{BusAddress, DmaBuffer}, - error::Error, -}; -use libk_util::sync::IrqSafeSpinlock; +use libk::{dma::BusAddress, error::Error}; +use ygg_driver_net_core::util::{GenericRxDescriptor, GenericTxDescriptor}; pub const RX_BUFFER_SIZE: usize = 4096; @@ -15,241 +8,80 @@ static_assertions::const_assert_ne!(RX_BUFFER_SIZE / super::RX_BUF_SIZE_MUL, 0); #[derive(Clone, Copy, Debug)] #[repr(C)] -struct Descriptor { +pub(crate) struct Descriptor { address: u32, control: u32, } -struct RxQueue { - entries: DmaBuffer<[Descriptor]>, - buffers: Vec]>>, +impl GenericTxDescriptor for Descriptor { + // USED = 0: owned by hardware + // USED = 1: owned by software + const EMPTY: Self = Self { + address: 0, + control: Self::DESC_1_USED, + }; + const EMPTY_LAST: Self = Self { + address: 0, + control: Self::DESC_1_USED | Self::DESC_1_TX_WRAP, + }; - rd: usize, + fn consume(&mut self) -> Option> { + if self.control & Self::DESC_1_USED != 0 && self.address != 0 { + Some(Ok(())) + } else { + None + } + } + + fn setup_tx( + &mut self, + buffer_address: BusAddress, + size: usize, + index: usize, + capacity: usize, + ) -> Result<(), Error> { + let buffer_address = buffer_address.try_into_u32()?; + if !(8..8192).contains(&size) { + return Err(Error::InvalidArgument); + } + let mut control = size as u32 | Self::DESC_1_TX_LAST; + if index == capacity - 1 { + control |= Self::DESC_1_TX_WRAP; + } + self.address = buffer_address; + self.control = control; + Ok(()) + } } -struct TxQueue { - entries: DmaBuffer<[Descriptor]>, - buffers: Vec>>, +impl GenericRxDescriptor for Descriptor { + fn consume(&mut self) -> Option> { + if self.address & Self::DESC_0_RX_OWNERSHIP != 0 { + Some(Ok((self.control & Self::LENGTH) as usize)) + } else { + None + } + } - wr: usize, - rd: usize, -} - -pub struct Queue { - rx_queue: IrqSafeSpinlock, - rx_queue_base: BusAddress, - tx_queue: IrqSafeSpinlock, - tx_queue_base: BusAddress, + fn setup_rx(buffer_address: BusAddress, _size: usize, last: bool) -> Result { + let mut address = buffer_address.try_into_u32()?; + if last { + address |= Self::DESC_0_RX_WRAP; + } + Ok(Self { + control: 0, + address, + }) + } } impl Descriptor { - const USED: u32 = 1 << 31; + const DESC_1_USED: u32 = 1 << 31; const LENGTH: u32 = 0x1FFF; - const TX_WRAP: u32 = 1 << 30; - const TX_LAST: u32 = 1 << 15; + const DESC_1_TX_WRAP: u32 = 1 << 30; + const DESC_1_TX_LAST: u32 = 1 << 15; - const RX_OWNERSHIP: u32 = 1 << 0; - const RX_WRAP: u32 = 1 << 1; - - const EMPTY_RX: Self = Self { - address: Self::RX_OWNERSHIP, - control: 0, - }; - const EMPTY_TX: Self = Self { - address: 0, - control: Self::USED, - }; - - fn setup_rx(&mut self, buffer: BusAddress, wrap: bool) { - let mut address = buffer.try_into_u32().unwrap(); - if wrap { - address |= Self::RX_WRAP; - } - self.address = address; - self.control = 0; - } - - fn setup_tx(&mut self, buffer: BusAddress, len: usize, wrap: bool) { - assert!(len < Self::LENGTH as usize - 1); - let mut control = len as u32; - if wrap { - control |= Self::TX_WRAP; - } - // LAST - control |= Self::TX_LAST; - self.address = buffer.try_into_u32().unwrap(); - self.control = control; - } -} - -impl RxQueue { - fn with_capacity(dma: &dyn DmaAllocator, capacity: usize) -> Result { - let mut entries = DmaBuffer::new_slice(dma, Descriptor::EMPTY_RX, capacity)?; - let buffers = (0..capacity) - .map(|_| DmaBuffer::new_uninit_slice(dma, RX_BUFFER_SIZE)) - .collect::, _>>()?; - for i in 0..capacity { - entries[i].setup_rx(buffers[i].bus_address(), i == capacity - 1); - } - Ok(Self { - buffers, - entries, - - rd: 0, - }) - } - - pub fn consume)>( - &mut self, - dma: &dyn DmaAllocator, - packet_handler: F, - ) -> Result { - let mut count = 0; - let capacity = self.entries.len(); - - loop { - let index = self.rd % self.entries.len(); - - self.entries.cache_flush_element(index, false); - - let entry = &mut self.entries[index]; - - if entry.address & Descriptor::RX_OWNERSHIP != 0 { - // if entry.rx_completed().is_some() { - // Grab the current buffer (the one just written to by the DMA), replace it - // with the newly allocated one, and mark the descriptor as DMA-owned again - let new_buffer = DmaBuffer::new_uninit_slice(dma, 4096)?; - let new_buffer_address = new_buffer.bus_address(); - let buffer = mem::replace(&mut self.buffers[index], new_buffer); - let buffer = unsafe { DmaBuffer::assume_init_slice(buffer) }; - // TODO packet size hint - packet_handler(buffer); - entry.setup_rx(new_buffer_address, index == capacity - 1); - // entry.setup_rx(new_buffer_address, true)?; - self.rd = self.rd.wrapping_add(1); - count += 1; - } else { - break; - } - } - - Ok(count) - } -} - -impl TxQueue { - pub fn with_capacity(dma: &dyn DmaAllocator, capacity: usize) -> Result { - let mut entries = DmaBuffer::new_slice(dma, Descriptor::EMPTY_TX, capacity)?; - entries[capacity - 1].control |= Descriptor::TX_WRAP; - let buffers = (0..capacity).map(|_| None).collect(); - Ok(Self { - entries, - buffers, - - wr: 0, - rd: 0, - }) - } - - pub fn can_xmit(&self) -> bool { - let capacity = self.entries.len(); - self.wr.wrapping_add(1) % capacity != self.rd % capacity - } - - pub fn push_xmit(&mut self, frame: DmaBuffer<[u8]>) -> Result { - if !self.can_xmit() { - return Err(Error::WouldBlock); - } - - let address = frame.bus_address(); - let frame_len = frame.len(); - - if !(8..0x2000).contains(&frame_len) { - return Err(Error::InvalidArgument); - } - - let capacity = self.entries.len(); - let index = self.wr % capacity; - assert!(self.buffers[index].is_none()); - - frame.cache_flush_all(true); - self.entries[index].setup_tx(address, frame_len, index == capacity - 1); - // self.entries[index].setup_tx(address, frame_len, true)?; - self.entries.cache_flush_element(index, true); - - self.buffers[index] = Some(frame); - self.wr = self.wr.wrapping_add(1); - - Ok(self.wr % capacity) - } - - pub fn consume(&mut self) -> Result { - let mut count = 0; - - loop { - let index = self.rd % self.entries.len(); - - self.entries.cache_flush_element(index, false); - - if self.rd == self.wr { - break; - } - - let entry = &self.entries[index]; - - if entry.control & Descriptor::USED != 0 { - let _ = self.buffers[index].take().unwrap(); - self.rd = self.rd.wrapping_add(1); - count += 1; - } else { - break; - } - } - - Ok(count) - } -} - -impl Queue { - pub fn with_capacity( - dma: &dyn DmaAllocator, - tx_capacity: usize, - rx_capacity: usize, - ) -> Result { - let rx_queue = RxQueue::with_capacity(dma, rx_capacity)?; - let rx_queue_base = rx_queue.entries.bus_address(); - let tx_queue = TxQueue::with_capacity(dma, tx_capacity)?; - let tx_queue_base = tx_queue.entries.bus_address(); - - Ok(Self { - rx_queue: IrqSafeSpinlock::new(rx_queue), - rx_queue_base, - tx_queue: IrqSafeSpinlock::new(tx_queue), - tx_queue_base, - }) - } - - pub fn rx_queue_base(&self) -> BusAddress { - self.rx_queue_base - } - pub fn tx_queue_base(&self) -> BusAddress { - self.tx_queue_base - } - - pub fn push_xmit(&self, packet: DmaBuffer<[u8]>) -> Result { - self.tx_queue.lock().push_xmit(packet) - } - - pub fn consume_rx)>( - &self, - dma: &dyn DmaAllocator, - packet_handler: F, - ) -> Result { - self.rx_queue.lock().consume(dma, packet_handler) - } - - pub fn consume_tx(&self) -> Result { - self.tx_queue.lock().consume() - } + const DESC_0_RX_OWNERSHIP: u32 = 1 << 0; + const DESC_0_RX_WRAP: u32 = 1 << 1; } diff --git a/kernel/driver/net/core/Cargo.toml b/kernel/driver/net/core/Cargo.toml index 9af3681b..9bf0cb03 100644 --- a/kernel/driver/net/core/Cargo.toml +++ b/kernel/driver/net/core/Cargo.toml @@ -8,6 +8,7 @@ yggdrasil-abi = { workspace = true, features = ["serde_kernel", "bytemuck"] } libk-mm.workspace = true libk-util.workspace = true libk.workspace = true +device-api.workspace = true kernel-fs = { path = "../../fs/kernel-fs" } diff --git a/kernel/driver/net/core/src/util/mod.rs b/kernel/driver/net/core/src/util/mod.rs new file mode 100644 index 00000000..e5cc96f9 --- /dev/null +++ b/kernel/driver/net/core/src/util/mod.rs @@ -0,0 +1,5 @@ +pub use queue::*; +pub use reassembler::*; + +mod queue; +mod reassembler; diff --git a/kernel/driver/net/core/src/util/queue.rs b/kernel/driver/net/core/src/util/queue.rs new file mode 100644 index 00000000..8177b484 --- /dev/null +++ b/kernel/driver/net/core/src/util/queue.rs @@ -0,0 +1,291 @@ +use core::mem::{self, MaybeUninit}; + +use alloc::vec::Vec; +use device_api::dma::DmaAllocator; +use libk::{ + dma::{BusAddress, DmaBuffer}, + error::Error, +}; +use libk_util::{sync::IrqSafeSpinlock, waker::QueueWaker}; + +pub trait GenericTxDescriptor: Sized { + const EMPTY: Self; + const EMPTY_LAST: Self = Self::EMPTY; + + fn consume(&mut self) -> Option>; + fn setup_tx( + &mut self, + buffer_address: BusAddress, + size: usize, + index: usize, + capacity: usize, + ) -> Result<(), Error>; +} + +pub trait GenericRxDescriptor: Sized { + // TODO Rx flags (short, runt, etc) + fn consume(&mut self) -> Option>; + fn setup_rx(buffer_address: BusAddress, buffer_size: usize, last: bool) -> Result; +} + +struct TxInFlight { + #[allow(unused)] + buffer: DmaBuffer<[u8]>, +} + +struct TxRingInner { + entries: DmaBuffer<[T]>, + in_flight: Vec>, + + wr: usize, + rd: usize, +} + +struct RxRingInner { + entries: DmaBuffer<[T]>, + buffers: Vec]>>, + + rd: usize, +} + +pub struct GenericTxRing { + inner: IrqSafeSpinlock>, + + buffer_base: BusAddress, + capacity: usize, + free_notify: QueueWaker, +} + +pub struct GenericRxRing { + inner: IrqSafeSpinlock>, + + rx_buffer_size: usize, + buffer_base: BusAddress, + capacity: usize, +} + +pub struct GenericQueue { + pub tx_ring: GenericTxRing, + pub rx_ring: GenericRxRing, +} + +impl GenericQueue { + pub fn with_capacity( + dma: &dyn DmaAllocator, + tx_capacity: usize, + rx_capacity: usize, + rx_buffer_size: usize, + ) -> Result { + let tx_ring = GenericTxRing::with_capacity(dma, tx_capacity)?; + let rx_ring = GenericRxRing::with_capacity(dma, rx_capacity, rx_buffer_size)?; + Ok(Self { tx_ring, rx_ring }) + } + + pub fn tx_buffer_base(&self) -> BusAddress { + self.tx_ring.buffer_base + } + pub fn rx_buffer_base(&self) -> BusAddress { + self.rx_ring.buffer_base + } + + pub fn try_push_xmit(&self, frame: DmaBuffer<[u8]>) -> Result { + self.tx_ring.try_push_xmit(frame) + } + + pub fn drop_tx_until(&self, head: usize) { + self.tx_ring.drop_until(head); + } + pub fn consume_tx(&self) -> Result { + self.tx_ring.consume() + } + pub fn consume_rx, usize)>( + &self, + dma: &dyn DmaAllocator, + head: Option, + handler: F, + ) -> Result { + self.rx_ring.consume(dma, head, handler) + } +} + +impl GenericTxRing { + pub fn with_capacity(dma: &dyn DmaAllocator, capacity: usize) -> Result { + let entries = DmaBuffer::new_slice_with( + dma, + |i| { + if i == capacity - 1 { + T::EMPTY_LAST + } else { + T::EMPTY + } + }, + capacity, + )?; + let in_flight = (0..capacity).map(|_| None).collect(); + let buffer_base = entries.bus_address(); + let inner = TxRingInner { + entries, + in_flight, + wr: 0, + rd: 0, + }; + Ok(Self { + inner: IrqSafeSpinlock::new(inner), + buffer_base, + capacity, + free_notify: QueueWaker::new(), + }) + } + + pub fn base(&self) -> BusAddress { + self.buffer_base + } + pub fn capacity(&self) -> usize { + self.capacity + } + + pub async fn push_xmit(&self, _frame: DmaBuffer<[u8]>) -> Result { + todo!() + } + + pub fn try_push_xmit(&self, frame: DmaBuffer<[u8]>) -> Result { + let mut inner = self.inner.lock(); + if inner.wr.wrapping_add(1) % self.capacity == inner.rd % self.capacity { + return Err(Error::WouldBlock); + } + + let buffer_address = frame.bus_address(); + let buffer_len = frame.len(); + + let index = inner.wr % self.capacity; + assert!(inner.in_flight[index].is_none()); + + frame.cache_flush_all(true); + inner.entries[index].setup_tx(buffer_address, buffer_len, index, self.capacity)?; + inner.entries.cache_flush_element(index, true); + + inner.in_flight[index] = Some(TxInFlight { buffer: frame }); + inner.wr = inner.wr.wrapping_add(1); + + Ok(inner.wr % self.capacity) + } + + pub fn drop_until(&self, head: usize) { + self.inner.lock().rd = head % self.capacity; + } + + pub fn consume(&self) -> Result { + let mut count = 0; + + { + let mut inner = self.inner.lock(); + + while inner.rd != inner.wr { + let index = inner.rd % self.capacity; + inner.entries.cache_flush_element(index, false); + let entry = &mut inner.entries[index]; + + if let Some(_status) = entry.consume() { + let _ = inner.in_flight[index].take().unwrap(); + inner.rd = inner.rd.wrapping_add(1); + count += 1; + } else { + break; + } + } + } + + if count != 0 { + self.free_notify.wake_all(); + } + + Ok(count) + } +} + +impl GenericRxRing { + pub fn with_capacity( + dma: &dyn DmaAllocator, + capacity: usize, + rx_buffer_size: usize, + ) -> Result { + let buffers = (0..capacity) + .map(|_| DmaBuffer::new_uninit_slice(dma, rx_buffer_size)) + .collect::, _>>()?; + let entries = DmaBuffer::new_slice_with( + dma, + |i| { + let buffer = buffers[i].bus_address(); + let last = i == capacity - 1; + T::setup_rx(buffer, rx_buffer_size, last).expect("Rx buffer descriptor setup error") + }, + capacity, + )?; + let buffer_base = entries.bus_address(); + let inner = RxRingInner { + entries, + buffers, + + rd: 0, + }; + Ok(Self { + inner: IrqSafeSpinlock::new(inner), + rx_buffer_size, + buffer_base, + capacity, + }) + } + + pub fn base(&self) -> BusAddress { + self.buffer_base + } + pub fn capacity(&self) -> usize { + self.capacity + } + + pub fn consume, usize)>( + &self, + dma: &dyn DmaAllocator, + head: Option, + handler: F, + ) -> Result { + let mut inner = self.inner.lock(); + + while head.map_or(true, |tail| tail != inner.rd) { + let index = inner.rd % self.capacity; + + inner.entries.cache_flush_element(index, false); + + if let Some(status) = inner.entries[index].consume() { + // Grab the current buffer (the one just written to by the DMA), replace it + // with the newly allocated one, and mark the descriptor as DMA-owned again + let new_buffer = DmaBuffer::new_uninit_slice(dma, self.rx_buffer_size)?; + let new_buffer_address = new_buffer.bus_address(); + let buffer = mem::replace(&mut inner.buffers[index], new_buffer); + let old_buffer_address = buffer.bus_address(); + let buffer = unsafe { DmaBuffer::assume_init_slice(buffer) }; + + match status { + Ok(len) => { + handler(buffer, len); + } + Err(_error) => { + log::warn!("Drop error packet {old_buffer_address:#x}"); + } + } + + inner.entries[index] = T::setup_rx( + new_buffer_address, + self.rx_buffer_size, + index == self.capacity - 1, + )?; + + inner.rd = inner.rd.wrapping_add(1); + } else { + break; + } + } + + Ok(inner.rd) + } +} diff --git a/kernel/driver/net/core/src/util.rs b/kernel/driver/net/core/src/util/reassembler.rs similarity index 100% rename from kernel/driver/net/core/src/util.rs rename to kernel/driver/net/core/src/util/reassembler.rs diff --git a/kernel/driver/net/igbe/Cargo.toml b/kernel/driver/net/igbe/Cargo.toml index ac947342..ea7fbb4e 100644 --- a/kernel/driver/net/igbe/Cargo.toml +++ b/kernel/driver/net/igbe/Cargo.toml @@ -14,4 +14,5 @@ ygg_driver_pci.path = "../../bus/pci" ygg_driver_net_core.path = "../core" log.workspace = true +futures-util.workspace = true tock-registers.workspace = true diff --git a/kernel/driver/net/igbe/src/lib.rs b/kernel/driver/net/igbe/src/lib.rs index ce5f93c9..36d560e3 100644 --- a/kernel/driver/net/igbe/src/lib.rs +++ b/kernel/driver/net/igbe/src/lib.rs @@ -8,17 +8,16 @@ use device_api::{ dma::DmaAllocator, interrupt::{InterruptAffinity, InterruptHandler, IrqVector}, }; -use libk::{dma::DmaBuffer, error::Error}; -use libk_util::{ - OneTimeInit, - sync::{IrqSafeSpinlock, Spinlock}, -}; -use regs::{ICR, Regs}; -use ring::{RxRing, TxRing}; +use futures_util::task::AtomicWaker; +use libk::{dma::DmaBuffer, error::Error, task::runtime}; +use libk_util::{OneTimeInit, event::BitmapEvent, sync::IrqSafeSpinlock}; +use regs::Regs; +use tock_registers::{LocalRegisterCopy, fields::FieldValue}; use ygg_driver_net_core::{ RxPacket, interface::{NetworkDevice, NetworkInterfaceType}, register_interface, + util::GenericQueue, }; use ygg_driver_pci::{ PciBaseAddress, PciConfigurationSpace, @@ -27,7 +26,10 @@ use ygg_driver_pci::{ }; use yggdrasil_abi::net::{MacAddress, link::LinkState}; -use crate::regs::Revision; +use crate::{ + regs::Revision, + ring::{RxDescriptor, TxDescriptor}, +}; extern crate alloc; @@ -41,8 +43,8 @@ struct Igbe { pci: PciDeviceInfo, mac: OneTimeInit, - rx_ring: OneTimeInit>, - tx_ring: OneTimeInit>, + queue: OneTimeInit>, + softirq: BitmapEvent, nic: OneTimeInit, } @@ -55,11 +57,55 @@ impl Igbe { mac: OneTimeInit::new(), regs: IrqSafeSpinlock::new(regs), - rx_ring: OneTimeInit::new(), - tx_ring: OneTimeInit::new(), + queue: OneTimeInit::new(), + softirq: BitmapEvent::new(AtomicWaker::new()), nic: OneTimeInit::new(), } } + + async fn softirq(self: Arc) { + let queue = self.queue.get(); + let rx_capacity = queue.rx_ring.capacity(); + let nic = *self.nic.get(); + + loop { + let event = self.softirq.wait().await as u32; + let event = LocalRegisterCopy::<_, regs::ICR::Register>::new(event); + + if event.matches_all(regs::ICR::LSC::SET) { + let status = self.regs.lock().read_link(); + log::info!("igbe: link is {status}"); + self.interrupt_handled(regs::ICR::LSC::SET); + } + if event.matches_any(&[regs::ICR::TXQE::SET, regs::ICR::TXDW::SET]) { + let head = self.regs.lock().tx_queue_head(); + queue.drop_tx_until(head as usize); + // queue.consume_tx(Some(head as usize)).ok(); + self.interrupt_handled(regs::ICR::TXQE::SET + regs::ICR::TXDW::SET); + } + if event.matches_any(&[regs::ICR::RXDMT0::SET, regs::ICR::RXT0::SET]) { + { + let mut regs = self.regs.lock(); + let head = regs.rx_queue_head() as usize; + let tail = queue + .consume_rx(&*self.dma, Some(head), |packet, _| { + let packet = RxPacket::new(packet, 0, nic); + ygg_driver_net_core::receive_packet(packet).ok(); + }) + .expect("Rx ring handle error"); + let tail = (tail + rx_capacity - 1) & (rx_capacity - 1); + regs.set_rx_queue_tail(tail as u16); + } + self.interrupt_handled(regs::ICR::RXDMT0::SET + regs::ICR::RXT0::SET); + } + } + } + + fn interrupt_handled(&self, icr: FieldValue) { + let mut regs = self.regs.lock(); + regs.clear_interrupts(icr.value); + regs.enable_interrupts(icr.value); + } } impl Device for Igbe { @@ -68,33 +114,44 @@ impl Device for Igbe { .pci .map_interrupt(InterruptAffinity::Any, self.clone())?; - let rx_ring = RxRing::with_capacity(&*self.dma, 128, 2048 + 16)?; - let tx_ring = TxRing::with_capacity(&*self.dma, 128)?; + let queue = self.queue.init(GenericQueue::with_capacity( + &*self.dma, + 128, + 128, + 2048 + 16, + )?); let mut regs = self.regs.lock(); - regs.disable_interrupts(); + regs.disable_interrupts(0xFFFFFFFF); let mac = regs.read_mac()?; self.mac.init(mac); regs.reset(Duration::from_millis(200))?; // Intel 8257x manuals say an additional interrupt disable is needed after a global reset - regs.disable_interrupts(); + regs.disable_interrupts(0xFFFFFFFF); regs.set_link_up(self.chip)?; // Initialize Rx - regs.initialize_receiver(&rx_ring); - regs.initialize_transmitter(&tx_ring); + regs.initialize_receiver(&queue.rx_ring); + regs.initialize_transmitter(&queue.tx_ring); // If MSI(-x) was initialized, notify the NIC about it if let Some(msi_info) = msi_info { regs.initialize_ivar(msi_info.vector); } - self.rx_ring.init(Spinlock::new(rx_ring)); - self.tx_ring.init(IrqSafeSpinlock::new(tx_ring)); let nic = register_interface(NetworkInterfaceType::Ethernet, self.clone()); self.nic.init(nic.id()); - regs.enable_interrupts(); + regs.enable_interrupts( + (regs::IMS::TXDW::SET + + regs::IMS::TXQE::SET + + regs::IMS::LSC::SET + + regs::IMS::RXT0::SET + + regs::IMS::RXDMT0::SET) + .value, + ); + + runtime::spawn(self.clone().softirq())?; Ok(()) } @@ -107,43 +164,13 @@ impl Device for Igbe { impl InterruptHandler for Igbe { fn handle_irq(self: Arc, _vector: IrqVector) -> bool { let mut regs = self.regs.lock(); - let cause = regs.interrupt_cause(); - if cause.get() == 0 { + let cause = regs.interrupt_cause().get(); + if cause == 0 { return false; } - regs.clear_interrupts(cause.get()); - let mut any = false; - - if cause.matches_all(ICR::LSC::SET) { - let status = regs.read_link(); - log::info!("igbe: link is {status}"); - any = true; - } - - if cause.matches_all(ICR::RXT0::SET) { - let mut rx = self.rx_ring.get().lock(); - let nic = *self.nic.get(); - let head = regs.rx_queue_head(); - let tail = rx.handle_rx(&*self.dma, head, |packet, _| { - let packet = RxPacket::new(packet, 0, nic); - ygg_driver_net_core::receive_packet(packet).ok(); - }); - regs.set_rx_queue_tail(tail); - any = true; - } - - if cause.matches_any(&[ICR::TXQE::SET, ICR::TXDW::SET]) { - let mut tx = self.tx_ring.get().lock(); - let head = regs.tx_queue_head(); - tx.handle_tx(head); - any = true; - } - - if !any { - log::info!("igbe: unhandled irq {:#x}", cause.get()); - } - - any + regs.disable_interrupts(cause); + self.softirq.signal(cause as u64); + true } } @@ -161,15 +188,10 @@ impl NetworkDevice for Igbe { } fn transmit_buffer(&self, buffer: DmaBuffer<[u8]>) -> Result<(), Error> { - let mut tx = self.tx_ring.get().lock(); - - let Ok(head) = tx.tx_now(buffer) else { - return Err(Error::WouldBlock); - }; - + let queue = self.queue.get(); + let tail = queue.try_push_xmit(buffer)?; let mut regs = self.regs.lock(); - regs.set_tx_queue_tail(head); - + regs.set_tx_queue_tail(tail as _); Ok(()) } diff --git a/kernel/driver/net/igbe/src/regs.rs b/kernel/driver/net/igbe/src/regs.rs index 3a53bee3..1156a828 100644 --- a/kernel/driver/net/igbe/src/regs.rs +++ b/kernel/driver/net/igbe/src/regs.rs @@ -7,14 +7,17 @@ use libk::{ }; use libk_mm::{address::PhysicalAddress, device::RawDeviceMemoryMapping}; use tock_registers::{LocalRegisterCopy, RegisterLongName, fields::FieldValue, register_bitfields}; -use ygg_driver_net_core::ephy::{GBESR, MdioBus, PhyAccess}; +use ygg_driver_net_core::{ + ephy::{GBESR, MdioBus, PhyAccess}, + util::{GenericRxRing, GenericTxRing}, +}; use ygg_driver_pci::PciBaseAddress; use yggdrasil_abi::net::{ MacAddress, link::{Duplex, EthernetLinkState, EthernetSpeed}, }; -use crate::{RxRing, TxRing}; +use crate::ring::{RxDescriptor, TxDescriptor}; enum Inner { Memory(RawDeviceMemoryMapping), @@ -79,12 +82,14 @@ register_bitfields! { TXDW OFFSET(0) NUMBITS(1) [], TXQE OFFSET(1) NUMBITS(1) [], LSC OFFSET(2) NUMBITS(1) [], + RXDMT0 OFFSET(4) NUMBITS(1) [], RXT0 OFFSET(7) NUMBITS(1) [], ], pub IMS [ TXDW OFFSET(0) NUMBITS(1) [], TXQE OFFSET(1) NUMBITS(1) [], LSC OFFSET(2) NUMBITS(1) [], + RXDMT0 OFFSET(4) NUMBITS(1) [], RXT0 OFFSET(7) NUMBITS(1) [], ], pub RCTL [ @@ -459,36 +464,47 @@ impl Regs { } } - pub fn disable_interrupts(&mut self) { - self.inner.set::(0xFFFFFFFF); - } - - pub fn clear_interrupts(&mut self, cause: u32) { - let _ = self.inner.get::(); - self.inner.set::(cause); - } - - pub fn enable_interrupts(&mut self) { - self.inner.set::(0xFFFFFFFF); - // self.inner - // .modify(IMS::LSC::SET + IMS::RXT0::SET + IMS::TXDW::SET + IMS::TXQE::SET); - } + // pub fn disable_interrupts(&mut self) { + // self.inner.set::(0xFFFFFFFF); + // } pub fn interrupt_cause(&mut self) -> LocalRegisterCopy { self.inner.extract() } - pub fn initialize_receiver(&mut self, rx_ring: &RxRing) { - let rx_queue_base = rx_ring.descriptors.bus_address().into_u64(); + pub fn clear_interrupts(&mut self, icr: u32) { + let _ = self.inner.get::(); + self.inner.set::(icr); + } + + pub fn enable_interrupts(&mut self, mask: u32) { + self.inner.set::(mask); + } + + pub fn disable_interrupts(&mut self, mask: u32) { + self.inner.set::(mask); + } + + // pub fn enable_interrupts(&mut self) { + // self.inner.set::(0xFFFFFFFF); + // // self.inner + // // .modify(IMS::LSC::SET + IMS::RXT0::SET + IMS::TXDW::SET + IMS::TXQE::SET); + // } + + pub fn initialize_receiver(&mut self, rx_ring: &GenericRxRing) { + let rx_queue_base = rx_ring.base().into_u64(); + let rx_queue_capacity = rx_ring.capacity(); + + // let rx_queue_base = rx_ring.descriptors.bus_address().into_u64(); self.inner.set::(rx_queue_base as u32); self.inner .set::((rx_queue_base >> 32) as u32); self.inner - .write(RDLEN0::LEN0.val((rx_ring.descriptors.len() / 8) as u32)); + .write(RDLEN0::LEN0.val((rx_queue_capacity / 8) as u32)); self.inner.set::(0); self.inner - .set::((rx_ring.descriptors.len() - 1) as u32); + .set::((rx_queue_capacity - 1) as u32); self.inner.write( RCTL::EN::SET @@ -499,14 +515,16 @@ impl Regs { ); } - pub fn initialize_transmitter(&mut self, tx_ring: &TxRing) { - let tx_queue_base = tx_ring.descriptors.bus_address().into_u64(); + pub fn initialize_transmitter(&mut self, tx_ring: &GenericTxRing) { + let tx_queue_base = tx_ring.base().into_u64(); + let tx_queue_capacity = tx_ring.capacity(); + self.inner.set::(tx_queue_base as u32); self.inner .set::((tx_queue_base >> 32) as u32); self.inner - .write(TDLEN::LEN.val((tx_ring.descriptors.len() / 8) as u32)); + .write(TDLEN::LEN.val((tx_queue_capacity / 8) as u32)); self.inner.set::(0); self.inner.set::(0); diff --git a/kernel/driver/net/igbe/src/ring.rs b/kernel/driver/net/igbe/src/ring.rs index 0c418206..a241189b 100644 --- a/kernel/driver/net/igbe/src/ring.rs +++ b/kernel/driver/net/igbe/src/ring.rs @@ -1,31 +1,10 @@ -use core::mem::{self, MaybeUninit}; - -use alloc::vec::Vec; -use device_api::dma::DmaAllocator; -use libk::{ - dma::{BusAddress, DmaBuffer}, - error::Error, -}; - -pub(crate) struct RxRing { - pub(crate) descriptors: DmaBuffer<[RxDescriptor]>, - buffers: Vec]>>, - buffer_size: usize, - tail: u16, -} - -pub(crate) struct TxRing { - pub(crate) descriptors: DmaBuffer<[TxDescriptor]>, - buffers: Vec>>, - // Consumer end - tail: u16, - // Producer end - head: u16, -} +use libk::{dma::BusAddress, error::Error}; +use ygg_driver_net_core::util::{GenericRxDescriptor, GenericTxDescriptor}; +#[derive(Debug)] #[repr(C)] pub(crate) struct RxDescriptor { - address: BusAddress, + address: u64, length: u16, checksum: u16, status: u8, @@ -33,9 +12,10 @@ pub(crate) struct RxDescriptor { special: u16, } +#[derive(Debug)] #[repr(C)] pub(crate) struct TxDescriptor { - address: BusAddress, + address: u64, length: u16, cso: u8, cmd: u8, @@ -45,117 +25,73 @@ pub(crate) struct TxDescriptor { special: u8, } -impl RxRing { - pub fn with_capacity( - dma: &dyn DmaAllocator, - capacity: usize, - buffer_size: usize, - ) -> Result { - let buffers = (0..capacity) - .map(|_| DmaBuffer::new_uninit_slice(dma, buffer_size)) - .collect::, _>>()?; - let descriptors = DmaBuffer::new_slice_with( - dma, - |i| RxDescriptor::new(buffers[i].bus_address(), buffer_size as u16), - capacity, - )?; +impl GenericTxDescriptor for TxDescriptor { + const EMPTY: Self = Self { + address: 0, + length: 0, + cso: 0, + cmd: 0, + sta: Self::STA_DD, + _0: 0, + css: 0, + special: 0, + }; - Ok(Self { - descriptors, - buffers, - tail: 0, - buffer_size, - }) + fn consume(&mut self) -> Option> { + unreachable!() } - // TODO move to background task/softirq to reduce amount of code run by the irq handler - pub fn handle_rx, usize)>( + fn setup_tx( &mut self, - dma: &dyn DmaAllocator, - head: u16, - mut handler: F, - ) -> u16 { - let capacity = self.descriptors.len(); - while self.tail != head { - let index = self.tail as usize; - // Replace the buffer - - let new_buffer = DmaBuffer::new_uninit_slice(dma, self.buffer_size).unwrap(); - let new_buffer_address = new_buffer.bus_address(); - let buffer = mem::replace(&mut self.buffers[index], new_buffer); - let buffer = unsafe { DmaBuffer::assume_init_slice(buffer) }; - - let descriptor = &mut self.descriptors[index]; - - if descriptor.errors & !1 != 0 { - log::warn!("igbe: drop erroneous packet {:#x}", descriptor.errors); - } else { - let len = descriptor.length as usize; - handler(buffer, len); - } - - // Replace the descriptor - *descriptor = RxDescriptor::new(new_buffer_address, self.buffer_size as u16); - - self.tail = (self.tail + 1) & (capacity as u16 - 1); + buffer_address: BusAddress, + size: usize, + index: usize, + capacity: usize, + ) -> Result<(), Error> { + if !(4..16288).contains(&size) { + return Err(Error::InvalidArgument); } - - (self.tail + capacity as u16 - 1) & (capacity as u16 - 1) - } -} - -impl TxRing { - pub fn with_capacity(dma: &dyn DmaAllocator, capacity: usize) -> Result { - let buffers = (0..capacity).map(|_| None).collect::>(); - let descriptors = DmaBuffer::new_slice_with(dma, |_| TxDescriptor::empty(), capacity)?; - Ok(Self { - descriptors, - buffers, - head: 0, - tail: 0, - }) - } - - pub fn handle_tx(&mut self, head: u16) { - self.tail = head; - } - - pub fn tx_now(&mut self, buffer: DmaBuffer<[u8]>) -> Result> { - // Queue full - let capacity = self.descriptors.len(); - if (self.head + 1) & (capacity as u16 - 1) == self.tail { - log::warn!("igbe: tx queue full"); - return Err(buffer); - } - let index = self.head as usize; - - let descriptor = &mut self.descriptors[index]; - // Only generate interrupts for every 1/4th of the buffer let quarter = capacity / 4; - descriptor.setup_tx( - buffer.bus_address(), - buffer.len() as u16 - 4, - index % quarter == quarter - 1, - ); - self.descriptors.cache_flush_element(index, true); - - self.buffers[index] = Some(buffer); - - self.head = (self.head + 1) & (capacity as u16 - 1); - Ok(self.head) + let ioc = index % quarter == quarter - 1; + let mut cmd = Self::CMD_EOP | Self::CMD_IFCS; + if ioc { + cmd |= Self::CMD_RS; + } + self.address = buffer_address.into_u64(); + self.length = size as u16 - 4; + self.css = 0; + self.cso = 0; + self.sta = 0; + self.cmd = cmd; + self.special = 0; + Ok(()) } } -impl RxDescriptor { - pub fn new(address: BusAddress, length: u16) -> Self { - Self { - address, - length, +impl GenericRxDescriptor for RxDescriptor { + fn consume(&mut self) -> Option> { + if self.status & Self::STA_DD == 0 { + None + } else if self.errors & !1 != 0 { + Some(Err(Error::InvalidArgument)) + } else { + Some(Ok(self.length as _)) + } + } + + fn setup_rx( + buffer_address: BusAddress, + buffer_size: usize, + _last: bool, + ) -> Result { + Ok(Self { + address: buffer_address.into_u64(), + length: buffer_size as _, checksum: 0, status: 0, errors: 0, special: 0, - } + }) } } @@ -170,31 +106,8 @@ impl TxDescriptor { // const CMD_IC: u8 = 1 << 2; // Report status const CMD_RS: u8 = 1 << 3; - - pub const fn empty() -> Self { - Self { - address: BusAddress::ZERO, - length: 0, - cso: 0, - cmd: 0, - sta: Self::STA_DD, - _0: 0, - css: 0, - special: 0, - } - } - - pub fn setup_tx(&mut self, address: BusAddress, length: u16, ioc: bool) { - let mut cmd = Self::CMD_EOP | Self::CMD_IFCS; - if ioc { - cmd |= Self::CMD_RS; - } - self.address = address; - self.length = length; - self.css = 0; - self.cso = 0; - self.sta = 0; - self.cmd = cmd; - self.special = 0; - } +} + +impl RxDescriptor { + const STA_DD: u8 = 1 << 0; }