alnyan/yggdrasil
1
0
Fork 0
Code Issues 32 Pull Requests Packages Projects Releases Wiki Activity

net: unify netdev tx/rx queue interface

Browse Source
This commit is contained in:
Mark Poliakov
2026-02-07 14:13:09 +02:00
parent 1a87bc3666
commit 7f46da9ebd
11 changed files with 570 additions and 478 deletions
Download Patch File Download Diff File Expand all files Collapse all files
kernel/driver/bsp/sifive/src/ethernet/mod.rs
+15 -8
View File
@@ -26,11 +26,12 @@ use ygg_driver_net_core::{
RxPacket,
ephy::PhyAccess,
interface::{NetworkDevice, NetworkInterfaceType},
util::GenericQueue,
};
use yggdrasil_abi::net::{MacAddress, link::LinkState};
use crate::ethernet::{
queue::Queue,
queue::Descriptor,
regs::{ControlRegs, Regs},
};
@@ -55,7 +56,7 @@ struct Ethernet {
regs: IrqSafeSpinlock<DeviceMemoryIo<'static, Regs>>,
// Operation
queue: OneTimeInit<Queue>,
queue: OneTimeInit<GenericQueue<Descriptor, Descriptor>>,
softirq: BitmapEvent<AtomicWaker>,
iface_id: OneTimeInit<u32>,
}
@@ -126,7 +127,7 @@ impl Ethernet {
if status.matches_all(regs::Interrupt::RCOMP::SET) {
queue
.consume_rx(&**self.dma.get(), |packet| {
.consume_rx(&**self.dma.get(), None, |packet, _len| {
let packet = RxPacket::new(packet, 0, iface);
ygg_driver_net_core::receive_packet(packet).ok();
})
@@ -152,7 +153,7 @@ impl Ethernet {
fn start_xmit(&self, frame: DmaBuffer<[u8]>) -> Result<(), Error> {
let queue = self.queue.get();
queue.push_xmit(frame)?;
queue.try_push_xmit(frame)?;
let regs = self.regs.lock();
regs.network_control
@@ -170,10 +171,17 @@ impl Device for Ethernet {
let dma = self.dma.init(cx.dma_allocator);
let regs = self.regs.lock();
let queue = self.queue.init(Queue::with_capacity(&**dma, 256, 64)?);
let queue = self.queue.init(GenericQueue::with_capacity(
&**dma,
256,
256,
queue::RX_BUFFER_SIZE,
)?);
let rx_queue_base = queue.rx_queue_base().try_into_u32().unwrap();
let tx_queue_base = queue.tx_queue_base().try_into_u32().unwrap();
let rx_queue_base = queue.rx_buffer_base().try_into_u32().unwrap();
let tx_queue_base = queue.tx_buffer_base().try_into_u32().unwrap();
regs.interrupt_disable.set(0xFFFFFFFF);
regs.rx_buffer_queue_base.set(rx_queue_base);
regs.tx_buffer_queue_base.set(tx_queue_base);
@@ -197,7 +205,6 @@ impl Device for Ethernet {
// if (GEM_BFEXT(DAW64, gem_readl(...)))
// hw_dma_cap |= HW_DMA_CAP_64B
regs.interrupt_disable.set(0xFFFFFFFF);
regs.interrupt_enable.write(
regs::Interrupt::LINK::SET + regs::Interrupt::RCOMP::SET + regs::Interrupt::TCOMP::SET,
);
kernel/driver/bsp/sifive/src/ethernet/queue.rs
+64 -232
View File
@@ -1,12 +1,5 @@
use core::mem::{self, MaybeUninit};
use alloc::vec::Vec;
use device_api::dma::DmaAllocator;
use libk::{
dma::{BusAddress, DmaBuffer},
error::Error,
};
use libk_util::sync::IrqSafeSpinlock;
use libk::{dma::BusAddress, error::Error};
use ygg_driver_net_core::util::{GenericRxDescriptor, GenericTxDescriptor};
pub const RX_BUFFER_SIZE: usize = 4096;
@@ -15,241 +8,80 @@ static_assertions::const_assert_ne!(RX_BUFFER_SIZE / super::RX_BUF_SIZE_MUL, 0);
#[derive(Clone, Copy, Debug)]
#[repr(C)]
struct Descriptor {
pub(crate) struct Descriptor {
address: u32,
control: u32,
}
struct RxQueue {
entries: DmaBuffer<[Descriptor]>,
buffers: Vec<DmaBuffer<[MaybeUninit<u8>]>>,
impl GenericTxDescriptor for Descriptor {
// USED = 0: owned by hardware
// USED = 1: owned by software
const EMPTY: Self = Self {
address: 0,
control: Self::DESC_1_USED,
};
const EMPTY_LAST: Self = Self {
address: 0,
control: Self::DESC_1_USED | Self::DESC_1_TX_WRAP,
};
rd: usize,
fn consume(&mut self) -> Option<Result<(), Error>> {
if self.control & Self::DESC_1_USED != 0 && self.address != 0 {
Some(Ok(()))
} else {
None
}
}
fn setup_tx(
&mut self,
buffer_address: BusAddress,
size: usize,
index: usize,
capacity: usize,
) -> Result<(), Error> {
let buffer_address = buffer_address.try_into_u32()?;
if !(8..8192).contains(&size) {
return Err(Error::InvalidArgument);
}
let mut control = size as u32 | Self::DESC_1_TX_LAST;
if index == capacity - 1 {
control |= Self::DESC_1_TX_WRAP;
}
self.address = buffer_address;
self.control = control;
Ok(())
}
}
struct TxQueue {
entries: DmaBuffer<[Descriptor]>,
buffers: Vec<Option<DmaBuffer<[u8]>>>,
impl GenericRxDescriptor for Descriptor {
fn consume(&mut self) -> Option<Result<usize, Error>> {
if self.address & Self::DESC_0_RX_OWNERSHIP != 0 {
Some(Ok((self.control & Self::LENGTH) as usize))
} else {
None
}
}
wr: usize,
rd: usize,
}
pub struct Queue {
rx_queue: IrqSafeSpinlock<RxQueue>,
rx_queue_base: BusAddress,
tx_queue: IrqSafeSpinlock<TxQueue>,
tx_queue_base: BusAddress,
fn setup_rx(buffer_address: BusAddress, _size: usize, last: bool) -> Result<Self, Error> {
let mut address = buffer_address.try_into_u32()?;
if last {
address |= Self::DESC_0_RX_WRAP;
}
Ok(Self {
control: 0,
address,
})
}
}
impl Descriptor {
const USED: u32 = 1 << 31;
const DESC_1_USED: u32 = 1 << 31;
const LENGTH: u32 = 0x1FFF;
const TX_WRAP: u32 = 1 << 30;
const TX_LAST: u32 = 1 << 15;
const DESC_1_TX_WRAP: u32 = 1 << 30;
const DESC_1_TX_LAST: u32 = 1 << 15;
const RX_OWNERSHIP: u32 = 1 << 0;
const RX_WRAP: u32 = 1 << 1;
const EMPTY_RX: Self = Self {
address: Self::RX_OWNERSHIP,
control: 0,
};
const EMPTY_TX: Self = Self {
address: 0,
control: Self::USED,
};
fn setup_rx(&mut self, buffer: BusAddress, wrap: bool) {
let mut address = buffer.try_into_u32().unwrap();
if wrap {
address |= Self::RX_WRAP;
}
self.address = address;
self.control = 0;
}
fn setup_tx(&mut self, buffer: BusAddress, len: usize, wrap: bool) {
assert!(len < Self::LENGTH as usize - 1);
let mut control = len as u32;
if wrap {
control |= Self::TX_WRAP;
}
// LAST
control |= Self::TX_LAST;
self.address = buffer.try_into_u32().unwrap();
self.control = control;
}
}
impl RxQueue {
fn with_capacity(dma: &dyn DmaAllocator, capacity: usize) -> Result<Self, Error> {
let mut entries = DmaBuffer::new_slice(dma, Descriptor::EMPTY_RX, capacity)?;
let buffers = (0..capacity)
.map(|_| DmaBuffer::new_uninit_slice(dma, RX_BUFFER_SIZE))
.collect::<Result<Vec<_>, _>>()?;
for i in 0..capacity {
entries[i].setup_rx(buffers[i].bus_address(), i == capacity - 1);
}
Ok(Self {
buffers,
entries,
rd: 0,
})
}
pub fn consume<F: Fn(DmaBuffer<[u8]>)>(
&mut self,
dma: &dyn DmaAllocator,
packet_handler: F,
) -> Result<usize, Error> {
let mut count = 0;
let capacity = self.entries.len();
loop {
let index = self.rd % self.entries.len();
self.entries.cache_flush_element(index, false);
let entry = &mut self.entries[index];
if entry.address & Descriptor::RX_OWNERSHIP != 0 {
// if entry.rx_completed().is_some() {
// Grab the current buffer (the one just written to by the DMA), replace it
// with the newly allocated one, and mark the descriptor as DMA-owned again
let new_buffer = DmaBuffer::new_uninit_slice(dma, 4096)?;
let new_buffer_address = new_buffer.bus_address();
let buffer = mem::replace(&mut self.buffers[index], new_buffer);
let buffer = unsafe { DmaBuffer::assume_init_slice(buffer) };
// TODO packet size hint
packet_handler(buffer);
entry.setup_rx(new_buffer_address, index == capacity - 1);
// entry.setup_rx(new_buffer_address, true)?;
self.rd = self.rd.wrapping_add(1);
count += 1;
} else {
break;
}
}
Ok(count)
}
}
impl TxQueue {
pub fn with_capacity(dma: &dyn DmaAllocator, capacity: usize) -> Result<Self, Error> {
let mut entries = DmaBuffer::new_slice(dma, Descriptor::EMPTY_TX, capacity)?;
entries[capacity - 1].control |= Descriptor::TX_WRAP;
let buffers = (0..capacity).map(|_| None).collect();
Ok(Self {
entries,
buffers,
wr: 0,
rd: 0,
})
}
pub fn can_xmit(&self) -> bool {
let capacity = self.entries.len();
self.wr.wrapping_add(1) % capacity != self.rd % capacity
}
pub fn push_xmit(&mut self, frame: DmaBuffer<[u8]>) -> Result<usize, Error> {
if !self.can_xmit() {
return Err(Error::WouldBlock);
}
let address = frame.bus_address();
let frame_len = frame.len();
if !(8..0x2000).contains(&frame_len) {
return Err(Error::InvalidArgument);
}
let capacity = self.entries.len();
let index = self.wr % capacity;
assert!(self.buffers[index].is_none());
frame.cache_flush_all(true);
self.entries[index].setup_tx(address, frame_len, index == capacity - 1);
// self.entries[index].setup_tx(address, frame_len, true)?;
self.entries.cache_flush_element(index, true);
self.buffers[index] = Some(frame);
self.wr = self.wr.wrapping_add(1);
Ok(self.wr % capacity)
}
pub fn consume(&mut self) -> Result<usize, Error> {
let mut count = 0;
loop {
let index = self.rd % self.entries.len();
self.entries.cache_flush_element(index, false);
if self.rd == self.wr {
break;
}
let entry = &self.entries[index];
if entry.control & Descriptor::USED != 0 {
let _ = self.buffers[index].take().unwrap();
self.rd = self.rd.wrapping_add(1);
count += 1;
} else {
break;
}
}
Ok(count)
}
}
impl Queue {
pub fn with_capacity(
dma: &dyn DmaAllocator,
tx_capacity: usize,
rx_capacity: usize,
) -> Result<Self, Error> {
let rx_queue = RxQueue::with_capacity(dma, rx_capacity)?;
let rx_queue_base = rx_queue.entries.bus_address();
let tx_queue = TxQueue::with_capacity(dma, tx_capacity)?;
let tx_queue_base = tx_queue.entries.bus_address();
Ok(Self {
rx_queue: IrqSafeSpinlock::new(rx_queue),
rx_queue_base,
tx_queue: IrqSafeSpinlock::new(tx_queue),
tx_queue_base,
})
}
pub fn rx_queue_base(&self) -> BusAddress {
self.rx_queue_base
}
pub fn tx_queue_base(&self) -> BusAddress {
self.tx_queue_base
}
pub fn push_xmit(&self, packet: DmaBuffer<[u8]>) -> Result<usize, Error> {
self.tx_queue.lock().push_xmit(packet)
}
pub fn consume_rx<F: Fn(DmaBuffer<[u8]>)>(
&self,
dma: &dyn DmaAllocator,
packet_handler: F,
) -> Result<usize, Error> {
self.rx_queue.lock().consume(dma, packet_handler)
}
pub fn consume_tx(&self) -> Result<usize, Error> {
self.tx_queue.lock().consume()
}
const DESC_0_RX_OWNERSHIP: u32 = 1 << 0;
const DESC_0_RX_WRAP: u32 = 1 << 1;
}
kernel/driver/net/core/Cargo.toml
+1
View File
@@ -8,6 +8,7 @@ yggdrasil-abi = { workspace = true, features = ["serde_kernel", "bytemuck"] }
libk-mm.workspace = true
libk-util.workspace = true
libk.workspace = true
device-api.workspace = true
kernel-fs = { path = "../../fs/kernel-fs" }
kernel/driver/net/core/src/util/mod.rs
+5
View File
@@ -0,0 +1,5 @@
pub use queue::*;
pub use reassembler::*;
mod queue;
mod reassembler;
kernel/driver/net/core/src/util/queue.rs
+291
View File
@@ -0,0 +1,291 @@
use core::mem::{self, MaybeUninit};
use alloc::vec::Vec;
use device_api::dma::DmaAllocator;
use libk::{
dma::{BusAddress, DmaBuffer},
error::Error,
};
use libk_util::{sync::IrqSafeSpinlock, waker::QueueWaker};
pub trait GenericTxDescriptor: Sized {
const EMPTY: Self;
const EMPTY_LAST: Self = Self::EMPTY;
fn consume(&mut self) -> Option<Result<(), Error>>;
fn setup_tx(
&mut self,
buffer_address: BusAddress,
size: usize,
index: usize,
capacity: usize,
) -> Result<(), Error>;
}
pub trait GenericRxDescriptor: Sized {
// TODO Rx flags (short, runt, etc)
fn consume(&mut self) -> Option<Result<usize, Error>>;
fn setup_rx(buffer_address: BusAddress, buffer_size: usize, last: bool) -> Result<Self, Error>;
}
struct TxInFlight {
#[allow(unused)]
buffer: DmaBuffer<[u8]>,
}
struct TxRingInner<T: GenericTxDescriptor> {
entries: DmaBuffer<[T]>,
in_flight: Vec<Option<TxInFlight>>,
wr: usize,
rd: usize,
}
struct RxRingInner<T: GenericRxDescriptor> {
entries: DmaBuffer<[T]>,
buffers: Vec<DmaBuffer<[MaybeUninit<u8>]>>,
rd: usize,
}
pub struct GenericTxRing<T: GenericTxDescriptor> {
inner: IrqSafeSpinlock<TxRingInner<T>>,
buffer_base: BusAddress,
capacity: usize,
free_notify: QueueWaker,
}
pub struct GenericRxRing<T: GenericRxDescriptor> {
inner: IrqSafeSpinlock<RxRingInner<T>>,
rx_buffer_size: usize,
buffer_base: BusAddress,
capacity: usize,
}
pub struct GenericQueue<Tx: GenericTxDescriptor, Rx: GenericRxDescriptor> {
pub tx_ring: GenericTxRing<Tx>,
pub rx_ring: GenericRxRing<Rx>,
}
impl<Tx: GenericTxDescriptor, Rx: GenericRxDescriptor> GenericQueue<Tx, Rx> {
pub fn with_capacity(
dma: &dyn DmaAllocator,
tx_capacity: usize,
rx_capacity: usize,
rx_buffer_size: usize,
) -> Result<Self, Error> {
let tx_ring = GenericTxRing::with_capacity(dma, tx_capacity)?;
let rx_ring = GenericRxRing::with_capacity(dma, rx_capacity, rx_buffer_size)?;
Ok(Self { tx_ring, rx_ring })
}
pub fn tx_buffer_base(&self) -> BusAddress {
self.tx_ring.buffer_base
}
pub fn rx_buffer_base(&self) -> BusAddress {
self.rx_ring.buffer_base
}
pub fn try_push_xmit(&self, frame: DmaBuffer<[u8]>) -> Result<usize, Error> {
self.tx_ring.try_push_xmit(frame)
}
pub fn drop_tx_until(&self, head: usize) {
self.tx_ring.drop_until(head);
}
pub fn consume_tx(&self) -> Result<usize, Error> {
self.tx_ring.consume()
}
pub fn consume_rx<F: Fn(DmaBuffer<[u8]>, usize)>(
&self,
dma: &dyn DmaAllocator,
head: Option<usize>,
handler: F,
) -> Result<usize, Error> {
self.rx_ring.consume(dma, head, handler)
}
}
impl<T: GenericTxDescriptor> GenericTxRing<T> {
pub fn with_capacity(dma: &dyn DmaAllocator, capacity: usize) -> Result<Self, Error> {
let entries = DmaBuffer::new_slice_with(
dma,
|i| {
if i == capacity - 1 {
T::EMPTY_LAST
} else {
T::EMPTY
}
},
capacity,
)?;
let in_flight = (0..capacity).map(|_| None).collect();
let buffer_base = entries.bus_address();
let inner = TxRingInner {
entries,
in_flight,
wr: 0,
rd: 0,
};
Ok(Self {
inner: IrqSafeSpinlock::new(inner),
buffer_base,
capacity,
free_notify: QueueWaker::new(),
})
}
pub fn base(&self) -> BusAddress {
self.buffer_base
}
pub fn capacity(&self) -> usize {
self.capacity
}
pub async fn push_xmit(&self, _frame: DmaBuffer<[u8]>) -> Result<usize, Error> {
todo!()
}
pub fn try_push_xmit(&self, frame: DmaBuffer<[u8]>) -> Result<usize, Error> {
let mut inner = self.inner.lock();
if inner.wr.wrapping_add(1) % self.capacity == inner.rd % self.capacity {
return Err(Error::WouldBlock);
}
let buffer_address = frame.bus_address();
let buffer_len = frame.len();
let index = inner.wr % self.capacity;
assert!(inner.in_flight[index].is_none());
frame.cache_flush_all(true);
inner.entries[index].setup_tx(buffer_address, buffer_len, index, self.capacity)?;
inner.entries.cache_flush_element(index, true);
inner.in_flight[index] = Some(TxInFlight { buffer: frame });
inner.wr = inner.wr.wrapping_add(1);
Ok(inner.wr % self.capacity)
}
pub fn drop_until(&self, head: usize) {
self.inner.lock().rd = head % self.capacity;
}
pub fn consume(&self) -> Result<usize, Error> {
let mut count = 0;
{
let mut inner = self.inner.lock();
while inner.rd != inner.wr {
let index = inner.rd % self.capacity;
inner.entries.cache_flush_element(index, false);
let entry = &mut inner.entries[index];
if let Some(_status) = entry.consume() {
let _ = inner.in_flight[index].take().unwrap();
inner.rd = inner.rd.wrapping_add(1);
count += 1;
} else {
break;
}
}
}
if count != 0 {
self.free_notify.wake_all();
}
Ok(count)
}
}
impl<T: GenericRxDescriptor> GenericRxRing<T> {
pub fn with_capacity(
dma: &dyn DmaAllocator,
capacity: usize,
rx_buffer_size: usize,
) -> Result<Self, Error> {
let buffers = (0..capacity)
.map(|_| DmaBuffer::new_uninit_slice(dma, rx_buffer_size))
.collect::<Result<Vec<_>, _>>()?;
let entries = DmaBuffer::new_slice_with(
dma,
|i| {
let buffer = buffers[i].bus_address();
let last = i == capacity - 1;
T::setup_rx(buffer, rx_buffer_size, last).expect("Rx buffer descriptor setup error")
},
capacity,
)?;
let buffer_base = entries.bus_address();
let inner = RxRingInner {
entries,
buffers,
rd: 0,
};
Ok(Self {
inner: IrqSafeSpinlock::new(inner),
rx_buffer_size,
buffer_base,
capacity,
})
}
pub fn base(&self) -> BusAddress {
self.buffer_base
}
pub fn capacity(&self) -> usize {
self.capacity
}
pub fn consume<F: Fn(DmaBuffer<[u8]>, usize)>(
&self,
dma: &dyn DmaAllocator,
head: Option<usize>,
handler: F,
) -> Result<usize, Error> {
let mut inner = self.inner.lock();
while head.map_or(true, |tail| tail != inner.rd) {
let index = inner.rd % self.capacity;
inner.entries.cache_flush_element(index, false);
if let Some(status) = inner.entries[index].consume() {
// Grab the current buffer (the one just written to by the DMA), replace it
// with the newly allocated one, and mark the descriptor as DMA-owned again
let new_buffer = DmaBuffer::new_uninit_slice(dma, self.rx_buffer_size)?;
let new_buffer_address = new_buffer.bus_address();
let buffer = mem::replace(&mut inner.buffers[index], new_buffer);
let old_buffer_address = buffer.bus_address();
let buffer = unsafe { DmaBuffer::assume_init_slice(buffer) };
match status {
Ok(len) => {
handler(buffer, len);
}
Err(_error) => {
log::warn!("Drop error packet {old_buffer_address:#x}");
}
}
inner.entries[index] = T::setup_rx(
new_buffer_address,
self.rx_buffer_size,
index == self.capacity - 1,
)?;
inner.rd = inner.rd.wrapping_add(1);
} else {
break;
}
}
Ok(inner.rd)
}
}
kernel/driver/net/core/src/util.rs → kernel/driver/net/core/src/util/reassembler.rs
View File
kernel/driver/net/igbe/Cargo.toml
+1
View File
@@ -14,4 +14,5 @@ ygg_driver_pci.path = "../../bus/pci"
ygg_driver_net_core.path = "../core"
log.workspace = true
futures-util.workspace = true
tock-registers.workspace = true
kernel/driver/net/igbe/src/lib.rs
+86 -64
View File
@@ -8,17 +8,16 @@ use device_api::{
dma::DmaAllocator,
interrupt::{InterruptAffinity, InterruptHandler, IrqVector},
};
use libk::{dma::DmaBuffer, error::Error};
use libk_util::{
OneTimeInit,
sync::{IrqSafeSpinlock, Spinlock},
};
use regs::{ICR, Regs};
use ring::{RxRing, TxRing};
use futures_util::task::AtomicWaker;
use libk::{dma::DmaBuffer, error::Error, task::runtime};
use libk_util::{OneTimeInit, event::BitmapEvent, sync::IrqSafeSpinlock};
use regs::Regs;
use tock_registers::{LocalRegisterCopy, fields::FieldValue};
use ygg_driver_net_core::{
RxPacket,
interface::{NetworkDevice, NetworkInterfaceType},
register_interface,
util::GenericQueue,
};
use ygg_driver_pci::{
PciBaseAddress, PciConfigurationSpace,
@@ -27,7 +26,10 @@ use ygg_driver_pci::{
};
use yggdrasil_abi::net::{MacAddress, link::LinkState};
use crate::regs::Revision;
use crate::{
regs::Revision,
ring::{RxDescriptor, TxDescriptor},
};
extern crate alloc;
@@ -41,8 +43,8 @@ struct Igbe {
pci: PciDeviceInfo,
mac: OneTimeInit<MacAddress>,
rx_ring: OneTimeInit<Spinlock<RxRing>>,
tx_ring: OneTimeInit<IrqSafeSpinlock<TxRing>>,
queue: OneTimeInit<GenericQueue<TxDescriptor, RxDescriptor>>,
softirq: BitmapEvent<AtomicWaker>,
nic: OneTimeInit<u32>,
}
@@ -55,11 +57,55 @@ impl Igbe {
mac: OneTimeInit::new(),
regs: IrqSafeSpinlock::new(regs),
rx_ring: OneTimeInit::new(),
tx_ring: OneTimeInit::new(),
queue: OneTimeInit::new(),
softirq: BitmapEvent::new(AtomicWaker::new()),
nic: OneTimeInit::new(),
}
}
async fn softirq(self: Arc<Self>) {
let queue = self.queue.get();
let rx_capacity = queue.rx_ring.capacity();
let nic = *self.nic.get();
loop {
let event = self.softirq.wait().await as u32;
let event = LocalRegisterCopy::<_, regs::ICR::Register>::new(event);
if event.matches_all(regs::ICR::LSC::SET) {
let status = self.regs.lock().read_link();
log::info!("igbe: link is {status}");
self.interrupt_handled(regs::ICR::LSC::SET);
}
if event.matches_any(&[regs::ICR::TXQE::SET, regs::ICR::TXDW::SET]) {
let head = self.regs.lock().tx_queue_head();
queue.drop_tx_until(head as usize);
// queue.consume_tx(Some(head as usize)).ok();
self.interrupt_handled(regs::ICR::TXQE::SET + regs::ICR::TXDW::SET);
}
if event.matches_any(&[regs::ICR::RXDMT0::SET, regs::ICR::RXT0::SET]) {
{
let mut regs = self.regs.lock();
let head = regs.rx_queue_head() as usize;
let tail = queue
.consume_rx(&*self.dma, Some(head), |packet, _| {
let packet = RxPacket::new(packet, 0, nic);
ygg_driver_net_core::receive_packet(packet).ok();
})
.expect("Rx ring handle error");
let tail = (tail + rx_capacity - 1) & (rx_capacity - 1);
regs.set_rx_queue_tail(tail as u16);
}
self.interrupt_handled(regs::ICR::RXDMT0::SET + regs::ICR::RXT0::SET);
}
}
}
fn interrupt_handled(&self, icr: FieldValue<u32, regs::ICR::Register>) {
let mut regs = self.regs.lock();
regs.clear_interrupts(icr.value);
regs.enable_interrupts(icr.value);
}
}
impl Device for Igbe {
@@ -68,33 +114,44 @@ impl Device for Igbe {
.pci
.map_interrupt(InterruptAffinity::Any, self.clone())?;
let rx_ring = RxRing::with_capacity(&*self.dma, 128, 2048 + 16)?;
let tx_ring = TxRing::with_capacity(&*self.dma, 128)?;
let queue = self.queue.init(GenericQueue::with_capacity(
&*self.dma,
128,
128,
2048 + 16,
)?);
let mut regs = self.regs.lock();
regs.disable_interrupts();
regs.disable_interrupts(0xFFFFFFFF);
let mac = regs.read_mac()?;
self.mac.init(mac);
regs.reset(Duration::from_millis(200))?;
// Intel 8257x manuals say an additional interrupt disable is needed after a global reset
regs.disable_interrupts();
regs.disable_interrupts(0xFFFFFFFF);
regs.set_link_up(self.chip)?;
// Initialize Rx
regs.initialize_receiver(&rx_ring);
regs.initialize_transmitter(&tx_ring);
regs.initialize_receiver(&queue.rx_ring);
regs.initialize_transmitter(&queue.tx_ring);
// If MSI(-x) was initialized, notify the NIC about it
if let Some(msi_info) = msi_info {
regs.initialize_ivar(msi_info.vector);
}
self.rx_ring.init(Spinlock::new(rx_ring));
self.tx_ring.init(IrqSafeSpinlock::new(tx_ring));
let nic = register_interface(NetworkInterfaceType::Ethernet, self.clone());
self.nic.init(nic.id());
regs.enable_interrupts();
regs.enable_interrupts(
(regs::IMS::TXDW::SET
+ regs::IMS::TXQE::SET
+ regs::IMS::LSC::SET
+ regs::IMS::RXT0::SET
+ regs::IMS::RXDMT0::SET)
.value,
);
runtime::spawn(self.clone().softirq())?;
Ok(())
}
@@ -107,43 +164,13 @@ impl Device for Igbe {
impl InterruptHandler for Igbe {
fn handle_irq(self: Arc<Self>, _vector: IrqVector) -> bool {
let mut regs = self.regs.lock();
let cause = regs.interrupt_cause();
if cause.get() == 0 {
let cause = regs.interrupt_cause().get();
if cause == 0 {
return false;
}
regs.clear_interrupts(cause.get());
let mut any = false;
if cause.matches_all(ICR::LSC::SET) {
let status = regs.read_link();
log::info!("igbe: link is {status}");
any = true;
}
if cause.matches_all(ICR::RXT0::SET) {
let mut rx = self.rx_ring.get().lock();
let nic = *self.nic.get();
let head = regs.rx_queue_head();
let tail = rx.handle_rx(&*self.dma, head, |packet, _| {
let packet = RxPacket::new(packet, 0, nic);
ygg_driver_net_core::receive_packet(packet).ok();
});
regs.set_rx_queue_tail(tail);
any = true;
}
if cause.matches_any(&[ICR::TXQE::SET, ICR::TXDW::SET]) {
let mut tx = self.tx_ring.get().lock();
let head = regs.tx_queue_head();
tx.handle_tx(head);
any = true;
}
if !any {
log::info!("igbe: unhandled irq {:#x}", cause.get());
}
any
regs.disable_interrupts(cause);
self.softirq.signal(cause as u64);
true
}
}
@@ -161,15 +188,10 @@ impl NetworkDevice for Igbe {
}
fn transmit_buffer(&self, buffer: DmaBuffer<[u8]>) -> Result<(), Error> {
let mut tx = self.tx_ring.get().lock();
let Ok(head) = tx.tx_now(buffer) else {
return Err(Error::WouldBlock);
};
let queue = self.queue.get();
let tail = queue.try_push_xmit(buffer)?;
let mut regs = self.regs.lock();
regs.set_tx_queue_tail(head);
regs.set_tx_queue_tail(tail as _);
Ok(())
}
kernel/driver/net/igbe/src/regs.rs
+41 -23
View File
@@ -7,14 +7,17 @@ use libk::{
};
use libk_mm::{address::PhysicalAddress, device::RawDeviceMemoryMapping};
use tock_registers::{LocalRegisterCopy, RegisterLongName, fields::FieldValue, register_bitfields};
use ygg_driver_net_core::ephy::{GBESR, MdioBus, PhyAccess};
use ygg_driver_net_core::{
ephy::{GBESR, MdioBus, PhyAccess},
util::{GenericRxRing, GenericTxRing},
};
use ygg_driver_pci::PciBaseAddress;
use yggdrasil_abi::net::{
MacAddress,
link::{Duplex, EthernetLinkState, EthernetSpeed},
};
use crate::{RxRing, TxRing};
use crate::ring::{RxDescriptor, TxDescriptor};
enum Inner {
Memory(RawDeviceMemoryMapping),
@@ -79,12 +82,14 @@ register_bitfields! {
TXDW OFFSET(0) NUMBITS(1) [],
TXQE OFFSET(1) NUMBITS(1) [],
LSC OFFSET(2) NUMBITS(1) [],
RXDMT0 OFFSET(4) NUMBITS(1) [],
RXT0 OFFSET(7) NUMBITS(1) [],
],
pub IMS [
TXDW OFFSET(0) NUMBITS(1) [],
TXQE OFFSET(1) NUMBITS(1) [],
LSC OFFSET(2) NUMBITS(1) [],
RXDMT0 OFFSET(4) NUMBITS(1) [],
RXT0 OFFSET(7) NUMBITS(1) [],
],
pub RCTL [
@@ -459,36 +464,47 @@ impl Regs {
}
}
pub fn disable_interrupts(&mut self) {
self.inner.set::<IMC::Register>(0xFFFFFFFF);
}
pub fn clear_interrupts(&mut self, cause: u32) {
let _ = self.inner.get::<CPUVEC::Register>();
self.inner.set::<ICR::Register>(cause);
}
pub fn enable_interrupts(&mut self) {
self.inner.set::<IMS::Register>(0xFFFFFFFF);
// self.inner
// .modify(IMS::LSC::SET + IMS::RXT0::SET + IMS::TXDW::SET + IMS::TXQE::SET);
}
// pub fn disable_interrupts(&mut self) {
// self.inner.set::<IMC::Register>(0xFFFFFFFF);
// }
pub fn interrupt_cause(&mut self) -> LocalRegisterCopy<u32, ICR::Register> {
self.inner.extract()
}
pub fn initialize_receiver(&mut self, rx_ring: &RxRing) {
let rx_queue_base = rx_ring.descriptors.bus_address().into_u64();
pub fn clear_interrupts(&mut self, icr: u32) {
let _ = self.inner.get::<CPUVEC::Register>();
self.inner.set::<ICR::Register>(icr);
}
pub fn enable_interrupts(&mut self, mask: u32) {
self.inner.set::<IMS::Register>(mask);
}
pub fn disable_interrupts(&mut self, mask: u32) {
self.inner.set::<IMC::Register>(mask);
}
// pub fn enable_interrupts(&mut self) {
// self.inner.set::<IMS::Register>(0xFFFFFFFF);
// // self.inner
// // .modify(IMS::LSC::SET + IMS::RXT0::SET + IMS::TXDW::SET + IMS::TXQE::SET);
// }
pub fn initialize_receiver(&mut self, rx_ring: &GenericRxRing<RxDescriptor>) {
let rx_queue_base = rx_ring.base().into_u64();
let rx_queue_capacity = rx_ring.capacity();
// let rx_queue_base = rx_ring.descriptors.bus_address().into_u64();
self.inner.set::<RDBAL0::Register>(rx_queue_base as u32);
self.inner
.set::<RDBAH0::Register>((rx_queue_base >> 32) as u32);
self.inner
.write(RDLEN0::LEN0.val((rx_ring.descriptors.len() / 8) as u32));
.write(RDLEN0::LEN0.val((rx_queue_capacity / 8) as u32));
self.inner.set::<RDH0::Register>(0);
self.inner
.set::<RDT0::Register>((rx_ring.descriptors.len() - 1) as u32);
.set::<RDT0::Register>((rx_queue_capacity - 1) as u32);
self.inner.write(
RCTL::EN::SET
@@ -499,14 +515,16 @@ impl Regs {
);
}
pub fn initialize_transmitter(&mut self, tx_ring: &TxRing) {
let tx_queue_base = tx_ring.descriptors.bus_address().into_u64();
pub fn initialize_transmitter(&mut self, tx_ring: &GenericTxRing<TxDescriptor>) {
let tx_queue_base = tx_ring.base().into_u64();
let tx_queue_capacity = tx_ring.capacity();
self.inner.set::<TDBAL::Register>(tx_queue_base as u32);
self.inner
.set::<TDBAH::Register>((tx_queue_base >> 32) as u32);
self.inner
.write(TDLEN::LEN.val((tx_ring.descriptors.len() / 8) as u32));
.write(TDLEN::LEN.val((tx_queue_capacity / 8) as u32));
self.inner.set::<TDT::Register>(0);
self.inner.set::<TDH::Register>(0);
kernel/driver/net/igbe/src/ring.rs
+64 -151
View File
@@ -1,31 +1,10 @@
use core::mem::{self, MaybeUninit};
use alloc::vec::Vec;
use device_api::dma::DmaAllocator;
use libk::{
dma::{BusAddress, DmaBuffer},
error::Error,
};
pub(crate) struct RxRing {
pub(crate) descriptors: DmaBuffer<[RxDescriptor]>,
buffers: Vec<DmaBuffer<[MaybeUninit<u8>]>>,
buffer_size: usize,
tail: u16,
}
pub(crate) struct TxRing {
pub(crate) descriptors: DmaBuffer<[TxDescriptor]>,
buffers: Vec<Option<DmaBuffer<[u8]>>>,
// Consumer end
tail: u16,
// Producer end
head: u16,
}
use libk::{dma::BusAddress, error::Error};
use ygg_driver_net_core::util::{GenericRxDescriptor, GenericTxDescriptor};
#[derive(Debug)]
#[repr(C)]
pub(crate) struct RxDescriptor {
address: BusAddress,
address: u64,
length: u16,
checksum: u16,
status: u8,
@@ -33,9 +12,10 @@ pub(crate) struct RxDescriptor {
special: u16,
}
#[derive(Debug)]
#[repr(C)]
pub(crate) struct TxDescriptor {
address: BusAddress,
address: u64,
length: u16,
cso: u8,
cmd: u8,
@@ -45,117 +25,73 @@ pub(crate) struct TxDescriptor {
special: u8,
}
impl RxRing {
pub fn with_capacity(
dma: &dyn DmaAllocator,
capacity: usize,
buffer_size: usize,
) -> Result<Self, Error> {
let buffers = (0..capacity)
.map(|_| DmaBuffer::new_uninit_slice(dma, buffer_size))
.collect::<Result<Vec<_>, _>>()?;
let descriptors = DmaBuffer::new_slice_with(
dma,
|i| RxDescriptor::new(buffers[i].bus_address(), buffer_size as u16),
capacity,
)?;
impl GenericTxDescriptor for TxDescriptor {
const EMPTY: Self = Self {
address: 0,
length: 0,
cso: 0,
cmd: 0,
sta: Self::STA_DD,
_0: 0,
css: 0,
special: 0,
};
Ok(Self {
descriptors,
buffers,
tail: 0,
buffer_size,
})
fn consume(&mut self) -> Option<Result<(), Error>> {
unreachable!()
}
// TODO move to background task/softirq to reduce amount of code run by the irq handler
pub fn handle_rx<F: FnMut(DmaBuffer<[u8]>, usize)>(
fn setup_tx(
&mut self,
dma: &dyn DmaAllocator,
head: u16,
mut handler: F,
) -> u16 {
let capacity = self.descriptors.len();
while self.tail != head {
let index = self.tail as usize;
// Replace the buffer
let new_buffer = DmaBuffer::new_uninit_slice(dma, self.buffer_size).unwrap();
let new_buffer_address = new_buffer.bus_address();
let buffer = mem::replace(&mut self.buffers[index], new_buffer);
let buffer = unsafe { DmaBuffer::assume_init_slice(buffer) };
let descriptor = &mut self.descriptors[index];
if descriptor.errors & !1 != 0 {
log::warn!("igbe: drop erroneous packet {:#x}", descriptor.errors);
} else {
let len = descriptor.length as usize;
handler(buffer, len);
}
// Replace the descriptor
*descriptor = RxDescriptor::new(new_buffer_address, self.buffer_size as u16);
self.tail = (self.tail + 1) & (capacity as u16 - 1);
buffer_address: BusAddress,
size: usize,
index: usize,
capacity: usize,
) -> Result<(), Error> {
if !(4..16288).contains(&size) {
return Err(Error::InvalidArgument);
}
(self.tail + capacity as u16 - 1) & (capacity as u16 - 1)
}
}
impl TxRing {
pub fn with_capacity(dma: &dyn DmaAllocator, capacity: usize) -> Result<Self, Error> {
let buffers = (0..capacity).map(|_| None).collect::<Vec<_>>();
let descriptors = DmaBuffer::new_slice_with(dma, |_| TxDescriptor::empty(), capacity)?;
Ok(Self {
descriptors,
buffers,
head: 0,
tail: 0,
})
}
pub fn handle_tx(&mut self, head: u16) {
self.tail = head;
}
pub fn tx_now(&mut self, buffer: DmaBuffer<[u8]>) -> Result<u16, DmaBuffer<[u8]>> {
// Queue full
let capacity = self.descriptors.len();
if (self.head + 1) & (capacity as u16 - 1) == self.tail {
log::warn!("igbe: tx queue full");
return Err(buffer);
}
let index = self.head as usize;
let descriptor = &mut self.descriptors[index];
// Only generate interrupts for every 1/4th of the buffer
let quarter = capacity / 4;
descriptor.setup_tx(
buffer.bus_address(),
buffer.len() as u16 - 4,
index % quarter == quarter - 1,
);
self.descriptors.cache_flush_element(index, true);
self.buffers[index] = Some(buffer);
self.head = (self.head + 1) & (capacity as u16 - 1);
Ok(self.head)
let ioc = index % quarter == quarter - 1;
let mut cmd = Self::CMD_EOP | Self::CMD_IFCS;
if ioc {
cmd |= Self::CMD_RS;
}
self.address = buffer_address.into_u64();
self.length = size as u16 - 4;
self.css = 0;
self.cso = 0;
self.sta = 0;
self.cmd = cmd;
self.special = 0;
Ok(())
}
}
impl RxDescriptor {
pub fn new(address: BusAddress, length: u16) -> Self {
Self {
address,
length,
impl GenericRxDescriptor for RxDescriptor {
fn consume(&mut self) -> Option<Result<usize, Error>> {
if self.status & Self::STA_DD == 0 {
None
} else if self.errors & !1 != 0 {
Some(Err(Error::InvalidArgument))
} else {
Some(Ok(self.length as _))
}
}
fn setup_rx(
buffer_address: BusAddress,
buffer_size: usize,
_last: bool,
) -> Result<Self, Error> {
Ok(Self {
address: buffer_address.into_u64(),
length: buffer_size as _,
checksum: 0,
status: 0,
errors: 0,
special: 0,
}
})
}
}
@@ -170,31 +106,8 @@ impl TxDescriptor {
// const CMD_IC: u8 = 1 << 2;
// Report status
const CMD_RS: u8 = 1 << 3;
pub const fn empty() -> Self {
Self {
address: BusAddress::ZERO,
length: 0,
cso: 0,
cmd: 0,
sta: Self::STA_DD,
_0: 0,
css: 0,
special: 0,
}
}
pub fn setup_tx(&mut self, address: BusAddress, length: u16, ioc: bool) {
let mut cmd = Self::CMD_EOP | Self::CMD_IFCS;
if ioc {
cmd |= Self::CMD_RS;
}
self.address = address;
self.length = length;
self.css = 0;
self.cso = 0;
self.sta = 0;
self.cmd = cmd;
self.special = 0;
}
}
impl RxDescriptor {
const STA_DD: u8 = 1 << 0;
}