yggdrasil/kernel/src/arch/aarch64/gic/mod.rs

//! ARM Generic Interrupt Controller v2 driver

use core::sync::atomic::Ordering;

use aarch64_cpu::asm::barrier;
use abi::error::Error;
use alloc::sync::Arc;
use device_api::{
    device::{Device, DeviceInitContext},
    interrupt::{
        ExternalInterruptController, FixedInterruptTable, FullIrq, InterruptHandler,
        InterruptTable, IpiDeliveryTarget, IpiMessage, Irq, IrqLevel, IrqOptions, IrqTrigger,
        IrqVector, LocalInterruptController,
    },
};
use device_tree::{
    driver::{device_tree_driver, DeviceTreeInterruptController, Node, ProbeContext},
    DeviceTreePropertyRead, TProp,
};
use kernel_arch_aarch64::{GicInterface, CPU_COUNT};
use libk::{arch::Cpu, device::register_external_interrupt_controller, task::cpu_index};
use libk_mm::{
    address::PhysicalAddress,
    device::{DeviceMemoryIo, RawDeviceMemoryMapping},
};
use libk_util::{sync::spin_rwlock::IrqSafeRwLock, OneTimeInit};

use self::{gicc::Gicc, gicd::Gicd};

use super::AArch64;

const MAX_IRQ: usize = 300;
const IPI_VECTOR: u64 = 1;

const GIC_PPI_START: u32 = 16;
const GIC_SPI_START: u32 = 32;

pub mod gicc;
pub mod gicd;
pub mod gicv2m;

/// ARM Generic Interrupt Controller v2
pub struct Gic {
    gicc: OneTimeInit<Gicc>,
    gicd: OneTimeInit<Gicd>,
    gicd_base: PhysicalAddress,
    gicc_base: PhysicalAddress,
    table: IrqSafeRwLock<FixedInterruptTable<MAX_IRQ>>,
}

/// Per-CPU GIC information
pub struct GicPerCpu {}

impl GicInterface for Gic {}

impl Device for Gic {
    fn display_name(&self) -> &str {
        "ARM Generic Interrupt Controller v2"
    }

    unsafe fn init(self: Arc<Self>, _cx: DeviceInitContext) -> Result<(), Error> {
        log::debug!(
            "Init GIC: gicd={:#x}, gicc={:#x}",
            self.gicd_base,
            self.gicc_base
        );
        let gicd_mmio = Arc::new(RawDeviceMemoryMapping::map(
            self.gicd_base.into_u64(),
            0x1000,
            Default::default(),
        )?);
        let gicd_mmio_shared = DeviceMemoryIo::from_raw(gicd_mmio.clone())?;
        let gicd_mmio_banked = DeviceMemoryIo::from_raw(gicd_mmio)?;
        let gicc_mmio = DeviceMemoryIo::map(self.gicc_base, Default::default())?;

        let gicd = Gicd::new(gicd_mmio_shared, gicd_mmio_banked);
        let gicc = Gicc::new(gicc_mmio);

        gicd.init();
        gicc.init();

        self.gicd.init(gicd);
        self.gicc.init(gicc);

        register_external_interrupt_controller(self.clone());
        AArch64::set_gic(self.clone());

        Ok(())
    }
}

impl ExternalInterruptController for Gic {
    fn register_irq(
        &self,
        irq: Irq,
        options: IrqOptions,
        handler: Arc<dyn InterruptHandler>,
    ) -> Result<(), Error> {
        let mut table = self.table.write();
        let gicd = self.gicd.get();

        let index = match irq {
            Irq::External(i) => i + GIC_SPI_START,
            Irq::Private(i) => i + GIC_PPI_START,
        } as usize;

        log::debug!(
            "Bound irq{} to {:?} {:?} {:?}",
            index,
            handler.display_name(),
            options.trigger,
            options.level
        );
        if index >= GIC_SPI_START as usize {
            gicd.configure_irq(index, options);
        }
        table.insert(index, handler)?;

        Ok(())
    }

    fn enable_irq(&self, irq: Irq) -> Result<(), Error> {
        let gicd = self.gicd.get();
        let index = match irq {
            Irq::External(i) => i + GIC_SPI_START,
            Irq::Private(i) => i + GIC_PPI_START,
        } as usize;
        log::debug!("Enable irq{index} ({irq:?})");
        gicd.enable_irq(index);
        Ok(())
    }

    fn handle_pending_irqs(&self) {
        let gicc = self.gicc.get();

        let irq_number = gicc.pending_irq_number();
        if irq_number >= MAX_IRQ {
            return;
        }

        gicc.clear_irq(irq_number);

        if irq_number == IPI_VECTOR as usize {
            // TODO pop entry
            crate::panic::panic_secondary();
        }

        if irq_number < GIC_PPI_START as usize {
            return;
        }

        let irq = if irq_number < GIC_SPI_START as usize {
            Irq::Private(irq_number as u32 - GIC_PPI_START)
        } else {
            Irq::External(irq_number as u32 - GIC_SPI_START)
        };

        {
            let table = self.table.read();
            let entry = match table.handler(irq_number) {
                Some(handler) => handler.clone(),
                None => {
                    log::warn!("No handler for irq{}", irq_number);
                    return;
                }
            };

            entry.handle_irq(IrqVector::Irq(irq));
        }
    }
}

impl LocalInterruptController for Gic {
    fn send_ipi(&self, target: IpiDeliveryTarget, msg: IpiMessage) -> Result<(), Error> {
        let local = cpu_index();
        let mask = match target {
            IpiDeliveryTarget::OtherCpus => usize::MAX & !(1 << local),
            IpiDeliveryTarget::Specific(mask) => mask,
            IpiDeliveryTarget::ThisCpu => 1 << local,
        };

        for i in 0..CPU_COUNT.load(Ordering::Acquire) {
            if mask & (1 << i) != 0 {
                Cpu::push_ipi_queue(i as _, msg);
            }
        }

        // Issue a memory barrier
        barrier::dsb(barrier::ISH);
        barrier::isb(barrier::SY);

        unsafe {
            self.gicd.get().set_sgir(target, IPI_VECTOR);
        }

        Ok(())
    }

    unsafe fn init_ap(&self) -> Result<(), Error> {
        self.gicc.get().init();
        Ok(())
    }
}

impl DeviceTreeInterruptController for Gic {
    fn map_interrupt(&self, property: &TProp, offset: usize) -> Option<FullIrq> {
        // IRQ_TYPE_NONE - 0
        // IRQ_TYPE_EDGE_RISING - 1
        // IRQ_TYPE_EDGE_FALLING - 2
        // IRQ_TYPE_EDGE_BOTH - 3
        // IRQ_TYPE_LEVEL_HIGH - 4
        // IRQ_TYPE_LEVEL_LOW - 8

        let sizes = (1, 1, 1);
        let (kind, number, options) = property.read_cells_at(offset, sizes)?;
        let number = number as u32;

        let (trigger, level) = match options & 0xF {
            0x01 => (IrqTrigger::Edge, IrqLevel::ActiveHigh),
            0x02 => (IrqTrigger::Edge, IrqLevel::ActiveLow),
            0x03 => return None, // TODO
            0x04 => (IrqTrigger::Level, IrqLevel::ActiveHigh),
            0x08 => (IrqTrigger::Level, IrqLevel::ActiveLow),
            _ => return None,
        };
        let irq = match kind {
            0x00 => Irq::External(number),
            0x01 => Irq::Private(number),
            _ => return None,
        };

        Some(FullIrq {
            irq,
            options: IrqOptions { trigger, level },
        })
    }

    fn as_interrupt_controller(self: Arc<Self>) -> Arc<dyn ExternalInterruptController> {
        self
    }
}

impl Gic {
    /// Constructs an instance of GICv2.
    ///
    /// # Safety
    ///
    /// The caller must ensure the addresses actually point to the GIC components.
    pub unsafe fn new(gicd_base: PhysicalAddress, gicc_base: PhysicalAddress) -> Self {
        Self {
            gicc: OneTimeInit::new(),
            gicd: OneTimeInit::new(),
            gicd_base,
            gicc_base,
            table: IrqSafeRwLock::new(FixedInterruptTable::new()),
        }
    }
}

impl GicPerCpu {
    /// Constructs per-CPU GIC data structure
    pub fn new() -> Self {
        Self {}
    }
}

device_tree_driver! {
    compatible: ["arm,cortex-a15-gic", "arm,gic-400"],
    driver: {
        fn probe(&self, node: &Arc<Node>, context: &ProbeContext) -> Option<Arc<dyn Device>> {
            let gicd_range = node.map_range(context, 0)?;
            let gicc_range = node.map_range(context, 1)?;

            let gicd_base = PhysicalAddress::from_u64(gicd_range.start);
            let gicc_base = PhysicalAddress::from_u64(gicc_range.start);

            let gic = Arc::new(unsafe { Gic::new(gicd_base, gicc_base) });

            // Register device-tree interrupt controller
            node.make_interrupt_controller(gic.clone());

            Some(gic)
        }
    }
}