diff --git a/kernel/src/arch/aarch64/irq/gic/gicd.rs b/kernel/src/arch/aarch64/irq/gic/gicd.rs index 389bf4e..af34da0 100644 --- a/kernel/src/arch/aarch64/irq/gic/gicd.rs +++ b/kernel/src/arch/aarch64/irq/gic/gicd.rs @@ -1,5 +1,5 @@ use crate::mem::virt::DeviceMemoryIo; -use crate::sync::IrqSafeNullLock; +use crate::sync::IrqSafeSpinLock; use tock_registers::interfaces::{Readable, Writeable}; use tock_registers::registers::{ReadOnly, ReadWrite}; use tock_registers::{register_bitfields, register_structs}; @@ -63,7 +63,7 @@ impl GicdSharedRegs { } pub(super) struct Gicd { - shared_regs: IrqSafeNullLock>, + shared_regs: IrqSafeSpinLock>, banked_regs: DeviceMemoryIo, } @@ -73,7 +73,7 @@ impl Gicd { banked_mmio: DeviceMemoryIo, ) -> Self { Self { - shared_regs: IrqSafeNullLock::new(shared_mmio), + shared_regs: IrqSafeSpinLock::new(shared_mmio), banked_regs: banked_mmio, } } diff --git a/kernel/src/arch/aarch64/irq/gic/mod.rs b/kernel/src/arch/aarch64/irq/gic/mod.rs index e81175e..aa70f78 100644 --- a/kernel/src/arch/aarch64/irq/gic/mod.rs +++ b/kernel/src/arch/aarch64/irq/gic/mod.rs @@ -5,7 +5,7 @@ use crate::dev::{ Device, }; use crate::mem::virt::{DeviceMemory, DeviceMemoryIo}; -use crate::sync::IrqSafeNullLock; +use crate::sync::IrqSafeSpinLock; use crate::util::InitOnce; use error::Errno; @@ -29,7 +29,7 @@ pub struct Gic { gicd_base: usize, gicc_base: usize, scheduler_irq: IrqNumber, - table: IrqSafeNullLock<[Option<&'static (dyn IntSource + Sync)>; MAX_IRQ]>, + table: IrqSafeSpinLock<[Option<&'static (dyn IntSource + Sync)>; MAX_IRQ]>, } impl IrqNumber { @@ -94,7 +94,10 @@ impl IntController for Gic { let table = self.table.lock(); match table[irq_number] { None => panic!("No handler registered for irq{}", irq_number), - Some(handler) => handler.handle_irq().expect("irq handler failed"), + Some(handler) => { + drop(table); + handler.handle_irq().expect("irq handler failed") + }, } } @@ -132,7 +135,7 @@ impl Gic { gicd_base, gicc_base, scheduler_irq, - table: IrqSafeNullLock::new([None; MAX_IRQ]), + table: IrqSafeSpinLock::new([None; MAX_IRQ]), } } } diff --git a/kernel/src/arch/aarch64/mach_orangepi3/gpio.rs b/kernel/src/arch/aarch64/mach_orangepi3/gpio.rs index 048ffb1..06db644 100644 --- a/kernel/src/arch/aarch64/mach_orangepi3/gpio.rs +++ b/kernel/src/arch/aarch64/mach_orangepi3/gpio.rs @@ -10,7 +10,7 @@ use crate::dev::{ Device, }; use crate::mem::virt::DeviceMemoryIo; -use crate::sync::IrqSafeNullLock; +use crate::sync::IrqSafeSpinLock; use crate::util::InitOnce; use error::Errno; use tock_registers::interfaces::{Readable, Writeable}; @@ -33,7 +33,7 @@ struct CpuxGpio { } pub struct Gpio { - cpux: InitOnce>, + cpux: InitOnce>, cpux_base: usize, } @@ -144,7 +144,7 @@ impl Device for Gpio { } unsafe fn enable(&self) -> Result<(), Errno> { - self.cpux.init(IrqSafeNullLock::new(CpuxGpio { + self.cpux.init(IrqSafeSpinLock::new(CpuxGpio { regs: DeviceMemoryIo::map(self.name(), self.cpux_base, 1)?, })); Ok(()) diff --git a/kernel/src/arch/aarch64/mach_orangepi3/rtc.rs b/kernel/src/arch/aarch64/mach_orangepi3/rtc.rs index 2ac5d54..0f75b49 100644 --- a/kernel/src/arch/aarch64/mach_orangepi3/rtc.rs +++ b/kernel/src/arch/aarch64/mach_orangepi3/rtc.rs @@ -5,7 +5,7 @@ use crate::dev::{ Device, }; use crate::mem::virt::DeviceMemoryIo; -use crate::sync::IrqSafeNullLock; +use crate::sync::IrqSafeSpinLock; use crate::util::InitOnce; use error::Errno; use tock_registers::{ @@ -47,7 +47,7 @@ register_structs! { } pub struct Rtc { - regs: InitOnce>>, + regs: InitOnce>>, base: usize, irq: IrqNumber, } @@ -90,7 +90,7 @@ impl Device for Rtc { } unsafe fn enable(&self) -> Result<(), Errno> { - self.regs.init(IrqSafeNullLock::new(DeviceMemoryIo::map( + self.regs.init(IrqSafeSpinLock::new(DeviceMemoryIo::map( self.name(), self.base, 1, diff --git a/kernel/src/arch/aarch64/mach_orangepi3/uart.rs b/kernel/src/arch/aarch64/mach_orangepi3/uart.rs index df8dff6..a2641a2 100644 --- a/kernel/src/arch/aarch64/mach_orangepi3/uart.rs +++ b/kernel/src/arch/aarch64/mach_orangepi3/uart.rs @@ -5,7 +5,7 @@ use crate::dev::{ Device, }; use crate::mem::virt::DeviceMemoryIo; -use crate::sync::IrqSafeNullLock; +use crate::sync::IrqSafeSpinLock; use crate::util::InitOnce; use error::Errno; use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; @@ -77,7 +77,7 @@ struct UartInner { } pub(super) struct Uart { - inner: InitOnce>, + inner: InitOnce>, base: usize, irq: IrqNumber, } @@ -92,7 +92,7 @@ impl Device for Uart { regs: DeviceMemoryIo::map(self.name(), self.base, 1)?, }; // TODO - self.inner.init(IrqSafeNullLock::new(inner)); + self.inner.init(IrqSafeSpinLock::new(inner)); Ok(()) } } diff --git a/kernel/src/arch/aarch64/mach_orangepi3/wdog.rs b/kernel/src/arch/aarch64/mach_orangepi3/wdog.rs index f759205..3bca2dd 100644 --- a/kernel/src/arch/aarch64/mach_orangepi3/wdog.rs +++ b/kernel/src/arch/aarch64/mach_orangepi3/wdog.rs @@ -1,6 +1,6 @@ use crate::dev::Device; use crate::mem::virt::DeviceMemoryIo; -use crate::sync::IrqSafeNullLock; +use crate::sync::IrqSafeSpinLock; use crate::util::InitOnce; use error::Errno; use tock_registers::{ @@ -39,7 +39,7 @@ register_structs! { } pub(super) struct RWdog { - inner: InitOnce>>, + inner: InitOnce>>, base: usize, } @@ -49,7 +49,7 @@ impl Device for RWdog { } unsafe fn enable(&self) -> Result<(), Errno> { - self.inner.init(IrqSafeNullLock::new(DeviceMemoryIo::map( + self.inner.init(IrqSafeSpinLock::new(DeviceMemoryIo::map( self.name(), self.base, 1, diff --git a/kernel/src/arch/aarch64/mach_rpi3/uart.rs b/kernel/src/arch/aarch64/mach_rpi3/uart.rs index f7e059e..0ab4312 100644 --- a/kernel/src/arch/aarch64/mach_rpi3/uart.rs +++ b/kernel/src/arch/aarch64/mach_rpi3/uart.rs @@ -1,6 +1,6 @@ use crate::dev::{serial::SerialDevice, Device}; use crate::mem::virt::DeviceMemoryIo; -use crate::sync::IrqSafeNullLock; +use crate::sync::IrqSafeSpinLock; use crate::util::InitOnce; use error::Errno; use tock_registers::{ @@ -17,7 +17,7 @@ register_structs! { } pub(super) struct Bcm283xMiniUart { - regs: InitOnce>>, + regs: InitOnce>>, base: usize, } @@ -27,7 +27,7 @@ impl Device for Bcm283xMiniUart { } unsafe fn enable(&self) -> Result<(), Errno> { - self.regs.init(IrqSafeNullLock::new(DeviceMemoryIo::map( + self.regs.init(IrqSafeSpinLock::new(DeviceMemoryIo::map( self.name(), self.base, 1, diff --git a/kernel/src/dev/rtc/pl031.rs b/kernel/src/dev/rtc/pl031.rs index ad025c0..cf6e6b6 100644 --- a/kernel/src/dev/rtc/pl031.rs +++ b/kernel/src/dev/rtc/pl031.rs @@ -6,7 +6,7 @@ use crate::dev::{ Device, }; use crate::mem::virt::DeviceMemoryIo; -use crate::sync::IrqSafeNullLock; +use crate::sync::IrqSafeSpinLock; use crate::util::InitOnce; use error::Errno; use tock_registers::{ @@ -49,7 +49,7 @@ struct Pl031Inner { /// Device struct for PL031 pub struct Pl031 { - inner: InitOnce>, + inner: InitOnce>, base: usize, irq: IrqNumber, } @@ -88,7 +88,7 @@ impl Device for Pl031 { inner.regs.MR.set(inner.regs.DR.get() + 1); inner.regs.CR.modify(CR::RTCStart::SET); - self.inner.init(IrqSafeNullLock::new(inner)); + self.inner.init(IrqSafeSpinLock::new(inner)); Ok(()) } diff --git a/kernel/src/dev/serial/pl011.rs b/kernel/src/dev/serial/pl011.rs index e9b8271..5d13d6e 100644 --- a/kernel/src/dev/serial/pl011.rs +++ b/kernel/src/dev/serial/pl011.rs @@ -7,7 +7,7 @@ use crate::dev::{ Device, }; use crate::mem::virt::DeviceMemoryIo; -use crate::sync::IrqSafeNullLock; +use crate::sync::IrqSafeSpinLock; use crate::util::InitOnce; use core::fmt; use error::Errno; @@ -77,7 +77,7 @@ struct Pl011Inner { /// Device struct for PL011 pub struct Pl011 { - inner: InitOnce>, + inner: InitOnce>, base: usize, irq: IrqNumber, } @@ -134,6 +134,7 @@ impl IntSource for Pl011 { inner.regs.ICR.write(ICR::ALL::CLEAR); let byte = inner.regs.DR.get(); + drop(inner); debugln!("irq byte = {:#04x}", byte); Ok(()) @@ -176,7 +177,7 @@ impl Device for Pl011 { }; inner.enable(); - self.inner.init(IrqSafeNullLock::new(inner)); + self.inner.init(IrqSafeSpinLock::new(inner)); Ok(()) } diff --git a/kernel/src/mem/heap.rs b/kernel/src/mem/heap.rs index 36011c6..539f8df 100644 --- a/kernel/src/mem/heap.rs +++ b/kernel/src/mem/heap.rs @@ -1,6 +1,6 @@ //! Kernel heap allocation facilities -use crate::sync::IrqSafeNullLock; +use crate::sync::IrqSafeSpinLock; use crate::util::InitOnce; use core::alloc::{GlobalAlloc, Layout}; use core::ptr::null_mut; @@ -49,7 +49,7 @@ fn alloc_error_handler(layout: Layout) -> ! { #[global_allocator] static SYSTEM_ALLOC: SystemAlloc = SystemAlloc; -static HEAP: InitOnce> = InitOnce::new(); +static HEAP: InitOnce> = InitOnce::new(); /// Initializes kernel heap with virtual `base` address and `size`. /// @@ -61,5 +61,5 @@ pub unsafe fn init(base: usize, size: usize) { infoln!("Kernel heap: {:#x}..{:#x}", base, base + size); - HEAP.init(IrqSafeNullLock::new(heap)); + HEAP.init(IrqSafeSpinLock::new(heap)); } diff --git a/kernel/src/mem/phys/manager.rs b/kernel/src/mem/phys/manager.rs index 3d34af5..09f574d 100644 --- a/kernel/src/mem/phys/manager.rs +++ b/kernel/src/mem/phys/manager.rs @@ -1,6 +1,6 @@ use super::{PageInfo, PageUsage}; use crate::mem::{virtualize, PAGE_SIZE}; -use crate::sync::IrqSafeNullLock; +use crate::sync::IrqSafeSpinLock; use core::mem; use error::Errno; @@ -73,4 +73,4 @@ unsafe impl Manager for SimpleManager { } } -pub(super) static MANAGER: IrqSafeNullLock> = IrqSafeNullLock::new(None); +pub(super) static MANAGER: IrqSafeSpinLock> = IrqSafeSpinLock::new(None); diff --git a/kernel/src/proc/mod.rs b/kernel/src/proc/mod.rs index 576364f..5962ca8 100644 --- a/kernel/src/proc/mod.rs +++ b/kernel/src/proc/mod.rs @@ -1,7 +1,7 @@ //! Process and thread manipulation facilities use crate::mem; -use crate::sync::IrqSafeNullLock; +use crate::sync::IrqSafeSpinLock; use alloc::{ boxed::Box, collections::{BTreeMap}, @@ -42,8 +42,8 @@ pub fn switch() { } /// Global list of all processes in the system -pub(self) static PROCESSES: IrqSafeNullLock> = - IrqSafeNullLock::new(BTreeMap::new()); +pub(self) static PROCESSES: IrqSafeSpinLock> = + IrqSafeSpinLock::new(BTreeMap::new()); /// Sets up initial process and enters it. /// diff --git a/kernel/src/proc/process.rs b/kernel/src/proc/process.rs index 672c6c4..7400e74 100644 --- a/kernel/src/proc/process.rs +++ b/kernel/src/proc/process.rs @@ -5,7 +5,7 @@ use crate::mem::{ virt::{MapAttributes, Space}, }; use crate::proc::{PROCESSES, SCHED}; -use crate::sync::IrqSafeNullLock; +use crate::sync::IrqSafeSpinLock; use alloc::rc::Rc; use core::cell::UnsafeCell; use core::fmt; @@ -51,7 +51,7 @@ struct ProcessInner { #[allow(dead_code)] pub struct Process { ctx: UnsafeCell, - inner: IrqSafeNullLock, + inner: IrqSafeSpinLock, } impl From for ExitCode { @@ -141,9 +141,6 @@ impl Process { proc.inner.lock().state = State::Running; let ctx = proc.ctx.get(); - // I don't think this is bad: process can't be dropped fully unless - // it's been reaped (and this function won't run for such process) - drop(proc); (&mut *ctx).enter() } @@ -171,10 +168,6 @@ impl Process { let src_ctx = src.ctx.get(); let dst_ctx = dst.ctx.get(); - // See "drop" note in Process::enter() - drop(src); - drop(dst); - (&mut *src_ctx).switch(&mut *dst_ctx); } @@ -188,7 +181,7 @@ impl Process { let id = Pid::new_kernel(); let res = Rc::new(Self { ctx: UnsafeCell::new(Context::kernel(entry as usize, arg)), - inner: IrqSafeNullLock::new(ProcessInner { + inner: IrqSafeSpinLock::new(ProcessInner { id, exit: None, space: None, @@ -238,7 +231,6 @@ impl Process { debugln!("r = {}", Rc::strong_count(&proc)); return Ok(r); } else { - drop(proc); cortex_a::asm::wfi(); } } @@ -312,7 +304,6 @@ impl Process { drop(lock); - drop(proc); (*ctx).enter(); } } diff --git a/kernel/src/proc/sched.rs b/kernel/src/proc/sched.rs index 3f2f598..2700fba 100644 --- a/kernel/src/proc/sched.rs +++ b/kernel/src/proc/sched.rs @@ -1,7 +1,7 @@ //! use crate::proc::{Pid, Process, ProcessRef, PROCESSES}; use crate::util::InitOnce; -use crate::sync::IrqSafeNullLock; +use crate::sync::IrqSafeSpinLock; use alloc::{rc::Rc, collections::VecDeque}; struct SchedulerInner { @@ -12,7 +12,7 @@ struct SchedulerInner { /// Process scheduler state and queues pub struct Scheduler { - inner: InitOnce>, + inner: InitOnce>, } impl SchedulerInner { @@ -35,7 +35,7 @@ impl Scheduler { /// * idle thread /// * process list/queue data structs pub fn init(&self) { - self.inner.init(IrqSafeNullLock::new(SchedulerInner::new())); + self.inner.init(IrqSafeSpinLock::new(SchedulerInner::new())); } /// Schedules a thread for execution diff --git a/kernel/src/sync.rs b/kernel/src/sync.rs index 5022ad0..4361065 100644 --- a/kernel/src/sync.rs +++ b/kernel/src/sync.rs @@ -1,63 +1,84 @@ //! Synchronization facilities module use crate::arch::platform::{irq_mask_save, irq_restore}; +use core::sync::atomic::{AtomicBool, Ordering}; use core::cell::UnsafeCell; use core::ops::{Deref, DerefMut}; /// Lock structure ensuring IRQs are disabled when inner value is accessed -pub struct IrqSafeNullLock { +pub struct IrqSafeSpinLock { value: UnsafeCell, + state: AtomicBool } -/// Guard-structure wrapping a reference to value owned by [IrqSafeNullLock]. +/// Guard-structure wrapping a reference to value owned by [IrqSafeSpinLock]. /// Restores saved IRQ state when dropped. -pub struct IrqSafeNullLockGuard<'a, T: ?Sized> { - value: &'a mut T, +pub struct IrqSafeSpinLockGuard<'a, T> { + lock: &'a IrqSafeSpinLock, irq_state: u64, } -impl IrqSafeNullLock { +impl IrqSafeSpinLock { /// Constructs a new instance of the lock, wrapping `value` #[inline(always)] pub const fn new(value: T) -> Self { Self { value: UnsafeCell::new(value), + state: AtomicBool::new(false) } } - /// Returns [IrqSafeNullLockGuard] for this lock + #[inline(always)] + fn try_lock(&self) -> Result { + self.state.compare_exchange_weak(false, true, Ordering::Acquire, Ordering::Relaxed) + } + + #[inline(always)] + unsafe fn force_release(&self) { + self.state.store(false, Ordering::Release); + cortex_a::asm::sev(); + } + + /// Returns [IrqSafeSpinLockGuard] for this lock #[inline] - pub fn lock(&self) -> IrqSafeNullLockGuard { + pub fn lock(&self) -> IrqSafeSpinLockGuard { + let irq_state = unsafe { irq_mask_save() }; + + while self.try_lock().is_err() { + cortex_a::asm::wfe(); + } + unsafe { - IrqSafeNullLockGuard { - value: &mut *self.value.get(), - irq_state: irq_mask_save(), + IrqSafeSpinLockGuard { + lock: self, + irq_state } } } } -impl Deref for IrqSafeNullLockGuard<'_, T> { +impl Deref for IrqSafeSpinLockGuard<'_, T> { type Target = T; fn deref(&self) -> &Self::Target { - self.value + unsafe { &*self.lock.value.get() } } } -impl DerefMut for IrqSafeNullLockGuard<'_, T> { +impl DerefMut for IrqSafeSpinLockGuard<'_, T> { fn deref_mut(&mut self) -> &mut Self::Target { - self.value + unsafe { &mut *self.lock.value.get() } } } -impl Drop for IrqSafeNullLockGuard<'_, T> { +impl Drop for IrqSafeSpinLockGuard<'_, T> { #[inline(always)] fn drop(&mut self) { unsafe { + self.lock.force_release(); irq_restore(self.irq_state); } } } -unsafe impl Sync for IrqSafeNullLock {} +unsafe impl Sync for IrqSafeSpinLock {}