From 2f405ef17b1d7989b5adbe9b574458de67aa4a4a Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Thu, 2 Sep 2021 10:31:34 +0300 Subject: [PATCH] Rewrite platform-specific stuff a bit --- Cargo.lock | 7 ++ build.sh | 2 +- kernel/Cargo.toml | 5 + kernel/src/arch/{ => mach_bcm283x}/mbox.rs | 47 ++++++--- kernel/src/arch/mach_bcm283x/mod.rs | 3 + kernel/src/arch/mach_bcm283x/smp.rs | 42 ++++++++ kernel/src/arch/mach_bcm283x/timer.rs | 87 ++++++++++++++++ kernel/src/arch/mod.rs | 9 +- kernel/src/arch/smp.rs | 89 ++++++++--------- kernel/src/arch/timer.rs | 110 ++++++--------------- kernel/src/dev/irq.rs | 15 ++- kernel/src/main.rs | 23 +++-- kernel/src/mem/phys/mod.rs | 6 +- 13 files changed, 286 insertions(+), 159 deletions(-) rename kernel/src/arch/{ => mach_bcm283x}/mbox.rs (72%) create mode 100644 kernel/src/arch/mach_bcm283x/mod.rs create mode 100644 kernel/src/arch/mach_bcm283x/smp.rs create mode 100644 kernel/src/arch/mach_bcm283x/timer.rs diff --git a/Cargo.lock b/Cargo.lock index 155c4a5..ea39a2e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9,6 +9,12 @@ dependencies = [ "error", ] +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + [[package]] name = "error" version = "0.1.0" @@ -18,6 +24,7 @@ name = "kernel" version = "0.1.0" dependencies = [ "address", + "cfg-if", "error", "spin", ] diff --git a/build.sh b/build.sh index d0861f5..c5c5a3b 100755 --- a/build.sh +++ b/build.sh @@ -5,5 +5,5 @@ if [ -z "${MACH}" ]; then fi set -e -cd kernel && cargo build --target ../etc/aarch64-unknown-none-$MACH.json +cd kernel && cargo build --target ../etc/aarch64-unknown-none-$MACH.json --features=mach_$MACH cd .. diff --git a/kernel/Cargo.toml b/kernel/Cargo.toml index 0c38770..fe9c9b6 100644 --- a/kernel/Cargo.toml +++ b/kernel/Cargo.toml @@ -9,3 +9,8 @@ edition = "2018" address = { path = "../address" } error = { path = "../error" } spin = "0.9.2" +cfg-if = "*" + +[features] +mach_rpi3b = [] +mach_virt = [] diff --git a/kernel/src/arch/mbox.rs b/kernel/src/arch/mach_bcm283x/mbox.rs similarity index 72% rename from kernel/src/arch/mbox.rs rename to kernel/src/arch/mach_bcm283x/mbox.rs index 4dbe2d4..ef81b80 100644 --- a/kernel/src/arch/mbox.rs +++ b/kernel/src/arch/mach_bcm283x/mbox.rs @@ -1,5 +1,7 @@ -use super::{cpu, smp, intrin, mmio_read, mmio_write}; -use crate::KernelSpace; +use crate::{ + arch::{cpu, intrin, mmio_read, mmio_write, smp::{self, IpiDelivery, IpiMessage}}, + KernelSpace, +}; use address::{PhysicalAddress, VirtualAddress}; const MBOX_BASE: usize = 0x3F00B880; @@ -32,27 +34,46 @@ pub struct CoreMailbox { index: usize, } +impl IpiDelivery for CoreMailbox { + fn enable(&self) { + let phys_core_id = cpu::get_phys_id(); + unsafe { + mmio_write(Self::REG_INTC + phys_core_id * 4, 1 << self.index); + } + } + + fn send_ipi(target_id: u32, message: IpiMessage) { + unsafe { + mmio_write(Self::REG_SET + target_id as usize * 16, 1 << (message as u32)); + } + } +} + impl CoreMailbox { const REG_INTC: usize = 0x40000050; const REG_SET: usize = 0x40000080; const REG_RDCLR: usize = 0x400000C0; - pub unsafe fn enable(&self) { - let phys_core_id = cpu::get_phys_id(); - mmio_write(Self::REG_INTC + phys_core_id * 4, 1 << self.index); - } - - pub unsafe fn send(core_id: u32, mbox_id: usize, value: u32) { - mmio_write(Self::REG_SET + core_id as usize * 16 + mbox_id * 4, value); - } - pub fn do_irq(&self) { let phys_core_id = cpu::get_phys_id(); let value = unsafe { mmio_read(Self::REG_RDCLR + phys_core_id * 16 + self.index * 4) }; if value != 0 { - smp::handle_ipi(value); + macro_rules! test_ipi { + ($value:expr, $msg:expr) => { + if $value & (1 << ($msg as u32)) != 0 { + smp::handle_ipi($msg); + } + } + } + + test_ipi!(value, IpiMessage::Halt); + test_ipi!(value, IpiMessage::Tick); + unsafe { - mmio_write(Self::REG_RDCLR + phys_core_id * 16 + self.index * 4, 0xFFFFFFFF); + mmio_write( + Self::REG_RDCLR + phys_core_id * 16 + self.index * 4, + 0xFFFFFFFF, + ); } } } diff --git a/kernel/src/arch/mach_bcm283x/mod.rs b/kernel/src/arch/mach_bcm283x/mod.rs new file mode 100644 index 0000000..7bab122 --- /dev/null +++ b/kernel/src/arch/mach_bcm283x/mod.rs @@ -0,0 +1,3 @@ +pub mod smp; +pub mod timer; +pub mod mbox; diff --git a/kernel/src/arch/mach_bcm283x/smp.rs b/kernel/src/arch/mach_bcm283x/smp.rs new file mode 100644 index 0000000..808c17e --- /dev/null +++ b/kernel/src/arch/mach_bcm283x/smp.rs @@ -0,0 +1,42 @@ +use crate::{ + arch::{cpu::CPU_COUNT, intrin}, + mem::phys::{self, PageUsage}, + KernelSpace, +}; +use address::VirtualAddress; +use core::hint; +use core::sync::atomic::Ordering; + +pub const MAX_CPU: usize = 4; + +pub fn cpu_ready(_index: usize) { + CPU_COUNT.fetch_add(1, Ordering::SeqCst); +} + +fn wakeup_single_ap() { + extern "C" { + static mut ap_wakeup_lock: u64; + static mut ap_init_value: u64; + } + + let stack_bottom_phys = phys::alloc_contiguous_pages(PageUsage::Kernel, 4).unwrap(); + let stack_bottom = VirtualAddress::::from(stack_bottom_phys); + + let old_count = CPU_COUNT.load(Ordering::SeqCst); + + unsafe { + core::ptr::write_volatile(&mut ap_init_value, stack_bottom.into()); + intrin::dsb_sy(); + core::ptr::write_volatile(&mut ap_wakeup_lock, 0); + } + + while CPU_COUNT.load(Ordering::SeqCst) == old_count { + hint::spin_loop(); + } +} + +pub fn wakeup_ap_cpus() { + for _ in 1..4 { + wakeup_single_ap(); + } +} diff --git a/kernel/src/arch/mach_bcm283x/timer.rs b/kernel/src/arch/mach_bcm283x/timer.rs new file mode 100644 index 0000000..8726e2b --- /dev/null +++ b/kernel/src/arch/mach_bcm283x/timer.rs @@ -0,0 +1,87 @@ +use crate::arch::{ + cpu, intrin, mmio_read, mmio_write, + timer::{global_tick, local_tick, GlobalTimer, LocalTimer, LocalTimerMode, GLOBAL_TIMER_FREQ}, +}; + +pub struct CoreTimer; +pub struct SocTimer; + +impl CoreTimer { + const REG_PSC: usize = 0x40000008; + const REG_INTC: usize = 0x40000040; + + const INTC_CNTPNSIRQ_FIQ: u32 = 1 << 5; + const INTC_CNTPNSIRQ_IRQ: u32 = 1 << 1; + + const PRESCALER: u32 = 0x80000000; + const RELOAD_VALUE: usize = 1 << 18; +} + +impl LocalTimer for CoreTimer { + unsafe fn enable(mode: LocalTimerMode) { + let phys_core_id = cpu::get_phys_id(); + + mmio_write(Self::REG_PSC + 4 * phys_core_id, Self::PRESCALER); + let tmp = mmio_read(Self::REG_INTC + 4 * phys_core_id); + let flags = match mode { + LocalTimerMode::Irq => Self::INTC_CNTPNSIRQ_IRQ, + LocalTimerMode::Fiq => Self::INTC_CNTPNSIRQ_FIQ, + }; + mmio_write(Self::REG_INTC + 4 * phys_core_id, tmp | flags); + + intrin::write_cntp_ctl_el0(1); + } + + fn do_irq() { + local_tick(); + Self::update(Self::RELOAD_VALUE); + } + + #[inline(always)] + fn update(value: usize) { + let current = intrin::read_cntp_cval_el0(); + unsafe { + intrin::write_cntp_cval_el0(current + value); + } + } +} + +impl SocTimer { + const REG_CTL: usize = 0x40000034; + const REG_IRQ_RT: usize = 0x40000024; + const REG_IRQ_CLR: usize = 0x40000038; + + const CTL_TIMER_EN: u32 = 1 << 28; + const CTL_IRQ_EN: u32 = 1 << 29; + const IRQ_RT_IRQ0: u32 = 0; + const IRQ_CLR_VALUE: u32 = (1 << 31) | (1 << 30); + + const BASE_FREQ: u32 = 38400000; + + pub const fn new() -> Self { + Self {} + } +} + +impl GlobalTimer for SocTimer { + unsafe fn enable(&self) { + let reload = Self::BASE_FREQ / (GLOBAL_TIMER_FREQ as u32); + + mmio_write(Self::REG_IRQ_RT, Self::IRQ_RT_IRQ0); + mmio_write( + Self::REG_CTL, + reload | Self::CTL_TIMER_EN | Self::CTL_IRQ_EN, + ); + } + + fn reset(&self) { + unsafe { + mmio_write(Self::REG_IRQ_CLR, Self::IRQ_CLR_VALUE); + } + } + + fn do_irq(&self) { + global_tick(); + self.reset(); + } +} diff --git a/kernel/src/arch/mod.rs b/kernel/src/arch/mod.rs index 012ab9f..35b2c80 100644 --- a/kernel/src/arch/mod.rs +++ b/kernel/src/arch/mod.rs @@ -5,9 +5,16 @@ pub mod cpu; pub mod smp; pub mod exception; pub mod intrin; -pub mod mbox; pub mod timer; +cfg_if! { + if #[cfg(feature = "mach_rpi3b")] { + pub mod mach_bcm283x; + + pub use mach_bcm283x::smp as smp_impl; + } +} + pub unsafe fn mmio_write(addr: usize, value: u32) { core::ptr::write_volatile( VirtualAddress::::from(PhysicalAddress::from(addr)).as_mut_ptr(), diff --git a/kernel/src/arch/smp.rs b/kernel/src/arch/smp.rs index ac6ff59..a7937af 100644 --- a/kernel/src/arch/smp.rs +++ b/kernel/src/arch/smp.rs @@ -1,19 +1,33 @@ use crate::{ arch::{ - cpu::{self, Cpu, CPUS, CPU_COUNT}, - mbox::CoreMailbox, - intrin, + cpu::{self, Cpu, CPU_COUNT}, + intrin, smp_impl, }, - mem::phys::{self, PageUsage}, - KernelSpace, entry_common, }; -use address::VirtualAddress; -use core::hint; use core::sync::atomic::Ordering; -pub const MAX_CPU: usize = 4; -pub const IPI_HALT: u32 = 1 << 0; +pub trait IpiDelivery { + fn enable(&self); + fn send_ipi(target_id: u32, message: IpiMessage); +} + +cfg_if! { + if #[cfg(feature = "mach_rpi3b")] { + use super::mach_bcm283x::mbox; + + pub type IpiDeliveryImpl = mbox::CoreMailbox; + use mbox::CORE_MBOX0 as IPI_IMPL; + } +} + +pub use smp_impl::MAX_CPU; +#[derive(Clone, Copy, PartialEq, Debug)] +#[repr(u32)] +pub enum IpiMessage { + Halt = 0, + Tick = 1, +} #[no_mangle] extern "C" fn kernel_ap_main() -> ! { @@ -21,64 +35,45 @@ extern "C" fn kernel_ap_main() -> ! { debugln!("cpu{}: ap wake up", index); cpu::init(index); - CPU_COUNT.fetch_add(1, Ordering::SeqCst); + smp_impl::cpu_ready(index); entry_common(); } -pub unsafe fn send_ipi(mask: usize, value: u32) { +pub unsafe fn send_ipi(mask: usize, message: IpiMessage) { let self_index = Cpu::get().index(); for index in 0..CPU_COUNT.load(Ordering::Relaxed) { if (1 << index) & mask != 0 && self_index != index as u32 { - let dst = CPUS[index].assume_init_ref(); - debugln!("cpu{}: send IPI to cpu{}", self_index, dst.index()); - CoreMailbox::send(index as u32, 0, value); + // TODO cpus must be in phys order? + IpiDeliveryImpl::send_ipi(index as u32, message); } } } -pub fn handle_ipi(mask: u32) { - debugln!("cpu{} received ipi: {}", Cpu::get().index(), mask); +pub fn handle_ipi(message: IpiMessage) { + debugln!("cpu{} received ipi: {:?}", Cpu::get().index(), message); - if mask & IPI_HALT != 0 { - unsafe { - intrin::disable_irq(); - } - loop { + match message { + IpiMessage::Halt => { unsafe { intrin::disable_irq(); } - intrin::nop(); + loop { + unsafe { + intrin::disable_irq(); + } + intrin::nop(); + } } + IpiMessage::Tick => {} } } -fn wakeup_single_ap() { - extern "C" { - static mut ap_wakeup_lock: u64; - static mut ap_init_value: u64; - } - - let stack_bottom_phys = phys::alloc_contiguous_pages(PageUsage::Kernel, 4).unwrap(); - let stack_bottom = VirtualAddress::::from(stack_bottom_phys); - - let old_count = CPU_COUNT.load(Ordering::SeqCst); - - unsafe { - core::ptr::write_volatile(&mut ap_init_value, stack_bottom.into()); - intrin::dsb_sy(); - core::ptr::write_volatile(&mut ap_wakeup_lock, 0); - } - - while CPU_COUNT.load(Ordering::SeqCst) == old_count { - hint::spin_loop(); - } +pub fn wakeup_ap_cpus() { + smp_impl::wakeup_ap_cpus(); } -pub fn wakeup_ap_cpus(count: usize) { - for _ in 1..count { - wakeup_single_ap(); - } - debugln!("Done waking up {} ap cpus", count); +pub fn init_ipi_delivery() { + IPI_IMPL.enable(); } diff --git a/kernel/src/arch/timer.rs b/kernel/src/arch/timer.rs index 747cb46..4232e4c 100644 --- a/kernel/src/arch/timer.rs +++ b/kernel/src/arch/timer.rs @@ -1,99 +1,53 @@ -use crate::{ - arch::{cpu, intrin, mmio_read, mmio_write}, -}; use core::sync::atomic::{AtomicU64, Ordering}; +use crate::arch::smp::{self, IpiMessage}; -pub struct LocalTimer; -pub struct GlobalTimer { - counter: AtomicU64, -} pub enum LocalTimerMode { Irq, Fiq, } -impl LocalTimer { - const REG_PSC: usize = 0x40000008; - const REG_INTC: usize = 0x40000040; +pub trait GlobalTimer { + unsafe fn enable(&self); + fn reset(&self); - const INTC_CNTPNSIRQ_FIQ: u32 = 1 << 5; - const INTC_CNTPNSIRQ_IRQ: u32 = 1 << 1; + fn do_irq(&self); +} - const PRESCALER: u32 = 0x80000000; - const RELOAD_VALUE: usize = 1 << 18; +pub trait LocalTimer { + unsafe fn enable(mode: LocalTimerMode); + fn do_irq(); + fn update(value: usize); +} - pub unsafe fn enable(mode: LocalTimerMode) { - let phys_core_id = cpu::get_phys_id(); +cfg_if! { + if #[cfg(feature = "mach_rpi3b")] { + use super::mach_bcm283x; - mmio_write(Self::REG_PSC + 4 * phys_core_id, Self::PRESCALER); - let tmp = mmio_read(Self::REG_INTC + 4 * phys_core_id); - let flags = match mode { - LocalTimerMode::Irq => Self::INTC_CNTPNSIRQ_IRQ, - LocalTimerMode::Fiq => Self::INTC_CNTPNSIRQ_FIQ, - }; - mmio_write(Self::REG_INTC + 4 * phys_core_id, tmp | flags); + pub type LocalTimerImpl = mach_bcm283x::timer::CoreTimer; + pub type GlobalTimerImpl = mach_bcm283x::timer::SocTimer; + } else if #[cfg(feature = "mach_virt")] { + use super::mach_virt; - intrin::write_cntp_ctl_el0(1); + pub type LocalTimerImpl = mach_virt::timer::LocalTimer; + pub type GlobalTimerImpl = mach_virt::timer::GlobalTimer; } +} - pub fn do_irq() { - //proc::sched_yield(); - Self::update(Self::RELOAD_VALUE); - } +pub fn local_tick() { +} - #[inline(always)] - pub fn update(value: usize) { - let current = intrin::read_cntp_cval_el0(); +pub fn global_tick() { + GLOBAL_TICK_COUNT.fetch_add(1, Ordering::Release); + + if GLOBAL_TICK_COUNT.load(Ordering::Acquire) % GLOBAL_TIMER_FREQ as u64 == 0 { + debugln!("Tick"); unsafe { - intrin::write_cntp_cval_el0(current + value); + smp::send_ipi(usize::MAX, IpiMessage::Tick); } } } -impl GlobalTimer { - const REG_CTL: usize = 0x40000034; - const REG_IRQ_RT: usize = 0x40000024; - const REG_IRQ_CLR: usize = 0x40000038; +pub const GLOBAL_TIMER_FREQ: u64 = 1000; - const CTL_TIMER_EN: u32 = 1 << 28; - const CTL_IRQ_EN: u32 = 1 << 29; - const IRQ_RT_IRQ0: u32 = 0; - const IRQ_CLR_VALUE: u32 = (1 << 31) | (1 << 30); - - const BASE_FRQ: u32 = 38400000; - const TICK: u32 = 1000; - - pub const fn new() -> Self { - Self { - counter: AtomicU64::new(0), - } - } - - pub unsafe fn enable(&self) { - let reload = Self::BASE_FRQ / Self::TICK; - - mmio_write(Self::REG_IRQ_RT, Self::IRQ_RT_IRQ0); - mmio_write( - Self::REG_CTL, - reload | Self::CTL_TIMER_EN | Self::CTL_IRQ_EN, - ); - } - - pub fn clear(&self) { - unsafe { - mmio_write(Self::REG_IRQ_CLR, Self::IRQ_CLR_VALUE); - } - } - - pub fn do_irq(&self) { - self.counter.fetch_add(1, Ordering::Release); - - if self.counter.load(Ordering::Acquire) % Self::TICK as u64 == 0 { - debugln!("Tick"); - } - - self.clear(); - } -} - -pub static GLOBAL_TIMER: GlobalTimer = GlobalTimer::new(); +pub static GLOBAL_TIMER: GlobalTimerImpl = GlobalTimerImpl::new(); +pub static GLOBAL_TICK_COUNT: AtomicU64 = AtomicU64::new(0); diff --git a/kernel/src/dev/irq.rs b/kernel/src/dev/irq.rs index d4c691a..f48f036 100644 --- a/kernel/src/dev/irq.rs +++ b/kernel/src/dev/irq.rs @@ -1,9 +1,8 @@ -use crate::{ - arch::{ - cpu, mmio_read, - timer::{GLOBAL_TIMER, LocalTimer}, - mbox::CORE_MBOX0, - }, +use crate::arch::{ + cpu, + mmio_read, + timer::{GlobalTimer, LocalTimer, LocalTimerImpl, GLOBAL_TIMER}, + mach_bcm283x::mbox::CORE_MBOX0 }; const INT_SRC_CNTPNSIRQ: u32 = 1 << 1; @@ -49,7 +48,7 @@ extern "C" fn do_irq(_regs: &mut IrqRegisters) { return; } if int_src & INT_SRC_CNTPNSIRQ != 0 { - LocalTimer::do_irq(); + LocalTimerImpl::do_irq(); return; } if int_src & INT_SRC_MBOX0 != 0 { @@ -62,5 +61,5 @@ extern "C" fn do_irq(_regs: &mut IrqRegisters) { #[no_mangle] extern "C" fn do_fiq() { // Only used by core timer - LocalTimer::do_irq(); + LocalTimerImpl::do_irq(); } diff --git a/kernel/src/main.rs b/kernel/src/main.rs index f593ffa..0b8c0a6 100644 --- a/kernel/src/main.rs +++ b/kernel/src/main.rs @@ -2,6 +2,9 @@ #![no_std] #![no_main] +#[macro_use] +extern crate cfg_if; + #[macro_use] pub mod debug; @@ -14,17 +17,16 @@ pub use mem::KernelSpace; use address::PhysicalAddress; use arch::{ - cpu, intrin, - mbox::{self, CORE_MBOX0}, - smp, - timer::{LocalTimer, LocalTimerMode, GLOBAL_TIMER}, + cpu, intrin, smp, + timer::{GlobalTimer, LocalTimer, LocalTimerImpl, LocalTimerMode, GLOBAL_TIMER}, }; use mem::phys::{SimpleMemoryIterator, UsableMemory}; pub fn entry_common() -> ! { + smp::init_ipi_delivery(); + unsafe { - CORE_MBOX0.enable(); - LocalTimer::enable(LocalTimerMode::Irq); + LocalTimerImpl::enable(LocalTimerMode::Irq); intrin::enable_irq(); } @@ -35,14 +37,15 @@ pub fn entry_common() -> ! { extern "C" fn kernel_bsp_main() -> ! { cpu::init(0); - let arm_memory = mbox::get_arm_memory().expect("Failed to determine ARM memory"); + let arm_memory = 0x3F000000usize; + //let arm_memory = mbox::get_arm_memory().expect("Failed to determine ARM memory"); let iter = SimpleMemoryIterator::new(UsableMemory { start: PhysicalAddress::from(0usize), end: PhysicalAddress::from(arm_memory), }); unsafe { - mem::phys::initialize(iter); + mem::phys::init_from_iter(iter); } debug!("BSP init finished\n"); @@ -50,7 +53,7 @@ extern "C" fn kernel_bsp_main() -> ! { unsafe { GLOBAL_TIMER.enable(); } - smp::wakeup_ap_cpus(4); + smp::wakeup_ap_cpus(); entry_common(); } @@ -62,7 +65,7 @@ fn panic_handler(pi: &PanicInfo) -> ! { } debug!("PANIC: {:?}\n", pi); unsafe { - smp::send_ipi(usize::MAX, smp::IPI_HALT); + smp::send_ipi(usize::MAX, smp::IpiMessage::Halt); } loop {} } diff --git a/kernel/src/mem/phys/mod.rs b/kernel/src/mem/phys/mod.rs index e0cce68..61439ac 100644 --- a/kernel/src/mem/phys/mod.rs +++ b/kernel/src/mem/phys/mod.rs @@ -98,7 +98,7 @@ fn find_contiguous>( None } -pub unsafe fn initialize + Clone>(iter: T) { +pub unsafe fn init_from_iter + Clone>(iter: T) { // Step 1. Count available memory let mut total_pages = 0usize; for reg in iter.clone() { @@ -136,3 +136,7 @@ pub unsafe fn initialize + Clone>(iter: T) { *MANAGER.lock() = Some(manager); } + +pub unsafe fn init_from_platform() { + todo!(); +}