diff --git a/address/src/phys.rs b/address/src/phys.rs index 4baff2a..971e291 100644 --- a/address/src/phys.rs +++ b/address/src/phys.rs @@ -2,7 +2,7 @@ use crate::{AddressSpace, TrivialConvert, VirtualAddress}; use core::convert::TryFrom; use core::fmt; use core::iter::Step; -use core::ops::{Add, AddAssign, Neg, Sub, SubAssign}; +use core::ops::{Add, AddAssign, Sub, SubAssign}; #[repr(transparent)] #[derive(PartialEq, PartialOrd, Copy, Clone)] diff --git a/build.sh b/build.sh new file mode 100755 index 0000000..d0861f5 --- /dev/null +++ b/build.sh @@ -0,0 +1,9 @@ +#!/bin/sh + +if [ -z "${MACH}" ]; then + MACH=rpi3b +fi + +set -e +cd kernel && cargo build --target ../etc/aarch64-unknown-none-$MACH.json +cd .. diff --git a/kernel/src/arch/cpu.rs b/kernel/src/arch/cpu.rs index e9848d1..fc66721 100644 --- a/kernel/src/arch/cpu.rs +++ b/kernel/src/arch/cpu.rs @@ -1,36 +1,65 @@ -use crate::{ - arch::intrin, - entry_common, - mem::phys::{self, PageUsage}, - mbox::CoreMailbox, - proc::Scheduler, - KernelSpace, -}; -use address::{PhysicalAddress, VirtualAddress}; +use crate::arch::{intrin, smp}; use core::mem::MaybeUninit; -use core::sync::atomic::{AtomicUsize, Ordering}; +use core::ops::{Deref, DerefMut}; +use core::sync::atomic::AtomicUsize; #[repr(C)] pub struct Cpu { - pub cpu_id: u32, // 0x00 - _pad0: u32, // 0x04 - stack_top: usize, // 0x08 - phys_id: u32, // 0x10 - // - pub scheduler: Scheduler, + index: u32, // 0x00 } -pub const MAX_CPU: usize = 4; -pub static mut CPUS: [MaybeUninit; MAX_CPU] = MaybeUninit::uninit_array(); +#[repr(transparent)] +pub struct CpuRef { + inner: &'static mut Cpu, +} + +impl Cpu { + fn new(index: u32) -> Self { + Self { index } + } + + #[inline(always)] + pub unsafe fn get_raw() -> &'static mut Cpu { + &mut *(intrin::read_tpidr_el1() as *mut _) + } + + #[inline(always)] + pub const fn index(&self) -> u32 { + self.index + } + + #[inline] + pub fn get() -> CpuRef { + // TODO lock + CpuRef { + inner: unsafe { Self::get_raw() }, + } + } +} + +impl Drop for CpuRef { + fn drop(&mut self) { + // TODO release + } +} + +impl Deref for CpuRef { + type Target = Cpu; + + fn deref(&self) -> &Self::Target { + self.inner + } +} + +impl DerefMut for CpuRef { + fn deref_mut(&mut self) -> &mut Self::Target { + self.inner + } +} + +pub static mut CPUS: [MaybeUninit; smp::MAX_CPU] = MaybeUninit::uninit_array(); pub static CPU_COUNT: AtomicUsize = AtomicUsize::new(1); -pub const IPI_HALT: u32 = 1 << 0; - -#[inline(always)] -pub fn get_cpu() -> &'static mut Cpu { - unsafe { &mut *(intrin::read_tpidr_el1() as *mut _) } -} - fn set_cpu(cpu: *mut Cpu) { unsafe { intrin::write_tpidr_el1(cpu as usize); @@ -42,98 +71,9 @@ pub fn get_phys_id() -> usize { intrin::read_mpidr_el1() & 0x3 } -#[no_mangle] -extern "C" fn kernel_ap_main() -> ! { - let cpu = get_cpu(); - cpu.phys_id = get_phys_id() as u32; - debug!("cpu{} awakened\n", cpu.cpu_id); - - CPU_COUNT.fetch_add(1, Ordering::SeqCst); - - entry_common(); -} - -pub unsafe fn send_ipi(mask: usize, value: u32) { - for index in 0..CPU_COUNT.load(Ordering::Relaxed) { - if (1 << index) & mask != 0 && get_cpu().cpu_id != index as u32 { - let dst = CPUS[index].assume_init_ref(); - debugln!("cpu{}: send IPI to cpu{}", get_cpu().cpu_id, dst.cpu_id); - CoreMailbox::send(dst.phys_id, 0, value); - } - } -} - -pub fn handle_ipi(mask: u32) { - debugln!("cpu{} received ipi: {}", get_cpu().cpu_id, mask); - - if mask & IPI_HALT != 0 { - unsafe { - intrin::disable_irq(); - } - loop { - unsafe { - intrin::disable_irq(); - } - intrin::nop(); - } - } -} - -pub fn wakeup_single_ap() { - extern "C" { - static mut ap_wakeup_lock: u64; - static mut ap_init_value: u64; - } - - let this_cpu = get_cpu(); - - // Allocate a stack for the AP - let stack_bottom_phys = phys::alloc_contiguous_pages(PageUsage::Kernel, 4).unwrap(); - let stack_bottom = VirtualAddress::::from(stack_bottom_phys); - - // Allocate a new CPU struct - let index = CPU_COUNT.load(Ordering::Acquire); - - if index == MAX_CPU { - debug!("cpu{}: reached cpu limit\n", this_cpu.cpu_id); - return; - } - - debug!("cpu{}: waking up cpu{}\n", this_cpu.cpu_id, index); - +pub fn init(index: usize) { unsafe { - CPUS[index].write(Cpu { - cpu_id: index as u32, - phys_id: 0, - _pad0: 0, - stack_top: (stack_bottom + 4 * 4096).into(), - scheduler: Scheduler::new(), - }); - - let cpu_addr = VirtualAddress::::from(CPUS[index].as_mut_ptr() as u64); - let cpu_addr_phys = PhysicalAddress::from(cpu_addr); - - // Wakeup a single AP for test - core::ptr::write_volatile(&mut ap_init_value, cpu_addr_phys.into()); - // Ensure all writes are complete before waking up an AP - intrin::dsb_sy(); - core::ptr::write_volatile(&mut ap_wakeup_lock, 0); - } - - // Any further CPUs will be waken up by the newly started AP -} - -pub fn bsp_init() { - unsafe { - CPUS[0].write(Cpu { - cpu_id: 0, - phys_id: get_phys_id() as u32, - _pad0: 0, - stack_top: 0, - scheduler: Scheduler::new(), - }); - set_cpu(CPUS[0].as_mut_ptr()); - - debug!("BSP cpu: {:p}\n", get_cpu()); + CPUS[index].write(Cpu::new(index as u32)); + set_cpu(CPUS[index].as_mut_ptr()); } } diff --git a/kernel/src/arch/mbox.rs b/kernel/src/arch/mbox.rs index 3ba8849..4dbe2d4 100644 --- a/kernel/src/arch/mbox.rs +++ b/kernel/src/arch/mbox.rs @@ -1,11 +1,10 @@ -use super::{cpu, intrin, mmio_read, mmio_write}; +use super::{cpu, smp, intrin, mmio_read, mmio_write}; use crate::KernelSpace; use address::{PhysicalAddress, VirtualAddress}; -use spin::Mutex; const MBOX_BASE: usize = 0x3F00B880; const MBOX_READ: usize = MBOX_BASE + 0x00; -const MBOX_POLL: usize = MBOX_BASE + 0x10; +//const MBOX_POLL: usize = MBOX_BASE + 0x10; const MBOX_STATUS: usize = MBOX_BASE + 0x18; const MBOX_WRITE: usize = MBOX_BASE + 0x20; @@ -51,7 +50,7 @@ impl CoreMailbox { let phys_core_id = cpu::get_phys_id(); let value = unsafe { mmio_read(Self::REG_RDCLR + phys_core_id * 16 + self.index * 4) }; if value != 0 { - cpu::handle_ipi(value); + smp::handle_ipi(value); unsafe { mmio_write(Self::REG_RDCLR + phys_core_id * 16 + self.index * 4, 0xFFFFFFFF); } diff --git a/kernel/src/arch/mod.rs b/kernel/src/arch/mod.rs index 7a833f0..012ab9f 100644 --- a/kernel/src/arch/mod.rs +++ b/kernel/src/arch/mod.rs @@ -2,6 +2,7 @@ use crate::KernelSpace; use address::{PhysicalAddress, VirtualAddress}; pub mod cpu; +pub mod smp; pub mod exception; pub mod intrin; pub mod mbox; diff --git a/kernel/src/arch/smp.rs b/kernel/src/arch/smp.rs new file mode 100644 index 0000000..ac6ff59 --- /dev/null +++ b/kernel/src/arch/smp.rs @@ -0,0 +1,84 @@ +use crate::{ + arch::{ + cpu::{self, Cpu, CPUS, CPU_COUNT}, + mbox::CoreMailbox, + intrin, + }, + mem::phys::{self, PageUsage}, + KernelSpace, + entry_common, +}; +use address::VirtualAddress; +use core::hint; +use core::sync::atomic::Ordering; + +pub const MAX_CPU: usize = 4; +pub const IPI_HALT: u32 = 1 << 0; + +#[no_mangle] +extern "C" fn kernel_ap_main() -> ! { + let index = cpu::get_phys_id(); + debugln!("cpu{}: ap wake up", index); + cpu::init(index); + + CPU_COUNT.fetch_add(1, Ordering::SeqCst); + + entry_common(); +} + +pub unsafe fn send_ipi(mask: usize, value: u32) { + let self_index = Cpu::get().index(); + + for index in 0..CPU_COUNT.load(Ordering::Relaxed) { + if (1 << index) & mask != 0 && self_index != index as u32 { + let dst = CPUS[index].assume_init_ref(); + debugln!("cpu{}: send IPI to cpu{}", self_index, dst.index()); + CoreMailbox::send(index as u32, 0, value); + } + } +} + +pub fn handle_ipi(mask: u32) { + debugln!("cpu{} received ipi: {}", Cpu::get().index(), mask); + + if mask & IPI_HALT != 0 { + unsafe { + intrin::disable_irq(); + } + loop { + unsafe { + intrin::disable_irq(); + } + intrin::nop(); + } + } +} + +fn wakeup_single_ap() { + extern "C" { + static mut ap_wakeup_lock: u64; + static mut ap_init_value: u64; + } + + let stack_bottom_phys = phys::alloc_contiguous_pages(PageUsage::Kernel, 4).unwrap(); + let stack_bottom = VirtualAddress::::from(stack_bottom_phys); + + let old_count = CPU_COUNT.load(Ordering::SeqCst); + + unsafe { + core::ptr::write_volatile(&mut ap_init_value, stack_bottom.into()); + intrin::dsb_sy(); + core::ptr::write_volatile(&mut ap_wakeup_lock, 0); + } + + while CPU_COUNT.load(Ordering::SeqCst) == old_count { + hint::spin_loop(); + } +} + +pub fn wakeup_ap_cpus(count: usize) { + for _ in 1..count { + wakeup_single_ap(); + } + debugln!("Done waking up {} ap cpus", count); +} diff --git a/kernel/src/arch/timer.rs b/kernel/src/arch/timer.rs index 9f29bbf..747cb46 100644 --- a/kernel/src/arch/timer.rs +++ b/kernel/src/arch/timer.rs @@ -1,8 +1,7 @@ use crate::{ arch::{cpu, intrin, mmio_read, mmio_write}, - proc, }; -use core::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; +use core::sync::atomic::{AtomicU64, Ordering}; pub struct LocalTimer; pub struct GlobalTimer { @@ -25,11 +24,6 @@ impl LocalTimer { pub unsafe fn enable(mode: LocalTimerMode) { let phys_core_id = cpu::get_phys_id(); - debug!( - "cpu{}: physical core id is {}\n", - cpu::get_cpu().cpu_id, - phys_core_id - ); mmio_write(Self::REG_PSC + 4 * phys_core_id, Self::PRESCALER); let tmp = mmio_read(Self::REG_INTC + 4 * phys_core_id); @@ -43,7 +37,7 @@ impl LocalTimer { } pub fn do_irq() { - proc::sched_yield(); + //proc::sched_yield(); Self::update(Self::RELOAD_VALUE); } @@ -95,17 +89,7 @@ impl GlobalTimer { self.counter.fetch_add(1, Ordering::Release); if self.counter.load(Ordering::Acquire) % Self::TICK as u64 == 0 { - proc::spawn_task(); - - let mut sum = 0; - for item in unsafe { proc::COUNTERS.iter() } { - let value = unsafe { item.assume_init_ref().load(Ordering::Relaxed) }; - debug!("{} ", value); - sum += value; - } - let diff = sum - OLD_COUNTER.load(Ordering::Relaxed); - OLD_COUNTER.store(sum, Ordering::Relaxed); - debugln!(" = {}", diff); + debugln!("Tick"); } self.clear(); @@ -113,4 +97,3 @@ impl GlobalTimer { } pub static GLOBAL_TIMER: GlobalTimer = GlobalTimer::new(); -static OLD_COUNTER: AtomicUsize = AtomicUsize::new(0); diff --git a/kernel/src/boot/entry.S b/kernel/src/boot/entry.S index c0b25d7..cf832a4 100644 --- a/kernel/src/boot/entry.S +++ b/kernel/src/boot/entry.S @@ -23,12 +23,13 @@ _ap_loop: cbnz x0, _ap_loop _ap_wakeup: adr x0, ap_init_value - ldr x0, [x0] mov x1, #KERNEL_OFFSET - ldr x10, [x0, #8] - add x0, x0, x1 - msr tpidr_el1, x0 + + // Kernel stack + ldr x10, [x0] + + // Entry adr x11, kernel_ap_main add x11, x11, x1 diff --git a/kernel/src/dev/irq.rs b/kernel/src/dev/irq.rs index 028d1a0..d4c691a 100644 --- a/kernel/src/dev/irq.rs +++ b/kernel/src/dev/irq.rs @@ -4,7 +4,6 @@ use crate::{ timer::{GLOBAL_TIMER, LocalTimer}, mbox::CORE_MBOX0, }, - proc, }; const INT_SRC_CNTPNSIRQ: u32 = 1 << 1; @@ -40,7 +39,7 @@ struct IrqRegisters { } #[no_mangle] -extern "C" fn do_irq(regs: &mut IrqRegisters) { +extern "C" fn do_irq(_regs: &mut IrqRegisters) { let phys_id = cpu::get_phys_id(); let int_src = unsafe { mmio_read(0x40000060 + phys_id * 4) }; diff --git a/kernel/src/main.rs b/kernel/src/main.rs index 7903b52..f593ffa 100644 --- a/kernel/src/main.rs +++ b/kernel/src/main.rs @@ -9,7 +9,6 @@ pub mod arch; pub mod boot; pub mod dev; pub mod mem; -pub mod proc; pub use mem::KernelSpace; @@ -17,25 +16,24 @@ use address::PhysicalAddress; use arch::{ cpu, intrin, mbox::{self, CORE_MBOX0}, + smp, timer::{LocalTimer, LocalTimerMode, GLOBAL_TIMER}, }; use mem::phys::{SimpleMemoryIterator, UsableMemory}; pub fn entry_common() -> ! { - cpu::wakeup_single_ap(); - unsafe { CORE_MBOX0.enable(); LocalTimer::enable(LocalTimerMode::Irq); intrin::enable_irq(); } - proc::enter(); + loop {} } #[no_mangle] extern "C" fn kernel_bsp_main() -> ! { - cpu::bsp_init(); + cpu::init(0); let arm_memory = mbox::get_arm_memory().expect("Failed to determine ARM memory"); @@ -52,6 +50,7 @@ extern "C" fn kernel_bsp_main() -> ! { unsafe { GLOBAL_TIMER.enable(); } + smp::wakeup_ap_cpus(4); entry_common(); } @@ -60,8 +59,10 @@ use core::panic::PanicInfo; fn panic_handler(pi: &PanicInfo) -> ! { unsafe { intrin::disable_irq(); - cpu::send_ipi(usize::MAX, cpu::IPI_HALT); } debug!("PANIC: {:?}\n", pi); + unsafe { + smp::send_ipi(usize::MAX, smp::IPI_HALT); + } loop {} } diff --git a/qemu.sh b/qemu.sh new file mode 100755 index 0000000..b7ce491 --- /dev/null +++ b/qemu.sh @@ -0,0 +1,27 @@ +#!/bin/sh + +set -e + +if [ -z "${MACH}" ]; then + MACH=rpi3b +fi + +ARCH=aarch64-unknown-none-${MACH} +KERNEL=target/${ARCH}/debug/kernel + +QEMU_OPTS="" +case ${MACH} in + rpi3b) + QEMU_OPTS="$QEMU_OPTS \ + -serial null \ + -serial stdio \ + -M raspi3b" + ;; +esac +QEMU_OPTS="$QEMU_OPTS \ + -kernel ${KERNEL} \ + -s" + +./build.sh + +qemu-system-aarch64 ${QEMU_OPTS}