alnyan/osdev4
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Better SMP init

Browse Source
This commit is contained in:
Mark Poliakov
2021-09-01 17:38:16 +03:00
parent aa4feff489
commit 6bfda36887
11 changed files with 198 additions and 154 deletions
Download Patch File Download Diff File Expand all files Collapse all files
address/src/phys.rs
+1 -1
View File
@@ -2,7 +2,7 @@ use crate::{AddressSpace, TrivialConvert, VirtualAddress};
use core::convert::TryFrom;
use core::fmt;
use core::iter::Step;
use core::ops::{Add, AddAssign, Neg, Sub, SubAssign};
use core::ops::{Add, AddAssign, Sub, SubAssign};
#[repr(transparent)]
#[derive(PartialEq, PartialOrd, Copy, Clone)]
build.sh
Executable
+9
View File
@@ -0,0 +1,9 @@
#!/bin/sh
if [ -z "${MACH}" ]; then
MACH=rpi3b
fi
set -e
cd kernel && cargo build --target ../etc/aarch64-unknown-none-$MACH.json
cd ..
kernel/src/arch/cpu.rs
+57 -117
View File
@@ -1,36 +1,65 @@
use crate::{
arch::intrin,
entry_common,
mem::phys::{self, PageUsage},
mbox::CoreMailbox,
proc::Scheduler,
KernelSpace,
};
use address::{PhysicalAddress, VirtualAddress};
use crate::arch::{intrin, smp};
use core::mem::MaybeUninit;
use core::sync::atomic::{AtomicUsize, Ordering};
use core::ops::{Deref, DerefMut};
use core::sync::atomic::AtomicUsize;
#[repr(C)]
pub struct Cpu {
pub cpu_id: u32, // 0x00
_pad0: u32, // 0x04
stack_top: usize, // 0x08
phys_id: u32, // 0x10
//
pub scheduler: Scheduler,
index: u32, // 0x00
}
pub const MAX_CPU: usize = 4;
pub static mut CPUS: [MaybeUninit<Cpu>; MAX_CPU] = MaybeUninit::uninit_array();
#[repr(transparent)]
pub struct CpuRef {
inner: &'static mut Cpu,
}
impl Cpu {
fn new(index: u32) -> Self {
Self { index }
}
#[inline(always)]
pub unsafe fn get_raw() -> &'static mut Cpu {
&mut *(intrin::read_tpidr_el1() as *mut _)
}
#[inline(always)]
pub const fn index(&self) -> u32 {
self.index
}
#[inline]
pub fn get() -> CpuRef {
// TODO lock
CpuRef {
inner: unsafe { Self::get_raw() },
}
}
}
impl Drop for CpuRef {
fn drop(&mut self) {
// TODO release
}
}
impl Deref for CpuRef {
type Target = Cpu;
fn deref(&self) -> &Self::Target {
self.inner
}
}
impl DerefMut for CpuRef {
fn deref_mut(&mut self) -> &mut Self::Target {
self.inner
}
}
pub static mut CPUS: [MaybeUninit<Cpu>; smp::MAX_CPU] = MaybeUninit::uninit_array();
pub static CPU_COUNT: AtomicUsize = AtomicUsize::new(1);
pub const IPI_HALT: u32 = 1 << 0;
#[inline(always)]
pub fn get_cpu() -> &'static mut Cpu {
unsafe { &mut *(intrin::read_tpidr_el1() as *mut _) }
}
fn set_cpu(cpu: *mut Cpu) {
unsafe {
intrin::write_tpidr_el1(cpu as usize);
@@ -42,98 +71,9 @@ pub fn get_phys_id() -> usize {
intrin::read_mpidr_el1() & 0x3
}
#[no_mangle]
extern "C" fn kernel_ap_main() -> ! {
let cpu = get_cpu();
cpu.phys_id = get_phys_id() as u32;
debug!("cpu{} awakened\n", cpu.cpu_id);
CPU_COUNT.fetch_add(1, Ordering::SeqCst);
entry_common();
}
pub unsafe fn send_ipi(mask: usize, value: u32) {
for index in 0..CPU_COUNT.load(Ordering::Relaxed) {
if (1 << index) & mask != 0 && get_cpu().cpu_id != index as u32 {
let dst = CPUS[index].assume_init_ref();
debugln!("cpu{}: send IPI to cpu{}", get_cpu().cpu_id, dst.cpu_id);
CoreMailbox::send(dst.phys_id, 0, value);
}
}
}
pub fn handle_ipi(mask: u32) {
debugln!("cpu{} received ipi: {}", get_cpu().cpu_id, mask);
if mask & IPI_HALT != 0 {
unsafe {
intrin::disable_irq();
}
loop {
unsafe {
intrin::disable_irq();
}
intrin::nop();
}
}
}
pub fn wakeup_single_ap() {
extern "C" {
static mut ap_wakeup_lock: u64;
static mut ap_init_value: u64;
}
let this_cpu = get_cpu();
// Allocate a stack for the AP
let stack_bottom_phys = phys::alloc_contiguous_pages(PageUsage::Kernel, 4).unwrap();
let stack_bottom = VirtualAddress::<KernelSpace>::from(stack_bottom_phys);
// Allocate a new CPU struct
let index = CPU_COUNT.load(Ordering::Acquire);
if index == MAX_CPU {
debug!("cpu{}: reached cpu limit\n", this_cpu.cpu_id);
return;
}
debug!("cpu{}: waking up cpu{}\n", this_cpu.cpu_id, index);
pub fn init(index: usize) {
unsafe {
CPUS[index].write(Cpu {
cpu_id: index as u32,
phys_id: 0,
_pad0: 0,
stack_top: (stack_bottom + 4 * 4096).into(),
scheduler: Scheduler::new(),
});
let cpu_addr = VirtualAddress::<KernelSpace>::from(CPUS[index].as_mut_ptr() as u64);
let cpu_addr_phys = PhysicalAddress::from(cpu_addr);
// Wakeup a single AP for test
core::ptr::write_volatile(&mut ap_init_value, cpu_addr_phys.into());
// Ensure all writes are complete before waking up an AP
intrin::dsb_sy();
core::ptr::write_volatile(&mut ap_wakeup_lock, 0);
}
// Any further CPUs will be waken up by the newly started AP
}
pub fn bsp_init() {
unsafe {
CPUS[0].write(Cpu {
cpu_id: 0,
phys_id: get_phys_id() as u32,
_pad0: 0,
stack_top: 0,
scheduler: Scheduler::new(),
});
set_cpu(CPUS[0].as_mut_ptr());
debug!("BSP cpu: {:p}\n", get_cpu());
CPUS[index].write(Cpu::new(index as u32));
set_cpu(CPUS[index].as_mut_ptr());
}
}
kernel/src/arch/mbox.rs
+3 -4
View File
@@ -1,11 +1,10 @@
use super::{cpu, intrin, mmio_read, mmio_write};
use super::{cpu, smp, intrin, mmio_read, mmio_write};
use crate::KernelSpace;
use address::{PhysicalAddress, VirtualAddress};
use spin::Mutex;
const MBOX_BASE: usize = 0x3F00B880;
const MBOX_READ: usize = MBOX_BASE + 0x00;
const MBOX_POLL: usize = MBOX_BASE + 0x10;
//const MBOX_POLL: usize = MBOX_BASE + 0x10;
const MBOX_STATUS: usize = MBOX_BASE + 0x18;
const MBOX_WRITE: usize = MBOX_BASE + 0x20;
@@ -51,7 +50,7 @@ impl CoreMailbox {
let phys_core_id = cpu::get_phys_id();
let value = unsafe { mmio_read(Self::REG_RDCLR + phys_core_id * 16 + self.index * 4) };
if value != 0 {
cpu::handle_ipi(value);
smp::handle_ipi(value);
unsafe {
mmio_write(Self::REG_RDCLR + phys_core_id * 16 + self.index * 4, 0xFFFFFFFF);
}
kernel/src/arch/mod.rs
+1
View File
@@ -2,6 +2,7 @@ use crate::KernelSpace;
use address::{PhysicalAddress, VirtualAddress};
pub mod cpu;
pub mod smp;
pub mod exception;
pub mod intrin;
pub mod mbox;
kernel/src/arch/smp.rs
+84
View File
@@ -0,0 +1,84 @@
use crate::{
arch::{
cpu::{self, Cpu, CPUS, CPU_COUNT},
mbox::CoreMailbox,
intrin,
},
mem::phys::{self, PageUsage},
KernelSpace,
entry_common,
};
use address::VirtualAddress;
use core::hint;
use core::sync::atomic::Ordering;
pub const MAX_CPU: usize = 4;
pub const IPI_HALT: u32 = 1 << 0;
#[no_mangle]
extern "C" fn kernel_ap_main() -> ! {
let index = cpu::get_phys_id();
debugln!("cpu{}: ap wake up", index);
cpu::init(index);
CPU_COUNT.fetch_add(1, Ordering::SeqCst);
entry_common();
}
pub unsafe fn send_ipi(mask: usize, value: u32) {
let self_index = Cpu::get().index();
for index in 0..CPU_COUNT.load(Ordering::Relaxed) {
if (1 << index) & mask != 0 && self_index != index as u32 {
let dst = CPUS[index].assume_init_ref();
debugln!("cpu{}: send IPI to cpu{}", self_index, dst.index());
CoreMailbox::send(index as u32, 0, value);
}
}
}
pub fn handle_ipi(mask: u32) {
debugln!("cpu{} received ipi: {}", Cpu::get().index(), mask);
if mask & IPI_HALT != 0 {
unsafe {
intrin::disable_irq();
}
loop {
unsafe {
intrin::disable_irq();
}
intrin::nop();
}
}
}
fn wakeup_single_ap() {
extern "C" {
static mut ap_wakeup_lock: u64;
static mut ap_init_value: u64;
}
let stack_bottom_phys = phys::alloc_contiguous_pages(PageUsage::Kernel, 4).unwrap();
let stack_bottom = VirtualAddress::<KernelSpace>::from(stack_bottom_phys);
let old_count = CPU_COUNT.load(Ordering::SeqCst);
unsafe {
core::ptr::write_volatile(&mut ap_init_value, stack_bottom.into());
intrin::dsb_sy();
core::ptr::write_volatile(&mut ap_wakeup_lock, 0);
}
while CPU_COUNT.load(Ordering::SeqCst) == old_count {
hint::spin_loop();
}
}
pub fn wakeup_ap_cpus(count: usize) {
for _ in 1..count {
wakeup_single_ap();
}
debugln!("Done waking up {} ap cpus", count);
}
kernel/src/arch/timer.rs
+3 -20
View File
@@ -1,8 +1,7 @@
use crate::{
arch::{cpu, intrin, mmio_read, mmio_write},
proc,
};
use core::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
use core::sync::atomic::{AtomicU64, Ordering};
pub struct LocalTimer;
pub struct GlobalTimer {
@@ -25,11 +24,6 @@ impl LocalTimer {
pub unsafe fn enable(mode: LocalTimerMode) {
let phys_core_id = cpu::get_phys_id();
debug!(
"cpu{}: physical core id is {}\n",
cpu::get_cpu().cpu_id,
phys_core_id
);
mmio_write(Self::REG_PSC + 4 * phys_core_id, Self::PRESCALER);
let tmp = mmio_read(Self::REG_INTC + 4 * phys_core_id);
@@ -43,7 +37,7 @@ impl LocalTimer {
}
pub fn do_irq() {
proc::sched_yield();
//proc::sched_yield();
Self::update(Self::RELOAD_VALUE);
}
@@ -95,17 +89,7 @@ impl GlobalTimer {
self.counter.fetch_add(1, Ordering::Release);
if self.counter.load(Ordering::Acquire) % Self::TICK as u64 == 0 {
proc::spawn_task();
let mut sum = 0;
for item in unsafe { proc::COUNTERS.iter() } {
let value = unsafe { item.assume_init_ref().load(Ordering::Relaxed) };
debug!("{} ", value);
sum += value;
}
let diff = sum - OLD_COUNTER.load(Ordering::Relaxed);
OLD_COUNTER.store(sum, Ordering::Relaxed);
debugln!(" = {}", diff);
debugln!("Tick");
}
self.clear();
@@ -113,4 +97,3 @@ impl GlobalTimer {
}
pub static GLOBAL_TIMER: GlobalTimer = GlobalTimer::new();
static OLD_COUNTER: AtomicUsize = AtomicUsize::new(0);
kernel/src/boot/entry.S
+5 -4
View File
@@ -23,12 +23,13 @@ _ap_loop:
cbnz x0, _ap_loop
_ap_wakeup:
adr x0, ap_init_value
ldr x0, [x0]
mov x1, #KERNEL_OFFSET
ldr x10, [x0, #8]
add x0, x0, x1
msr tpidr_el1, x0
// Kernel stack
ldr x10, [x0]
// Entry
adr x11, kernel_ap_main
add x11, x11, x1
kernel/src/dev/irq.rs
+1 -2
View File
@@ -4,7 +4,6 @@ use crate::{
timer::{GLOBAL_TIMER, LocalTimer},
mbox::CORE_MBOX0,
},
proc,
};
const INT_SRC_CNTPNSIRQ: u32 = 1 << 1;
@@ -40,7 +39,7 @@ struct IrqRegisters {
}
#[no_mangle]
extern "C" fn do_irq(regs: &mut IrqRegisters) {
extern "C" fn do_irq(_regs: &mut IrqRegisters) {
let phys_id = cpu::get_phys_id();
let int_src = unsafe { mmio_read(0x40000060 + phys_id * 4) };
kernel/src/main.rs
+7 -6
View File
@@ -9,7 +9,6 @@ pub mod arch;
pub mod boot;
pub mod dev;
pub mod mem;
pub mod proc;
pub use mem::KernelSpace;
@@ -17,25 +16,24 @@ use address::PhysicalAddress;
use arch::{
cpu, intrin,
mbox::{self, CORE_MBOX0},
smp,
timer::{LocalTimer, LocalTimerMode, GLOBAL_TIMER},
};
use mem::phys::{SimpleMemoryIterator, UsableMemory};
pub fn entry_common() -> ! {
cpu::wakeup_single_ap();
unsafe {
CORE_MBOX0.enable();
LocalTimer::enable(LocalTimerMode::Irq);
intrin::enable_irq();
}
proc::enter();
loop {}
}
#[no_mangle]
extern "C" fn kernel_bsp_main() -> ! {
cpu::bsp_init();
cpu::init(0);
let arm_memory = mbox::get_arm_memory().expect("Failed to determine ARM memory");
@@ -52,6 +50,7 @@ extern "C" fn kernel_bsp_main() -> ! {
unsafe {
GLOBAL_TIMER.enable();
}
smp::wakeup_ap_cpus(4);
entry_common();
}
@@ -60,8 +59,10 @@ use core::panic::PanicInfo;
fn panic_handler(pi: &PanicInfo) -> ! {
unsafe {
intrin::disable_irq();
cpu::send_ipi(usize::MAX, cpu::IPI_HALT);
}
debug!("PANIC: {:?}\n", pi);
unsafe {
smp::send_ipi(usize::MAX, smp::IPI_HALT);
}
loop {}
}
qemu.sh
Executable
+27
View File
@@ -0,0 +1,27 @@
#!/bin/sh
set -e
if [ -z "${MACH}" ]; then
MACH=rpi3b
fi
ARCH=aarch64-unknown-none-${MACH}
KERNEL=target/${ARCH}/debug/kernel
QEMU_OPTS=""
case ${MACH} in
rpi3b)
QEMU_OPTS="$QEMU_OPTS \
-serial null \
-serial stdio \
-M raspi3b"
;;
esac
QEMU_OPTS="$QEMU_OPTS \
-kernel ${KERNEL} \
-s"
./build.sh
qemu-system-aarch64 ${QEMU_OPTS}