From d45a8adc34490becc81bef390417435dfb5b8e27 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Wed, 26 Jan 2022 15:15:38 +0200 Subject: [PATCH] refactor: move asm! usage to intrin module --- kernel/src/arch/aarch64/exception.rs | 6 +- kernel/src/arch/aarch64/intrin.rs | 17 +++++ .../src/arch/aarch64/mach_orangepi3/wdog.rs | 3 +- kernel/src/arch/aarch64/mod.rs | 5 +- kernel/src/arch/aarch64/reg/cntkctl_el1.rs | 50 -------------- kernel/src/arch/aarch64/reg/cpacr_el1.rs | 58 ---------------- kernel/src/arch/aarch64/reg/mod.rs | 67 +++++++++++++++++-- kernel/src/arch/mod.rs | 2 + kernel/src/main.rs | 2 +- kernel/src/mem/mod.rs | 15 +++++ kernel/src/proc/process.rs | 27 ++++---- kernel/src/proc/sched.rs | 6 +- kernel/src/syscall/arg.rs | 20 ++---- 13 files changed, 127 insertions(+), 151 deletions(-) create mode 100644 kernel/src/arch/aarch64/intrin.rs delete mode 100644 kernel/src/arch/aarch64/reg/cntkctl_el1.rs delete mode 100644 kernel/src/arch/aarch64/reg/cpacr_el1.rs diff --git a/kernel/src/arch/aarch64/exception.rs b/kernel/src/arch/aarch64/exception.rs index 4d905d4..00d09fd 100644 --- a/kernel/src/arch/aarch64/exception.rs +++ b/kernel/src/arch/aarch64/exception.rs @@ -1,6 +1,6 @@ //! AArch64 exception handling -use crate::arch::machine; +use crate::arch::{machine, intrin}; use crate::debug::Level; use crate::dev::irq::{IntController, IrqContext}; use crate::mem; @@ -98,7 +98,9 @@ extern "C" fn __aa64_exc_sync_handler(exc: &mut ExceptionFrame) { let res = proc.manipulate_space(|space| { space.try_cow_copy(far)?; - Process::invalidate_asid(asid); + unsafe { + intrin::flush_tlb_asid(asid); + } Result::<(), Errno>::Ok(()) }); diff --git a/kernel/src/arch/aarch64/intrin.rs b/kernel/src/arch/aarch64/intrin.rs new file mode 100644 index 0000000..b8f0c34 --- /dev/null +++ b/kernel/src/arch/aarch64/intrin.rs @@ -0,0 +1,17 @@ +use core::arch::asm; + +#[inline(always)] +pub unsafe fn irq_disable() { + asm!("msr daifset, {bits}", bits = const 2, options(nomem, nostack, preserves_flags)); +} + +#[inline(always)] +pub unsafe fn flush_tlb_virt(addr: usize) { +} + +// TODO non-portable +#[inline(always)] +pub unsafe fn flush_tlb_asid(asid: usize) { + asm!("tlbi aside1, {}", in(reg) asid); +} + diff --git a/kernel/src/arch/aarch64/mach_orangepi3/wdog.rs b/kernel/src/arch/aarch64/mach_orangepi3/wdog.rs index 5178b9d..82b24d7 100644 --- a/kernel/src/arch/aarch64/mach_orangepi3/wdog.rs +++ b/kernel/src/arch/aarch64/mach_orangepi3/wdog.rs @@ -1,5 +1,6 @@ use crate::dev::Device; use crate::mem::virt::DeviceMemoryIo; +use crate::arch::intrin; use crate::sync::IrqSafeSpinLock; use crate::util::InitOnce; use libsys::error::Errno; @@ -72,7 +73,7 @@ impl RWdog { regs.CTRL.write(CTRL::KEY::Value + CTRL::RESTART::SET); loop { - asm!("wfe"); + intrin::hang(); } } diff --git a/kernel/src/arch/aarch64/mod.rs b/kernel/src/arch/aarch64/mod.rs index 5939518..01ebded 100644 --- a/kernel/src/arch/aarch64/mod.rs +++ b/kernel/src/arch/aarch64/mod.rs @@ -1,12 +1,13 @@ //! aarch64 architecture implementation +use core::arch::asm; use cortex_a::registers::DAIF; use tock_registers::interfaces::{Readable, Writeable}; -use core::arch::asm; pub mod boot; pub mod context; pub mod exception; +pub mod intrin; pub mod irq; pub mod reg; pub mod timer; @@ -35,7 +36,7 @@ cfg_if! { #[inline(always)] pub unsafe fn irq_mask_save() -> u64 { let state = DAIF.get(); - asm!("msr daifset, {bits}", bits = const 2, options(nomem, nostack, preserves_flags)); + intrin::irq_disable(); state } diff --git a/kernel/src/arch/aarch64/reg/cntkctl_el1.rs b/kernel/src/arch/aarch64/reg/cntkctl_el1.rs deleted file mode 100644 index c4b515c..0000000 --- a/kernel/src/arch/aarch64/reg/cntkctl_el1.rs +++ /dev/null @@ -1,50 +0,0 @@ -//! CNTKCTL_EL1 register -#![allow(missing_docs)] - -use core::arch::asm; - -use tock_registers::{ - interfaces::{Readable, Writeable}, - register_bitfields, -}; - -register_bitfields! { - u64, - /// Counter-timer Kernel Control Register - pub CNTKCTL_EL1 [ - /// If set, disables CNTPCT and CNTFRQ trapping from EL0 - EL0PCTEN OFFSET(0) NUMBITS(1) [] - ] -} - -/// CNTKCTL_EL1 register -pub struct Reg; - -impl Readable for Reg { - type T = u64; - type R = CNTKCTL_EL1::Register; - - #[inline(always)] - fn get(&self) -> Self::T { - let mut tmp; - unsafe { - asm!("mrs {}, cntkctl_el1", out(reg) tmp); - } - tmp - } -} - -impl Writeable for Reg { - type T = u64; - type R = CNTKCTL_EL1::Register; - - #[inline(always)] - fn set(&self, value: Self::T) { - unsafe { - asm!("msr cntkctl_el1, {}", in(reg) value); - } - } -} - -/// CNTKCTL_EL1 register -pub const CNTKCTL_EL1: Reg = Reg; diff --git a/kernel/src/arch/aarch64/reg/cpacr_el1.rs b/kernel/src/arch/aarch64/reg/cpacr_el1.rs deleted file mode 100644 index 9a0ef2c..0000000 --- a/kernel/src/arch/aarch64/reg/cpacr_el1.rs +++ /dev/null @@ -1,58 +0,0 @@ -//! CPACR_EL1 register -#![allow(missing_docs)] - -use core::arch::asm; -use tock_registers::{ - interfaces::{Readable, Writeable}, - register_bitfields, -}; - -register_bitfields! { - u64, - /// EL1 Architectural Feature Access Control Register - pub CPACR_EL1 [ - /// Enable EL0 and EL1 SIMD/FP accesses to EL1 - FPEN OFFSET(20) NUMBITS(2) [ - /// Trap both EL0 and EL1 - TrapAll = 0, - /// Trap EL0 - TrapEl0 = 1, - /// Trap EL1 - TrapEl1 = 2, - /// Do not trap any SIMD/FP instructions - TrapNone = 3 - ] - ] -} - -/// CPACR_EL1 register -pub struct Reg; - -impl Readable for Reg { - type T = u64; - type R = CPACR_EL1::Register; - - #[inline(always)] - fn get(&self) -> Self::T { - let mut tmp; - unsafe { - asm!("mrs {}, cpacr_el1", out(reg) tmp); - } - tmp - } -} - -impl Writeable for Reg { - type T = u64; - type R = CPACR_EL1::Register; - - #[inline(always)] - fn set(&self, value: Self::T) { - unsafe { - asm!("msr cpacr_el1, {}", in(reg) value); - } - } -} - -/// CPACR_EL1 register -pub const CPACR_EL1: Reg = Reg; diff --git a/kernel/src/arch/aarch64/reg/mod.rs b/kernel/src/arch/aarch64/reg/mod.rs index 01666e3..05c01b6 100644 --- a/kernel/src/arch/aarch64/reg/mod.rs +++ b/kernel/src/arch/aarch64/reg/mod.rs @@ -1,7 +1,66 @@ //! AArch64 architectural registers -pub mod cpacr_el1; -pub use cpacr_el1::CPACR_EL1; +use core::arch::asm; +use tock_registers::{ + interfaces::{Readable, Writeable}, + register_bitfields, +}; -pub mod cntkctl_el1; -pub use cntkctl_el1::CNTKCTL_EL1; +macro_rules! wrap_msr { + ($struct_name:ident, $name:ident, $reg:literal, $fields:tt) => { + register_bitfields! { + u64, + pub $name $fields + } + + pub struct $struct_name; + + impl Readable for $struct_name { + type T = u64; + type R = $name::Register; + + #[inline(always)] + fn get(&self) -> Self::T { + let mut value; + unsafe { + asm!(concat!("mrs {}, ", $reg), out(reg) value) + } + value + } + } + + impl Writeable for $struct_name { + type T = u64; + type R = $name::Register; + + #[inline(always)] + fn set(&self, value: Self::T) { + unsafe { + asm!(concat!("msr ", $reg, ", {}"), in(reg) value); + } + } + } + + pub const $name: $struct_name = $struct_name; + }; +} + +/// EL1 Architectural Feature Access Control Register +wrap_msr!(CpacrEl1, CPACR_EL1, "cpacr_el1", [ + /// Enable EL0 and EL1 SIMD/FP accesses to EL1 + FPEN OFFSET(20) NUMBITS(2) [ + /// Trap both EL0 and EL1 + TrapAll = 0, + /// Trap EL0 + TrapEl0 = 1, + /// Trap EL1 + TrapEl1 = 2, + /// Do not trap any SIMD/FP instructions + TrapNone = 3 + ] +]); + +wrap_msr!(CntkctlEl1, CNTKCTL_EL1, "cntkctl_el1", [ + /// If set, disables CNTPCT and CNTFRQ trapping from EL0 + EL0PCTEN OFFSET(0) NUMBITS(1) [] +]); diff --git a/kernel/src/arch/mod.rs b/kernel/src/arch/mod.rs index d8db6d8..2a0ae98 100644 --- a/kernel/src/arch/mod.rs +++ b/kernel/src/arch/mod.rs @@ -15,6 +15,8 @@ cfg_if! { pub use aarch64 as platform; pub use aarch64::machine; + + pub use aarch64::intrin; } } diff --git a/kernel/src/main.rs b/kernel/src/main.rs index e6b2633..2426870 100644 --- a/kernel/src/main.rs +++ b/kernel/src/main.rs @@ -42,7 +42,7 @@ use core::arch::asm; #[panic_handler] fn panic_handler(pi: &core::panic::PanicInfo) -> ! { unsafe { - asm!("msr daifset, #2"); + arch::intrin::irq_disable(); } errorln!("Panic: {:?}", pi); diff --git a/kernel/src/mem/mod.rs b/kernel/src/mem/mod.rs index 3f2433e..34df055 100644 --- a/kernel/src/mem/mod.rs +++ b/kernel/src/mem/mod.rs @@ -26,3 +26,18 @@ pub fn kernel_end_phys() -> usize { } unsafe { &__kernel_end as *const _ as usize - KERNEL_OFFSET } } + +// TODO cross-platform variant +#[inline(always)] +pub fn is_el0_accessible(virt: usize, write: bool) -> bool { + use core::arch::asm; + let mut res: usize; + unsafe { + if write { + asm!("at s1e0w, {}; mrs {}, par_el1", in(reg) virt, out(reg) res); + } else { + asm!("at s1e0r, {}; mrs {}, par_el1", in(reg) virt, out(reg) res); + } + } + res & 1 == 0 +} diff --git a/kernel/src/proc/process.rs b/kernel/src/proc/process.rs index ac83904..27092f6 100644 --- a/kernel/src/proc/process.rs +++ b/kernel/src/proc/process.rs @@ -1,15 +1,16 @@ //! Process data and control -use crate::arch::aarch64::exception::ExceptionFrame; +use crate::arch::{aarch64::exception::ExceptionFrame, intrin}; use crate::mem::{ self, phys::{self, PageUsage}, virt::{MapAttributes, Space}, }; use crate::proc::{ - wait::Wait, Context, ProcessIo, Thread, ThreadRef, ThreadState, PROCESSES, SCHED, Tid, + wait::Wait, Context, ProcessIo, Thread, ThreadRef, ThreadState, Tid, PROCESSES, SCHED, }; -use crate::sync::{IrqSafeSpinLock}; +use crate::sync::IrqSafeSpinLock; use alloc::{rc::Rc, vec::Vec}; +use core::arch::asm; use core::sync::atomic::{AtomicU32, Ordering}; use libsys::{ error::Errno, @@ -18,7 +19,6 @@ use libsys::{ signal::Signal, ProgramArgs, }; -use core::arch::asm; /// Wrapper type for a process struct reference pub type ProcessRef = Rc; @@ -166,7 +166,8 @@ impl Process { loop { let state = self.signal_state.load(Ordering::Acquire); if let Some(signal) = Self::find1(state).map(|e| Signal::try_from(e as u32).unwrap()) { - self.signal_state.fetch_and(!(1 << (signal as u32)), Ordering::Release); + self.signal_state + .fetch_and(!(1 << (signal as u32)), Ordering::Release); main_thread.clone().enter_signal(signal, ttbr0); } else { break; @@ -190,7 +191,8 @@ impl Process { main_thread.enter_signal(signal, ttbr0); } ThreadState::Waiting => { - self.signal_state.fetch_or(1 << (signal as u32), Ordering::Release); + self.signal_state + .fetch_or(1 << (signal as u32), Ordering::Release); main_thread.interrupt_wait(true); } ThreadState::Ready => { @@ -289,7 +291,7 @@ impl Process { if let Some(space) = lock.space.take() { unsafe { Space::release(space); - Process::invalidate_asid((lock.id.asid() as usize) << 48); + intrin::flush_tlb_asid((lock.id.asid() as usize) << 48); } } @@ -466,13 +468,8 @@ impl Process { } pub fn invalidate_tlb(&self) { - Process::invalidate_asid(self.asid()); - } - - #[inline] - pub fn invalidate_asid(asid: usize) { unsafe { - asm!("tlbi aside1, {}", in(reg) asid); + intrin::flush_tlb_asid(self.asid()); } } @@ -483,7 +480,7 @@ impl Process { ) -> Result<(), Errno> { unsafe { // Run with interrupts disabled - asm!("msr daifset, #2"); + intrin::irq_disable(); } let proc = Process::current(); @@ -540,7 +537,7 @@ impl Process { // TODO drop old context let ctx = thread.ctx.get(); let asid = (process_lock.id.asid() as usize) << 48; - Process::invalidate_asid(asid); + intrin::flush_tlb_asid(asid); ctx.write(Context::user( entry, diff --git a/kernel/src/proc/sched.rs b/kernel/src/proc/sched.rs index 7b9336e..176280b 100644 --- a/kernel/src/proc/sched.rs +++ b/kernel/src/proc/sched.rs @@ -1,10 +1,10 @@ //! use crate::proc::{Thread, ThreadRef, THREADS}; use crate::sync::IrqSafeSpinLock; +use crate::arch::intrin; use crate::util::InitOnce; use libsys::proc::Tid; use alloc::{collections::VecDeque, rc::Rc}; -use core::arch::asm; struct SchedulerInner { queue: VecDeque, @@ -70,7 +70,7 @@ impl Scheduler { THREADS.lock().get(&id).unwrap().clone() }; - asm!("msr daifset, #2"); + intrin::irq_disable(); Thread::enter(thread) } @@ -122,7 +122,7 @@ impl Scheduler { if !Rc::ptr_eq(&from, &to) { unsafe { - asm!("msr daifset, #2"); + intrin::irq_disable(); Thread::switch(from, to, discard); } } diff --git a/kernel/src/syscall/arg.rs b/kernel/src/syscall/arg.rs index 871cf42..bdda7c0 100644 --- a/kernel/src/syscall/arg.rs +++ b/kernel/src/syscall/arg.rs @@ -1,6 +1,7 @@ //! System call argument ABI helpers use crate::mem; +use crate::arch::intrin; use core::alloc::Layout; use libsys::error::Errno; use crate::proc::Process; @@ -24,19 +25,6 @@ macro_rules! invalid_memory { } } -#[inline(always)] -fn is_el0_accessible(virt: usize, write: bool) -> bool { - let mut res: usize; - unsafe { - if write { - asm!("at s1e0w, {}; mrs {}, par_el1", in(reg) virt, out(reg) res); - } else { - asm!("at s1e0r, {}; mrs {}, par_el1", in(reg) virt, out(reg) res); - } - } - res & 1 == 0 -} - /// Checks given argument and interprets it as a `T` reference pub fn struct_ref<'a, T>(base: usize) -> Result<&'a T, Errno> { let layout = Layout::new::(); @@ -126,13 +114,15 @@ pub fn validate_ptr(base: usize, len: usize, write: bool) -> Result<(), Errno> { let asid = process.asid(); for i in (base / mem::PAGE_SIZE)..((base + len + mem::PAGE_SIZE - 1) / mem::PAGE_SIZE) { - if !is_el0_accessible(i * mem::PAGE_SIZE, write) { + if !mem::is_el0_accessible(i * mem::PAGE_SIZE, write) { // It's possible a CoW page hasn't yet been cloned when trying // a write access let res = if write { process.manipulate_space(|space| { space.try_cow_copy(i * mem::PAGE_SIZE)?; - Process::invalidate_asid(asid); + unsafe { + intrin::flush_tlb_asid(asid); + } Ok(()) }) } else {