diff --git a/kernel/arch/aarch64/src/mem/mod.rs b/kernel/arch/aarch64/src/mem/mod.rs index 34d123e0..5a845a2f 100644 --- a/kernel/arch/aarch64/src/mem/mod.rs +++ b/kernel/arch/aarch64/src/mem/mod.rs @@ -6,7 +6,7 @@ use core::{ use aarch64_cpu::{ asm::barrier, - registers::{PAR_EL1, TTBR0_EL1, TTBR1_EL1}, + registers::{MAIR_EL1, PAR_EL1, SCTLR_EL1, TTBR0_EL1, TTBR1_EL1}, }; use kernel_arch_interface::{ mem::{DeviceMemoryAttributes, KernelTableManager, RawDeviceMemoryMapping}, @@ -18,7 +18,7 @@ use libk_mm_interface::{ }; use memtables::aarch64::{FixedTables, KERNEL_L3_COUNT}; use static_assertions::const_assert_eq; -use tock_registers::interfaces::{Readable, Writeable}; +use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; use yggdrasil_abi::error::Error; use crate::ArchitectureImpl; @@ -496,3 +496,47 @@ pub unsafe fn init_fixed_tables() { tlb_flush_all(); } + +pub fn setup_memory_attributes() { + // TODO: Figure out why WriteBack_NonTransient_ReadWriteAlloc doesn't work on Pi 4B + + MAIR_EL1.write( + //// Attribute 0 -- normal memory + MAIR_EL1::Attr0_Normal_Inner::WriteBack_NonTransient + + MAIR_EL1::Attr0_Normal_Outer::WriteBack_NonTransient + + //// Attribute 1 -- normal non-cacheable memory + MAIR_EL1::Attr0_Normal_Inner::NonCacheable + + MAIR_EL1::Attr0_Normal_Outer::NonCacheable + + //// Attribute 2 -- device memory + MAIR_EL1::Attr1_Device::nonGathering_nonReordering_EarlyWriteAck, + ); +} + +pub unsafe fn enable_dcache() { + barrier::dsb(barrier::ISHST); + barrier::isb(barrier::SY); + + SCTLR_EL1.modify(SCTLR_EL1::C::Cacheable); + + barrier::dsb(barrier::ISH); + barrier::isb(barrier::SY); +} + +pub unsafe fn enable_icache() { + barrier::isb(barrier::SY); + + SCTLR_EL1.modify(SCTLR_EL1::I::Cacheable); + + barrier::dsb(barrier::ISH); + barrier::isb(barrier::SY); +} + +pub unsafe fn disable_icache() { + barrier::isb(barrier::SY); + + ic_iallu(); + SCTLR_EL1.modify(SCTLR_EL1::I::NonCacheable); + + barrier::dsb(barrier::ISH); + barrier::isb(barrier::SY); +} diff --git a/kernel/arch/aarch64/src/mem/table.rs b/kernel/arch/aarch64/src/mem/table.rs index 6b15b3df..82a5eeed 100644 --- a/kernel/arch/aarch64/src/mem/table.rs +++ b/kernel/arch/aarch64/src/mem/table.rs @@ -41,7 +41,8 @@ bitflags! { const SH_INNER = 3 << 8; const PAGE_ATTR_NORMAL = 0 << 2; - const PAGE_ATTR_DEVICE = 1 << 2; + const PAGE_ATTR_NORMAL_NC = 1 << 2; + const PAGE_ATTR_DEVICE = 2 << 2; const NON_GLOBAL = 1 << 11; diff --git a/kernel/src/arch/aarch64/boot/mod.rs b/kernel/src/arch/aarch64/boot/mod.rs index 0f86ba85..35013e14 100644 --- a/kernel/src/arch/aarch64/boot/mod.rs +++ b/kernel/src/arch/aarch64/boot/mod.rs @@ -3,10 +3,10 @@ use core::arch::global_asm; use aarch64_cpu::{ asm::barrier, - registers::{CPACR_EL1, ID_AA64MMFR0_EL1, MAIR_EL1, SCTLR_EL1, TCR_EL1, TTBR0_EL1}, + registers::{CPACR_EL1, ID_AA64MMFR0_EL1, SCTLR_EL1, TCR_EL1, TTBR0_EL1}, }; use kernel_arch::{absolute_address, Architecture, ArchitectureImpl}; -use kernel_arch_aarch64::mem::ic_iallu; +use kernel_arch_aarch64::mem; use libk::{devfs, task::runtime}; use libk_mm::address::PhysicalAddress; use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; @@ -22,26 +22,17 @@ unsafe fn check_mmu_features() { } unsafe fn pre_init_mmu() { - // TODO: Figure out why WriteBack_NonTransient_ReadWriteAlloc doesn't work on Pi 4B + mem::setup_memory_attributes(); - MAIR_EL1.write( - //// Attribute 0 -- normal memory - // Inner - MAIR_EL1::Attr0_Normal_Inner::WriteBack_NonTransient + - // Outer - MAIR_EL1::Attr0_Normal_Outer::WriteBack_NonTransient + - //// Attribute 1 -- device memory - MAIR_EL1::Attr1_Device::nonGathering_nonReordering_EarlyWriteAck, - ); TCR_EL1.write( TCR_EL1::AS::ASID8Bits + TCR_EL1::A1::TTBR0 + // General TCR_EL1::IPS::Bits_48 + // TTBR0 - TCR_EL1::TG0::KiB_4 + TCR_EL1::T0SZ.val(25) + TCR_EL1::SH0::Outer + + TCR_EL1::TG0::KiB_4 + TCR_EL1::T0SZ.val(25) + TCR_EL1::SH0::Inner + // TTBR1 - TCR_EL1::TG1::KiB_4 + TCR_EL1::T1SZ.val(25) + TCR_EL1::SH1::Outer, + TCR_EL1::TG1::KiB_4 + TCR_EL1::T1SZ.val(25) + TCR_EL1::SH1::Inner, ); } @@ -66,18 +57,9 @@ unsafe fn enable_mmu() { // Enable translation SCTLR_EL1.modify(SCTLR_EL1::M::Enable); - barrier::dsb(barrier::ISH); - barrier::isb(barrier::SY); - - ic_iallu(); - SCTLR_EL1.modify(SCTLR_EL1::I::Cacheable); - - barrier::dsb(barrier::ISH); - barrier::isb(barrier::SY); - - SCTLR_EL1.modify(SCTLR_EL1::C::Cacheable); - - barrier::isb(barrier::SY); + // Enable caches + mem::enable_icache(); + mem::enable_dcache(); } unsafe fn enter_higher_half(sp: usize, elr: usize, x0: usize) -> ! { diff --git a/kernel/src/arch/aarch64/mod.rs b/kernel/src/arch/aarch64/mod.rs index 8b4e76c6..de945338 100644 --- a/kernel/src/arch/aarch64/mod.rs +++ b/kernel/src/arch/aarch64/mod.rs @@ -2,10 +2,7 @@ use core::sync::atomic::{self, Ordering}; -use aarch64_cpu::{ - asm::barrier, - registers::{CNTP_CTL_EL0, CNTP_TVAL_EL0, SCTLR_EL1}, -}; +use aarch64_cpu::registers::{CNTP_CTL_EL0, CNTP_TVAL_EL0}; use abi::error::Error; use alloc::sync::Arc; use device_api::{ @@ -16,7 +13,7 @@ use device_tree::dt::{DevTreeIndexNodePropGet, DeviceTree, FdtMemoryRegionIter}; use kernel_arch::Architecture; use kernel_arch_aarch64::{ mem::{ - ic_iallu, + self, table::{L1, L3}, EarlyMapping, MEMORY_LIMIT, RAM_MAPPING_L1_COUNT, }, @@ -34,7 +31,7 @@ use libk_mm::{ table::EntryLevelExt, }; use libk_util::OneTimeInit; -use tock_registers::interfaces::{ReadWriteable, Writeable}; +use tock_registers::interfaces::Writeable; use ygg_driver_pci::PciBusManager; use crate::{ @@ -237,14 +234,7 @@ impl AArch64 { "raspberrypi,4-model-b" => { // XXX Workaround for Raspberry Pi 4B: // Instruction cache totally freaks out at EL0 and I don't know why - barrier::dsb(barrier::ISH); - barrier::isb(barrier::SY); - - ic_iallu(); - SCTLR_EL1.modify(SCTLR_EL1::I::NonCacheable); - - barrier::dsb(barrier::ISH); - barrier::isb(barrier::SY); + unsafe { mem::disable_icache() }; } _ => (), } diff --git a/kernel/src/device/serial/bcm2835_aux_uart.rs b/kernel/src/device/serial/bcm2835_aux_uart.rs index 524667be..5075db3c 100644 --- a/kernel/src/device/serial/bcm2835_aux_uart.rs +++ b/kernel/src/device/serial/bcm2835_aux_uart.rs @@ -71,6 +71,10 @@ pub struct Bcm2835AuxUart { impl Regs { fn write_byte(&self, byte: u8) -> Result<(), Error> { + if byte == b'\n' { + self.write_byte(b'\r').ok(); + } + while !self .AUX_MU_LSR_REG .matches_all(AUX_MU_LSR_REG::TX_EMPTY::SET)