aarch64: prettify cache init

kernel/arch/aarch64/src/mem/mod.rs

@@ -6,7 +6,7 @@ use core::{
 
 use aarch64_cpu::{
     asm::barrier,
-    registers::{PAR_EL1, TTBR0_EL1, TTBR1_EL1},
+    registers::{MAIR_EL1, PAR_EL1, SCTLR_EL1, TTBR0_EL1, TTBR1_EL1},
 };
 use kernel_arch_interface::{
     mem::{DeviceMemoryAttributes, KernelTableManager, RawDeviceMemoryMapping},
@@ -18,7 +18,7 @@ use libk_mm_interface::{
 };
 use memtables::aarch64::{FixedTables, KERNEL_L3_COUNT};
 use static_assertions::const_assert_eq;
-use tock_registers::interfaces::{Readable, Writeable};
+use tock_registers::interfaces::{ReadWriteable, Readable, Writeable};
 use yggdrasil_abi::error::Error;
 
 use crate::ArchitectureImpl;
@@ -496,3 +496,47 @@ pub unsafe fn init_fixed_tables() {
 
     tlb_flush_all();
 }
+
+pub fn setup_memory_attributes() {
+    // TODO: Figure out why WriteBack_NonTransient_ReadWriteAlloc doesn't work on Pi 4B
+
+    MAIR_EL1.write(
+        //// Attribute 0 -- normal memory
+        MAIR_EL1::Attr0_Normal_Inner::WriteBack_NonTransient +
+        MAIR_EL1::Attr0_Normal_Outer::WriteBack_NonTransient +
+        //// Attribute 1 -- normal non-cacheable memory
+        MAIR_EL1::Attr0_Normal_Inner::NonCacheable +
+        MAIR_EL1::Attr0_Normal_Outer::NonCacheable +
+        //// Attribute 2 -- device memory
+        MAIR_EL1::Attr1_Device::nonGathering_nonReordering_EarlyWriteAck,
+    );
+}
+
+pub unsafe fn enable_dcache() {
+    barrier::dsb(barrier::ISHST);
+    barrier::isb(barrier::SY);
+
+    SCTLR_EL1.modify(SCTLR_EL1::C::Cacheable);
+
+    barrier::dsb(barrier::ISH);
+    barrier::isb(barrier::SY);
+}
+
+pub unsafe fn enable_icache() {
+    barrier::isb(barrier::SY);
+
+    SCTLR_EL1.modify(SCTLR_EL1::I::Cacheable);
+
+    barrier::dsb(barrier::ISH);
+    barrier::isb(barrier::SY);
+}
+
+pub unsafe fn disable_icache() {
+    barrier::isb(barrier::SY);
+
+    ic_iallu();
+    SCTLR_EL1.modify(SCTLR_EL1::I::NonCacheable);
+
+    barrier::dsb(barrier::ISH);
+    barrier::isb(barrier::SY);
+}

kernel/arch/aarch64/src/mem/table.rs

@@ -41,7 +41,8 @@ bitflags! {
         const SH_INNER = 3 << 8;
 
         const PAGE_ATTR_NORMAL = 0 << 2;
-        const PAGE_ATTR_DEVICE = 1 << 2;
+        const PAGE_ATTR_NORMAL_NC = 1 << 2;
+        const PAGE_ATTR_DEVICE = 2 << 2;
 
         const NON_GLOBAL = 1 << 11;
 

kernel/src/arch/aarch64/boot/mod.rs

@@ -3,10 +3,10 @@ use core::arch::global_asm;
 
 use aarch64_cpu::{
     asm::barrier,
-    registers::{CPACR_EL1, ID_AA64MMFR0_EL1, MAIR_EL1, SCTLR_EL1, TCR_EL1, TTBR0_EL1},
+    registers::{CPACR_EL1, ID_AA64MMFR0_EL1, SCTLR_EL1, TCR_EL1, TTBR0_EL1},
 };
 use kernel_arch::{absolute_address, Architecture, ArchitectureImpl};
-use kernel_arch_aarch64::mem::ic_iallu;
+use kernel_arch_aarch64::mem;
 use libk::{devfs, task::runtime};
 use libk_mm::address::PhysicalAddress;
 use tock_registers::interfaces::{ReadWriteable, Readable, Writeable};
@@ -22,26 +22,17 @@ unsafe fn check_mmu_features() {
 }
 
 unsafe fn pre_init_mmu() {
-    // TODO: Figure out why WriteBack_NonTransient_ReadWriteAlloc doesn't work on Pi 4B
+    mem::setup_memory_attributes();
 
-    MAIR_EL1.write(
-        //// Attribute 0 -- normal memory
-        // Inner
-        MAIR_EL1::Attr0_Normal_Inner::WriteBack_NonTransient +
-        // Outer
-        MAIR_EL1::Attr0_Normal_Outer::WriteBack_NonTransient +
-        //// Attribute 1 -- device memory
-        MAIR_EL1::Attr1_Device::nonGathering_nonReordering_EarlyWriteAck,
-    );
     TCR_EL1.write(
         TCR_EL1::AS::ASID8Bits +
         TCR_EL1::A1::TTBR0 +
         // General
         TCR_EL1::IPS::Bits_48 +
         // TTBR0
-        TCR_EL1::TG0::KiB_4 + TCR_EL1::T0SZ.val(25) + TCR_EL1::SH0::Outer +
+        TCR_EL1::TG0::KiB_4 + TCR_EL1::T0SZ.val(25) + TCR_EL1::SH0::Inner +
         // TTBR1
-        TCR_EL1::TG1::KiB_4 + TCR_EL1::T1SZ.val(25) + TCR_EL1::SH1::Outer,
+        TCR_EL1::TG1::KiB_4 + TCR_EL1::T1SZ.val(25) + TCR_EL1::SH1::Inner,
     );
 }
 
@@ -66,18 +57,9 @@ unsafe fn enable_mmu() {
     // Enable translation
     SCTLR_EL1.modify(SCTLR_EL1::M::Enable);
 
-    barrier::dsb(barrier::ISH);
+    // Enable caches
-    barrier::isb(barrier::SY);
+    mem::enable_icache();
-
+    mem::enable_dcache();
-    ic_iallu();
-    SCTLR_EL1.modify(SCTLR_EL1::I::Cacheable);
-
-    barrier::dsb(barrier::ISH);
-    barrier::isb(barrier::SY);
-
-    SCTLR_EL1.modify(SCTLR_EL1::C::Cacheable);
-
-    barrier::isb(barrier::SY);
 }
 
 unsafe fn enter_higher_half(sp: usize, elr: usize, x0: usize) -> ! {

kernel/src/arch/aarch64/mod.rs

@@ -2,10 +2,7 @@
 
 use core::sync::atomic::{self, Ordering};
 
-use aarch64_cpu::{
+use aarch64_cpu::registers::{CNTP_CTL_EL0, CNTP_TVAL_EL0};
-    asm::barrier,
-    registers::{CNTP_CTL_EL0, CNTP_TVAL_EL0, SCTLR_EL1},
-};
 use abi::error::Error;
 use alloc::sync::Arc;
 use device_api::{
@@ -16,7 +13,7 @@ use device_tree::dt::{DevTreeIndexNodePropGet, DeviceTree, FdtMemoryRegionIter};
 use kernel_arch::Architecture;
 use kernel_arch_aarch64::{
     mem::{
-        ic_iallu,
+        self,
         table::{L1, L3},
         EarlyMapping, MEMORY_LIMIT, RAM_MAPPING_L1_COUNT,
     },
@@ -34,7 +31,7 @@ use libk_mm::{
     table::EntryLevelExt,
 };
 use libk_util::OneTimeInit;
-use tock_registers::interfaces::{ReadWriteable, Writeable};
+use tock_registers::interfaces::Writeable;
 use ygg_driver_pci::PciBusManager;
 
 use crate::{
@@ -237,14 +234,7 @@ impl AArch64 {
             "raspberrypi,4-model-b" => {
                 // XXX Workaround for Raspberry Pi 4B:
                 //  Instruction cache totally freaks out at EL0 and I don't know why
-                barrier::dsb(barrier::ISH);
+                unsafe { mem::disable_icache() };
-                barrier::isb(barrier::SY);
-
-                ic_iallu();
-                SCTLR_EL1.modify(SCTLR_EL1::I::NonCacheable);
-
-                barrier::dsb(barrier::ISH);
-                barrier::isb(barrier::SY);
             }
             _ => (),
         }

kernel/src/device/serial/bcm2835_aux_uart.rs

@@ -71,6 +71,10 @@ pub struct Bcm2835AuxUart {
 
 impl Regs {
     fn write_byte(&self, byte: u8) -> Result<(), Error> {
+        if byte == b'\n' {
+            self.write_byte(b'\r').ok();
+        }
+
         while !self
             .AUX_MU_LSR_REG
             .matches_all(AUX_MU_LSR_REG::TX_EMPTY::SET)