diff --git a/Cargo.lock b/Cargo.lock index 8cf66ba..f3d6bff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,6 +8,12 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + [[package]] name = "cfg-if" version = "1.0.0" @@ -59,6 +65,7 @@ dependencies = [ name = "kernel" version = "0.1.0" dependencies = [ + "bitflags", "cfg-if", "cortex-a", "error", diff --git a/kernel/Cargo.toml b/kernel/Cargo.toml index 9fb88f9..f981f4b 100644 --- a/kernel/Cargo.toml +++ b/kernel/Cargo.toml @@ -14,6 +14,7 @@ error = { path = "../error" } cfg-if = "1.x.x" tock-registers = "0.7.x" fdt-rs = { version = "0.x.x", default-features = false } +bitflags = "^1.3.0" [target.'cfg(target_arch = "aarch64")'.dependencies] cortex-a = { version = "6.x.x" } diff --git a/kernel/src/main.rs b/kernel/src/main.rs index 389a261..4405c5e 100644 --- a/kernel/src/main.rs +++ b/kernel/src/main.rs @@ -17,6 +17,8 @@ #[macro_use] extern crate cfg_if; +#[macro_use] +extern crate bitflags; extern crate alloc; #[macro_use] diff --git a/kernel/src/mem/virt/fixed.rs b/kernel/src/mem/virt/fixed.rs new file mode 100644 index 0000000..28e32c2 --- /dev/null +++ b/kernel/src/mem/virt/fixed.rs @@ -0,0 +1,111 @@ +// +//#[repr(C, align(0x1000))] +//pub struct OldTable([u64; 512]); +// +//#[no_mangle] +//static mut KERNEL_TTBR1: OldTable = OldTable([0; 512]); +//// 1GiB +//static mut KERNEL_L1: OldTable = OldTable([0; 512]); +//// 2MiB +//static mut KERNEL_L2: OldTable = OldTable([0; 512]); +//static mut COUNT: usize = 0; +//static mut BIG_COUNT: usize = 1; +//static mut HUGE_COUNT: usize = 1; +// + +use crate::mem::{ + self, + virt::{Entry, MapAttributes, Table}, +}; +use cortex_a::asm::barrier::{self, dsb, isb}; +use error::Errno; + +const DEVICE_MAP_OFFSET: usize = mem::KERNEL_OFFSET + (256usize << 30); + +#[repr(C, align(0x1000))] +pub struct FixedTableGroup { + l0: Table, + l1: Table, + l2: Table, + + pages_4k: usize, + pages_2m: usize, + pages_1g: usize, +} + +impl FixedTableGroup { + pub const fn empty() -> Self { + Self { + l0: Table::empty(), + l1: Table::empty(), + l2: Table::empty(), + + pages_4k: 0, + pages_2m: 1, + pages_1g: 1, + } + } + + pub fn map_region(&mut self, phys: usize, count: usize) -> Result { + // TODO generalize region allocation + let phys_page = phys & !0xFFF; + let attrs = MapAttributes::SH_OUTER | MapAttributes::DEVICE | MapAttributes::ACCESS; + + match count { + 262144 => { + let count = self.pages_1g; + if count == 512 { + return Err(Errno::OutOfMemory); + } + self.pages_1g += 1; + + self.l0[count + 256] = Entry::block(phys_page, attrs); + unsafe { + dsb(barrier::SY); + isb(barrier::SY); + } + + Ok(DEVICE_MAP_OFFSET + (count << 30) + (phys & 0xFFF)) + } + 512 => { + let count = self.pages_2m; + if count == 512 { + return Err(Errno::OutOfMemory); + } + self.pages_2m += 1; + + self.l1[count] = Entry::block(phys_page, attrs); + unsafe { + dsb(barrier::SY); + isb(barrier::SY); + } + + Ok(DEVICE_MAP_OFFSET + (count << 21) + (phys & 0xFFF)) + } + 1 => { + let count = self.pages_4k; + if count == 512 { + return Err(Errno::OutOfMemory); + } + self.pages_4k += 1; + + self.l2[count] = Entry::table(phys_page, attrs); + unsafe { + dsb(barrier::SY); + isb(barrier::SY); + } + + Ok(DEVICE_MAP_OFFSET + (count << 12) + (phys & 0xFFF)) + } + _ => unimplemented!(), + } + } + + pub fn init_device_map(&mut self) { + let l1_phys = (&self.l1 as *const _) as usize - mem::KERNEL_OFFSET; + let l2_phys = (&self.l2 as *const _) as usize - mem::KERNEL_OFFSET; + + self.l0[256] = Entry::table(l1_phys, MapAttributes::empty()); + self.l1[0] = Entry::table(l2_phys, MapAttributes::empty()); + } +} diff --git a/kernel/src/mem/virt/mod.rs b/kernel/src/mem/virt/mod.rs index bc5dafb..688fd69 100644 --- a/kernel/src/mem/virt/mod.rs +++ b/kernel/src/mem/virt/mod.rs @@ -1,10 +1,5 @@ #![allow(missing_docs)] -use crate::mem::{ - self, - phys::{self, PageUsage}, - KERNEL_OFFSET, -}; use core::marker::PhantomData; use core::ops::Deref; use cortex_a::asm::barrier::{self, dsb, isb}; @@ -12,29 +7,13 @@ use cortex_a::registers::TTBR0_EL1; use error::Errno; use tock_registers::interfaces::Writeable; -const PTE_BLOCK_AF: u64 = 1 << 10; -const PTE_BLOCK_OSH: u64 = 2 << 8; -const PTE_TABLE: u64 = 1 << 1; -const PTE_PRESENT: u64 = 1 << 0; -const PTE_ATTR1: u64 = 1 << 2; -const PTE_BLOCK_NG: u64 = 1 << 11; - -#[repr(C, align(0x1000))] -pub struct Table([u64; 512]); - -pub use Table as Space; +pub mod table; +pub use table::{Entry, MapAttributes, Space, Table}; +pub mod fixed; +pub use fixed::FixedTableGroup; #[no_mangle] -static mut KERNEL_TTBR1: Table = Table([0; 512]); -// 1GiB -static mut KERNEL_L1: Table = Table([0; 512]); -// 2MiB -static mut KERNEL_L2: Table = Table([0; 512]); -static mut COUNT: usize = 0; -static mut BIG_COUNT: usize = 1; -static mut HUGE_COUNT: usize = 1; - -const DEVICE_MAP_OFFSET: usize = KERNEL_OFFSET + (256 << 30); +static mut KERNEL_TTBR1: FixedTableGroup = FixedTableGroup::empty(); #[derive(Debug)] #[allow(dead_code)] @@ -48,101 +27,6 @@ pub struct DeviceMemoryIo { mmio: DeviceMemory, _0: PhantomData, } - -impl Table { - pub fn empty() -> Result<&'static mut Self, Errno> { - let phys = phys::alloc_page(PageUsage::Paging)?; - let virt = mem::virtualize(phys); - let res = unsafe { &mut *(virt as *mut Self) }; - res.0.fill(0); - Ok(res) - } - - pub fn dump(&self) { - debugln!("Paging table dump:"); - for l0i in 0usize..512 { - let l0e = self.0[l0i]; - if l0e & PTE_PRESENT == 0 { - continue; - } - - let l1_phys = (l0e & 0xfffffffff000) as usize; - let l1t = unsafe { &mut *(mem::virtualize(l1_phys) as *mut Self) }; - - for l1i in 0usize..512 { - let l1e = l1t.0[l1i]; - if l1e & PTE_PRESENT == 0 { - continue; - } - - let l2_phys = (l1e & 0xfffffffff000) as usize; - let l2t = unsafe { &mut *(mem::virtualize(l2_phys) as *mut Self) }; - - for l2i in 0usize..512 { - let l2e = l2t.0[l2i]; - if l2e & PTE_PRESENT == 0 { - continue; - } - - let virt = (l0i << 30) | (l1i << 21) | (l2i << 12); - debugln!("{:#x} -> {:#x}", virt, l2e & 0xfffffffff000); - } - } - } - } - - pub unsafe fn map(&mut self, virt: usize, phys: usize) -> Result<(), Errno> { - let l0i = virt >> 30; - let l1i = (virt >> 21) & 0x1FF; - let l2i = (virt >> 12) & 0x1FF; - - debugln!("l0i = {}, l1i = {}, l2i = {}", l0i, l1i, l2i); - - let l0e = self.0[l0i]; - let l1_phys = if l0e & PTE_PRESENT != 0 { - assert!(l0e & PTE_TABLE != 0); - (l0e & 0xfffffffff000) as usize - } else { - let page = phys::alloc_page(PageUsage::Paging)?; - self.0[l0i] = (page as u64) | (PTE_PRESENT | PTE_TABLE); - - let virt = mem::virtualize(phys); - let res = &mut *(virt as *mut Self); - res.0.fill(0); - page - }; - let l1t = &mut *(mem::virtualize(l1_phys) as *mut Self); - - let l1e = l1t.0[l1i]; - let l2_phys = if l1e & PTE_PRESENT != 0 { - assert!(l1e & PTE_TABLE != 0); - (l1e & 0xfffffffff000) as usize - } else { - let page = phys::alloc_page(PageUsage::Paging)?; - l1t.0[l1i] = (page as u64) | (PTE_PRESENT | PTE_TABLE); - - let virt = mem::virtualize(phys); - let res = &mut *(virt as *mut Self); - res.0.fill(0); - page - }; - let l2t = &mut *(mem::virtualize(l2_phys) as *mut Self); - - if l2t.0[l2i] & PTE_PRESENT != 0 { - panic!( - "Page is already mapped: {:#x} (tried {:#x}, got {:#x})", - virt, phys, l2t.0[l2i] - ); - } - - debugln!("{:p} map {:#x} -> {:#x}", self, virt, phys); - l2t.0[l2i] = - (phys as u64) | (PTE_BLOCK_NG | PTE_PRESENT | PTE_TABLE | PTE_BLOCK_OSH | PTE_BLOCK_AF); - - Ok(()) - } -} - impl DeviceMemory { #[inline(always)] pub const fn base(&self) -> usize { @@ -150,66 +34,14 @@ impl DeviceMemory { } pub fn map(name: &'static str, phys: usize, count: usize) -> Result { - // TODO generalize this - let phys_page = phys & !0xFFF; - - let base = unsafe { - match count { - 262144 => { - let count = HUGE_COUNT; - if count == 512 { - return Err(Errno::OutOfMemory); - } - HUGE_COUNT += 1; - - KERNEL_TTBR1.0[count + 256] = - (phys_page as u64) | PTE_PRESENT | PTE_BLOCK_OSH | PTE_BLOCK_AF | PTE_ATTR1; - asm!("dsb ish; isb"); - - DEVICE_MAP_OFFSET + (count << 30) + (phys & 0xFFF) - } - 512 => { - let count = BIG_COUNT; - if count == 512 { - return Err(Errno::OutOfMemory); - } - BIG_COUNT += 1; - - KERNEL_L1.0[count] = - (phys_page as u64) | PTE_PRESENT | PTE_BLOCK_OSH | PTE_BLOCK_AF | PTE_ATTR1; - asm!("dsb ish; isb"); - - DEVICE_MAP_OFFSET + (count << 21) + (phys & 0xFFF) - } - 1 => { - let count = COUNT; - if count == 512 { - return Err(Errno::OutOfMemory); - } - COUNT += 1; - - KERNEL_L2.0[count] = (phys_page as u64) - | PTE_TABLE - | PTE_BLOCK_OSH - | PTE_PRESENT - | PTE_BLOCK_AF - | PTE_ATTR1; - asm!("dsb ish; isb"); - - DEVICE_MAP_OFFSET + (count << 12) + (phys & 0xFFF) - } - _ => unimplemented!(), - } - }; - + let base = unsafe { KERNEL_TTBR1.map_region(phys, count) }?; debugln!( - "Mapping {:#x}..{:#x} -> {:#x} ({:?})", + "Mapping {:#x}..{:#x} -> {:#x} for {:?}", base, base + count * 0x1000, phys, name ); - Ok(Self { name, base, count }) } @@ -247,17 +79,8 @@ impl Deref for DeviceMemoryIo { pub fn enable() -> Result<(), Errno> { unsafe { - // TODO function to translate kernel addresses to physical ones - let l1_base = (&KERNEL_L1 as *const _ as u64) - KERNEL_OFFSET as u64; - let l2_base = (&KERNEL_L2 as *const _ as u64) - KERNEL_OFFSET as u64; + KERNEL_TTBR1.init_device_map(); - KERNEL_L1.0[0] = l2_base | PTE_TABLE | PTE_PRESENT; - KERNEL_TTBR1.0[256] = l1_base | PTE_TABLE | PTE_PRESENT; - - // NOTE don't think tlb needs to be invalidated when new entries are created - } - - unsafe { dsb(barrier::ISH); isb(barrier::SY); } diff --git a/kernel/src/mem/virt/table.rs b/kernel/src/mem/virt/table.rs new file mode 100644 index 0000000..4610f85 --- /dev/null +++ b/kernel/src/mem/virt/table.rs @@ -0,0 +1,125 @@ +use crate::mem::{ + self, + phys::{self, PageUsage}, +}; +use error::Errno; +use core::ops::{Index, IndexMut}; + +#[derive(Clone, Copy)] +#[repr(transparent)] +pub struct Entry(u64); + +#[repr(C, align(0x1000))] +pub struct Table { + entries: [Entry; 512], +} + +#[repr(transparent)] +pub struct Space(Table); + +bitflags! { + pub struct MapAttributes: u64 { + // TODO use 2 lower bits to determine mapping size? + const NOT_GLOBAL = 1 << 11; + const ACCESS = 1 << 10; + const SH_OUTER = 2 << 8; + const DEVICE = 1 << 2; + + const UXN = 1 << 54; + const PXN = 1 << 53; + } +} + +impl Table { + pub fn next_level_table_or_alloc(&mut self, index: usize) -> Result<&'static mut Table, Errno> { + let entry = self[index]; + if entry.is_present() { + if !entry.is_table() { + return Err(Errno::InvalidArgument); + } + + Ok(unsafe { &mut *(mem::virtualize(entry.address_unchecked()) as *mut _) }) + } else { + let phys = phys::alloc_page(PageUsage::Paging)?; + debugln!("Allocated new page table at {:#x}", phys); + let res = unsafe { &mut *(mem::virtualize(phys) as *mut Self) }; + self[index] = Entry::table(phys, MapAttributes::empty()); + res.entries.fill(Entry::invalid()); + Ok(res) + } + } + + pub const fn empty() -> Table { + Table { entries: [Entry::invalid(); 512] } + } +} + +impl Index for Table { + type Output = Entry; + + fn index(&self, index: usize) -> &Self::Output { + &self.entries[index] + } +} + +impl IndexMut for Table { + fn index_mut(&mut self, index: usize) -> &mut Self::Output { + &mut self.entries[index] + } +} + +impl Entry { + const PRESENT: u64 = 1 << 0; + const TABLE: u64 = 1 << 1; + const PHYS_MASK: u64 = 0x0000FFFFFFFFF000; + + pub const fn invalid() -> Self { + Self(0) + } + + pub const fn block(phys: usize, attrs: MapAttributes) -> Self { + Self((phys as u64 & Self::PHYS_MASK) | attrs.bits() | Self::PRESENT) + } + + pub const fn table(phys: usize, attrs: MapAttributes) -> Self { + Self((phys as u64 & Self::PHYS_MASK) | attrs.bits() | Self::PRESENT | Self::TABLE) + } + + pub const fn is_present(self) -> bool { + self.0 & Self::PRESENT != 0 + } + + pub const fn is_table(self) -> bool { + self.0 & Self::TABLE != 0 + } + + pub const unsafe fn address_unchecked(self) -> usize { + (self.0 & Self::PHYS_MASK) as usize + } +} + +impl Space { + pub fn alloc_empty() -> Result<&'static mut Self, Errno> { + let phys = phys::alloc_page(PageUsage::Paging)?; + let res = unsafe { &mut *(mem::virtualize(phys) as *mut Self) }; + res.0.entries.fill(Entry::invalid()); + Ok(res) + } + + pub fn map(&mut self, virt: usize, phys: usize, flags: MapAttributes) -> Result<(), Errno> { + let l0i = virt >> 30; + let l1i = (virt >> 21) & 0x1FF; + let l2i = (virt >> 12) & 0x1FF; + + let l1_table = self.0.next_level_table_or_alloc(l0i)?; + let l2_table = l1_table.next_level_table_or_alloc(l1i)?; + + if l2_table[l2i].is_present() { + Err(Errno::AlreadyExists) + } else { + l2_table[l2i] = Entry::table(phys, flags | MapAttributes::ACCESS); + debugln!("Map {:#x} -> {:#x}", virt, phys); + Ok(()) + } + } +} diff --git a/kernel/src/proc/mod.rs b/kernel/src/proc/mod.rs index 1e7dbe9..766ffda 100644 --- a/kernel/src/proc/mod.rs +++ b/kernel/src/proc/mod.rs @@ -3,7 +3,7 @@ use crate::mem::{ self, phys::{self, PageUsage}, - virt::Space, + virt::{MapAttributes, Space}, }; use crate::sync::IrqSafeNullLock; use crate::util::InitOnce; @@ -45,15 +45,20 @@ impl SchedulerInner { if id == 256 { panic!("Ran out of ASIDs (TODO FIXME)"); } - let space = Space::empty().unwrap(); + let space = Space::alloc_empty().unwrap(); - unsafe { - for i in 0..USTACK_PAGE_COUNT { - let page = phys::alloc_page(PageUsage::Kernel).unwrap(); - space - .map(USTACK_VIRT_BASE + i * mem::PAGE_SIZE, page) - .unwrap(); - } + for i in 0..USTACK_PAGE_COUNT { + let page = phys::alloc_page(PageUsage::Kernel).unwrap(); + space + .map( + USTACK_VIRT_BASE + i * mem::PAGE_SIZE, + page, + MapAttributes::SH_OUTER + | MapAttributes::NOT_GLOBAL + | MapAttributes::UXN + | MapAttributes::PXN, + ) + .unwrap(); } let proc = Process { @@ -131,6 +136,7 @@ impl Scheduler { inner.queue.pop_front().unwrap() }; + debugln!("{} -> {}", current, next); inner.current = Some(next); ( inner.processes.get(¤t).unwrap().clone(), @@ -155,6 +161,7 @@ extern "C" fn idle_fn(_a: usize) -> ! { extern "C" fn f1(u: usize) { let mut x = u; while x != 0 { + cortex_a::asm::nop(); x -= 1; } } @@ -165,7 +172,7 @@ extern "C" fn f0(a: usize) -> ! { unsafe { asm!("svc #0", in("x0") a, in("x1") &a); } - f1(1000000); + f1(10000000); } } @@ -179,7 +186,7 @@ static SCHED: Scheduler = Scheduler { pub unsafe fn enter() -> ! { SCHED.init(); - for i in 0..10 { + for i in 0..4 { SCHED.enqueue(SCHED.new_kernel(f0 as usize, i)); } SCHED.enter();