alnyan/yggdrasil
1
0
Fork 0
Code Issues 32 Pull Requests Packages Projects Releases Wiki Activity

x86_64: position-independent kernel

Browse Source
This commit is contained in:
Mark Poliakov
2025-07-17 16:45:57 +03:00
parent 019146e9ff
commit 511d1e45c0
16 changed files with 578 additions and 650 deletions
Download Patch File Download Diff File Expand all files Collapse all files
boot/yboot/src/elf.rs
+110 -38
View File
@@ -1,7 +1,8 @@
use core::mem::size_of;
use bytemuck::Zeroable;
use log::{debug, error, info};
use log::{error, info};
use types::{Rela, SHT_RELA};
// TODO use 'elf' crate
use uefi::{
prelude::BootServices,
@@ -23,14 +24,18 @@ mod types {
pub type Half = u16;
pub type Word = u32;
pub type XWord = u64;
pub type SXWord = i64;
pub const PT_LOAD: Word = 1;
pub const SHT_PROGBITS: Word = 1;
pub const SHT_RELA: Word = 4;
pub const SHF_WRITE: XWord = 1 << 0;
pub const SHF_ALLOC: XWord = 1 << 1;
pub const R_X86_64_RELATIVE: u32 = 8;
#[derive(Clone, Copy, Zeroable, Pod)]
#[repr(C)]
pub struct Ehdr {
@@ -77,6 +82,20 @@ mod types {
pub memsz: XWord,
pub align: XWord,
}
#[derive(Clone, Copy, Zeroable, Pod)]
#[repr(C)]
pub struct Rela {
pub offset: Addr,
pub info: XWord,
pub addend: SXWord,
}
impl Rela {
pub fn r_type(&self) -> u32 {
self.info as u32
}
}
}
// Maximum address this loader can map in the target kernel
@@ -92,6 +111,8 @@ pub struct LoadedObject {
pub image_start: u64,
pub image_end: u64,
pub load_address: u64,
pub entry: u64,
pub protocol_struct_paddr: u64,
@@ -105,6 +126,12 @@ struct LocatedProtocol {
size: usize,
}
struct RelaSection {
offset: u64,
entry_count: usize,
entry_size: usize,
}
trait ReadExact {
fn read_exact(&mut self, buf: &mut [u8]) -> Result<(), Error>;
}
@@ -119,6 +146,23 @@ impl ReadExact for RegularFile {
}
}
impl RelaSection {
pub fn from_shdr(shdr: &Shdr) -> Option<Self> {
if shdr.type_ != SHT_RELA {
return None;
}
let entry_size = shdr.entsize as usize;
let entry_count = shdr.size as usize / entry_size;
Some(Self {
offset: shdr.offset,
entry_size,
entry_count,
})
}
}
impl Object {
pub fn open<D: File>(root: &mut D, path: &CStr16) -> Result<Self, Error> {
let file = root.open(path, FileMode::Read, FileAttribute::empty())?;
@@ -140,11 +184,11 @@ impl Object {
return Err(Error::new(Status::LOAD_ERROR, ()));
}
// Check that the entry point is set
if ehdr.entry == 0 {
error!("Image does not have a valid entry point");
return Err(Error::new(Status::LOAD_ERROR, ()));
}
// // Check that the entry point is set
// if ehdr.entry == 0 {
// error!("Image does not have a valid entry point");
// return Err(Error::new(Status::LOAD_ERROR, ()));
// }
Ok(Self { file, ehdr })
}
@@ -170,11 +214,6 @@ impl Object {
self.file
.read_exact(bytemuck::bytes_of_mut(&mut proto_data))?;
info!(
"Kernel is virtually mapped at {:#x}",
proto_data.kernel_virt_offset
);
// 2. Find the kernel's range and check that the loaded physical addresses are actually
// usable from UEFI
let mut image_start = u64::MAX;
@@ -209,16 +248,22 @@ impl Object {
assert_eq!(image_start & 0xFFF, 0);
assert_eq!(image_end & 0xFFF, 0);
info!("Image start: {:#x}, end: {:#x}", image_start, image_end);
// Reserve the kernel memory
let reserved_addr = bs
// Allocate memory to load the kernel into
let kernel_load_address = bs
.allocate_pages(
AllocateType::Address(image_start),
AllocateType::MaxAddress(0xFFFFFFFF),
MemoryType::LOADER_DATA,
(image_end - image_start) as usize / 0x1000,
)
.expect("Could not allocate memory for kernel image");
assert_eq!(reserved_addr, image_start);
.expect("Could not allocate memory for the kernel");
// Print info
info!("Image start: {:#x}, end: {:#x}", image_start, image_end);
info!(
"Kernel virtual offset: {:#x}",
proto_data.kernel_virt_offset
);
info!("Kernel load address: {kernel_load_address:#x}");
// 3. Load the segments
for i in 0..self.ehdr.phnum {
@@ -228,53 +273,80 @@ impl Object {
continue;
}
let segment_load_base = phdr.paddr + kernel_load_address;
info!(
"Load segment {}: {:#x?}",
i,
phdr.paddr..phdr.paddr + phdr.memsz
"[{i}] Load {:#x?}",
segment_load_base..segment_load_base + phdr.memsz
);
if phdr.filesz > 0 {
// The section has load data
let dst = unsafe {
core::slice::from_raw_parts_mut(phdr.paddr as *mut u8, phdr.filesz as usize)
let dst_slice = unsafe {
core::slice::from_raw_parts_mut(
segment_load_base as *mut u8,
phdr.filesz as usize,
)
};
debug!(
"Load {:#x?} from ELF offset {:#x}",
phdr.paddr..phdr.paddr + phdr.filesz,
phdr.offset
);
self.file.set_position(phdr.offset)?;
self.file.read_exact(dst)?;
self.file.read_exact(dst_slice)?;
}
if phdr.memsz > phdr.filesz {
let dst = unsafe {
let dst_slice = unsafe {
core::slice::from_raw_parts_mut(
(phdr.paddr + phdr.filesz) as *mut u8,
(segment_load_base + phdr.filesz) as *mut u8,
(phdr.memsz - phdr.filesz) as usize,
)
};
debug!(
"Zero data {:#x?}",
phdr.paddr + phdr.filesz..phdr.paddr + phdr.memsz
);
dst_slice.fill(0);
}
}
dst.fill(0);
// 4. Perform kernel relocation
let mut rela_section = None;
for i in 0..self.ehdr.shnum as usize {
let shdr = self.read_shdr(i)?;
if let Some(rela) = RelaSection::from_shdr(&shdr) {
rela_section = Some(rela);
break;
}
}
if let Some(rela_section) = rela_section {
info!("Relocating kernel: {image_start:#x} -> {kernel_load_address:#x}");
info!("({} relocations)", rela_section.entry_count);
let b = (kernel_load_address + proto_data.kernel_virt_offset) as i64;
for i in 0..rela_section.entry_count {
let mut rela = Rela::zeroed();
self.file
.set_position(rela_section.offset + (i * rela_section.entry_size) as u64)?;
self.file.read_exact(bytemuck::bytes_of_mut(&mut rela))?;
match rela.r_type() {
types::R_X86_64_RELATIVE => {
let qword = (rela.offset + kernel_load_address) as *mut i64;
let value = rela.addend + b;
unsafe { qword.write_volatile(value) };
}
other => todo!("Unsupported relocation type: {other}"),
}
}
}
// Now that the image is in memory, protocol structure can be written in the further steps
let protocol_struct_paddr = (loc_proto.address as u64) - proto_data.kernel_virt_offset;
let protocol_struct_paddr = loc_proto.address as u64 + kernel_load_address; // (loc_proto.address as u64) - proto_data.kernel_virt_offset;
let protocol_version = proto_data.header.version;
let entry = self.ehdr.entry;
let entry = self.ehdr.entry + kernel_load_address;
Ok(LoadedObject {
image_start,
image_end,
load_address: kernel_load_address,
entry,
protocol_struct_paddr,
protocol_version,
boot/yboot/src/main.rs
+14 -12
View File
@@ -93,7 +93,7 @@ fn locate_rsdp(st: &SystemTable<Boot>) -> Option<u64> {
fn boot_partition(
image: Handle,
bs: &BootServices,
) -> Result<ScopedProtocol<SimpleFileSystem>, Error> {
) -> Result<ScopedProtocol<'_, SimpleFileSystem>, Error> {
let loaded_image = bs.open_protocol_exclusive::<LoadedImage>(image)?;
let device_handle = loaded_image.device();
@@ -113,7 +113,7 @@ fn load_kernel<'a>(
config: &Config,
root: &mut Directory,
st: &SystemTable<Boot>,
) -> Result<(u64, u64, &'a mut LoadProtocolV1), Error> {
) -> Result<(u64, u64, u64, &'a mut LoadProtocolV1), Error> {
let bs = st.boot_services();
let mut kernel_obj = Object::open(root, cstr16!("kernel.elf"))?;
@@ -183,13 +183,14 @@ fn load_kernel<'a>(
let entry = loaded_obj.entry + proto_data.kernel_virt_offset;
Ok((entry, mmap_memory, proto_data))
Ok((entry, loaded_obj.load_address, mmap_memory, proto_data))
}
unsafe fn map_and_enter_kernel(
st: SystemTable<Boot>,
proto_data: &mut LoadProtocolV1,
mmap_memory: u64,
load_base: u64,
entry: u64,
) -> ! {
let (_, mmap) = st.exit_boot_services();
@@ -216,7 +217,7 @@ unsafe fn map_and_enter_kernel(
let cr3 = mem::map_image();
asm!("cli; wbinvd; mov {0}, %cr3", in(reg) cr3, options(att_syntax));
asm!("jmp *{0}", in(reg) entry, in("eax") LOADER_MAGIC, options(noreturn, att_syntax));
asm!("jmp *{0}", in(reg) entry, in("eax") LOADER_MAGIC, in("ecx") load_base, options(noreturn, att_syntax));
}
#[entry]
@@ -243,15 +244,16 @@ fn efi_main(image_handle: Handle, mut system_table: SystemTable<Boot>) -> Status
}
};
let (entry, mmap_memory, proto_data) = match load_kernel(&config, &mut root, &system_table) {
Ok(e) => e,
Err(error) => {
error!("Failed to load the kernel/initrd: {error:?}");
return Status::LOAD_ERROR;
}
};
let (entry, load_base, mmap_memory, proto_data) =
match load_kernel(&config, &mut root, &system_table) {
Ok(e) => e,
Err(error) => {
error!("Failed to load the kernel/initrd: {error:?}");
return Status::LOAD_ERROR;
}
};
unsafe {
map_and_enter_kernel(system_table, proto_data, mmap_memory, entry);
map_and_enter_kernel(system_table, proto_data, mmap_memory, load_base, entry);
}
}
etc/ld/x86/x86_64-unknown-none.ld
-52
View File
@@ -1,52 +0,0 @@
ENTRY(__x86_64_entry);
KERNEL_PHYS_BASE = 0x200000;
KERNEL_VIRT_OFFSET = 0xFFFFFF8000000000;
SECTIONS {
. = KERNEL_PHYS_BASE;
PROVIDE(__kernel_start = . + KERNEL_VIRT_OFFSET);
.text.entry : {
*(.text.entry)
}
. = ALIGN(16);
. = . + KERNEL_VIRT_OFFSET;
.text : AT(. - KERNEL_VIRT_OFFSET) {
*(.text*)
}
.export.text : AT(. - KERNEL_VIRT_OFFSET) {
KEEP(*(.export.text*))
}
. = ALIGN(4K);
.rodata : AT(. - KERNEL_VIRT_OFFSET) {
*(.eh_frame*)
*(.rodata*)
}
. = ALIGN(4K);
.data.tables : AT (. - KERNEL_VIRT_OFFSET) {
KEEP(*(.data.tables))
}
.data : AT(. - KERNEL_VIRT_OFFSET) {
KEEP(*(.data.yboot))
*(.data*)
*(.got*)
}
. = ALIGN(4K);
PROVIDE(__bss_start_phys = . - KERNEL_VIRT_OFFSET);
.bss : AT(. - KERNEL_VIRT_OFFSET) {
*(COMMON)
*(.bss*)
}
. = ALIGN(4K);
PROVIDE(__bss_end_phys = . - KERNEL_VIRT_OFFSET);
PROVIDE(__kernel_end = .);
};
etc/ld/x86_64-unknown-none.ld
+73
View File
@@ -0,0 +1,73 @@
ENTRY(__x86_64_entry);
SECTIONS {
. = 0x0;
PROVIDE(__kernel_start = .);
.text : {
*(.text.entry)
*(.text*)
}
. = ALIGN(4K);
.rodata : {
*(.rodata*)
*(.eh_frame*)
. = ALIGN(16);
PROVIDE(__init_array_start = .);
KEEP(*(.init_array*));
PROVIDE(__init_array_end = .);
}
. = ALIGN(4K);
.rela : {
PROVIDE(__rela_start = .);
*(.rela*)
PROVIDE(__rela_end = .);
}
.dynamic : {
*(.dynamic)
}
. = ALIGN(4K);
.data : {
KEEP(*(.data.yboot))
*(.data*)
*(.got*)
}
. = ALIGN(4K);
PROVIDE(__bss_start = .);
.bss : {
*(COMMON)
*(.bss*)
}
. = ALIGN(4K);
PROVIDE(__bss_end = .);
PROVIDE(__kernel_end = .);
PROVIDE(__kernel_size = __kernel_end - __kernel_start);
.dynsym : {
*(.dynsym)
}
.gnu.hash : {
*(.gnu.hash)
}
.hash : {
*(.hash)
}
.dynstr : {
*(.dynstr)
}
}
etc/x86_64-unknown-none.json
+1
View File
@@ -15,6 +15,7 @@
"panic-strategy": "abort",
"dynamic-linking": true,
"relocation-model": "pic",
"position-independent-executables": true,
"has-thread-local": false,
kernel/arch/x86_64/src/context.rs
+5 -2
View File
@@ -9,7 +9,10 @@ use libk_mm_interface::address::{AsPhysicalAddress, PhysicalAddress};
use tock_registers::interfaces::Writeable;
use yggdrasil_abi::{arch::SavedFrame, error::Error};
use crate::{mem::KERNEL_TABLES, ArchitectureImpl};
use crate::{
mem::{auto_lower_address, fixed},
ArchitectureImpl,
};
/// Frame saved onto the stack when taking an IRQ
#[derive(Debug)]
@@ -431,7 +434,7 @@ impl<K: KernelTableManager, PA: PhysicalMemoryAllocator<Address = PhysicalAddres
fn kernel(entry: extern "C" fn(usize) -> !, arg: usize) -> Result<Self, Error> {
const KERNEL_TASK_PAGES: usize = 32;
let cr3: usize = unsafe { KERNEL_TABLES.lock().as_physical_address() }.into();
let cr3: usize = auto_lower_address(&raw const fixed::KERNEL_PML4); // unsafe { KERNEL_TABLES.lock().as_physical_address() }.into();
let stack_base_phys = PA::allocate_contiguous_pages(KERNEL_TASK_PAGES)?;
let stack_base = stack_base_phys.raw_virtualize::<K>();
kernel/arch/x86_64/src/mem/fixed.rs
+132
View File
@@ -0,0 +1,132 @@
use core::ops::Range;
use kernel_arch_interface::mem::DeviceMemoryAttributes;
use kernel_arch_x86::registers::CR3;
use libk_mm_interface::{
address::PhysicalAddress,
device::{DevicePageManager, DevicePageTableLevel},
table::{page_index, EntryLevel},
};
use crate::{
mem::{
auto_lower_address,
table::{PageAttributes, PageEntry, PageTable, L0, L1, L2, L3},
},
KERNEL_VIRT_OFFSET,
};
pub const IDENTITY_SIZE_L1: usize = 64;
pub const KERNEL_L0I: usize = page_index::<L0>(KERNEL_VIRT_OFFSET);
pub const DEVICE_L1: usize = IDENTITY_SIZE_L1;
pub const DEVICE_MAPPING_L3_COUNT: usize = 32;
pub const DEVICE_MAPPING_OFFSET: usize = KERNEL_VIRT_OFFSET + (DEVICE_L1 << L1::SHIFT);
pub static mut KERNEL_PDPT: PageTable<L1> = PageTable::zeroed();
pub static mut KERNEL_PML4: PageTable<L0> = PageTable::zeroed();
pub(super) static mut DEVICE_MEMORY: DevicePageManager<L3DeviceMemory, L2DeviceMemory> =
DevicePageManager::new(
L3DeviceMemory([PageTable::zeroed(); DEVICE_MAPPING_L3_COUNT]),
L2DeviceMemory(PageTable::zeroed()),
);
#[repr(transparent)]
pub struct L2DeviceMemory(pub PageTable<L2>);
#[repr(transparent)]
pub struct L3DeviceMemory(pub [PageTable<L3>; DEVICE_MAPPING_L3_COUNT]);
impl DevicePageTableLevel for L2DeviceMemory {
type Level = L2;
const VIRTUAL_BASE: usize = DEVICE_MAPPING_OFFSET;
const INDEX_RANGE: Range<usize> = DEVICE_MAPPING_L3_COUNT..512;
fn map_page(
&mut self,
index: usize,
physical: PhysicalAddress,
attrs: &DeviceMemoryAttributes,
) {
let _ = attrs;
self.0[index] = PageEntry::<L2>::block(physical, PageAttributes::WRITABLE);
}
fn unmap_page(&mut self, index: usize) {
self.0[index - DEVICE_MAPPING_L3_COUNT] = PageEntry::INVALID;
}
fn is_mapped(&self, index: usize) -> bool {
self.0[index - DEVICE_MAPPING_L3_COUNT].is_present()
}
fn flush_range(range: Range<usize>) {
// TODO
}
}
impl DevicePageTableLevel for L3DeviceMemory {
type Level = L3;
const VIRTUAL_BASE: usize = DEVICE_MAPPING_OFFSET;
const INDEX_RANGE: Range<usize> = 0..512 * DEVICE_MAPPING_L3_COUNT;
fn map_page(
&mut self,
index: usize,
physical: PhysicalAddress,
attrs: &DeviceMemoryAttributes,
) {
let _ = attrs;
let l2i = index / 512;
let l3i = index % 512;
self.0[l2i][l3i] = PageEntry::page(physical, PageAttributes::WRITABLE);
}
fn unmap_page(&mut self, index: usize) {
let l2i = index / 512;
let l3i = index % 512;
self.0[l2i][l3i] = PageEntry::INVALID;
}
fn is_mapped(&self, index: usize) -> bool {
let l2i = index / 512;
let l3i = index % 512;
self.0[l2i][l3i].is_present()
}
fn flush_range(range: Range<usize>) {
// TODO
// let start = range.start * L3::SIZE + Self::VIRTUAL_BASE;
// let size = (range.end - range.start) * L3::SIZE;
// tlb_flush_range_va(start, size);
}
}
pub(super) unsafe fn setup(have_1gib_pages: bool) {
let phys = PhysicalAddress::from_usize(auto_lower_address(&raw const KERNEL_PDPT));
KERNEL_PML4[KERNEL_L0I] = PageEntry::table(phys, PageAttributes::WRITABLE);
if have_1gib_pages {
for i in 0..IDENTITY_SIZE_L1 {
let phys = PhysicalAddress::from_usize(i * L1::SIZE);
KERNEL_PDPT[i] = PageEntry::<L1>::block(phys, PageAttributes::WRITABLE);
}
} else {
loop {}
}
// DEVICE_L1 -> Device L2 table
// 0..DEVICE_MAPPING_L3_COUNT -> Device L3 tables -> Device L3 pages
// ..512 -> Device L2 pages
for i in 0..DEVICE_MAPPING_L3_COUNT {
let phys =
PhysicalAddress::from_usize(auto_lower_address(&raw const DEVICE_MEMORY.normal.0[i]));
DEVICE_MEMORY.large.0[i] = PageEntry::table(phys, PageAttributes::WRITABLE);
}
let phys = PhysicalAddress::from_usize(auto_lower_address(&raw const DEVICE_MEMORY.large.0));
KERNEL_PDPT[DEVICE_L1] = PageEntry::table(phys, PageAttributes::WRITABLE);
}
pub(super) unsafe fn load() {
CR3.set_address(auto_lower_address(&raw const KERNEL_PML4));
}
kernel/arch/x86_64/src/mem/mod.rs
+66 -315
View File
@@ -13,86 +13,85 @@ use libk_mm_interface::{
address::PhysicalAddress,
table::{page_index, EntryLevel, EntryLevelExt},
};
use static_assertions::{const_assert_eq, const_assert_ne};
use yggdrasil_abi::error::Error;
use crate::KERNEL_VIRT_OFFSET;
use self::table::{PageAttributes, PageEntry, PageTable, L0, L1, L2, L3};
pub mod fixed;
pub mod process;
pub mod table;
#[derive(Debug)]
pub struct KernelTableManagerImpl;
const CANONICAL_ADDRESS_MASK: usize = 0xFFFF000000000000;
const KERNEL_PHYS_BASE: usize = 0x200000;
// Mapped at compile time
const KERNEL_MAPPING_BASE: usize = KERNEL_VIRT_OFFSET + KERNEL_PHYS_BASE;
const KERNEL_L0_INDEX: usize = page_index::<L0>(KERNEL_MAPPING_BASE);
const KERNEL_L1_INDEX: usize = page_index::<L1>(KERNEL_MAPPING_BASE);
const KERNEL_START_L2_INDEX: usize = page_index::<L2>(KERNEL_MAPPING_BASE);
// Must not be zero, should be at 4MiB
const_assert_ne!(KERNEL_START_L2_INDEX, 0);
// From static mapping
const_assert_eq!(KERNEL_L0_INDEX, 511);
const_assert_eq!(KERNEL_L1_INDEX, 0);
// Mapped at boot
const EARLY_MAPPING_L2I: usize = KERNEL_START_L2_INDEX - 1;
const DEVICE_MAPPING_L1I: usize = KERNEL_L1_INDEX + 2;
const RAM_MAPPING_L0I: usize = KERNEL_L0_INDEX - 1;
const DEVICE_MAPPING_L3_COUNT: usize = 4;
split_spinlock! {
use libk_mm_interface::KernelImageObject;
use memtables::x86_64::FixedTables;
use crate::ArchitectureImpl;
#[link_section = ".data.tables"]
static KERNEL_TABLES: KernelImageObject<FixedTables> =
unsafe { KernelImageObject::new(FixedTables::zeroed()) };
}
// 2MiB for early mappings
const EARLY_MAPPING_OFFSET: usize = CANONICAL_ADDRESS_MASK
| (KERNEL_L0_INDEX * L0::SIZE)
| (KERNEL_L1_INDEX * L1::SIZE)
| (EARLY_MAPPING_L2I * L2::SIZE);
static mut EARLY_MAPPING_L3: PageTable<L3> = PageTable::zeroed();
// 1GiB for device MMIO mapping
const DEVICE_MAPPING_OFFSET: usize =
CANONICAL_ADDRESS_MASK | (KERNEL_L0_INDEX * L0::SIZE) | (DEVICE_MAPPING_L1I * L1::SIZE);
static mut DEVICE_MAPPING_L2: PageTable<L2> = PageTable::zeroed();
static mut DEVICE_MAPPING_L3S: [PageTable<L3>; DEVICE_MAPPING_L3_COUNT] =
[PageTable::zeroed(); DEVICE_MAPPING_L3_COUNT];
// 512GiB for whole RAM mapping
pub const RAM_MAPPING_OFFSET: usize = CANONICAL_ADDRESS_MASK | (RAM_MAPPING_L0I * L0::SIZE);
pub static MEMORY_LIMIT: AtomicUsize = AtomicUsize::new(0);
pub static mut RAM_MAPPING_L1: PageTable<L1> = PageTable::zeroed();
// const CANONICAL_ADDRESS_MASK: usize = 0xFFFF000000000000;
// const KERNEL_PHYS_BASE: usize = 0x200000;
//
// // Mapped at compile time
// const KERNEL_MAPPING_BASE: usize = KERNEL_VIRT_OFFSET + KERNEL_PHYS_BASE;
// const KERNEL_L0_INDEX: usize = page_index::<L0>(KERNEL_MAPPING_BASE);
// const KERNEL_L1_INDEX: usize = page_index::<L1>(KERNEL_MAPPING_BASE);
// const KERNEL_START_L2_INDEX: usize = page_index::<L2>(KERNEL_MAPPING_BASE);
//
// // Must not be zero, should be at 4MiB
// const_assert_ne!(KERNEL_START_L2_INDEX, 0);
// // From static mapping
// const_assert_eq!(KERNEL_L0_INDEX, 511);
// const_assert_eq!(KERNEL_L1_INDEX, 0);
//
// // Mapped at boot
// const EARLY_MAPPING_L2I: usize = KERNEL_START_L2_INDEX - 1;
// const DEVICE_MAPPING_L1I: usize = KERNEL_L1_INDEX + 2;
// const RAM_MAPPING_L0I: usize = KERNEL_L0_INDEX - 1;
//
// const DEVICE_MAPPING_L3_COUNT: usize = 4;
//
// split_spinlock! {
// use libk_mm_interface::KernelImageObject;
// use memtables::x86_64::FixedTables;
// use crate::ArchitectureImpl;
//
// #[link_section = ".data.tables"]
// static KERNEL_TABLES: KernelImageObject<FixedTables> =
// unsafe { KernelImageObject::new(FixedTables::zeroed()) };
// }
//
// // 2MiB for early mappings
// const EARLY_MAPPING_OFFSET: usize = CANONICAL_ADDRESS_MASK
// | (KERNEL_L0_INDEX * L0::SIZE)
// | (KERNEL_L1_INDEX * L1::SIZE)
// | (EARLY_MAPPING_L2I * L2::SIZE);
// static mut EARLY_MAPPING_L3: PageTable<L3> = PageTable::zeroed();
// // 1GiB for device MMIO mapping
// const DEVICE_MAPPING_OFFSET: usize =
// CANONICAL_ADDRESS_MASK | (KERNEL_L0_INDEX * L0::SIZE) | (DEVICE_MAPPING_L1I * L1::SIZE);
// static mut DEVICE_MAPPING_L2: PageTable<L2> = PageTable::zeroed();
// static mut DEVICE_MAPPING_L3S: [PageTable<L3>; DEVICE_MAPPING_L3_COUNT] =
// [PageTable::zeroed(); DEVICE_MAPPING_L3_COUNT];
// // 512GiB for whole RAM mapping
// pub const RAM_MAPPING_OFFSET: usize = CANONICAL_ADDRESS_MASK | (RAM_MAPPING_L0I * L0::SIZE);
// pub static MEMORY_LIMIT: AtomicUsize = AtomicUsize::new(0);
// pub static mut RAM_MAPPING_L1: PageTable<L1> = PageTable::zeroed();
impl KernelTableManager for KernelTableManagerImpl {
fn virtualize(address: u64) -> usize {
let address = address as usize;
if address < MEMORY_LIMIT.load(Ordering::Acquire) {
address + RAM_MAPPING_OFFSET
if address < fixed::IDENTITY_SIZE_L1 * L1::SIZE {
address + KERNEL_VIRT_OFFSET
} else {
panic!("Invalid physical address: {:#x}", address);
panic!("Invalid physical address: {address:#x}");
}
}
fn physicalize(address: usize) -> u64 {
if address < RAM_MAPPING_OFFSET
|| address - RAM_MAPPING_OFFSET >= MEMORY_LIMIT.load(Ordering::Acquire)
if address < KERNEL_VIRT_OFFSET
|| address - KERNEL_VIRT_OFFSET >= fixed::IDENTITY_SIZE_L1 * L1::SIZE
{
panic!("Not a virtualized physical address: {:#x}", address);
panic!("Invalid virtualized address: {address:#x}");
}
(address - RAM_MAPPING_OFFSET) as _
(address - KERNEL_VIRT_OFFSET) as u64
}
unsafe fn map_device_pages(
@@ -100,245 +99,20 @@ impl KernelTableManager for KernelTableManagerImpl {
count: usize,
attrs: DeviceMemoryAttributes,
) -> Result<RawDeviceMemoryMapping<Self>, Error> {
map_device_memory(PhysicalAddress::from_u64(base), count, attrs)
#[allow(static_mut_refs)]
fixed::DEVICE_MEMORY.map_device_pages(PhysicalAddress::from_u64(base), count, attrs)
}
unsafe fn unmap_device_pages(mapping: &RawDeviceMemoryMapping<Self>) {
unmap_device_memory(mapping)
}
}
// Early mappings
unsafe fn map_early_pages(physical: PhysicalAddress, count: usize) -> Result<usize, Error> {
for l3i in 0..512 {
let mut taken = false;
for i in 0..count {
if EARLY_MAPPING_L3[i + l3i].is_present() {
taken = true;
break;
}
}
if taken {
continue;
}
for i in 0..count {
// TODO NX, NC
EARLY_MAPPING_L3[i + l3i] =
PageEntry::page(physical.add(i * L3::SIZE), PageAttributes::WRITABLE);
flush_tlb_entry(EARLY_MAPPING_OFFSET + (i + l3i) * L3::SIZE);
}
return Ok(EARLY_MAPPING_OFFSET + l3i * L3::SIZE);
}
Err(Error::OutOfMemory)
}
unsafe fn unmap_early_page(address: usize) {
if !(EARLY_MAPPING_OFFSET..EARLY_MAPPING_OFFSET + L2::SIZE).contains(&address) {
panic!("Tried to unmap invalid early mapping: {:#x}", address);
}
let l3i = (address - EARLY_MAPPING_OFFSET).page_index::<L3>();
assert!(EARLY_MAPPING_L3[l3i].is_present());
EARLY_MAPPING_L3[l3i] = PageEntry::INVALID;
}
// Device mappings
unsafe fn map_device_memory_l3(
base: PhysicalAddress,
count: usize,
_attrs: DeviceMemoryAttributes,
) -> Result<usize, Error> {
// TODO don't map pages if already mapped
'l0: for i in 0..DEVICE_MAPPING_L3_COUNT * 512 {
for j in 0..count {
let l2i = (i + j) / 512;
let l3i = (i + j) % 512;
if DEVICE_MAPPING_L3S[l2i][l3i].is_present() {
continue 'l0;
}
}
for j in 0..count {
let l2i = (i + j) / 512;
let l3i = (i + j) % 512;
// TODO NX, NC
DEVICE_MAPPING_L3S[l2i][l3i] =
PageEntry::page(base.add(j * L3::SIZE), PageAttributes::WRITABLE);
}
return Ok(DEVICE_MAPPING_OFFSET + i * L3::SIZE);
}
Err(Error::OutOfMemory)
}
unsafe fn map_device_memory_l2(
base: PhysicalAddress,
count: usize,
_attrs: DeviceMemoryAttributes,
) -> Result<usize, Error> {
'l0: for i in DEVICE_MAPPING_L3_COUNT..512 {
for j in 0..count {
if DEVICE_MAPPING_L2[i + j].is_present() {
continue 'l0;
}
}
for j in 0..count {
DEVICE_MAPPING_L2[i + j] =
PageEntry::<L2>::block(base.add(j * L2::SIZE), PageAttributes::WRITABLE);
}
return Ok(DEVICE_MAPPING_OFFSET + i * L2::SIZE);
}
Err(Error::OutOfMemory)
}
unsafe fn map_device_memory(
base: PhysicalAddress,
size: usize,
attrs: DeviceMemoryAttributes,
) -> Result<RawDeviceMemoryMapping<KernelTableManagerImpl>, Error> {
let l3_aligned = base.page_align_down::<L3>();
let l3_offset = base.page_offset::<L3>();
let page_count = (l3_offset + size).page_count::<L3>();
if page_count > 256 {
// Large mapping, use L2 mapping instead
let l2_aligned = base.page_align_down::<L2>();
let l2_offset = base.page_offset::<L2>();
let page_count = (l2_offset + size).page_count::<L2>();
let base_address = map_device_memory_l2(l2_aligned, page_count, attrs)?;
let address = base_address + l2_offset;
Ok(RawDeviceMemoryMapping::from_raw_parts(
l2_aligned.into_u64(),
address,
base_address,
page_count,
L2::SIZE,
))
} else {
// Just map the pages directly
let base_address = map_device_memory_l3(l3_aligned, page_count, attrs)?;
let address = base_address + l3_offset;
Ok(RawDeviceMemoryMapping::from_raw_parts(
l3_aligned.into_u64(),
address,
base_address,
page_count,
L3::SIZE,
))
}
}
unsafe fn unmap_device_memory(map: &RawDeviceMemoryMapping<KernelTableManagerImpl>) {
match map.page_size {
L3::SIZE => {
for i in 0..map.page_count {
let page = map.base_address + i * L3::SIZE;
let l2i = page.page_index::<L2>();
let l3i = page.page_index::<L3>();
assert!(DEVICE_MAPPING_L3S[l2i][l3i].is_present());
DEVICE_MAPPING_L3S[l2i][l3i] = PageEntry::INVALID;
flush_tlb_entry(page);
}
}
L2::SIZE => todo!(),
_ => unimplemented!(),
}
}
/// Memory mapping which may be used for performing early kernel initialization
pub struct EarlyMapping<'a, T: ?Sized> {
value: &'a mut T,
page_count: usize,
}
impl<'a, T: Sized> EarlyMapping<'a, T> {
/// # Safety
///
/// `physical` address provided must be a valid non-NULL address actually containing `T`.
pub unsafe fn map(physical: PhysicalAddress) -> Result<EarlyMapping<'a, T>, Error> {
let layout = Layout::new::<T>();
let aligned = physical.page_align_down::<L3>();
let offset = physical.page_offset::<L3>();
let page_count = (offset + layout.size()).div_ceil(L3::SIZE);
let virt = map_early_pages(aligned, page_count)?;
let value = &mut *((virt + offset) as *mut T);
Ok(EarlyMapping { value, page_count })
}
/// # Safety
///
/// `physical` address provided must be a valid non-NULL address actually containing a `T`
/// slice of given `len`.
pub unsafe fn map_slice(
physical: PhysicalAddress,
len: usize,
) -> Result<EarlyMapping<'a, [T]>, Error> {
let layout = Layout::array::<T>(len).unwrap();
let aligned = physical.page_align_down::<L3>();
let offset = physical.page_offset::<L3>();
let page_count = (offset + layout.size()).div_ceil(L3::SIZE);
let virt = map_early_pages(aligned, page_count)?;
let value = core::slice::from_raw_parts_mut((virt + offset) as *mut T, len);
Ok(EarlyMapping { value, page_count })
}
}
impl<T: ?Sized> Deref for EarlyMapping<'_, T> {
type Target = T;
fn deref(&self) -> &Self::Target {
self.value
}
}
impl<T: ?Sized> DerefMut for EarlyMapping<'_, T> {
fn deref_mut(&mut self) -> &mut Self::Target {
self.value
}
}
impl<T: ?Sized> Drop for EarlyMapping<'_, T> {
fn drop(&mut self) {
let address = (self.value as *mut T).addr() & !(L3::SIZE - 1);
for i in 0..self.page_count {
let page = address + i * L3::SIZE;
unsafe {
unmap_early_page(page);
}
}
}
unsafe fn unmap_device_pages(mapping: &RawDeviceMemoryMapping<Self>) {}
}
pub fn clone_kernel_tables(dst: &mut PageTable<L0>) {
let tables = KERNEL_TABLES.lock();
unsafe {
dst[KERNEL_L0_INDEX] = PageEntry::from_raw(tables.l0.data[KERNEL_L0_INDEX]);
dst[RAM_MAPPING_L0I] = PageEntry::from_raw(tables.l0.data[RAM_MAPPING_L0I]);
dst[fixed::KERNEL_L0I] = fixed::KERNEL_PML4[fixed::KERNEL_L0I];
}
}
pub fn auto_address<T>(pointer: *const T) -> usize {
pub fn auto_lower_address<T>(pointer: *const T) -> usize {
let address = pointer.addr();
if address < KERNEL_VIRT_OFFSET {
address
@@ -363,35 +137,12 @@ pub fn auto_address<T>(pointer: *const T) -> usize {
/// # Safety
///
/// Unsafe, must only be called by BSP during its early init, must already be in "higher-half"
pub unsafe fn init_fixed_tables() {
let mut tables = KERNEL_TABLES.lock();
// TODO this could be built in compile-time too?
let early_mapping_l3_phys = auto_address(&raw const EARLY_MAPPING_L3);
let device_mapping_l2_phys = auto_address(&raw const DEVICE_MAPPING_L2);
let ram_mapping_l1_phys = auto_address(&raw const RAM_MAPPING_L1);
for i in 0..DEVICE_MAPPING_L3_COUNT {
let device_mapping_l3_phys =
PhysicalAddress::from_usize(auto_address(&raw const DEVICE_MAPPING_L3S[i]));
DEVICE_MAPPING_L2[i] = PageEntry::table(device_mapping_l3_phys, PageAttributes::WRITABLE);
#[inline(never)]
pub unsafe fn init_fixed_tables(have_1gib_pages: bool, bsp: bool) {
fixed::setup(have_1gib_pages);
if bsp {
fixed::load();
}
assert_eq!(tables.kernel_l2.data[EARLY_MAPPING_L2I], 0);
tables.kernel_l2.data[EARLY_MAPPING_L2I] = (early_mapping_l3_phys as u64)
| (PageAttributes::WRITABLE | PageAttributes::PRESENT).bits();
assert_eq!(tables.kernel_l1.data[DEVICE_MAPPING_L1I], 0);
tables.kernel_l1.data[DEVICE_MAPPING_L1I] = (device_mapping_l2_phys as u64)
| (PageAttributes::WRITABLE | PageAttributes::PRESENT).bits();
assert_eq!(tables.l0.data[RAM_MAPPING_L0I], 0);
tables.l0.data[RAM_MAPPING_L0I] =
(ram_mapping_l1_phys as u64) | (PageAttributes::WRITABLE | PageAttributes::PRESENT).bits();
// TODO ENABLE EFER.NXE
let cr3 = auto_address(&raw const tables.l0);
CR3.set_address(cr3);
}
/// # Safety
kernel/src/arch/mod.rs
+15 -16
View File
@@ -6,27 +6,26 @@ use kernel_arch::{Architecture, ArchitectureImpl};
pub mod aarch64;
#[cfg(any(target_arch = "aarch64", rust_analyzer))]
pub use aarch64::{AArch64 as PlatformImpl, L3, PLATFORM};
//
// #[cfg(any(target_arch = "x86_64", target_arch = "x86", rust_analyzer))]
// pub mod x86;
//
// #[cfg(any(target_arch = "x86_64", rust_analyzer))]
// pub mod x86_64;
// #[cfg(any(target_arch = "x86_64", rust_analyzer))]
// pub use x86_64::{PLATFORM, X86_64 as PlatformImpl};
#[cfg(any(target_arch = "x86_64", target_arch = "x86", rust_analyzer))]
pub mod x86;
#[cfg(any(target_arch = "x86_64", rust_analyzer))]
pub mod x86_64;
#[cfg(any(target_arch = "x86_64", rust_analyzer))]
pub use x86_64::{L3, PLATFORM, X86_64 as PlatformImpl};
#[cfg(any(target_arch = "riscv64", rust_analyzer))]
pub mod riscv64;
#[cfg(any(target_arch = "riscv64", rust_analyzer))]
pub use riscv64::{Riscv64 as PlatformImpl, L3, PLATFORM};
//
// #[cfg(not(any(
// target_arch = "x86_64",
// target_arch = "aarch64",
// target_arch = "riscv64",
// target_arch = "x86"
// )))]
// compile_error!("Unsupported architecture");
#[cfg(not(any(
target_arch = "x86_64",
target_arch = "aarch64",
target_arch = "riscv64",
)))]
compile_error!("Unsupported architecture");
// Architecture interfaces
kernel/src/arch/x86/mod.rs
+5 -19
View File
@@ -20,7 +20,7 @@ use libk_mm::{
phys::{self, PhysicalMemoryRegion},
table::EntryLevelExt,
};
use peripherals::{i8253::I8253, i8259::I8259, ps2::PS2Controller, rtc::Rtc, serial::ComPort};
use peripherals::{i8253::I8253, ps2::PS2Controller, rtc::Rtc, serial::ComPort};
use ygg_driver_pci::PciBusManager;
use crate::fs::{Initrd, INITRD_DATA};
@@ -44,7 +44,6 @@ struct ProbeClockSource {
pub struct EarlyPlatformDevices {
pub com1_3: Arc<ComPort>,
pub i8259: Arc<I8259>,
clock_sources: Vec<ProbeClockSource>,
}
@@ -79,24 +78,12 @@ pub fn init_platform_early(cmdline: &str) -> Result<EarlyPlatformDevices, Error>
0x3E8,
Irq::External(ISA_IRQ_OFFSET + 4),
)?;
let i8259 = I8259::setup().expect("Could not initialize i8259 PIC");
// let i8259 = I8259::setup().expect("Could not initialize i8259 PIC");
#[cfg(any(target_arch = "x86", rust_analyzer))]
{
use libk::device::register_external_interrupt_controller;
// No other interrupt handling options
unsafe { i8259.clone().init(dummy_init_context()) }?;
register_external_interrupt_controller(i8259.clone());
}
#[cfg(any(target_arch = "x86_64", rust_analyzer))]
{
i8259.disable();
}
// disable_i8259();
// I8259.disable();
Ok(EarlyPlatformDevices {
i8259,
com1_3,
clock_sources: Vec::new(),
})
@@ -116,9 +103,8 @@ pub fn init_platform_devices(early: EarlyPlatformDevices) {
source.display_name()
);
}
Ok(SelectedClockSource::Fallback(i8253)) => {
Ok(SelectedClockSource::Fallback(_i8253)) => {
log::info!("Selected i8253 as the primary clock source");
early.i8259.set_i8253(i8253);
}
Err(error) => {
log::error!("Could not select a clock source: {error:?}");
kernel/src/arch/x86_64/apic/vectors.S
+2 -1
View File
@@ -202,7 +202,7 @@ dummy_vector:
IRQ_VECTORS {irq_vector_offset}, {irq_vector_offset} + {irq_vector_count}
MSI_VECTORS {msi_vector_offset}, {msi_vector_offset} + {msi_vector_count}
.section .rodata
.pushsection .data.rel
// 224 vectors: 256 - 32 (exceptions)
.p2align 4
.type __x86_64_apic_vectors, @object
@@ -221,3 +221,4 @@ __x86_64_apic_vectors:
// Spurious interrupt vector: 223
.quad dummy_vector
.size __x86_64_apic_vectors, . - __x86_64_apic_vectors
.popsection // .data.rel
kernel/src/arch/x86_64/boot/entry.S
+18
View File
@@ -0,0 +1,18 @@
.global __x86_64_entry
.pushsection .text.entry
__x86_64_entry:
// %eax - loader magic
// %rcx - load base address
mov ${yboot_loader_magic}, %edi
cmp %edi, %eax
jne .not_yboot
leaq ({stack_bottom} + {stack_size})(%rip), %rsp
call {entry}
.not_yboot:
cli
hlt
jmp .not_yboot
.popsection // .text.entry
kernel/src/arch/x86_64/boot/mod.rs
+75 -51
View File
@@ -1,4 +1,5 @@
//! x86-64 boot and entry functions
use core::{arch::global_asm, sync::atomic::Ordering};
use kernel_arch_x86::registers::MSR_IA32_KERNEL_GS_BASE;
@@ -10,10 +11,23 @@ use yboot_proto::{
LoadProtocolHeader, LoadProtocolV1, KERNEL_MAGIC, LOADER_MAGIC, PROTOCOL_VERSION_1,
};
use crate::{kernel_main, kernel_secondary_main, mem::KERNEL_VIRT_OFFSET};
use super::PLATFORM;
use crate::{arch::PLATFORM, kernel_main, kernel_secondary_main, mem::KERNEL_VIRT_OFFSET};
// use core::{arch::global_asm, sync::atomic::Ordering};
//
// use kernel_arch_x86::registers::MSR_IA32_KERNEL_GS_BASE;
// use kernel_arch_x86_64::CPU_COUNT;
// use libk_mm::address::{PhysicalAddress, Virtualize};
// use tock_registers::interfaces::Writeable;
// use yboot_proto::{
// v1::{FramebufferOption, MemoryMap},
// LoadProtocolHeader, LoadProtocolV1, KERNEL_MAGIC, LOADER_MAGIC, PROTOCOL_VERSION_1,
// };
//
// use crate::{kernel_main, kernel_secondary_main, mem::KERNEL_VIRT_OFFSET};
//
// use super::PLATFORM;
//
pub enum BootData {
YBoot(&'static LoadProtocolV1),
}
@@ -48,7 +62,6 @@ struct BootStack {
data: [u8; BOOT_STACK_SIZE],
}
#[link_section = ".bss"]
static mut BSP_STACK: BootStack = BootStack {
data: [0; BOOT_STACK_SIZE],
};
@@ -97,25 +110,8 @@ unsafe fn init_dummy_cpu() {
core::arch::asm!("swapgs");
}
extern "C" fn __x86_64_upper_entry() -> ! {
// Safety: ok, CPU hasn't been initialized yet and it's the early kernel entry
unsafe {
init_dummy_cpu();
}
PLATFORM.set_boot_data(BootData::YBoot(&YBOOT_DATA));
unsafe {
PLATFORM
.init_platform(0)
.expect("Could not initialize the platform");
}
kernel_main()
}
/// Application processor entry point
pub extern "C" fn __x86_64_ap_entry() -> ! {
pub extern "C" fn ap_entry() -> ! {
let cpu_id = CPU_COUNT.load(Ordering::Acquire);
unsafe {
@@ -133,37 +129,65 @@ pub extern "C" fn __x86_64_ap_entry() -> ! {
kernel_secondary_main()
}
unsafe extern "C" fn bsp_entry() -> ! {
// Safety: ok, CPU hasn't been initialized yet and it's the early kernel entry
unsafe {
init_dummy_cpu();
}
PLATFORM.set_boot_data(BootData::YBoot(&YBOOT_DATA));
unsafe {
PLATFORM
.init_platform(0)
.expect("Could not initialize the platform");
}
kernel_main()
}
global_asm!(
r#"
// {boot_data}
.global __x86_64_entry
.section .text.entry
__x86_64_entry:
cli
mov ${yboot_loader_magic}, %edi
cmp %edi, %eax
je 2f
// (Currently) unsupported bootloader
1:
cli
hlt
jmp 1b
2:
// yboot entry method
movabsq ${stack_bottom} + {stack_size}, %rax
movabsq ${entry}, %rcx
mov %rax, %rsp
callq *%rcx
.section .text
"#,
include_str!("entry.S"),
yboot_loader_magic = const LOADER_MAGIC,
stack_size = const BOOT_STACK_SIZE,
stack_bottom = sym BSP_STACK,
boot_data = sym YBOOT_DATA,
entry = sym __x86_64_upper_entry,
stack_size = const BOOT_STACK_SIZE,
entry = sym bsp_entry,
options(att_syntax)
);
// global_asm!(
// r#"
// // {boot_data}
// .global __x86_64_entry
//
// .section .text.entry
// __x86_64_entry:
// cli
// mov ${yboot_loader_magic}, %edi
// cmp %edi, %eax
// je 2f
//
// // (Currently) unsupported bootloader
// 1:
// cli
// hlt
// jmp 1b
//
// 2:
// // yboot entry method
// movabsq ${stack_bottom} + {stack_size}, %rax
// movabsq ${entry}, %rcx
// mov %rax, %rsp
// callq *%rcx
//
// .section .text
// "#,
// yboot_loader_magic = const LOADER_MAGIC,
// stack_size = const BOOT_STACK_SIZE,
// stack_bottom = sym BSP_STACK,
// boot_data = sym YBOOT_DATA,
// entry = sym __x86_64_upper_entry,
// options(att_syntax)
// );
kernel/src/arch/x86_64/mod.rs
+48 -128
View File
@@ -1,23 +1,16 @@
//! x86-64 architecture implementation
use core::{mem::size_of, ops::DerefMut, ptr::null_mut, sync::atomic::Ordering};
use ::acpi::{mcfg::Mcfg, AcpiTables, HpetInfo, InterruptModel};
use core::ptr::null_mut;
use abi::error::Error;
use acpi::{mcfg::Mcfg, AcpiTables, HpetInfo, InterruptModel};
use alloc::sync::Arc;
use apic::{ioapic::IoApic, local::LocalApic};
use device_api::device::Device;
use kernel_arch_x86::{
cpuid::{self, CpuFeatures, EcxFeatures, EdxFeatures, ExtEdxFeatures},
gdt, intrinsics,
};
use kernel_arch_x86_64::{
mem::{
flush_tlb_entry, init_fixed_tables,
table::{PageAttributes, PageEntry, PageTable, L1, L2, L3},
EarlyMapping, MEMORY_LIMIT, RAM_MAPPING_L1, RAM_MAPPING_OFFSET,
},
LocalApicInterface, PerCpuData,
gdt,
};
use kernel_arch_x86_64::{mem, LocalApicInterface, PerCpuData};
use libk::{
arch::Cpu,
config, debug,
@@ -31,7 +24,8 @@ use libk::{
use libk_mm::{
address::PhysicalAddress,
phys::{self, reserved::reserve_region, PhysicalMemoryRegion},
table::{EntryLevel, EntryLevelExt},
pointer::PhysicalRef,
table::EntryLevel,
};
use libk_util::OneTimeInit;
use yboot_proto::{
@@ -41,22 +35,28 @@ use yboot_proto::{
use ygg_driver_acpi::{AcpiAllocator, AcpiHandlerImpl, EventAction, FixedEvent};
use ygg_driver_pci::PciBusManager;
use crate::{
arch::{
x86::{self, peripherals::hpet::Hpet, InitrdSource},
x86_64::{
apic::{ioapic::IoApic, local::LocalApic},
boot::BootData,
},
Platform,
},
device::display::linear_fb::LinearFramebuffer,
util::call_init_array,
};
/// Alias for terminal-level page tables
pub type L3 = mem::table::L3;
mod apic;
mod boot;
mod exception;
mod smp;
mod syscall;
use crate::{
arch::x86::{self, peripherals::hpet::Hpet},
device::display::linear_fb::LinearFramebuffer,
util::call_init_array,
};
use self::boot::BootData;
use super::{x86::InitrdSource, Platform};
/// x86-64 architecture implementation
pub struct X86_64 {
boot_data: OneTimeInit<BootData>,
@@ -72,11 +72,8 @@ pub static PLATFORM: X86_64 = X86_64 {
fbconsole: OneTimeInit::new(),
};
//
impl Platform for X86_64 {
const KERNEL_VIRT_OFFSET: usize = 0xFFFFFF8000000000;
type L3 = kernel_arch_x86_64::mem::table::L3;
unsafe fn start_application_processors(&self) {
if let Some(acpi) = self.acpi.try_get() {
let Some(pinfo) = acpi
@@ -124,107 +121,30 @@ impl X86_64 {
self.boot_data.init(data);
}
fn map_physical_memory<I: Iterator<Item = PhysicalMemoryRegion> + Clone>(
it: I,
_memory_start: PhysicalAddress,
memory_end: PhysicalAddress,
have_1gib_pages: bool,
) -> Result<(), Error> {
let end_l1i = memory_end
.into_usize()
.page_align_up::<L1>()
.page_index::<L1>();
if end_l1i > 512 {
panic!(
"Cannot handle {}GiB of RAM",
end_l1i * L1::SIZE / (1024 * 1024 * 1024)
);
}
MEMORY_LIMIT.store(memory_end.into_usize(), Ordering::Release);
if have_1gib_pages {
// Map RAM a gigabyte at a time
for l1i in 0..end_l1i {
// TODO NX
unsafe {
RAM_MAPPING_L1[l1i] = PageEntry::<L1>::block(
PhysicalAddress::from_usize(l1i * L1::SIZE),
PageAttributes::WRITABLE,
);
flush_tlb_entry(RAM_MAPPING_OFFSET + (l1i << L1::SHIFT));
}
}
} else {
// Allocate the intermediate tables first
let l2_tables_start = phys::find_contiguous_region(it, end_l1i)
.expect("Could not allocate the memory for RAM mapping L2 tables");
reserve_region(
"ram-l2-tables",
PhysicalMemoryRegion {
base: l2_tables_start,
size: end_l1i * L3::SIZE,
},
);
// Fill in the tables
for l1i in 0..end_l1i {
let l2_phys_addr = l2_tables_start.add(l1i * L3::SIZE);
// Safety: ok, the mapping is done to the memory obtained from
// find_contiguous_region()
let mut l2_data =
unsafe { EarlyMapping::<[PageEntry<L2>; 512]>::map(l2_phys_addr)? };
// Safety: ok, the slice comes from EarlyMapping of a page-aligned region
let l2 = unsafe { PageTable::from_raw_slice_mut(l2_data.deref_mut()) };
for l2i in 0..512 {
// TODO NX
l2[l2i] = PageEntry::<L2>::block(
PhysicalAddress::from_usize((l1i * L1::SIZE) | (l2i * L2::SIZE)),
PageAttributes::WRITABLE,
);
}
// Point the L1 entry to the L2 table
unsafe {
RAM_MAPPING_L1[l1i] =
PageEntry::<L1>::table(l2_phys_addr, PageAttributes::WRITABLE)
};
unsafe { flush_tlb_entry(RAM_MAPPING_OFFSET + (l1i << L1::SHIFT)) };
intrinsics::flush_cpu_cache();
// The EarlyMapping is then dropped
}
}
Ok(())
}
unsafe fn init_physical_memory_from_yboot(
data: &LoadProtocolV1,
have_1gib_pages: bool,
) -> Result<(), Error> {
let mmap = EarlyMapping::<AvailableMemoryRegion>::map_slice(
unsafe fn init_physical_memory_from_yboot(data: &LoadProtocolV1) -> Result<(), Error> {
let mmap = PhysicalRef::<AvailableMemoryRegion>::map_slice(
PhysicalAddress::from_u64(data.memory_map.address),
data.memory_map.len as usize,
)?;
);
phys::init_from_iter(
mmap.as_ref().iter().map(|reg| PhysicalMemoryRegion {
base: PhysicalAddress::from_u64(reg.start_address),
size: reg.page_count as usize * L3::SIZE,
}),
|it, start, end| Self::map_physical_memory(it, start, end, have_1gib_pages),
)
phys::init_from_iter(mmap.as_ref().iter().map(|reg| PhysicalMemoryRegion {
base: PhysicalAddress::from_u64(reg.start_address),
size: reg.page_count as usize * L3::SIZE,
}))
}
unsafe fn init_memory_management(&self, enabled_features: &CpuFeatures) -> Result<(), Error> {
unsafe fn init_memory_management(
&self,
enabled_features: &CpuFeatures,
is_bsp: bool,
) -> Result<(), Error> {
let have_1gib_pages = enabled_features.ext_edx.contains(ExtEdxFeatures::PDPE1GB);
init_fixed_tables();
mem::init_fixed_tables(have_1gib_pages, is_bsp);
if !is_bsp {
return Ok(());
}
// Reserve lower 4MiB just in case
reserve_region(
@@ -236,7 +156,7 @@ impl X86_64 {
);
match self.boot_data.get() {
&BootData::YBoot(data) => Self::init_physical_memory_from_yboot(data, have_1gib_pages)?,
&BootData::YBoot(data) => Self::init_physical_memory_from_yboot(data)?,
}
Ok(())
@@ -245,12 +165,9 @@ impl X86_64 {
unsafe fn init_platform(&'static self, cpu_id: usize) -> Result<(), Error> {
let (available_features, enabled_features) = self.init_cpu_features();
if cpu_id == 0 {
PLATFORM
.init_memory_management(&enabled_features)
.expect("Could not initialize memory management");
}
PLATFORM
.init_memory_management(&enabled_features, cpu_id == 0)
.expect("Could not initialize memory management");
let local_apic = self.init_local_cpu(cpu_id, available_features, enabled_features);
if cpu_id == 0 {
@@ -377,6 +294,9 @@ impl X86_64 {
unreachable!("The processor does not support APIC");
};
for _ in 0..10 {
log::info!("TICK");
}
let ioapic = IoApic::from_acpi(&apic_info)?;
register_external_interrupt_controller(ioapic);
kernel/src/arch/x86_64/smp.rs
+6 -12
View File
@@ -5,9 +5,8 @@ use ::acpi::platform::{ProcessorInfo, ProcessorState};
use kernel_arch::{Architecture, ArchitectureImpl};
use kernel_arch_x86_64::{
mem::{
flush_tlb_entry,
auto_lower_address, fixed, flush_tlb_entry,
table::{PageAttributes, PageEntry, PageTable, L1, L2},
KERNEL_TABLES,
},
CPU_COUNT,
};
@@ -20,7 +19,7 @@ use libk_mm::{
};
use ygg_driver_acpi::AcpiAllocator;
use crate::arch::x86_64::boot::__x86_64_ap_entry;
use crate::arch::x86_64::boot::ap_entry;
static AP_BOOTSTRAP_BIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/__x86_64_ap_boot.bin"));
@@ -43,7 +42,7 @@ unsafe fn start_ap_core(apic_id: u32) {
let bsp_cpu = Cpu::local();
let bsp_apic = bsp_cpu.local_apic();
let cr3 = KERNEL_TABLES.lock().as_physical_address();
let cr3 = PhysicalAddress::from_usize(auto_lower_address(&raw const fixed::KERNEL_PML4)); // KERNEL_TABLES.lock().as_physical_address();
let stack_base = phys::alloc_pages_contiguous(AP_STACK_PAGES)
.unwrap()
.virtualize();
@@ -53,7 +52,7 @@ unsafe fn start_ap_core(apic_id: u32) {
cr3,
stack_base,
stack_size,
entry: __x86_64_ap_entry as usize,
entry: ap_entry as usize,
};
let mut data_ref = PhysicalRefMut::<ApBootstrapData>::map(AP_BOOTSTRAP_DATA);
@@ -92,12 +91,8 @@ pub unsafe fn start_ap_cores(info: &ProcessorInfo<AcpiAllocator>) {
PageEntry::<L1>::table(identity_l2.as_physical_address(), PageAttributes::WRITABLE);
identity_l2[0] = PageEntry::<L2>::block(PhysicalAddress::ZERO, PageAttributes::WRITABLE);
{
let mut tables = KERNEL_TABLES.lock();
assert_eq!(tables.l0.data[0], 0);
tables.l0.data[0] = identity_l1.as_physical_address().into_u64()
| (PageAttributes::WRITABLE | PageAttributes::PRESENT).bits();
}
fixed::KERNEL_PML4[0] =
PageEntry::table(identity_l1.as_physical_address(), PageAttributes::WRITABLE);
// Load AP_BOOTSTRAP_CODE
let mut code_ref = PhysicalRefMut::map_slice(AP_BOOTSTRAP_CODE, AP_BOOTSTRAP_BIN.len());
@@ -112,7 +107,6 @@ pub unsafe fn start_ap_cores(info: &ProcessorInfo<AcpiAllocator>) {
// Remove the identity-map
identity_l2[0] = PageEntry::INVALID;
flush_tlb_entry(0);
KERNEL_TABLES.lock().l0.data[0] = 0;
PageTable::free::<TableAllocatorImpl>(identity_l1);
PageTable::free::<TableAllocatorImpl>(identity_l2);
kernel/src/arch/x86_64/vectors.S
+8 -4
View File
@@ -44,6 +44,7 @@
.endm
.macro ISR_NERR, n
.hidden __x86_64_exc_\n
__x86_64_exc_\n:
cli
pushq $0
@@ -52,6 +53,7 @@ __x86_64_exc_\n:
.endm
.macro ISR_YERR, n
.hidden __x86_64_exc_\n
__x86_64_exc_\n:
cli
pushq $\n
@@ -66,8 +68,9 @@ __x86_64_exc_\n:
.endm
.global __x86_64_exception_vectors
.global __kernel_start
.section .text
.pushsection .text
__x86_64_exc_common:
// %rsp + 0: error number
// %rsp + 8: error code
@@ -138,7 +141,9 @@ __x86_64_exc_2:
cli
jmp {nmi_handler}
.section .rodata
.popsection // .text
.pushsection .data.rel
.global __x86_64_exception_vectors
.p2align 4
__x86_64_exception_vectors:
@@ -174,5 +179,4 @@ __x86_64_exception_vectors:
.quad __x86_64_exc_29
.quad __x86_64_exc_30
.quad __x86_64_exc_31
.section .text
.popsection // .data.rel