diff --git a/boot/yboot/src/elf.rs b/boot/yboot/src/elf.rs index 4947ece8..804fb135 100644 --- a/boot/yboot/src/elf.rs +++ b/boot/yboot/src/elf.rs @@ -1,7 +1,8 @@ use core::mem::size_of; use bytemuck::Zeroable; -use log::{debug, error, info}; +use log::{error, info}; +use types::{Rela, SHT_RELA}; // TODO use 'elf' crate use uefi::{ prelude::BootServices, @@ -23,14 +24,18 @@ mod types { pub type Half = u16; pub type Word = u32; pub type XWord = u64; + pub type SXWord = i64; pub const PT_LOAD: Word = 1; pub const SHT_PROGBITS: Word = 1; + pub const SHT_RELA: Word = 4; pub const SHF_WRITE: XWord = 1 << 0; pub const SHF_ALLOC: XWord = 1 << 1; + pub const R_X86_64_RELATIVE: u32 = 8; + #[derive(Clone, Copy, Zeroable, Pod)] #[repr(C)] pub struct Ehdr { @@ -77,6 +82,20 @@ mod types { pub memsz: XWord, pub align: XWord, } + + #[derive(Clone, Copy, Zeroable, Pod)] + #[repr(C)] + pub struct Rela { + pub offset: Addr, + pub info: XWord, + pub addend: SXWord, + } + + impl Rela { + pub fn r_type(&self) -> u32 { + self.info as u32 + } + } } // Maximum address this loader can map in the target kernel @@ -92,6 +111,8 @@ pub struct LoadedObject { pub image_start: u64, pub image_end: u64, + pub load_address: u64, + pub entry: u64, pub protocol_struct_paddr: u64, @@ -105,6 +126,12 @@ struct LocatedProtocol { size: usize, } +struct RelaSection { + offset: u64, + entry_count: usize, + entry_size: usize, +} + trait ReadExact { fn read_exact(&mut self, buf: &mut [u8]) -> Result<(), Error>; } @@ -119,6 +146,23 @@ impl ReadExact for RegularFile { } } +impl RelaSection { + pub fn from_shdr(shdr: &Shdr) -> Option { + if shdr.type_ != SHT_RELA { + return None; + } + + let entry_size = shdr.entsize as usize; + let entry_count = shdr.size as usize / entry_size; + + Some(Self { + offset: shdr.offset, + entry_size, + entry_count, + }) + } +} + impl Object { pub fn open(root: &mut D, path: &CStr16) -> Result { let file = root.open(path, FileMode::Read, FileAttribute::empty())?; @@ -140,11 +184,11 @@ impl Object { return Err(Error::new(Status::LOAD_ERROR, ())); } - // Check that the entry point is set - if ehdr.entry == 0 { - error!("Image does not have a valid entry point"); - return Err(Error::new(Status::LOAD_ERROR, ())); - } + // // Check that the entry point is set + // if ehdr.entry == 0 { + // error!("Image does not have a valid entry point"); + // return Err(Error::new(Status::LOAD_ERROR, ())); + // } Ok(Self { file, ehdr }) } @@ -170,11 +214,6 @@ impl Object { self.file .read_exact(bytemuck::bytes_of_mut(&mut proto_data))?; - info!( - "Kernel is virtually mapped at {:#x}", - proto_data.kernel_virt_offset - ); - // 2. Find the kernel's range and check that the loaded physical addresses are actually // usable from UEFI let mut image_start = u64::MAX; @@ -209,16 +248,22 @@ impl Object { assert_eq!(image_start & 0xFFF, 0); assert_eq!(image_end & 0xFFF, 0); - info!("Image start: {:#x}, end: {:#x}", image_start, image_end); - // Reserve the kernel memory - let reserved_addr = bs + // Allocate memory to load the kernel into + let kernel_load_address = bs .allocate_pages( - AllocateType::Address(image_start), + AllocateType::MaxAddress(0xFFFFFFFF), MemoryType::LOADER_DATA, (image_end - image_start) as usize / 0x1000, ) - .expect("Could not allocate memory for kernel image"); - assert_eq!(reserved_addr, image_start); + .expect("Could not allocate memory for the kernel"); + + // Print info + info!("Image start: {:#x}, end: {:#x}", image_start, image_end); + info!( + "Kernel virtual offset: {:#x}", + proto_data.kernel_virt_offset + ); + info!("Kernel load address: {kernel_load_address:#x}"); // 3. Load the segments for i in 0..self.ehdr.phnum { @@ -228,53 +273,80 @@ impl Object { continue; } + let segment_load_base = phdr.paddr + kernel_load_address; + info!( - "Load segment {}: {:#x?}", - i, - phdr.paddr..phdr.paddr + phdr.memsz + "[{i}] Load {:#x?}", + segment_load_base..segment_load_base + phdr.memsz ); if phdr.filesz > 0 { - // The section has load data - let dst = unsafe { - core::slice::from_raw_parts_mut(phdr.paddr as *mut u8, phdr.filesz as usize) + let dst_slice = unsafe { + core::slice::from_raw_parts_mut( + segment_load_base as *mut u8, + phdr.filesz as usize, + ) }; - debug!( - "Load {:#x?} from ELF offset {:#x}", - phdr.paddr..phdr.paddr + phdr.filesz, - phdr.offset - ); self.file.set_position(phdr.offset)?; - self.file.read_exact(dst)?; + self.file.read_exact(dst_slice)?; } if phdr.memsz > phdr.filesz { - let dst = unsafe { + let dst_slice = unsafe { core::slice::from_raw_parts_mut( - (phdr.paddr + phdr.filesz) as *mut u8, + (segment_load_base + phdr.filesz) as *mut u8, (phdr.memsz - phdr.filesz) as usize, ) }; - debug!( - "Zero data {:#x?}", - phdr.paddr + phdr.filesz..phdr.paddr + phdr.memsz - ); + dst_slice.fill(0); + } + } - dst.fill(0); + // 4. Perform kernel relocation + let mut rela_section = None; + for i in 0..self.ehdr.shnum as usize { + let shdr = self.read_shdr(i)?; + if let Some(rela) = RelaSection::from_shdr(&shdr) { + rela_section = Some(rela); + break; + } + } + + if let Some(rela_section) = rela_section { + info!("Relocating kernel: {image_start:#x} -> {kernel_load_address:#x}"); + info!("({} relocations)", rela_section.entry_count); + + let b = (kernel_load_address + proto_data.kernel_virt_offset) as i64; + + for i in 0..rela_section.entry_count { + let mut rela = Rela::zeroed(); + self.file + .set_position(rela_section.offset + (i * rela_section.entry_size) as u64)?; + self.file.read_exact(bytemuck::bytes_of_mut(&mut rela))?; + + match rela.r_type() { + types::R_X86_64_RELATIVE => { + let qword = (rela.offset + kernel_load_address) as *mut i64; + let value = rela.addend + b; + unsafe { qword.write_volatile(value) }; + } + other => todo!("Unsupported relocation type: {other}"), + } } } // Now that the image is in memory, protocol structure can be written in the further steps - let protocol_struct_paddr = (loc_proto.address as u64) - proto_data.kernel_virt_offset; + let protocol_struct_paddr = loc_proto.address as u64 + kernel_load_address; // (loc_proto.address as u64) - proto_data.kernel_virt_offset; let protocol_version = proto_data.header.version; - let entry = self.ehdr.entry; + let entry = self.ehdr.entry + kernel_load_address; Ok(LoadedObject { image_start, image_end, + load_address: kernel_load_address, entry, protocol_struct_paddr, protocol_version, diff --git a/boot/yboot/src/main.rs b/boot/yboot/src/main.rs index 5324aed2..8546d435 100644 --- a/boot/yboot/src/main.rs +++ b/boot/yboot/src/main.rs @@ -93,7 +93,7 @@ fn locate_rsdp(st: &SystemTable) -> Option { fn boot_partition( image: Handle, bs: &BootServices, -) -> Result, Error> { +) -> Result, Error> { let loaded_image = bs.open_protocol_exclusive::(image)?; let device_handle = loaded_image.device(); @@ -113,7 +113,7 @@ fn load_kernel<'a>( config: &Config, root: &mut Directory, st: &SystemTable, -) -> Result<(u64, u64, &'a mut LoadProtocolV1), Error> { +) -> Result<(u64, u64, u64, &'a mut LoadProtocolV1), Error> { let bs = st.boot_services(); let mut kernel_obj = Object::open(root, cstr16!("kernel.elf"))?; @@ -183,13 +183,14 @@ fn load_kernel<'a>( let entry = loaded_obj.entry + proto_data.kernel_virt_offset; - Ok((entry, mmap_memory, proto_data)) + Ok((entry, loaded_obj.load_address, mmap_memory, proto_data)) } unsafe fn map_and_enter_kernel( st: SystemTable, proto_data: &mut LoadProtocolV1, mmap_memory: u64, + load_base: u64, entry: u64, ) -> ! { let (_, mmap) = st.exit_boot_services(); @@ -216,7 +217,7 @@ unsafe fn map_and_enter_kernel( let cr3 = mem::map_image(); asm!("cli; wbinvd; mov {0}, %cr3", in(reg) cr3, options(att_syntax)); - asm!("jmp *{0}", in(reg) entry, in("eax") LOADER_MAGIC, options(noreturn, att_syntax)); + asm!("jmp *{0}", in(reg) entry, in("eax") LOADER_MAGIC, in("ecx") load_base, options(noreturn, att_syntax)); } #[entry] @@ -243,15 +244,16 @@ fn efi_main(image_handle: Handle, mut system_table: SystemTable) -> Status } }; - let (entry, mmap_memory, proto_data) = match load_kernel(&config, &mut root, &system_table) { - Ok(e) => e, - Err(error) => { - error!("Failed to load the kernel/initrd: {error:?}"); - return Status::LOAD_ERROR; - } - }; + let (entry, load_base, mmap_memory, proto_data) = + match load_kernel(&config, &mut root, &system_table) { + Ok(e) => e, + Err(error) => { + error!("Failed to load the kernel/initrd: {error:?}"); + return Status::LOAD_ERROR; + } + }; unsafe { - map_and_enter_kernel(system_table, proto_data, mmap_memory, entry); + map_and_enter_kernel(system_table, proto_data, mmap_memory, load_base, entry); } } diff --git a/etc/ld/x86/x86_64-unknown-none.ld b/etc/ld/x86/x86_64-unknown-none.ld deleted file mode 100644 index add8309b..00000000 --- a/etc/ld/x86/x86_64-unknown-none.ld +++ /dev/null @@ -1,52 +0,0 @@ -ENTRY(__x86_64_entry); - -KERNEL_PHYS_BASE = 0x200000; -KERNEL_VIRT_OFFSET = 0xFFFFFF8000000000; - -SECTIONS { - . = KERNEL_PHYS_BASE; - PROVIDE(__kernel_start = . + KERNEL_VIRT_OFFSET); - - .text.entry : { - *(.text.entry) - } - - . = ALIGN(16); - . = . + KERNEL_VIRT_OFFSET; - - .text : AT(. - KERNEL_VIRT_OFFSET) { - *(.text*) - } - - .export.text : AT(. - KERNEL_VIRT_OFFSET) { - KEEP(*(.export.text*)) - } - - . = ALIGN(4K); - .rodata : AT(. - KERNEL_VIRT_OFFSET) { - *(.eh_frame*) - *(.rodata*) - } - - . = ALIGN(4K); - .data.tables : AT (. - KERNEL_VIRT_OFFSET) { - KEEP(*(.data.tables)) - } - - .data : AT(. - KERNEL_VIRT_OFFSET) { - KEEP(*(.data.yboot)) - *(.data*) - *(.got*) - } - - . = ALIGN(4K); - PROVIDE(__bss_start_phys = . - KERNEL_VIRT_OFFSET); - .bss : AT(. - KERNEL_VIRT_OFFSET) { - *(COMMON) - *(.bss*) - } - . = ALIGN(4K); - PROVIDE(__bss_end_phys = . - KERNEL_VIRT_OFFSET); - - PROVIDE(__kernel_end = .); -}; diff --git a/etc/ld/x86_64-unknown-none.ld b/etc/ld/x86_64-unknown-none.ld new file mode 100644 index 00000000..cfa5344c --- /dev/null +++ b/etc/ld/x86_64-unknown-none.ld @@ -0,0 +1,73 @@ +ENTRY(__x86_64_entry); + +SECTIONS { + . = 0x0; + + PROVIDE(__kernel_start = .); + + .text : { + *(.text.entry) + *(.text*) + } + + . = ALIGN(4K); + + .rodata : { + *(.rodata*) + *(.eh_frame*) + + . = ALIGN(16); + PROVIDE(__init_array_start = .); + KEEP(*(.init_array*)); + PROVIDE(__init_array_end = .); + } + + . = ALIGN(4K); + + .rela : { + PROVIDE(__rela_start = .); + *(.rela*) + PROVIDE(__rela_end = .); + } + + .dynamic : { + *(.dynamic) + } + + . = ALIGN(4K); + + .data : { + KEEP(*(.data.yboot)) + *(.data*) + *(.got*) + } + + . = ALIGN(4K); + PROVIDE(__bss_start = .); + + .bss : { + *(COMMON) + *(.bss*) + } + + . = ALIGN(4K); + PROVIDE(__bss_end = .); + PROVIDE(__kernel_end = .); + PROVIDE(__kernel_size = __kernel_end - __kernel_start); + + .dynsym : { + *(.dynsym) + } + + .gnu.hash : { + *(.gnu.hash) + } + + .hash : { + *(.hash) + } + + .dynstr : { + *(.dynstr) + } +} diff --git a/etc/x86_64-unknown-none.json b/etc/x86_64-unknown-none.json index d48f5369..87a75541 100644 --- a/etc/x86_64-unknown-none.json +++ b/etc/x86_64-unknown-none.json @@ -15,6 +15,7 @@ "panic-strategy": "abort", "dynamic-linking": true, "relocation-model": "pic", + "position-independent-executables": true, "has-thread-local": false, diff --git a/kernel/arch/x86_64/src/context.rs b/kernel/arch/x86_64/src/context.rs index b6e8f07b..61a51097 100644 --- a/kernel/arch/x86_64/src/context.rs +++ b/kernel/arch/x86_64/src/context.rs @@ -9,7 +9,10 @@ use libk_mm_interface::address::{AsPhysicalAddress, PhysicalAddress}; use tock_registers::interfaces::Writeable; use yggdrasil_abi::{arch::SavedFrame, error::Error}; -use crate::{mem::KERNEL_TABLES, ArchitectureImpl}; +use crate::{ + mem::{auto_lower_address, fixed}, + ArchitectureImpl, +}; /// Frame saved onto the stack when taking an IRQ #[derive(Debug)] @@ -431,7 +434,7 @@ impl !, arg: usize) -> Result { const KERNEL_TASK_PAGES: usize = 32; - let cr3: usize = unsafe { KERNEL_TABLES.lock().as_physical_address() }.into(); + let cr3: usize = auto_lower_address(&raw const fixed::KERNEL_PML4); // unsafe { KERNEL_TABLES.lock().as_physical_address() }.into(); let stack_base_phys = PA::allocate_contiguous_pages(KERNEL_TASK_PAGES)?; let stack_base = stack_base_phys.raw_virtualize::(); diff --git a/kernel/arch/x86_64/src/mem/fixed.rs b/kernel/arch/x86_64/src/mem/fixed.rs new file mode 100644 index 00000000..61e26ef8 --- /dev/null +++ b/kernel/arch/x86_64/src/mem/fixed.rs @@ -0,0 +1,132 @@ +use core::ops::Range; + +use kernel_arch_interface::mem::DeviceMemoryAttributes; +use kernel_arch_x86::registers::CR3; +use libk_mm_interface::{ + address::PhysicalAddress, + device::{DevicePageManager, DevicePageTableLevel}, + table::{page_index, EntryLevel}, +}; + +use crate::{ + mem::{ + auto_lower_address, + table::{PageAttributes, PageEntry, PageTable, L0, L1, L2, L3}, + }, + KERNEL_VIRT_OFFSET, +}; + +pub const IDENTITY_SIZE_L1: usize = 64; +pub const KERNEL_L0I: usize = page_index::(KERNEL_VIRT_OFFSET); + +pub const DEVICE_L1: usize = IDENTITY_SIZE_L1; +pub const DEVICE_MAPPING_L3_COUNT: usize = 32; +pub const DEVICE_MAPPING_OFFSET: usize = KERNEL_VIRT_OFFSET + (DEVICE_L1 << L1::SHIFT); + +pub static mut KERNEL_PDPT: PageTable = PageTable::zeroed(); +pub static mut KERNEL_PML4: PageTable = PageTable::zeroed(); + +pub(super) static mut DEVICE_MEMORY: DevicePageManager = + DevicePageManager::new( + L3DeviceMemory([PageTable::zeroed(); DEVICE_MAPPING_L3_COUNT]), + L2DeviceMemory(PageTable::zeroed()), + ); + +#[repr(transparent)] +pub struct L2DeviceMemory(pub PageTable); +#[repr(transparent)] +pub struct L3DeviceMemory(pub [PageTable; DEVICE_MAPPING_L3_COUNT]); + +impl DevicePageTableLevel for L2DeviceMemory { + type Level = L2; + const VIRTUAL_BASE: usize = DEVICE_MAPPING_OFFSET; + const INDEX_RANGE: Range = DEVICE_MAPPING_L3_COUNT..512; + + fn map_page( + &mut self, + index: usize, + physical: PhysicalAddress, + attrs: &DeviceMemoryAttributes, + ) { + let _ = attrs; + self.0[index] = PageEntry::::block(physical, PageAttributes::WRITABLE); + } + + fn unmap_page(&mut self, index: usize) { + self.0[index - DEVICE_MAPPING_L3_COUNT] = PageEntry::INVALID; + } + + fn is_mapped(&self, index: usize) -> bool { + self.0[index - DEVICE_MAPPING_L3_COUNT].is_present() + } + + fn flush_range(range: Range) { + // TODO + } +} + +impl DevicePageTableLevel for L3DeviceMemory { + type Level = L3; + const VIRTUAL_BASE: usize = DEVICE_MAPPING_OFFSET; + const INDEX_RANGE: Range = 0..512 * DEVICE_MAPPING_L3_COUNT; + + fn map_page( + &mut self, + index: usize, + physical: PhysicalAddress, + attrs: &DeviceMemoryAttributes, + ) { + let _ = attrs; + let l2i = index / 512; + let l3i = index % 512; + self.0[l2i][l3i] = PageEntry::page(physical, PageAttributes::WRITABLE); + } + + fn unmap_page(&mut self, index: usize) { + let l2i = index / 512; + let l3i = index % 512; + self.0[l2i][l3i] = PageEntry::INVALID; + } + + fn is_mapped(&self, index: usize) -> bool { + let l2i = index / 512; + let l3i = index % 512; + self.0[l2i][l3i].is_present() + } + + fn flush_range(range: Range) { + // TODO + // let start = range.start * L3::SIZE + Self::VIRTUAL_BASE; + // let size = (range.end - range.start) * L3::SIZE; + // tlb_flush_range_va(start, size); + } +} + +pub(super) unsafe fn setup(have_1gib_pages: bool) { + let phys = PhysicalAddress::from_usize(auto_lower_address(&raw const KERNEL_PDPT)); + KERNEL_PML4[KERNEL_L0I] = PageEntry::table(phys, PageAttributes::WRITABLE); + + if have_1gib_pages { + for i in 0..IDENTITY_SIZE_L1 { + let phys = PhysicalAddress::from_usize(i * L1::SIZE); + KERNEL_PDPT[i] = PageEntry::::block(phys, PageAttributes::WRITABLE); + } + } else { + loop {} + } + + // DEVICE_L1 -> Device L2 table + // 0..DEVICE_MAPPING_L3_COUNT -> Device L3 tables -> Device L3 pages + // ..512 -> Device L2 pages + for i in 0..DEVICE_MAPPING_L3_COUNT { + let phys = + PhysicalAddress::from_usize(auto_lower_address(&raw const DEVICE_MEMORY.normal.0[i])); + DEVICE_MEMORY.large.0[i] = PageEntry::table(phys, PageAttributes::WRITABLE); + } + let phys = PhysicalAddress::from_usize(auto_lower_address(&raw const DEVICE_MEMORY.large.0)); + KERNEL_PDPT[DEVICE_L1] = PageEntry::table(phys, PageAttributes::WRITABLE); +} + +pub(super) unsafe fn load() { + CR3.set_address(auto_lower_address(&raw const KERNEL_PML4)); +} diff --git a/kernel/arch/x86_64/src/mem/mod.rs b/kernel/arch/x86_64/src/mem/mod.rs index 1709d82d..13ed9389 100644 --- a/kernel/arch/x86_64/src/mem/mod.rs +++ b/kernel/arch/x86_64/src/mem/mod.rs @@ -13,86 +13,85 @@ use libk_mm_interface::{ address::PhysicalAddress, table::{page_index, EntryLevel, EntryLevelExt}, }; -use static_assertions::{const_assert_eq, const_assert_ne}; use yggdrasil_abi::error::Error; use crate::KERNEL_VIRT_OFFSET; use self::table::{PageAttributes, PageEntry, PageTable, L0, L1, L2, L3}; +pub mod fixed; pub mod process; pub mod table; #[derive(Debug)] pub struct KernelTableManagerImpl; -const CANONICAL_ADDRESS_MASK: usize = 0xFFFF000000000000; -const KERNEL_PHYS_BASE: usize = 0x200000; - -// Mapped at compile time -const KERNEL_MAPPING_BASE: usize = KERNEL_VIRT_OFFSET + KERNEL_PHYS_BASE; -const KERNEL_L0_INDEX: usize = page_index::(KERNEL_MAPPING_BASE); -const KERNEL_L1_INDEX: usize = page_index::(KERNEL_MAPPING_BASE); -const KERNEL_START_L2_INDEX: usize = page_index::(KERNEL_MAPPING_BASE); - -// Must not be zero, should be at 4MiB -const_assert_ne!(KERNEL_START_L2_INDEX, 0); -// From static mapping -const_assert_eq!(KERNEL_L0_INDEX, 511); -const_assert_eq!(KERNEL_L1_INDEX, 0); - -// Mapped at boot -const EARLY_MAPPING_L2I: usize = KERNEL_START_L2_INDEX - 1; -const DEVICE_MAPPING_L1I: usize = KERNEL_L1_INDEX + 2; -const RAM_MAPPING_L0I: usize = KERNEL_L0_INDEX - 1; - -const DEVICE_MAPPING_L3_COUNT: usize = 4; - -split_spinlock! { - use libk_mm_interface::KernelImageObject; - use memtables::x86_64::FixedTables; - use crate::ArchitectureImpl; - - #[link_section = ".data.tables"] - static KERNEL_TABLES: KernelImageObject = - unsafe { KernelImageObject::new(FixedTables::zeroed()) }; -} - -// 2MiB for early mappings -const EARLY_MAPPING_OFFSET: usize = CANONICAL_ADDRESS_MASK - | (KERNEL_L0_INDEX * L0::SIZE) - | (KERNEL_L1_INDEX * L1::SIZE) - | (EARLY_MAPPING_L2I * L2::SIZE); -static mut EARLY_MAPPING_L3: PageTable = PageTable::zeroed(); -// 1GiB for device MMIO mapping -const DEVICE_MAPPING_OFFSET: usize = - CANONICAL_ADDRESS_MASK | (KERNEL_L0_INDEX * L0::SIZE) | (DEVICE_MAPPING_L1I * L1::SIZE); -static mut DEVICE_MAPPING_L2: PageTable = PageTable::zeroed(); -static mut DEVICE_MAPPING_L3S: [PageTable; DEVICE_MAPPING_L3_COUNT] = - [PageTable::zeroed(); DEVICE_MAPPING_L3_COUNT]; -// 512GiB for whole RAM mapping -pub const RAM_MAPPING_OFFSET: usize = CANONICAL_ADDRESS_MASK | (RAM_MAPPING_L0I * L0::SIZE); -pub static MEMORY_LIMIT: AtomicUsize = AtomicUsize::new(0); -pub static mut RAM_MAPPING_L1: PageTable = PageTable::zeroed(); +// const CANONICAL_ADDRESS_MASK: usize = 0xFFFF000000000000; +// const KERNEL_PHYS_BASE: usize = 0x200000; +// +// // Mapped at compile time +// const KERNEL_MAPPING_BASE: usize = KERNEL_VIRT_OFFSET + KERNEL_PHYS_BASE; +// const KERNEL_L0_INDEX: usize = page_index::(KERNEL_MAPPING_BASE); +// const KERNEL_L1_INDEX: usize = page_index::(KERNEL_MAPPING_BASE); +// const KERNEL_START_L2_INDEX: usize = page_index::(KERNEL_MAPPING_BASE); +// +// // Must not be zero, should be at 4MiB +// const_assert_ne!(KERNEL_START_L2_INDEX, 0); +// // From static mapping +// const_assert_eq!(KERNEL_L0_INDEX, 511); +// const_assert_eq!(KERNEL_L1_INDEX, 0); +// +// // Mapped at boot +// const EARLY_MAPPING_L2I: usize = KERNEL_START_L2_INDEX - 1; +// const DEVICE_MAPPING_L1I: usize = KERNEL_L1_INDEX + 2; +// const RAM_MAPPING_L0I: usize = KERNEL_L0_INDEX - 1; +// +// const DEVICE_MAPPING_L3_COUNT: usize = 4; +// +// split_spinlock! { +// use libk_mm_interface::KernelImageObject; +// use memtables::x86_64::FixedTables; +// use crate::ArchitectureImpl; +// +// #[link_section = ".data.tables"] +// static KERNEL_TABLES: KernelImageObject = +// unsafe { KernelImageObject::new(FixedTables::zeroed()) }; +// } +// +// // 2MiB for early mappings +// const EARLY_MAPPING_OFFSET: usize = CANONICAL_ADDRESS_MASK +// | (KERNEL_L0_INDEX * L0::SIZE) +// | (KERNEL_L1_INDEX * L1::SIZE) +// | (EARLY_MAPPING_L2I * L2::SIZE); +// static mut EARLY_MAPPING_L3: PageTable = PageTable::zeroed(); +// // 1GiB for device MMIO mapping +// const DEVICE_MAPPING_OFFSET: usize = +// CANONICAL_ADDRESS_MASK | (KERNEL_L0_INDEX * L0::SIZE) | (DEVICE_MAPPING_L1I * L1::SIZE); +// static mut DEVICE_MAPPING_L2: PageTable = PageTable::zeroed(); +// static mut DEVICE_MAPPING_L3S: [PageTable; DEVICE_MAPPING_L3_COUNT] = +// [PageTable::zeroed(); DEVICE_MAPPING_L3_COUNT]; +// // 512GiB for whole RAM mapping +// pub const RAM_MAPPING_OFFSET: usize = CANONICAL_ADDRESS_MASK | (RAM_MAPPING_L0I * L0::SIZE); +// pub static MEMORY_LIMIT: AtomicUsize = AtomicUsize::new(0); +// pub static mut RAM_MAPPING_L1: PageTable = PageTable::zeroed(); impl KernelTableManager for KernelTableManagerImpl { fn virtualize(address: u64) -> usize { let address = address as usize; - if address < MEMORY_LIMIT.load(Ordering::Acquire) { - address + RAM_MAPPING_OFFSET + if address < fixed::IDENTITY_SIZE_L1 * L1::SIZE { + address + KERNEL_VIRT_OFFSET } else { - panic!("Invalid physical address: {:#x}", address); + panic!("Invalid physical address: {address:#x}"); } } fn physicalize(address: usize) -> u64 { - if address < RAM_MAPPING_OFFSET - || address - RAM_MAPPING_OFFSET >= MEMORY_LIMIT.load(Ordering::Acquire) + if address < KERNEL_VIRT_OFFSET + || address - KERNEL_VIRT_OFFSET >= fixed::IDENTITY_SIZE_L1 * L1::SIZE { - panic!("Not a virtualized physical address: {:#x}", address); + panic!("Invalid virtualized address: {address:#x}"); } - - (address - RAM_MAPPING_OFFSET) as _ + (address - KERNEL_VIRT_OFFSET) as u64 } unsafe fn map_device_pages( @@ -100,245 +99,20 @@ impl KernelTableManager for KernelTableManagerImpl { count: usize, attrs: DeviceMemoryAttributes, ) -> Result, Error> { - map_device_memory(PhysicalAddress::from_u64(base), count, attrs) + #[allow(static_mut_refs)] + fixed::DEVICE_MEMORY.map_device_pages(PhysicalAddress::from_u64(base), count, attrs) } - unsafe fn unmap_device_pages(mapping: &RawDeviceMemoryMapping) { - unmap_device_memory(mapping) - } -} - -// Early mappings -unsafe fn map_early_pages(physical: PhysicalAddress, count: usize) -> Result { - for l3i in 0..512 { - let mut taken = false; - for i in 0..count { - if EARLY_MAPPING_L3[i + l3i].is_present() { - taken = true; - break; - } - } - - if taken { - continue; - } - - for i in 0..count { - // TODO NX, NC - EARLY_MAPPING_L3[i + l3i] = - PageEntry::page(physical.add(i * L3::SIZE), PageAttributes::WRITABLE); - flush_tlb_entry(EARLY_MAPPING_OFFSET + (i + l3i) * L3::SIZE); - } - - return Ok(EARLY_MAPPING_OFFSET + l3i * L3::SIZE); - } - - Err(Error::OutOfMemory) -} - -unsafe fn unmap_early_page(address: usize) { - if !(EARLY_MAPPING_OFFSET..EARLY_MAPPING_OFFSET + L2::SIZE).contains(&address) { - panic!("Tried to unmap invalid early mapping: {:#x}", address); - } - - let l3i = (address - EARLY_MAPPING_OFFSET).page_index::(); - - assert!(EARLY_MAPPING_L3[l3i].is_present()); - EARLY_MAPPING_L3[l3i] = PageEntry::INVALID; -} - -// Device mappings -unsafe fn map_device_memory_l3( - base: PhysicalAddress, - count: usize, - _attrs: DeviceMemoryAttributes, -) -> Result { - // TODO don't map pages if already mapped - - 'l0: for i in 0..DEVICE_MAPPING_L3_COUNT * 512 { - for j in 0..count { - let l2i = (i + j) / 512; - let l3i = (i + j) % 512; - - if DEVICE_MAPPING_L3S[l2i][l3i].is_present() { - continue 'l0; - } - } - - for j in 0..count { - let l2i = (i + j) / 512; - let l3i = (i + j) % 512; - - // TODO NX, NC - DEVICE_MAPPING_L3S[l2i][l3i] = - PageEntry::page(base.add(j * L3::SIZE), PageAttributes::WRITABLE); - } - - return Ok(DEVICE_MAPPING_OFFSET + i * L3::SIZE); - } - - Err(Error::OutOfMemory) -} - -unsafe fn map_device_memory_l2( - base: PhysicalAddress, - count: usize, - _attrs: DeviceMemoryAttributes, -) -> Result { - 'l0: for i in DEVICE_MAPPING_L3_COUNT..512 { - for j in 0..count { - if DEVICE_MAPPING_L2[i + j].is_present() { - continue 'l0; - } - } - - for j in 0..count { - DEVICE_MAPPING_L2[i + j] = - PageEntry::::block(base.add(j * L2::SIZE), PageAttributes::WRITABLE); - } - - return Ok(DEVICE_MAPPING_OFFSET + i * L2::SIZE); - } - - Err(Error::OutOfMemory) -} - -unsafe fn map_device_memory( - base: PhysicalAddress, - size: usize, - attrs: DeviceMemoryAttributes, -) -> Result, Error> { - let l3_aligned = base.page_align_down::(); - let l3_offset = base.page_offset::(); - let page_count = (l3_offset + size).page_count::(); - - if page_count > 256 { - // Large mapping, use L2 mapping instead - let l2_aligned = base.page_align_down::(); - let l2_offset = base.page_offset::(); - let page_count = (l2_offset + size).page_count::(); - - let base_address = map_device_memory_l2(l2_aligned, page_count, attrs)?; - let address = base_address + l2_offset; - - Ok(RawDeviceMemoryMapping::from_raw_parts( - l2_aligned.into_u64(), - address, - base_address, - page_count, - L2::SIZE, - )) - } else { - // Just map the pages directly - let base_address = map_device_memory_l3(l3_aligned, page_count, attrs)?; - let address = base_address + l3_offset; - - Ok(RawDeviceMemoryMapping::from_raw_parts( - l3_aligned.into_u64(), - address, - base_address, - page_count, - L3::SIZE, - )) - } -} - -unsafe fn unmap_device_memory(map: &RawDeviceMemoryMapping) { - match map.page_size { - L3::SIZE => { - for i in 0..map.page_count { - let page = map.base_address + i * L3::SIZE; - let l2i = page.page_index::(); - let l3i = page.page_index::(); - assert!(DEVICE_MAPPING_L3S[l2i][l3i].is_present()); - DEVICE_MAPPING_L3S[l2i][l3i] = PageEntry::INVALID; - flush_tlb_entry(page); - } - } - L2::SIZE => todo!(), - _ => unimplemented!(), - } -} - -/// Memory mapping which may be used for performing early kernel initialization -pub struct EarlyMapping<'a, T: ?Sized> { - value: &'a mut T, - page_count: usize, -} - -impl<'a, T: Sized> EarlyMapping<'a, T> { - /// # Safety - /// - /// `physical` address provided must be a valid non-NULL address actually containing `T`. - pub unsafe fn map(physical: PhysicalAddress) -> Result, Error> { - let layout = Layout::new::(); - let aligned = physical.page_align_down::(); - let offset = physical.page_offset::(); - let page_count = (offset + layout.size()).div_ceil(L3::SIZE); - - let virt = map_early_pages(aligned, page_count)?; - let value = &mut *((virt + offset) as *mut T); - - Ok(EarlyMapping { value, page_count }) - } - - /// # Safety - /// - /// `physical` address provided must be a valid non-NULL address actually containing a `T` - /// slice of given `len`. - pub unsafe fn map_slice( - physical: PhysicalAddress, - len: usize, - ) -> Result, Error> { - let layout = Layout::array::(len).unwrap(); - let aligned = physical.page_align_down::(); - let offset = physical.page_offset::(); - let page_count = (offset + layout.size()).div_ceil(L3::SIZE); - - let virt = map_early_pages(aligned, page_count)?; - let value = core::slice::from_raw_parts_mut((virt + offset) as *mut T, len); - - Ok(EarlyMapping { value, page_count }) - } -} - -impl Deref for EarlyMapping<'_, T> { - type Target = T; - - fn deref(&self) -> &Self::Target { - self.value - } -} - -impl DerefMut for EarlyMapping<'_, T> { - fn deref_mut(&mut self) -> &mut Self::Target { - self.value - } -} - -impl Drop for EarlyMapping<'_, T> { - fn drop(&mut self) { - let address = (self.value as *mut T).addr() & !(L3::SIZE - 1); - - for i in 0..self.page_count { - let page = address + i * L3::SIZE; - - unsafe { - unmap_early_page(page); - } - } - } + unsafe fn unmap_device_pages(mapping: &RawDeviceMemoryMapping) {} } pub fn clone_kernel_tables(dst: &mut PageTable) { - let tables = KERNEL_TABLES.lock(); unsafe { - dst[KERNEL_L0_INDEX] = PageEntry::from_raw(tables.l0.data[KERNEL_L0_INDEX]); - dst[RAM_MAPPING_L0I] = PageEntry::from_raw(tables.l0.data[RAM_MAPPING_L0I]); + dst[fixed::KERNEL_L0I] = fixed::KERNEL_PML4[fixed::KERNEL_L0I]; } } -pub fn auto_address(pointer: *const T) -> usize { +pub fn auto_lower_address(pointer: *const T) -> usize { let address = pointer.addr(); if address < KERNEL_VIRT_OFFSET { address @@ -363,35 +137,12 @@ pub fn auto_address(pointer: *const T) -> usize { /// # Safety /// /// Unsafe, must only be called by BSP during its early init, must already be in "higher-half" -pub unsafe fn init_fixed_tables() { - let mut tables = KERNEL_TABLES.lock(); - - // TODO this could be built in compile-time too? - let early_mapping_l3_phys = auto_address(&raw const EARLY_MAPPING_L3); - let device_mapping_l2_phys = auto_address(&raw const DEVICE_MAPPING_L2); - let ram_mapping_l1_phys = auto_address(&raw const RAM_MAPPING_L1); - - for i in 0..DEVICE_MAPPING_L3_COUNT { - let device_mapping_l3_phys = - PhysicalAddress::from_usize(auto_address(&raw const DEVICE_MAPPING_L3S[i])); - DEVICE_MAPPING_L2[i] = PageEntry::table(device_mapping_l3_phys, PageAttributes::WRITABLE); +#[inline(never)] +pub unsafe fn init_fixed_tables(have_1gib_pages: bool, bsp: bool) { + fixed::setup(have_1gib_pages); + if bsp { + fixed::load(); } - - assert_eq!(tables.kernel_l2.data[EARLY_MAPPING_L2I], 0); - tables.kernel_l2.data[EARLY_MAPPING_L2I] = (early_mapping_l3_phys as u64) - | (PageAttributes::WRITABLE | PageAttributes::PRESENT).bits(); - - assert_eq!(tables.kernel_l1.data[DEVICE_MAPPING_L1I], 0); - tables.kernel_l1.data[DEVICE_MAPPING_L1I] = (device_mapping_l2_phys as u64) - | (PageAttributes::WRITABLE | PageAttributes::PRESENT).bits(); - - assert_eq!(tables.l0.data[RAM_MAPPING_L0I], 0); - tables.l0.data[RAM_MAPPING_L0I] = - (ram_mapping_l1_phys as u64) | (PageAttributes::WRITABLE | PageAttributes::PRESENT).bits(); - - // TODO ENABLE EFER.NXE - let cr3 = auto_address(&raw const tables.l0); - CR3.set_address(cr3); } /// # Safety diff --git a/kernel/src/arch/mod.rs b/kernel/src/arch/mod.rs index 119b9c90..dbaa3ad9 100644 --- a/kernel/src/arch/mod.rs +++ b/kernel/src/arch/mod.rs @@ -6,27 +6,26 @@ use kernel_arch::{Architecture, ArchitectureImpl}; pub mod aarch64; #[cfg(any(target_arch = "aarch64", rust_analyzer))] pub use aarch64::{AArch64 as PlatformImpl, L3, PLATFORM}; -// -// #[cfg(any(target_arch = "x86_64", target_arch = "x86", rust_analyzer))] -// pub mod x86; -// -// #[cfg(any(target_arch = "x86_64", rust_analyzer))] -// pub mod x86_64; -// #[cfg(any(target_arch = "x86_64", rust_analyzer))] -// pub use x86_64::{PLATFORM, X86_64 as PlatformImpl}; + +#[cfg(any(target_arch = "x86_64", target_arch = "x86", rust_analyzer))] +pub mod x86; + +#[cfg(any(target_arch = "x86_64", rust_analyzer))] +pub mod x86_64; +#[cfg(any(target_arch = "x86_64", rust_analyzer))] +pub use x86_64::{L3, PLATFORM, X86_64 as PlatformImpl}; #[cfg(any(target_arch = "riscv64", rust_analyzer))] pub mod riscv64; #[cfg(any(target_arch = "riscv64", rust_analyzer))] pub use riscv64::{Riscv64 as PlatformImpl, L3, PLATFORM}; -// -// #[cfg(not(any( -// target_arch = "x86_64", -// target_arch = "aarch64", -// target_arch = "riscv64", -// target_arch = "x86" -// )))] -// compile_error!("Unsupported architecture"); + +#[cfg(not(any( + target_arch = "x86_64", + target_arch = "aarch64", + target_arch = "riscv64", +)))] +compile_error!("Unsupported architecture"); // Architecture interfaces diff --git a/kernel/src/arch/x86/mod.rs b/kernel/src/arch/x86/mod.rs index 84d4488b..60a3f25e 100644 --- a/kernel/src/arch/x86/mod.rs +++ b/kernel/src/arch/x86/mod.rs @@ -20,7 +20,7 @@ use libk_mm::{ phys::{self, PhysicalMemoryRegion}, table::EntryLevelExt, }; -use peripherals::{i8253::I8253, i8259::I8259, ps2::PS2Controller, rtc::Rtc, serial::ComPort}; +use peripherals::{i8253::I8253, ps2::PS2Controller, rtc::Rtc, serial::ComPort}; use ygg_driver_pci::PciBusManager; use crate::fs::{Initrd, INITRD_DATA}; @@ -44,7 +44,6 @@ struct ProbeClockSource { pub struct EarlyPlatformDevices { pub com1_3: Arc, - pub i8259: Arc, clock_sources: Vec, } @@ -79,24 +78,12 @@ pub fn init_platform_early(cmdline: &str) -> Result 0x3E8, Irq::External(ISA_IRQ_OFFSET + 4), )?; - let i8259 = I8259::setup().expect("Could not initialize i8259 PIC"); + // let i8259 = I8259::setup().expect("Could not initialize i8259 PIC"); - #[cfg(any(target_arch = "x86", rust_analyzer))] - { - use libk::device::register_external_interrupt_controller; - - // No other interrupt handling options - unsafe { i8259.clone().init(dummy_init_context()) }?; - register_external_interrupt_controller(i8259.clone()); - } - - #[cfg(any(target_arch = "x86_64", rust_analyzer))] - { - i8259.disable(); - } + // disable_i8259(); + // I8259.disable(); Ok(EarlyPlatformDevices { - i8259, com1_3, clock_sources: Vec::new(), }) @@ -116,9 +103,8 @@ pub fn init_platform_devices(early: EarlyPlatformDevices) { source.display_name() ); } - Ok(SelectedClockSource::Fallback(i8253)) => { + Ok(SelectedClockSource::Fallback(_i8253)) => { log::info!("Selected i8253 as the primary clock source"); - early.i8259.set_i8253(i8253); } Err(error) => { log::error!("Could not select a clock source: {error:?}"); diff --git a/kernel/src/arch/x86_64/apic/vectors.S b/kernel/src/arch/x86_64/apic/vectors.S index 2ff7a27b..04e02386 100644 --- a/kernel/src/arch/x86_64/apic/vectors.S +++ b/kernel/src/arch/x86_64/apic/vectors.S @@ -202,7 +202,7 @@ dummy_vector: IRQ_VECTORS {irq_vector_offset}, {irq_vector_offset} + {irq_vector_count} MSI_VECTORS {msi_vector_offset}, {msi_vector_offset} + {msi_vector_count} -.section .rodata +.pushsection .data.rel // 224 vectors: 256 - 32 (exceptions) .p2align 4 .type __x86_64_apic_vectors, @object @@ -221,3 +221,4 @@ __x86_64_apic_vectors: // Spurious interrupt vector: 223 .quad dummy_vector .size __x86_64_apic_vectors, . - __x86_64_apic_vectors +.popsection // .data.rel diff --git a/kernel/src/arch/x86_64/boot/entry.S b/kernel/src/arch/x86_64/boot/entry.S new file mode 100644 index 00000000..5d82cf64 --- /dev/null +++ b/kernel/src/arch/x86_64/boot/entry.S @@ -0,0 +1,18 @@ +.global __x86_64_entry + +.pushsection .text.entry +__x86_64_entry: + // %eax - loader magic + // %rcx - load base address + mov ${yboot_loader_magic}, %edi + cmp %edi, %eax + jne .not_yboot + + leaq ({stack_bottom} + {stack_size})(%rip), %rsp + call {entry} + +.not_yboot: + cli + hlt + jmp .not_yboot +.popsection // .text.entry diff --git a/kernel/src/arch/x86_64/boot/mod.rs b/kernel/src/arch/x86_64/boot/mod.rs index 1d030d77..454b49dc 100644 --- a/kernel/src/arch/x86_64/boot/mod.rs +++ b/kernel/src/arch/x86_64/boot/mod.rs @@ -1,4 +1,5 @@ //! x86-64 boot and entry functions + use core::{arch::global_asm, sync::atomic::Ordering}; use kernel_arch_x86::registers::MSR_IA32_KERNEL_GS_BASE; @@ -10,10 +11,23 @@ use yboot_proto::{ LoadProtocolHeader, LoadProtocolV1, KERNEL_MAGIC, LOADER_MAGIC, PROTOCOL_VERSION_1, }; -use crate::{kernel_main, kernel_secondary_main, mem::KERNEL_VIRT_OFFSET}; - -use super::PLATFORM; +use crate::{arch::PLATFORM, kernel_main, kernel_secondary_main, mem::KERNEL_VIRT_OFFSET}; +// use core::{arch::global_asm, sync::atomic::Ordering}; +// +// use kernel_arch_x86::registers::MSR_IA32_KERNEL_GS_BASE; +// use kernel_arch_x86_64::CPU_COUNT; +// use libk_mm::address::{PhysicalAddress, Virtualize}; +// use tock_registers::interfaces::Writeable; +// use yboot_proto::{ +// v1::{FramebufferOption, MemoryMap}, +// LoadProtocolHeader, LoadProtocolV1, KERNEL_MAGIC, LOADER_MAGIC, PROTOCOL_VERSION_1, +// }; +// +// use crate::{kernel_main, kernel_secondary_main, mem::KERNEL_VIRT_OFFSET}; +// +// use super::PLATFORM; +// pub enum BootData { YBoot(&'static LoadProtocolV1), } @@ -48,7 +62,6 @@ struct BootStack { data: [u8; BOOT_STACK_SIZE], } -#[link_section = ".bss"] static mut BSP_STACK: BootStack = BootStack { data: [0; BOOT_STACK_SIZE], }; @@ -97,25 +110,8 @@ unsafe fn init_dummy_cpu() { core::arch::asm!("swapgs"); } -extern "C" fn __x86_64_upper_entry() -> ! { - // Safety: ok, CPU hasn't been initialized yet and it's the early kernel entry - unsafe { - init_dummy_cpu(); - } - - PLATFORM.set_boot_data(BootData::YBoot(&YBOOT_DATA)); - - unsafe { - PLATFORM - .init_platform(0) - .expect("Could not initialize the platform"); - } - - kernel_main() -} - /// Application processor entry point -pub extern "C" fn __x86_64_ap_entry() -> ! { +pub extern "C" fn ap_entry() -> ! { let cpu_id = CPU_COUNT.load(Ordering::Acquire); unsafe { @@ -133,37 +129,65 @@ pub extern "C" fn __x86_64_ap_entry() -> ! { kernel_secondary_main() } +unsafe extern "C" fn bsp_entry() -> ! { + // Safety: ok, CPU hasn't been initialized yet and it's the early kernel entry + unsafe { + init_dummy_cpu(); + } + + PLATFORM.set_boot_data(BootData::YBoot(&YBOOT_DATA)); + + unsafe { + PLATFORM + .init_platform(0) + .expect("Could not initialize the platform"); + } + + kernel_main() +} + global_asm!( - r#" -// {boot_data} -.global __x86_64_entry - -.section .text.entry -__x86_64_entry: - cli - mov ${yboot_loader_magic}, %edi - cmp %edi, %eax - je 2f - - // (Currently) unsupported bootloader -1: - cli - hlt - jmp 1b - -2: - // yboot entry method - movabsq ${stack_bottom} + {stack_size}, %rax - movabsq ${entry}, %rcx - mov %rax, %rsp - callq *%rcx - -.section .text -"#, + include_str!("entry.S"), yboot_loader_magic = const LOADER_MAGIC, - stack_size = const BOOT_STACK_SIZE, + stack_bottom = sym BSP_STACK, - boot_data = sym YBOOT_DATA, - entry = sym __x86_64_upper_entry, + stack_size = const BOOT_STACK_SIZE, + + entry = sym bsp_entry, options(att_syntax) ); + +// global_asm!( +// r#" +// // {boot_data} +// .global __x86_64_entry +// +// .section .text.entry +// __x86_64_entry: +// cli +// mov ${yboot_loader_magic}, %edi +// cmp %edi, %eax +// je 2f +// +// // (Currently) unsupported bootloader +// 1: +// cli +// hlt +// jmp 1b +// +// 2: +// // yboot entry method +// movabsq ${stack_bottom} + {stack_size}, %rax +// movabsq ${entry}, %rcx +// mov %rax, %rsp +// callq *%rcx +// +// .section .text +// "#, +// yboot_loader_magic = const LOADER_MAGIC, +// stack_size = const BOOT_STACK_SIZE, +// stack_bottom = sym BSP_STACK, +// boot_data = sym YBOOT_DATA, +// entry = sym __x86_64_upper_entry, +// options(att_syntax) +// ); diff --git a/kernel/src/arch/x86_64/mod.rs b/kernel/src/arch/x86_64/mod.rs index 9e432344..59d09a06 100644 --- a/kernel/src/arch/x86_64/mod.rs +++ b/kernel/src/arch/x86_64/mod.rs @@ -1,23 +1,16 @@ //! x86-64 architecture implementation -use core::{mem::size_of, ops::DerefMut, ptr::null_mut, sync::atomic::Ordering}; -use ::acpi::{mcfg::Mcfg, AcpiTables, HpetInfo, InterruptModel}; +use core::ptr::null_mut; + use abi::error::Error; +use acpi::{mcfg::Mcfg, AcpiTables, HpetInfo, InterruptModel}; use alloc::sync::Arc; -use apic::{ioapic::IoApic, local::LocalApic}; use device_api::device::Device; use kernel_arch_x86::{ cpuid::{self, CpuFeatures, EcxFeatures, EdxFeatures, ExtEdxFeatures}, - gdt, intrinsics, -}; -use kernel_arch_x86_64::{ - mem::{ - flush_tlb_entry, init_fixed_tables, - table::{PageAttributes, PageEntry, PageTable, L1, L2, L3}, - EarlyMapping, MEMORY_LIMIT, RAM_MAPPING_L1, RAM_MAPPING_OFFSET, - }, - LocalApicInterface, PerCpuData, + gdt, }; +use kernel_arch_x86_64::{mem, LocalApicInterface, PerCpuData}; use libk::{ arch::Cpu, config, debug, @@ -31,7 +24,8 @@ use libk::{ use libk_mm::{ address::PhysicalAddress, phys::{self, reserved::reserve_region, PhysicalMemoryRegion}, - table::{EntryLevel, EntryLevelExt}, + pointer::PhysicalRef, + table::EntryLevel, }; use libk_util::OneTimeInit; use yboot_proto::{ @@ -41,22 +35,28 @@ use yboot_proto::{ use ygg_driver_acpi::{AcpiAllocator, AcpiHandlerImpl, EventAction, FixedEvent}; use ygg_driver_pci::PciBusManager; +use crate::{ + arch::{ + x86::{self, peripherals::hpet::Hpet, InitrdSource}, + x86_64::{ + apic::{ioapic::IoApic, local::LocalApic}, + boot::BootData, + }, + Platform, + }, + device::display::linear_fb::LinearFramebuffer, + util::call_init_array, +}; + +/// Alias for terminal-level page tables +pub type L3 = mem::table::L3; + mod apic; mod boot; mod exception; mod smp; mod syscall; -use crate::{ - arch::x86::{self, peripherals::hpet::Hpet}, - device::display::linear_fb::LinearFramebuffer, - util::call_init_array, -}; - -use self::boot::BootData; - -use super::{x86::InitrdSource, Platform}; - /// x86-64 architecture implementation pub struct X86_64 { boot_data: OneTimeInit, @@ -72,11 +72,8 @@ pub static PLATFORM: X86_64 = X86_64 { fbconsole: OneTimeInit::new(), }; - +// impl Platform for X86_64 { - const KERNEL_VIRT_OFFSET: usize = 0xFFFFFF8000000000; - type L3 = kernel_arch_x86_64::mem::table::L3; - unsafe fn start_application_processors(&self) { if let Some(acpi) = self.acpi.try_get() { let Some(pinfo) = acpi @@ -124,107 +121,30 @@ impl X86_64 { self.boot_data.init(data); } - fn map_physical_memory + Clone>( - it: I, - _memory_start: PhysicalAddress, - memory_end: PhysicalAddress, - have_1gib_pages: bool, - ) -> Result<(), Error> { - let end_l1i = memory_end - .into_usize() - .page_align_up::() - .page_index::(); - - if end_l1i > 512 { - panic!( - "Cannot handle {}GiB of RAM", - end_l1i * L1::SIZE / (1024 * 1024 * 1024) - ); - } - - MEMORY_LIMIT.store(memory_end.into_usize(), Ordering::Release); - - if have_1gib_pages { - // Map RAM a gigabyte at a time - for l1i in 0..end_l1i { - // TODO NX - unsafe { - RAM_MAPPING_L1[l1i] = PageEntry::::block( - PhysicalAddress::from_usize(l1i * L1::SIZE), - PageAttributes::WRITABLE, - ); - flush_tlb_entry(RAM_MAPPING_OFFSET + (l1i << L1::SHIFT)); - } - } - } else { - // Allocate the intermediate tables first - let l2_tables_start = phys::find_contiguous_region(it, end_l1i) - .expect("Could not allocate the memory for RAM mapping L2 tables"); - - reserve_region( - "ram-l2-tables", - PhysicalMemoryRegion { - base: l2_tables_start, - size: end_l1i * L3::SIZE, - }, - ); - - // Fill in the tables - for l1i in 0..end_l1i { - let l2_phys_addr = l2_tables_start.add(l1i * L3::SIZE); - - // Safety: ok, the mapping is done to the memory obtained from - // find_contiguous_region() - let mut l2_data = - unsafe { EarlyMapping::<[PageEntry; 512]>::map(l2_phys_addr)? }; - // Safety: ok, the slice comes from EarlyMapping of a page-aligned region - let l2 = unsafe { PageTable::from_raw_slice_mut(l2_data.deref_mut()) }; - - for l2i in 0..512 { - // TODO NX - l2[l2i] = PageEntry::::block( - PhysicalAddress::from_usize((l1i * L1::SIZE) | (l2i * L2::SIZE)), - PageAttributes::WRITABLE, - ); - } - - // Point the L1 entry to the L2 table - unsafe { - RAM_MAPPING_L1[l1i] = - PageEntry::::table(l2_phys_addr, PageAttributes::WRITABLE) - }; - - unsafe { flush_tlb_entry(RAM_MAPPING_OFFSET + (l1i << L1::SHIFT)) }; - intrinsics::flush_cpu_cache(); - // The EarlyMapping is then dropped - } - } - - Ok(()) - } - - unsafe fn init_physical_memory_from_yboot( - data: &LoadProtocolV1, - have_1gib_pages: bool, - ) -> Result<(), Error> { - let mmap = EarlyMapping::::map_slice( + unsafe fn init_physical_memory_from_yboot(data: &LoadProtocolV1) -> Result<(), Error> { + let mmap = PhysicalRef::::map_slice( PhysicalAddress::from_u64(data.memory_map.address), data.memory_map.len as usize, - )?; + ); - phys::init_from_iter( - mmap.as_ref().iter().map(|reg| PhysicalMemoryRegion { - base: PhysicalAddress::from_u64(reg.start_address), - size: reg.page_count as usize * L3::SIZE, - }), - |it, start, end| Self::map_physical_memory(it, start, end, have_1gib_pages), - ) + phys::init_from_iter(mmap.as_ref().iter().map(|reg| PhysicalMemoryRegion { + base: PhysicalAddress::from_u64(reg.start_address), + size: reg.page_count as usize * L3::SIZE, + })) } - unsafe fn init_memory_management(&self, enabled_features: &CpuFeatures) -> Result<(), Error> { + unsafe fn init_memory_management( + &self, + enabled_features: &CpuFeatures, + is_bsp: bool, + ) -> Result<(), Error> { let have_1gib_pages = enabled_features.ext_edx.contains(ExtEdxFeatures::PDPE1GB); - init_fixed_tables(); + mem::init_fixed_tables(have_1gib_pages, is_bsp); + + if !is_bsp { + return Ok(()); + } // Reserve lower 4MiB just in case reserve_region( @@ -236,7 +156,7 @@ impl X86_64 { ); match self.boot_data.get() { - &BootData::YBoot(data) => Self::init_physical_memory_from_yboot(data, have_1gib_pages)?, + &BootData::YBoot(data) => Self::init_physical_memory_from_yboot(data)?, } Ok(()) @@ -245,12 +165,9 @@ impl X86_64 { unsafe fn init_platform(&'static self, cpu_id: usize) -> Result<(), Error> { let (available_features, enabled_features) = self.init_cpu_features(); - if cpu_id == 0 { - PLATFORM - .init_memory_management(&enabled_features) - .expect("Could not initialize memory management"); - } - + PLATFORM + .init_memory_management(&enabled_features, cpu_id == 0) + .expect("Could not initialize memory management"); let local_apic = self.init_local_cpu(cpu_id, available_features, enabled_features); if cpu_id == 0 { @@ -377,6 +294,9 @@ impl X86_64 { unreachable!("The processor does not support APIC"); }; + for _ in 0..10 { + log::info!("TICK"); + } let ioapic = IoApic::from_acpi(&apic_info)?; register_external_interrupt_controller(ioapic); diff --git a/kernel/src/arch/x86_64/smp.rs b/kernel/src/arch/x86_64/smp.rs index cdb9ace1..6fffa07e 100644 --- a/kernel/src/arch/x86_64/smp.rs +++ b/kernel/src/arch/x86_64/smp.rs @@ -5,9 +5,8 @@ use ::acpi::platform::{ProcessorInfo, ProcessorState}; use kernel_arch::{Architecture, ArchitectureImpl}; use kernel_arch_x86_64::{ mem::{ - flush_tlb_entry, + auto_lower_address, fixed, flush_tlb_entry, table::{PageAttributes, PageEntry, PageTable, L1, L2}, - KERNEL_TABLES, }, CPU_COUNT, }; @@ -20,7 +19,7 @@ use libk_mm::{ }; use ygg_driver_acpi::AcpiAllocator; -use crate::arch::x86_64::boot::__x86_64_ap_entry; +use crate::arch::x86_64::boot::ap_entry; static AP_BOOTSTRAP_BIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/__x86_64_ap_boot.bin")); @@ -43,7 +42,7 @@ unsafe fn start_ap_core(apic_id: u32) { let bsp_cpu = Cpu::local(); let bsp_apic = bsp_cpu.local_apic(); - let cr3 = KERNEL_TABLES.lock().as_physical_address(); + let cr3 = PhysicalAddress::from_usize(auto_lower_address(&raw const fixed::KERNEL_PML4)); // KERNEL_TABLES.lock().as_physical_address(); let stack_base = phys::alloc_pages_contiguous(AP_STACK_PAGES) .unwrap() .virtualize(); @@ -53,7 +52,7 @@ unsafe fn start_ap_core(apic_id: u32) { cr3, stack_base, stack_size, - entry: __x86_64_ap_entry as usize, + entry: ap_entry as usize, }; let mut data_ref = PhysicalRefMut::::map(AP_BOOTSTRAP_DATA); @@ -92,12 +91,8 @@ pub unsafe fn start_ap_cores(info: &ProcessorInfo) { PageEntry::::table(identity_l2.as_physical_address(), PageAttributes::WRITABLE); identity_l2[0] = PageEntry::::block(PhysicalAddress::ZERO, PageAttributes::WRITABLE); - { - let mut tables = KERNEL_TABLES.lock(); - assert_eq!(tables.l0.data[0], 0); - tables.l0.data[0] = identity_l1.as_physical_address().into_u64() - | (PageAttributes::WRITABLE | PageAttributes::PRESENT).bits(); - } + fixed::KERNEL_PML4[0] = + PageEntry::table(identity_l1.as_physical_address(), PageAttributes::WRITABLE); // Load AP_BOOTSTRAP_CODE let mut code_ref = PhysicalRefMut::map_slice(AP_BOOTSTRAP_CODE, AP_BOOTSTRAP_BIN.len()); @@ -112,7 +107,6 @@ pub unsafe fn start_ap_cores(info: &ProcessorInfo) { // Remove the identity-map identity_l2[0] = PageEntry::INVALID; flush_tlb_entry(0); - KERNEL_TABLES.lock().l0.data[0] = 0; PageTable::free::(identity_l1); PageTable::free::(identity_l2); diff --git a/kernel/src/arch/x86_64/vectors.S b/kernel/src/arch/x86_64/vectors.S index d04ee00b..6b6715d3 100644 --- a/kernel/src/arch/x86_64/vectors.S +++ b/kernel/src/arch/x86_64/vectors.S @@ -44,6 +44,7 @@ .endm .macro ISR_NERR, n +.hidden __x86_64_exc_\n __x86_64_exc_\n: cli pushq $0 @@ -52,6 +53,7 @@ __x86_64_exc_\n: .endm .macro ISR_YERR, n +.hidden __x86_64_exc_\n __x86_64_exc_\n: cli pushq $\n @@ -66,8 +68,9 @@ __x86_64_exc_\n: .endm .global __x86_64_exception_vectors +.global __kernel_start -.section .text +.pushsection .text __x86_64_exc_common: // %rsp + 0: error number // %rsp + 8: error code @@ -138,7 +141,9 @@ __x86_64_exc_2: cli jmp {nmi_handler} -.section .rodata +.popsection // .text + +.pushsection .data.rel .global __x86_64_exception_vectors .p2align 4 __x86_64_exception_vectors: @@ -174,5 +179,4 @@ __x86_64_exception_vectors: .quad __x86_64_exc_29 .quad __x86_64_exc_30 .quad __x86_64_exc_31 - -.section .text +.popsection // .data.rel