diff --git a/include/sys/amd64/mm/map.h b/include/sys/amd64/mm/map.h index eb9ccca..662fed4 100644 --- a/include/sys/amd64/mm/map.h +++ b/include/sys/amd64/mm/map.h @@ -46,4 +46,4 @@ uintptr_t amd64_map_umap(mm_space_t space, uintptr_t vaddr, uint32_t size); * * AMD64_MAP_USER * @return 0 on success, -1 otherwise */ -int amd64_map_single(mm_space_t pml4, uintptr_t virt_addr, uintptr_t phys, uint32_t flags); +int amd64_map_single(mm_space_t pml4, uintptr_t virt_addr, uintptr_t phys, uint64_t flags); diff --git a/include/sys/amd64/mm/mm.h b/include/sys/amd64/mm/mm.h index 9961bd0..707544c 100644 --- a/include/sys/amd64/mm/mm.h +++ b/include/sys/amd64/mm/mm.h @@ -4,17 +4,48 @@ #include "sys/amd64/loader/data.h" /// The place where the kernel pages are virtually mapped to -#define KERNEL_VIRT_BASE 0xFFFFFF0000000000 +#define KERNEL_VIRT_BASE 0xFFFFFF0000000000 /// amd64 standard states that addresses' upper bits are copies of the 47th bit, so these need to be /// stripped down to only 48 bits -#define AMD64_MM_STRIPSX(a) ((uintptr_t) (a) & 0xFFFFFFFFFFFF) -#define AMD64_MM_ADDRSX(a) (((uintptr_t) (a) & (1ULL << 47)) ? \ - (0xFFFFFF0000000000 | ((uintptr_t) (a))) : \ - ((uintptr_t) (a))) +#define AMD64_MM_STRIPSX(a) ((uintptr_t) (a) & 0xFFFFFFFFFFFF) +#define AMD64_MM_ADDRSX(a) (((uintptr_t) (a) & (1ULL << 47)) ? \ + (0xFFFFFF0000000000 | ((uintptr_t) (a))) : \ + ((uintptr_t) (a))) -#define MM_VIRTUALIZE(a) ((uintptr_t) (a) + 0xFFFFFF0000000000) -#define MM_PHYS(a) ((uintptr_t) (a) - 0xFFFFFF0000000000) +#define MM_VIRTUALIZE(a) ((uintptr_t) (a) + 0xFFFFFF0000000000) +#define MM_PHYS(a) ((uintptr_t) (a) - 0xFFFFFF0000000000) + +#define MM_PAGE_SIZE 0x1000 + +#define MM_PTE_INDEX_MASK 0x1FF +#define MM_PTE_COUNT 512 +#define MM_PTE_FLAGS_MASK 0xFFF +#define MM_PTE_MASK (~0xFFF) + +#define MM_PAGE_MASK (~0xFFF) + +// Same as L1 mask +#define MM_PAGE_OFFSET_MASK 0xFFF +#define MM_PAGE_L2_OFFSET_MASK ((1 << MM_PDI_SHIFT) - 1) +#define MM_PAGE_L3_OFFSET_MASK ((1 << MM_PDPTI_SHIFT) - 1) +#define MM_PAGE_L4_OFFSET_MASK ((1 << MM_PML4I_SHIFT) - 1) + +#define MM_PTI_SHIFT 12 +#define MM_PDI_SHIFT 21 +#define MM_PDPTI_SHIFT 30 +#define MM_PML4I_SHIFT 39 + +#define MM_PAGE_PRESENT (1 << 0) +#define MM_PAGE_WRITE (1 << 1) +#define MM_PAGE_USER (1 << 2) +#define MM_PAGE_WT (1 << 3) +#define MM_PAGE_NOCACHE (1 << 4) +#define MM_PAGE_ACCESSED (1 << 5) +#define MM_PAGE_DIRTY (1 << 6) +#define MM_PAGE_HUGE (1 << 7) +#define MM_PAGE_GLOBAL (1 << 8) +#define MM_PAGE_NOEXEC (1 << 63) /// Page map level 4 typedef uint64_t *mm_pml4_t; diff --git a/include/sys/vmalloc.h b/include/sys/vmalloc.h index 9380c4d..e17d02c 100644 --- a/include/sys/vmalloc.h +++ b/include/sys/vmalloc.h @@ -5,8 +5,8 @@ #pragma once #include "sys/mm.h" -#define VM_ALLOC_USER (1 << 0) -#define VM_ALLOC_WRITE (1 << 1) +#define VM_ALLOC_WRITE (MM_PAGE_WRITE) +#define VM_ALLOC_USER (MM_PAGE_USER) /** * @brief Find a free contiguous memory range inside a given one diff --git a/sys/amd64/mm/map.c b/sys/amd64/mm/map.c index 3c8824c..313892c 100644 --- a/sys/amd64/mm/map.c +++ b/sys/amd64/mm/map.c @@ -9,179 +9,195 @@ uintptr_t amd64_map_get(const mm_space_t pml4, uintptr_t vaddr, uint64_t *flags) { vaddr = AMD64_MM_STRIPSX(vaddr); - size_t pml4i = (vaddr >> 39) & 0x1FF; - size_t pdpti = (vaddr >> 30) & 0x1FF; - size_t pdi = (vaddr >> 21) & 0x1FF; - size_t pti = (vaddr >> 12) & 0x1FF; + size_t pml4i = (vaddr >> MM_PML4I_SHIFT) & MM_PTE_INDEX_MASK; + size_t pdpti = (vaddr >> MM_PDPTI_SHIFT) & MM_PTE_INDEX_MASK; + size_t pdi = (vaddr >> MM_PDI_SHIFT) & MM_PTE_INDEX_MASK; + size_t pti = (vaddr >> MM_PTI_SHIFT) & MM_PTE_INDEX_MASK; mm_pdpt_t pdpt; mm_pagedir_t pd; mm_pagetab_t pt; - if (!(pml4[pml4i] & 1)) { + // L4: + if (!(pml4[pml4i] & MM_PAGE_PRESENT)) { return MM_NADDR; } - if (pml4[pml4i] & (1 << 7)) { + if (pml4[pml4i] & MM_PAGE_HUGE) { panic("NYI\n"); } - pdpt = (mm_pdpt_t) MM_VIRTUALIZE(pml4[pml4i] & ~0xFFF); + // L3: + pdpt = (mm_pdpt_t) MM_VIRTUALIZE(pml4[pml4i] & MM_PTE_MASK); - if (!(pdpt[pdpti] & 1)) { + if (!(pdpt[pdpti] & MM_PAGE_PRESENT)) { return MM_NADDR; } - if (pdpt[pdpti] & (1 << 7)) { - if (flags) { - *flags = 2; - } - return (pdpt[pdpti] & ~0xFFF) | (vaddr & ((1 << 30) - 1)); + if (pdpt[pdpti] & MM_PAGE_HUGE) { + return (pdpt[pdpti] & MM_PTE_MASK) | (vaddr & MM_PAGE_L3_OFFSET_MASK); } - pd = (mm_pagedir_t) MM_VIRTUALIZE(pdpt[pdpti] & ~0xFFF); + // L2: + pd = (mm_pagedir_t) MM_VIRTUALIZE(pdpt[pdpti] & MM_PTE_MASK); - if (!(pd[pdi] & 1)) { + if (!(pd[pdi] & MM_PAGE_PRESENT)) { return MM_NADDR; } - if (pd[pdi] & (1 << 7)) { - if (flags) { - *flags = 1; - } - return (pd[pti] & ~0xFFF) | (vaddr & ((1 << 21) - 1)); + if (pd[pdi] & MM_PAGE_HUGE) { + // Page size is 2MiB (1 << 21) + return (pd[pti] & MM_PTE_MASK) | (vaddr & MM_PAGE_L2_OFFSET_MASK); } - pt = (mm_pagetab_t) MM_VIRTUALIZE(pd[pdi] & ~0xFFF); + // L1: + pt = (mm_pagetab_t) MM_VIRTUALIZE(pd[pdi] & MM_PTE_MASK); - if (!(pt[pti] & 1)) { + if (!(pt[pti] & MM_PAGE_PRESENT)) { return MM_NADDR; } - if (flags) { - *flags = 0; - } - - return (pt[pti] & ~0xFFF) | (vaddr & 0xFFF); + return (pt[pti] & MM_PTE_MASK) | (vaddr & MM_PAGE_OFFSET_MASK); } uintptr_t amd64_map_umap(mm_space_t pml4, uintptr_t vaddr, uint32_t size) { vaddr = AMD64_MM_STRIPSX(vaddr); // TODO: support page sizes other than 4KiB // (Though I can't think of any reason to use it) - size_t pml4i = (vaddr >> 39) & 0x1FF; - size_t pdpti = (vaddr >> 30) & 0x1FF; - size_t pdi = (vaddr >> 21) & 0x1FF; - size_t pti = (vaddr >> 12) & 0x1FF; + size_t pml4i = (vaddr >> MM_PML4I_SHIFT) & MM_PTE_INDEX_MASK; + size_t pdpti = (vaddr >> MM_PDPTI_SHIFT) & MM_PTE_INDEX_MASK; + size_t pdi = (vaddr >> MM_PDI_SHIFT) & MM_PTE_INDEX_MASK; + size_t pti = (vaddr >> MM_PTI_SHIFT) & MM_PTE_INDEX_MASK; mm_pdpt_t pdpt; mm_pagedir_t pd; mm_pagetab_t pt; - if (!(pml4[pml4i] & 1)) { + // L4: + if (!(pml4[pml4i] & MM_PAGE_PRESENT)) { return MM_NADDR; } - if (pml4[pml4i] & (1 << 7)) { + if (pml4[pml4i] & MM_PAGE_HUGE) { panic("NYI\n"); } - pdpt = (mm_pdpt_t) MM_VIRTUALIZE(pml4[pml4i] & ~0xFFF); + // L3: + pdpt = (mm_pdpt_t) MM_VIRTUALIZE(pml4[pml4i] & MM_PTE_MASK); - if (!(pdpt[pdpti] & 1)) { + if (!(pdpt[pdpti] & MM_PAGE_PRESENT)) { return MM_NADDR; } - if (pdpt[pdpti] & (1 << 7)) { + if (pdpt[pdpti] & MM_PAGE_HUGE) { panic("NYI\n"); } - pd = (mm_pagedir_t) MM_VIRTUALIZE(pdpt[pdpti] & ~0xFFF); + // L2: + pd = (mm_pagedir_t) MM_VIRTUALIZE(pdpt[pdpti] & MM_PTE_MASK); - if (!(pd[pdi] & 1)) { + if (!(pd[pdi] & MM_PAGE_PRESENT)) { return MM_NADDR; } - if (pd[pdi] & (1 << 7)) { + if (pd[pdi] & MM_PAGE_HUGE) { panic("NYI\n"); } - pt = (mm_pagetab_t) MM_VIRTUALIZE(pd[pdi] & ~0xFFF); + // L1: + pt = (mm_pagetab_t) MM_VIRTUALIZE(pd[pdi] & MM_PTE_MASK); - if (!(pt[pti] & 1)) { + if (!(pt[pti] & MM_PAGE_PRESENT)) { return MM_NADDR; } - uint64_t old = pt[pti] & ~0xFFF; + uint64_t old = pt[pti] & MM_PTE_MASK; pt[pti] = 0; - asm volatile("invlpg (%0)"::"a"(vaddr):"memory"); + + asm volatile("invlpg (%0)"::"r"(vaddr)); + return old; } -int amd64_map_single(mm_space_t pml4, uintptr_t virt_addr, uintptr_t phys, uint32_t flags) { +int amd64_map_single(mm_space_t pml4, uintptr_t virt_addr, uintptr_t phys, uint64_t flags) { virt_addr = AMD64_MM_STRIPSX(virt_addr); // TODO: support page sizes other than 4KiB // (Though I can't think of any reason to use it) - size_t pml4i = (virt_addr >> 39) & 0x1FF; - size_t pdpti = (virt_addr >> 30) & 0x1FF; - size_t pdi = (virt_addr >> 21) & 0x1FF; - size_t pti = (virt_addr >> 12) & 0x1FF; + size_t pml4i = (virt_addr >> MM_PML4I_SHIFT) & MM_PTE_INDEX_MASK; + size_t pdpti = (virt_addr >> MM_PDPTI_SHIFT) & MM_PTE_INDEX_MASK; + size_t pdi = (virt_addr >> MM_PDI_SHIFT) & MM_PTE_INDEX_MASK; + size_t pti = (virt_addr >> MM_PTI_SHIFT) & MM_PTE_INDEX_MASK; mm_pdpt_t pdpt; mm_pagedir_t pd; mm_pagetab_t pt; - if (!(pml4[pml4i] & 1)) { + if (!(pml4[pml4i] & MM_PAGE_PRESENT)) { // Allocate PDPT pdpt = (mm_pdpt_t) amd64_mm_pool_alloc(); assert(pdpt, "PDPT alloc failed\n"); kdebug("Allocated PDPT = %p\n", pdpt); - pml4[pml4i] = MM_PHYS(pdpt) | (1 << 2) | (1 << 1) | 1; + pml4[pml4i] = MM_PHYS(pdpt) | + MM_PAGE_PRESENT | + MM_PAGE_USER | + MM_PAGE_WRITE; } else { - pdpt = (mm_pdpt_t) MM_VIRTUALIZE(pml4[pml4i] & ~0xFFF); + pdpt = (mm_pdpt_t) MM_VIRTUALIZE(pml4[pml4i] & MM_PTE_MASK); } - if (!(pdpt[pdpti] & 1)) { + if (!(pdpt[pdpti] & MM_PAGE_PRESENT)) { // Allocate PD pd = (mm_pagedir_t) amd64_mm_pool_alloc(); assert(pd, "PD alloc failed\n"); kdebug("Allocated PD = %p\n", pd); - pdpt[pdpti] = MM_PHYS(pd) | (1 << 2) | (1 << 1) | 1; + pdpt[pdpti] = MM_PHYS(pd) | + MM_PAGE_PRESENT | + MM_PAGE_USER | + MM_PAGE_WRITE; } else { - pd = (mm_pagedir_t) MM_VIRTUALIZE(pdpt[pdpti] & ~0xFFF); + pd = (mm_pagedir_t) MM_VIRTUALIZE(pdpt[pdpti] & MM_PTE_MASK); } - if (!(pd[pdi] & 1)) { + if (!(pd[pdi] & MM_PAGE_PRESENT)) { // Allocate PT pt = (mm_pagetab_t) amd64_mm_pool_alloc(); assert(pt, "PT alloc failed\n"); kdebug("Allocated PT = %p\n", pt); - pd[pdi] = MM_PHYS(pt) | (1 << 2) | (1 << 1) | 1; + pd[pdi] = MM_PHYS(pt) | + MM_PAGE_PRESENT | + MM_PAGE_USER | + MM_PAGE_WRITE; } else { - pt = (mm_pagetab_t) MM_VIRTUALIZE(pd[pdi] & ~0xFFF); + pt = (mm_pagetab_t) MM_VIRTUALIZE(pd[pdi] & MM_PTE_MASK); } - if (!(flags & (1 << 31))) { - assert(!(pt[pti] & 1), "Entry already present for %p\n", virt_addr); - } + // Disallow overwriting without unmapping entries first + assert(!(pt[pti] & MM_PAGE_PRESENT), "Entry already present for %p\n", virt_addr); #if defined(KERNEL_TEST_MODE) - kdebug("map %p -> %p %cr%c\n", virt_addr, phys, (flags & (1 << 2)) ? 'u' : '-', (flags & (1 << 1)) ? 'w' : '-'); + kdebug("map %p -> %p %cr%c\n", virt_addr, phys, + (flags & MM_PAGE_USER) ? 'u' : '-', + (flags & MM_PAGE_WRITE) ? 'w' : '-', + (flags & MM_PAGE_GLOBAL) ? 'G' : '-'); #endif - pt[pti] = (phys & ~0xFFF) | (flags & 0xFFF) | 1; - asm volatile("invlpg (%0)"::"a"(virt_addr):"memory"); + + pt[pti] = (phys & MM_PAGE_MASK) | + (flags & MM_PTE_FLAGS_MASK) | + MM_PAGE_PRESENT; + + asm volatile("invlpg (%0)"::"r"(virt_addr)); return 0; } int mm_map_pages_contiguous(mm_space_t pml4, uintptr_t virt_base, uintptr_t phys_base, size_t count, uint32_t flags) { for (size_t i = 0; i < count; ++i) { - uintptr_t virt_addr = virt_base + (i << 12); + uintptr_t virt_addr = virt_base + i * MM_PAGE_SIZE; - if (amd64_map_single(pml4, virt_addr, phys_base + (i << 12), flags) != 0) { + if (amd64_map_single(pml4, virt_addr, phys_base + i * MM_PAGE_SIZE, flags) != 0) { return -1; } } @@ -198,7 +214,7 @@ int mm_space_clone(mm_space_t dst_pml4, const mm_space_t src_pml4, uint32_t flag // Kernel table references may be cloned verbatim, as they're guarannteed to be // shared across all the spaces. // This allows to save some resources on allocating the actual PDPT/PD/PTs - for (size_t i = AMD64_PML4I_USER_END; i < 512; ++i) { + for (size_t i = AMD64_PML4I_USER_END; i < MM_PTE_COUNT; ++i) { dst_pml4[i] = src_pml4[i]; } } @@ -211,66 +227,66 @@ int mm_space_fork(mm_space_t dst_pml4, const mm_space_t src_pml4, uint32_t flags // Copy user pages: // TODO: CoW for (size_t pml4i = 0; pml4i < AMD64_PML4I_USER_END; ++pml4i) { - if (!(src_pml4[pml4i] & 1)) { + if (!(src_pml4[pml4i] & MM_PAGE_PRESENT)) { continue; } - if (src_pml4[pml4i] & (1 << 7)) { + if (src_pml4[pml4i] & MM_PAGE_HUGE) { panic("PML4 page has PS bit set\n"); } - mm_pdpt_t src_pdpt = (mm_pdpt_t) MM_VIRTUALIZE(src_pml4[pml4i] & ~0xFFF); + mm_pdpt_t src_pdpt = (mm_pdpt_t) MM_VIRTUALIZE(src_pml4[pml4i] & MM_PTE_MASK); // Make sure we've got a clean table - _assert(!(dst_pml4[pml4i] & 1)); + _assert(!(dst_pml4[pml4i] & MM_PAGE_PRESENT)); mm_pdpt_t dst_pdpt = (mm_pdpt_t) amd64_mm_pool_alloc(); _assert(dst_pdpt); - dst_pml4[pml4i] = MM_PHYS(dst_pdpt) | (src_pml4[pml4i] & 0xFFF); + dst_pml4[pml4i] = MM_PHYS(dst_pdpt) | (src_pml4[pml4i] & MM_PTE_FLAGS_MASK); - for (size_t pdpti = 0; pdpti < 512; ++pdpti) { - if (!(src_pdpt[pdpti] & 1)) { + for (size_t pdpti = 0; pdpti < MM_PTE_COUNT; ++pdpti) { + if (!(src_pdpt[pdpti] & MM_PAGE_PRESENT)) { continue; } - if (src_pdpt[pdpti] & (1 << 7)) { + if (src_pdpt[pdpti] & MM_PAGE_HUGE) { // Not allowed in U/S panic("1GiB pages not supported in userspace\n"); } - mm_pagedir_t src_pd = (mm_pagedir_t) MM_VIRTUALIZE(src_pdpt[pdpti] & ~0xFFF); + mm_pagedir_t src_pd = (mm_pagedir_t) MM_VIRTUALIZE(src_pdpt[pdpti] & MM_PTE_MASK); mm_pagedir_t dst_pd = (mm_pagedir_t) amd64_mm_pool_alloc(); _assert(dst_pd); - dst_pdpt[pdpti] = MM_PHYS(dst_pd) | (src_pdpt[pdpti] & 0xFFF); + dst_pdpt[pdpti] = MM_PHYS(dst_pd) | (src_pdpt[pdpti] & MM_PTE_FLAGS_MASK); - for (size_t pdi = 0; pdi < 512; ++pdi) { - if (!(src_pd[pdi] & 1)) { + for (size_t pdi = 0; pdi < MM_PTE_COUNT; ++pdi) { + if (!(src_pd[pdi] & MM_PAGE_PRESENT)) { continue; } - if (src_pd[pdi] & (1 << 7)) { + if (src_pd[pdi] & MM_PAGE_HUGE) { panic("2MiB pages not supported in userspace\n"); } - mm_pagetab_t src_pt = (mm_pagetab_t) MM_VIRTUALIZE(src_pd[pdi] & ~0xFFF); + mm_pagetab_t src_pt = (mm_pagetab_t) MM_VIRTUALIZE(src_pd[pdi] & MM_PTE_MASK); mm_pagetab_t dst_pt = (mm_pagetab_t) amd64_mm_pool_alloc(); _assert(dst_pt); - dst_pd[pdi] = MM_PHYS(dst_pt) | (src_pd[pdi] & 0xFFF); + dst_pd[pdi] = MM_PHYS(dst_pt) | (src_pd[pdi] & MM_PTE_FLAGS_MASK); - for (size_t pti = 0; pti < 512; ++pti) { - if (!(src_pt[pti] & 1)) { + for (size_t pti = 0; pti < MM_PTE_COUNT; ++pti) { + if (!(src_pt[pti] & MM_PAGE_PRESENT)) { continue; } - uintptr_t src_page_phys = src_pt[pti] & ~0xFFF; + uintptr_t src_page_phys = src_pt[pti] & MM_PAGE_MASK; uintptr_t dst_page_phys = amd64_phys_alloc_page(); _assert(dst_page_phys != MM_NADDR); // SLOOOOOW UUUSEE COOOOW memcpy((void *) MM_VIRTUALIZE(dst_page_phys), (const void *) MM_VIRTUALIZE(src_page_phys), - 0x1000); + MM_PAGE_SIZE); kdebug("Cloning %p <- %p\n", dst_page_phys, src_page_phys); - dst_pt[pti] = dst_page_phys | (src_pt[pti] & 0xFFF); + dst_pt[pti] = dst_page_phys | (src_pt[pti] & MM_PTE_FLAGS_MASK); } } } @@ -283,32 +299,32 @@ int mm_space_fork(mm_space_t dst_pml4, const mm_space_t src_pml4, uint32_t flags void mm_space_release(mm_space_t pml4) { for (size_t pml4i = 0; pml4i < AMD64_PML4I_USER_END; ++pml4i) { - if (!(pml4[pml4i] & 1)) { + if (!(pml4[pml4i] & MM_PAGE_PRESENT)) { continue; } - mm_pdpt_t pdpt = (mm_pdpt_t) MM_VIRTUALIZE(pml4[pml4i] & ~0xFFF); + mm_pdpt_t pdpt = (mm_pdpt_t) MM_VIRTUALIZE(pml4[pml4i] & MM_PTE_MASK); - for (size_t pdpti = 0; pdpti < 512; ++pdpti) { - if (!(pdpt[pdpti] & 1)) { + for (size_t pdpti = 0; pdpti < MM_PTE_COUNT; ++pdpti) { + if (!(pdpt[pdpti] & MM_PAGE_PRESENT)) { continue; } - mm_pagedir_t pd = (mm_pagedir_t) MM_VIRTUALIZE(pdpt[pdpti] & ~0xFFF); + mm_pagedir_t pd = (mm_pagedir_t) MM_VIRTUALIZE(pdpt[pdpti] & MM_PTE_MASK); - for (size_t pdi = 0; pdi < 512; ++pdi) { - if (!(pd[pdi] & 1)) { + for (size_t pdi = 0; pdi < MM_PTE_COUNT; ++pdi) { + if (!(pd[pdi] & MM_PAGE_PRESENT)) { continue; } - mm_space_t pt = (mm_space_t) MM_VIRTUALIZE(pd[pdi] & ~0xFFF); + mm_space_t pt = (mm_space_t) MM_VIRTUALIZE(pd[pdi] & MM_PTE_MASK); - for (size_t pti = 0; pti < 512; ++pti) { - if (!(pt[pti] & 1)) { + for (size_t pti = 0; pti < MM_PTE_COUNT; ++pti) { + if (!(pt[pti] & MM_PAGE_PRESENT)) { continue; } - uintptr_t page_phys = pt[pti] & ~0xFFF; + uintptr_t page_phys = pt[pti] & MM_PTE_MASK; amd64_phys_free(page_phys); } @@ -329,79 +345,6 @@ void mm_space_free(mm_space_t pml4) { amd64_mm_pool_free(pml4); } -static void amd64_mm_describe_range(const mm_space_t pml4, uintptr_t start_addr, uintptr_t end_addr) { - uintptr_t addr = AMD64_MM_STRIPSX(start_addr); - - size_t range_length = 0; - uintptr_t virt_range_begin = MM_NADDR; - uint32_t range_flags = 0; - - mm_pdpt_t pdpt; - - while (addr < AMD64_MM_STRIPSX(end_addr)) { - size_t pml4i = (addr >> 39) & 0x1FF; - size_t pdpti = (addr >> 30) & 0x1FF; - size_t pdi = (addr >> 21) & 0x1FF; - size_t pti = (addr >> 12) & 0x1FF; - size_t page_size = 1ULL << 39; - - if (pml4[pml4i] & 1) { - if (pml4[pml4i] & (1 << 7)) { - panic("Found a huge page in PML4 (shouldn't be possible): %p\n", AMD64_MM_ADDRSX(addr)); - } - - page_size = 1ULL << 30; - pdpt = (mm_pdpt_t) MM_VIRTUALIZE(pml4[pml4i] & ~0xFFF); - - if (pdpt[pdpti] & 1) { - if (pdpt[pdpti] & (1 << 7)) { - if (virt_range_begin == MM_NADDR) { - virt_range_begin = addr; - range_length = 0; - range_flags = pdpt[pdpti] & 5; - } - - goto found; - } - - panic("TODO: implement smaller page description\n"); - } - } - - if (virt_range_begin != MM_NADDR) { - kdebug("Range %p .. %p (%c%c) %S\n", - virt_range_begin, - virt_range_begin + range_length, - (range_flags & (1 << 2) ? 'u' : '-'), - (range_flags & (1 << 1) ? 'w' : '-'), - range_length); - } - -found: - addr += page_size; - range_length += page_size; - } - - if (virt_range_begin != MM_NADDR) { - kdebug("Range %p .. %p (%c%c) %S\n", - virt_range_begin, - virt_range_begin + range_length, - (range_flags & (1 << 2) ? 'u' : '-'), - (range_flags & (1 << 1) ? 'w' : '-'), - range_length); - } -} - void mm_describe(const mm_space_t pml4) { - kdebug("Memory space V:%p:\n", pml4); - - mm_pdpt_t pdpt; - mm_pagedir_t pd; - mm_pagetab_t pt; - - // Dump everything except kernel-space mappings - kdebug("- Userspace:\n"); - amd64_mm_describe_range(pml4, 0, 0xFFFFFF0000000000); - kdebug("- Kernelspace:\n"); - amd64_mm_describe_range(pml4, 0xFFFFFF0000000000, 0xFFFFFF00FFFFFFFF); + kwarn("mm_describe was removed until I reimplement it properly\n"); } diff --git a/sys/amd64/mm/mm.c b/sys/amd64/mm/mm.c index 278a440..6585955 100644 --- a/sys/amd64/mm/mm.c +++ b/sys/amd64/mm/mm.c @@ -30,16 +30,23 @@ void amd64_mm_init(struct amd64_loader_data *data) { memset((void *) (0x200000 - 2 * 0x1000), 0, 0x1000 * 2); // 0x0000000000000000 -> 0 Mapping for AP bootstrapping - pml4[0] = ((uintptr_t) pdpt) | 1 | 2 | 4; + // pml4[0] = ((uintptr_t) pdpt) | 1 | 2 | 4; // 0xFFFFFF0000000000 -> 0 (512GiB) mapping - pml4[AMD64_MM_STRIPSX(KERNEL_VIRT_BASE) >> 39] = ((uintptr_t) pdpt) | 1 | 2 | 4; + pml4[AMD64_MM_STRIPSX(KERNEL_VIRT_BASE) >> 39] = ((uintptr_t) pdpt) | + MM_PAGE_PRESENT | + MM_PAGE_GLOBAL | + MM_PAGE_WRITE; for (uint64_t i = 0; i < 4; ++i) { kdebug("Mapping %p -> %p\n", KERNEL_VIRT_BASE | (i << 30), i << 30); - pdpt[((AMD64_MM_STRIPSX(KERNEL_VIRT_BASE) >> 30) + i) & 0x1FF] = (i << 30) | 1 | 2 | 4 | (1 << 7); + pdpt[((AMD64_MM_STRIPSX(KERNEL_VIRT_BASE) >> 30) + i) & 0x1FF] = (i << 30) | + MM_PAGE_WRITE | + MM_PAGE_GLOBAL | + MM_PAGE_PRESENT | + MM_PAGE_HUGE; } // Load the new table - asm volatile ("mov %0, %%cr3"::"a"(pml4):"memory"); + asm volatile ("movq %0, %%cr3"::"a"(pml4):"memory"); // Create a pool located right after kernel image amd64_mm_pool_init((uintptr_t) &_kernel_end, MM_POOL_SIZE); diff --git a/sys/amd64/mm/vmalloc.c b/sys/amd64/mm/vmalloc.c index c511e8f..7cd9511 100644 --- a/sys/amd64/mm/vmalloc.c +++ b/sys/amd64/mm/vmalloc.c @@ -8,16 +8,16 @@ uintptr_t vmfind(const mm_space_t pml4, uintptr_t from, uintptr_t to, size_t npages) { // XXX: The slowest approach I could think of // Though the easiest one - size_t page_index = from >> 12; + size_t page_index = from / MM_PAGE_SIZE; - while ((page_index + npages) <= (to >> 12)) { + while ((page_index + npages) <= (to / MM_PAGE_SIZE)) { for (size_t i = 0; i < npages; ++i) { - if (amd64_map_get(pml4, (page_index + i) << 12, 0) != MM_NADDR) { + if (amd64_map_get(pml4, (page_index + i) * MM_PAGE_SIZE, 0) != MM_NADDR) { goto no_match; } } - return page_index << 12; + return page_index * MM_PAGE_SIZE; no_match: ++page_index; continue; @@ -29,28 +29,21 @@ no_match: uintptr_t vmalloc(mm_space_t pml4, uintptr_t from, uintptr_t to, size_t npages, int flags) { uintptr_t addr = vmfind(pml4, from, to, npages); uintptr_t virt_page, phys_page; - uint32_t rflags = 0; - - if (flags & VM_ALLOC_WRITE) { - rflags |= 1 << 1; - } - if (flags & VM_ALLOC_USER) { - rflags |= 1 << 2; - } + uint64_t rflags = flags & (MM_PAGE_USER | MM_PAGE_WRITE); if (addr == MM_NADDR) { return MM_NADDR; } for (size_t i = 0; i < npages; ++i) { - virt_page = addr + (i << 12); + virt_page = addr + i * MM_PAGE_SIZE; phys_page = amd64_phys_alloc_page(); // Allocation of physical page failed, clean up if (phys_page == MM_NADDR) { // Unmap previously allocated pages for (size_t j = 0; j < i; ++j) { - virt_page = addr + (j << 12); + virt_page = addr + j * MM_PAGE_SIZE; // Deallocate physical pages that've already been mapped // We've mapped only 4KiB pages, so expect to unmap only // 4KiB pages @@ -72,8 +65,8 @@ uintptr_t vmalloc(mm_space_t pml4, uintptr_t from, uintptr_t to, size_t npages, void vmfree(mm_space_t pml4, uintptr_t addr, size_t npages) { uintptr_t phys; for (size_t i = 0; i < npages; ++i) { - if ((phys = amd64_map_umap(pml4, addr + (i << 12), 1)) == MM_NADDR) { - panic("Double vmfree error: %p is not an allocated page\n", addr + (i << 12)); + if ((phys = amd64_map_umap(pml4, addr + i * MM_PAGE_SIZE, 1)) == MM_NADDR) { + panic("Double vmfree error: %p is not an allocated page\n", addr + i * MM_PAGE_SIZE); } amd64_phys_free(phys); }