diff --git a/arch/amd64/hw/exc.c b/arch/amd64/hw/exc.c index 59a6ed5..872b99e 100644 --- a/arch/amd64/hw/exc.c +++ b/arch/amd64/hw/exc.c @@ -3,6 +3,9 @@ #include "arch/amd64/smp/smp.h" #endif #include "arch/amd64/cpu.h" +#include "sys/mem/phys.h" +#include "sys/thread.h" +#include "sys/string.h" #include "sys/types.h" #include "sys/debug.h" #include "sys/panic.h" @@ -40,18 +43,92 @@ struct amd64_exception_frame { uint64_t rip, cs, rflags, rsp, ss; }; +int do_pfault(struct amd64_exception_frame *frame, uintptr_t cr2, uintptr_t cr3) { + mm_space_t space = (mm_space_t) MM_VIRTUALIZE(cr3); + + if (!cr2) { + // Don't even try to resolve NULL references + return -1; + } + + if (frame->cs == 0x23 && cr2 < KERNEL_VIRT_BASE) { + // Userspace fault + uint64_t flags; + uintptr_t phys = mm_map_get(space, cr2 & MM_PAGE_MASK, &flags); + struct thread *thr = thread_self; + + if (phys != MM_NADDR) { + // If the exception was caused by write operation + if ((frame->exc_no & X86_PF_WRITE) && // Error was caused by write + (flags & MM_PAGE_USER) && // Page is user-accessible + (!(flags & MM_PAGE_WRITE))) { // Page is not writable + struct page *page = PHYS2PAGE(phys); + _assert(page); + _assert(page->refcount); + + if (page->refcount == 2) { + kdebug("[%d] Cloning page @ %p\n", thr->pid, cr2 & MM_PAGE_MASK); + uintptr_t new_phys = mm_phys_alloc_page(); + _assert(new_phys != MM_NADDR); + memcpy((void *) MM_VIRTUALIZE(new_phys), (const void *) MM_VIRTUALIZE(phys), MM_PAGE_SIZE); + _assert(mm_umap_single(space, cr2 & MM_PAGE_MASK, 1) == phys); + _assert(mm_map_single(space, cr2 & MM_PAGE_MASK, new_phys, MM_PAGE_USER | MM_PAGE_WRITE) == 0); + } else if (page->refcount == 1) { + kdebug("[%d] Only one referring to %p now, claiming ownership\n", thr->pid, cr2 & MM_PAGE_MASK); + _assert(mm_umap_single(space, cr2 & MM_PAGE_MASK, 1) == phys); + _assert(mm_map_single(space, cr2 & MM_PAGE_MASK, phys, MM_PAGE_USER | MM_PAGE_WRITE) == 0); + } else { + panic("???\n"); + } + + return 0; + } + } + + return -1; + } else { + // Kernel page faults are not resolvable + return -1; + } +} + void amd64_exception(struct amd64_exception_frame *frame) { - kfatal("Error\n"); -#if defined(AMD64_SMP) - // Send PANIC IPIs to all other CPUs - size_t cpu = get_cpu()->processor_id; - kfatal("cpu%u initiates panic sequence\n", cpu); - for (size_t i = 0; i < smp_ncpus; ++i) { - if (i != cpu) { - amd64_ipi_send(i, IPI_VECTOR_PANIC); + if (frame->exc_no == X86_EXCEPTION_PF) { + uintptr_t cr2, cr3; + asm volatile ("movq %%cr2, %0":"=r"(cr2)); + asm volatile ("movq %%cr3, %0":"=r"(cr3)); + + if (do_pfault(frame, cr2, cr3) != 0) { + kfatal("Page fault without resolution:\n"); + kfatal("%%cr3 = %p\n", cr3); + if (MM_VIRTUALIZE(cr3) == (uintptr_t) mm_kernel) { + kfatal("(Kernel)\n"); + } + kfatal("Fault address: %p\n", cr2); + + if (frame->exc_code & X86_PF_RESVD) { + kfatal(" - Page structure has reserved bit set\n"); + } + if (frame->exc_code & X86_PF_EXEC) { + kfatal(" - Instruction fetch\n"); + } else { + if (frame->exc_code & X86_PF_WRITE) { + kfatal(" - Write operation\n"); + } else { + kfatal(" - Read operation\n"); + } + } + if (!(frame->exc_code & X86_PF_PRESENT)) { + kfatal(" - Refers to non-present page\n"); + } + if (frame->exc_code & X86_PF_USER) { + kfatal(" - Userspace\n"); + } + } else { + return; } } -#endif + // Dump frame kfatal("CPU raised exception #%u\n", frame->exc_no); @@ -80,39 +157,6 @@ void amd64_exception(struct amd64_exception_frame *frame) { (frame->rflags & X86_FLAGS_OF) ? 'O' : '-', (frame->rflags & X86_FLAGS_IF) ? 'I' : '-', (frame->rflags & X86_FLAGS_TF) ? 'T' : '-'); - - if (frame->exc_no == X86_EXCEPTION_PF) { - uintptr_t cr2, cr3; - asm volatile ("movq %%cr2, %0":"=r"(cr2)); - asm volatile ("movq %%cr3, %0":"=r"(cr3)); - - kfatal("Page fault info:\n"); - kfatal("%%cr3 = %p\n", cr3); - if (MM_VIRTUALIZE(cr3) == (uintptr_t) mm_kernel) { - kfatal("(Kernel)\n"); - } - kfatal("Fault address: %p\n", cr2); - - if (frame->exc_code & X86_PF_RESVD) { - kfatal(" - Page structure has reserved bit set\n"); - } - if (frame->exc_code & X86_PF_EXEC) { - kfatal(" - Instruction fetch\n"); - } else { - if (frame->exc_code & X86_PF_WRITE) { - kfatal(" - Write operation\n"); - } else { - kfatal(" - Read operation\n"); - } - } - if (!(frame->exc_code & X86_PF_PRESENT)) { - kfatal(" - Refers to non-present page\n"); - } - if (frame->exc_code & X86_PF_USER) { - kfatal(" - Userspace\n"); - } - } - uintptr_t sym_base; const char *sym_name; if (debug_symbol_find(frame->rip, &sym_name, &sym_base) == 0) { @@ -123,6 +167,17 @@ void amd64_exception(struct amd64_exception_frame *frame) { kfatal("%rip is in unknown location\n"); } +#if defined(AMD64_SMP) + // Send PANIC IPIs to all other CPUs + size_t cpu = get_cpu()->processor_id; + kfatal("cpu%u initiates panic sequence\n", cpu); + for (size_t i = 0; i < smp_ncpus; ++i) { + if (i != cpu) { + amd64_ipi_send(i, IPI_VECTOR_PANIC); + } + } +#endif + while (1) { asm volatile ("cli; hlt"); } diff --git a/arch/amd64/mm/map.c b/arch/amd64/mm/map.c index 17b68ef..cb3406a 100644 --- a/arch/amd64/mm/map.c +++ b/arch/amd64/mm/map.c @@ -59,6 +59,10 @@ uintptr_t mm_map_get(const mm_space_t pml4, uintptr_t vaddr, uint64_t *flags) { return MM_NADDR; } + if (flags) { + *flags = pt[pti] & MM_PTE_FLAGS_MASK; + } + return (pt[pti] & MM_PTE_MASK) | (vaddr & MM_PAGE_OFFSET_MASK); } @@ -117,6 +121,10 @@ uintptr_t mm_umap_single(mm_space_t pml4, uintptr_t vaddr, uint32_t size) { pt[pti] = 0; asm volatile("invlpg (%0)"::"r"(vaddr)); + struct page *page = PHYS2PAGE(old); + _assert(page); + _assert(page->refcount); + --page->refcount; return old; } @@ -187,6 +195,11 @@ int mm_map_single(mm_space_t pml4, uintptr_t virt_addr, uintptr_t phys, uint64_t (flags & MM_PAGE_GLOBAL) ? 'G' : '-'); #endif + // Increase refcount on physical page + struct page *pg = PHYS2PAGE(phys); + _assert(pg); + ++pg->refcount; + pt[pti] = (phys & MM_PAGE_MASK) | (flags & MM_PTE_FLAGS_MASK) | MM_PAGE_PRESENT; @@ -196,18 +209,6 @@ int mm_map_single(mm_space_t pml4, uintptr_t virt_addr, uintptr_t phys, uint64_t return 0; } -//int mm_map_pages_contiguous(mm_space_t pml4, uintptr_t virt_base, uintptr_t phys_base, size_t count, uint32_t flags) { -// for (size_t i = 0; i < count; ++i) { -// uintptr_t virt_addr = virt_base + i * MM_PAGE_SIZE; -// -// if (amd64_map_single(pml4, virt_addr, phys_base + i * MM_PAGE_SIZE, flags) != 0) { -// return -1; -// } -// } -// -// return 0; -//} - int mm_space_clone(mm_space_t dst_pml4, const mm_space_t src_pml4, uint32_t flags) { if ((flags & MM_CLONE_FLG_USER)) { panic("NYI\n"); @@ -230,8 +231,6 @@ int mm_space_fork(struct thread *dst, const struct thread *src, uint32_t flags) const mm_pml4_t src_pml4 = src->space; if (flags & MM_CLONE_FLG_USER) { - // Copy user pages: - // TODO: CoW for (size_t pml4i = 0; pml4i < AMD64_PML4I_USER_END; ++pml4i) { if (!(src_pml4[pml4i] & MM_PAGE_PRESENT)) { continue; @@ -286,19 +285,22 @@ int mm_space_fork(struct thread *dst, const struct thread *src, uint32_t flags) (pdpti << MM_PDPTI_SHIFT) | (pdi << MM_PDI_SHIFT) | (pti << MM_PTI_SHIFT); + uintptr_t src_page_phys = src_pt[pti] & MM_PTE_MASK; + struct page *src_page = PHYS2PAGE(src_page_phys); - if (!shm_region_find((struct thread *) src, src_page_virt)) { - uintptr_t src_page_phys = src_pt[pti] & MM_PTE_MASK; - uintptr_t dst_page_phys = mm_phys_alloc_page(); //amd64_phys_alloc_page(); - _assert(dst_page_phys != MM_NADDR); + // TODO: differentiate between shared and anonymous mappings + _assert(src_page->refcount); + ++src_page->refcount; - // SLOOOOOW UUUSEE COOOOW - kdebug("Cloning %p <- %p\n", dst_page_phys, src_page_phys); - memcpy((void *) MM_VIRTUALIZE(dst_page_phys), - (const void *) MM_VIRTUALIZE(src_page_phys), - MM_PAGE_SIZE); - - dst_pt[pti] = dst_page_phys | (src_pt[pti] & MM_PTE_FLAGS_MASK); + if (src_pt[pti] & MM_PAGE_WRITE) { + // Clone the mapping, use CoW + uint64_t access = src_pt[pti] & (MM_PTE_FLAGS_MASK & ~MM_PAGE_WRITE); + dst_pt[pti] = src_page_phys | access; + src_pt[pti] &= ~MM_PAGE_WRITE; + asm volatile("invlpg (%0)"::"r"(src_page_virt)); + } else { + // Just clone the mapping - it's readonly + dst_pt[pti] = src_page_phys | (src_pt[pti] & MM_PTE_FLAGS_MASK); } } } @@ -306,9 +308,6 @@ int mm_space_fork(struct thread *dst, const struct thread *src, uint32_t flags) } } - kdebug("Forking shared memory:\n"); - shm_space_fork(dst, (struct thread *) src); - // Kernel pages don't need to be copied - just use mm_space_clone(, , MM_CLONE_FLG_KERNEL) return mm_space_clone(dst_pml4, src_pml4, MM_CLONE_FLG_KERNEL & flags); } @@ -342,13 +341,12 @@ void mm_space_release(struct thread *thr) { continue; } - uintptr_t page_virt = (pml4i << MM_PML4I_SHIFT) | - (pdpti << MM_PDPTI_SHIFT) | - (pdi << MM_PDI_SHIFT) | - (pti << MM_PTI_SHIFT); + uintptr_t page_phys = pt[pti] & MM_PTE_MASK; + struct page *page = PHYS2PAGE(page_phys); + _assert(page->refcount); + --page->refcount; - if (!shm_region_find(thr, page_virt)) { - uintptr_t page_phys = pt[pti] & MM_PTE_MASK; + if (!page->refcount) { mm_phys_free_page(page_phys); } } @@ -363,9 +361,6 @@ void mm_space_release(struct thread *thr) { pml4[pml4i] = 0; } - - kdebug("Releasing shared memory:\n"); - shm_region_release_all(thr, 1); } void mm_space_free(struct thread *thr) { diff --git a/arch/amd64/mm/phys.c b/arch/amd64/mm/phys.c index c5e134b..c488d63 100644 --- a/arch/amd64/mm/phys.c +++ b/arch/amd64/mm/phys.c @@ -5,154 +5,81 @@ #include "sys/debug.h" #include "sys/string.h" #include "sys/spin.h" +#include "sys/mem/phys.h" #include "sys/mm.h" extern int _kernel_end_phys; // Roughly 36MiB of lower memory is occupied by kernel so far: // The rest is available for both kernel and user allocation #define PHYS_ALLOWED_BEGIN ((((uintptr_t) &_kernel_end_phys + MM_POOL_SIZE) + 0xFFF) & ~0xFFF) +#define PHYS_ALLOC_START_INDEX (PHYS_ALLOWED_BEGIN / MM_PAGE_SIZE) +#define PHYS_MAX_PAGES ((1U << 30) / 0x1000) -// TODO: move to sys/util.h -#define MAX(x, y) ((x) > (y) ? (x) : (y)) - -// TODO: this could be a better allocator -// + Support more memory -// 1 index is 64 pages - 256KiB -#define PHYS_MAX_INDEX 16384 // Gives exactly 4GiB available for allocation -#define PHYS_TRACK_INDEX(page) ((page) >> 18) -#define PHYS_TRACK_BIT(page) (((page) >> 12) & 0x3F) - -static uint64_t amd64_phys_memory_track[PHYS_MAX_INDEX]; -static uint64_t amd64_phys_last_index = 0; -static uint64_t amd64_phys_ceiling = 0xFFFFFFFF; -static spin_t alloc_spin = 0; +static struct page _pages[PHYS_MAX_PAGES]; +static size_t _total_pages = 0, _alloc_pages = 0; +struct page *mm_pages = _pages; void amd64_phys_stat(struct amd64_phys_stat *st) { - uintptr_t irq; - spin_lock_irqsave(&alloc_spin, &irq); - - st->pages_free = 0; - st->pages_used = 0; - st->limit = amd64_phys_ceiling; - - for (uint64_t i = 0; i < PHYS_MAX_INDEX; ++i) { - if ((i << 18) >= amd64_phys_ceiling) { - break; - } - - for (uint64_t j = 0; j < 64; ++j) { - if (((i << 18) | (j << 12)) >= amd64_phys_ceiling) { - break; - } - - if (amd64_phys_memory_track[i] & (1ULL << j)) { - ++st->pages_used; - } else { - ++st->pages_free; - } - } - } - - spin_release_irqrestore(&alloc_spin, &irq); + st->limit = _total_pages * MM_PAGE_SIZE; + st->pages_free = _total_pages - _alloc_pages; + st->pages_used = _alloc_pages; } -// Allocate a single 4K page uintptr_t mm_phys_alloc_page(void) { - uintptr_t irq; - spin_lock_irqsave(&alloc_spin, &irq); - for (uint64_t i = amd64_phys_last_index; i < PHYS_MAX_INDEX; ++i) { - for (uint64_t j = 0; j < 64; ++j) { - if (!(amd64_phys_memory_track[i] & (1ULL << j))) { - amd64_phys_memory_track[i] |= (1ULL << j); - amd64_phys_last_index = i; - spin_release_irqrestore(&alloc_spin, &irq); - return PHYS_ALLOWED_BEGIN + ((i << 18) | (j << 12)); - } + for (size_t i = PHYS_ALLOC_START_INDEX; i < PHYS_MAX_PAGES; ++i) { + if (!(_pages[i].flags & PG_ALLOC)) { + _assert(!_pages[i].refcount); + _pages[i].flags |= PG_ALLOC; + ++_alloc_pages; + return i * MM_PAGE_SIZE; } } - for (uint64_t i = 0; i < amd64_phys_last_index; ++i) { - for (uint64_t j = 0; j < 64; ++j) { - if (!(amd64_phys_memory_track[i] & (1ULL << j))) { - amd64_phys_memory_track[i] |= (1ULL << j); - amd64_phys_last_index = i; - spin_release_irqrestore(&alloc_spin, &irq); - return PHYS_ALLOWED_BEGIN + ((i << 18) | (j << 12)); - } - } - } - - spin_release_irqrestore(&alloc_spin, &irq); return MM_NADDR; } -void mm_phys_free_page(uintptr_t page) { - uintptr_t irq; - spin_lock_irqsave(&alloc_spin, &irq); - - // Address is too low - assert(page >= PHYS_ALLOWED_BEGIN, "The page is outside the physical range: %p\n", page); - page -= PHYS_ALLOWED_BEGIN; - // Address is too high - assert(PHYS_TRACK_INDEX(page) < PHYS_MAX_INDEX, "The page is outside the physical range: %p\n", page + PHYS_ALLOWED_BEGIN); - - uint64_t bit = 1ULL << PHYS_TRACK_BIT(page); - uint64_t index = PHYS_TRACK_INDEX(page); - - // Double free error - assert(amd64_phys_memory_track[index] & bit, "Double free error (phys): %p\n", page + PHYS_ALLOWED_BEGIN); - - amd64_phys_memory_track[index] &= ~bit; - spin_release_irqrestore(&alloc_spin, &irq); +void mm_phys_free_page(uintptr_t addr) { + struct page *page = PHYS2PAGE(addr); + _assert(page); + _assert(!page->refcount); + _assert(page->flags & PG_ALLOC); + --_alloc_pages; + page->flags &= ~PG_ALLOC; } -// XXX: very slow impl. uintptr_t mm_phys_alloc_contiguous(size_t count) { - uintptr_t addr = 0; - kdebug("Requested %S\n", count << 12); - - if (count >= (PHYS_MAX_INDEX << 6)) { - // The requested range is too large - return MM_NADDR; - } - uintptr_t irq; - spin_lock_irqsave(&alloc_spin, &irq); - - while (addr < (((uint64_t) (PHYS_MAX_INDEX - (count >> 6) - 1)) << 18)) { - // Check if we can allocate these pages - for (size_t i = 0; i < count; ++i) { - uintptr_t page = (i << 12) + addr; - if (amd64_phys_memory_track[PHYS_TRACK_INDEX(page)] & (1ULL << PHYS_TRACK_BIT(page))) { - // When reaching unavailable page, just start searching beyond its address - addr = page + 0x1000; - goto no_match; + for (size_t i = PHYS_ALLOC_START_INDEX; i < PHYS_MAX_PAGES - count; ++i) { + for (size_t j = 0; j < count; ++j) { + if (_pages[i + j].flags & PG_ALLOC) { + goto fail; } } - // All pages are available - for (size_t i = 0; i < count; ++i) { - uintptr_t page = (i << 12) + addr; - amd64_phys_memory_track[PHYS_TRACK_INDEX(page)] |= (1ULL << PHYS_TRACK_BIT(page)); + + for (size_t j = 0; j < count; ++j) { + _assert(!_pages[i + j].refcount); + _pages[i + j].flags |= PG_ALLOC; } - kdebug("== %p\n", addr + PHYS_ALLOWED_BEGIN); - spin_release_irqrestore(&alloc_spin, &irq); - return PHYS_ALLOWED_BEGIN + addr; - - // Found unavailable page in the range, continue searching -no_match: + return i * MM_PAGE_SIZE; +fail: continue; } - - spin_release_irqrestore(&alloc_spin, &irq); return MM_NADDR; } +static void phys_add_page(size_t index) { + _pages[index].flags &= ~PG_ALLOC; + _pages[index].refcount = 0; +} + void amd64_phys_memory_map(const multiboot_memory_map_t *mmap, size_t length) { - kdebug("Kernel table pool ends @ %p\n", PHYS_ALLOWED_BEGIN); + //kdebug("Kernel table pool ends @ %p\n", PHYS_ALLOWED_BEGIN); kdebug("Memory map @ %p\n", mmap); - memset(amd64_phys_memory_track, 0xFF, sizeof(amd64_phys_memory_track)); uintptr_t curr_item = (uintptr_t) mmap; uintptr_t mmap_end = length + curr_item; - size_t total_phys = 0; + + memset(_pages, 0xFF, sizeof(_pages)); + _total_pages = 0; + _alloc_pages = 0; // Collect usable physical memory information while (curr_item < mmap_end) { @@ -166,19 +93,15 @@ void amd64_phys_memory_map(const multiboot_memory_map_t *mmap, size_t length) { for (uintptr_t addr = page_aligned_begin - PHYS_ALLOWED_BEGIN; addr < (page_aligned_end - PHYS_ALLOWED_BEGIN); addr += 0x1000) { - uintptr_t index = PHYS_TRACK_INDEX(addr); - if (index >= PHYS_MAX_INDEX) { - kdebug("Too high: %p\n", addr); - break; - } - amd64_phys_memory_track[index] &= ~(1ULL << PHYS_TRACK_BIT(addr)); - ++total_phys; - amd64_phys_ceiling = addr + 0x1000; + size_t index = addr / MM_PAGE_SIZE; + _assert(index < PHYS_MAX_PAGES); + phys_add_page(index); + ++_total_pages; } } curr_item += entry->size + sizeof(uint32_t); } - kdebug("%S available\n", total_phys << 12); + kdebug("%S available\n", _total_pages << 12); } diff --git a/include/sys/mem/phys.h b/include/sys/mem/phys.h index 89b9a4b..fdc203e 100644 --- a/include/sys/mem/phys.h +++ b/include/sys/mem/phys.h @@ -5,6 +5,26 @@ #pragma once #include "sys/types.h" +extern struct page *mm_pages; + +#define PHYS2PAGE(phys) \ + (&mm_pages[((uintptr_t) (phys)) / MM_PAGE_SIZE]) +#define PAGE2PHYS(page) \ + (MM_PAGE_SIZE * ((uintptr_t) (phys) - (uintptr_t) pages) / sizeof(struct page)) + +#define PG_ALLOC (1 << 0) +#define PG_MMAPED (1 << 1) + +struct page { + uint64_t flags; + enum page_usage { + PU_UNKNOWN = 0, + PU_PRIVATE, // Program data: image + mmap()ed anonymous regions + PU_SHARED, // Shared memory mapping + } usage; + size_t refcount; +}; + /** * @brief Allocate a single physical memory region of MM_PAGE_SIZE bytes * @return MM_NADDR on failure, a page-aligned physical address otherwise diff --git a/include/sys/mem/shmem.h b/include/sys/mem/shmem.h index 538ae89..180a9e8 100644 --- a/include/sys/mem/shmem.h +++ b/include/sys/mem/shmem.h @@ -4,61 +4,61 @@ #include "sys/spin.h" struct thread; -struct blkdev; - -struct shm_region { - spin_t lock; - - size_t ref_count; - // List head for tracking ownership of the region - struct list_head owners; - // Link for tracking all the regions in the system - struct list_head link; - - size_t page_count; - uintptr_t phys[0]; -}; - -// region -> thread ownership link -struct shm_owner_ref { - struct thread *owner; - uintptr_t virt; - struct list_head link; -}; - -// thread -> region mapping link -struct shm_region_ref { - enum { - // -> memory pages - SHM_TYPE_PHYS = 0, - // -> block device mapping - SHM_TYPE_BLOCK, - } type; - uintptr_t virt; - struct list_head link; - union { - // Physical mapping - struct shm_region *region; - // Block device mapping - struct { - size_t page_count; - uintptr_t phys_base; - struct blkdev *blk; - } block; - } map; -}; - -struct shm_region *shm_region_create(size_t count); -void shm_region_free(struct shm_region *region); - -uintptr_t shm_region_map(struct shm_region *region, struct thread *thr); -void shm_region_release_all(struct thread *thr, int noumap); -struct shm_region_ref *shm_region_find(struct thread *thr, uintptr_t ptr); - -void shm_space_fork(struct thread *dst, struct thread *src); - -void shm_region_describe(struct shm_region *region); -void shm_thread_describe(struct thread *thr); +//struct blkdev; +// +//struct shm_region { +// spin_t lock; +// +// size_t ref_count; +// // List head for tracking ownership of the region +// struct list_head owners; +// // Link for tracking all the regions in the system +// struct list_head link; +// +// size_t page_count; +// uintptr_t phys[0]; +//}; +// +//// region -> thread ownership link +//struct shm_owner_ref { +// struct thread *owner; +// uintptr_t virt; +// struct list_head link; +//}; +// +//// thread -> region mapping link +//struct shm_region_ref { +// enum { +// // -> memory pages +// SHM_TYPE_PHYS = 0, +// // -> block device mapping +// SHM_TYPE_BLOCK, +// } type; +// uintptr_t virt; +// struct list_head link; +// union { +// // Physical mapping +// struct shm_region *region; +// // Block device mapping +// struct { +// size_t page_count; +// uintptr_t phys_base; +// struct blkdev *blk; +// } block; +// } map; +//}; +// +//struct shm_region *shm_region_create(size_t count); +//void shm_region_free(struct shm_region *region); +// +//uintptr_t shm_region_map(struct shm_region *region, struct thread *thr); +//void shm_region_release_all(struct thread *thr, int noumap); +//struct shm_region_ref *shm_region_find(struct thread *thr, uintptr_t ptr); +// +//void shm_space_fork(struct thread *dst, struct thread *src); +// +//void shm_region_describe(struct shm_region *region); +//void shm_thread_describe(struct thread *thr); void *sys_mmap(void *hint, size_t length, int prot, int flags, int fd, off_t offset); int sys_munmap(void *addr, size_t length); diff --git a/sys/mem/shmem.c b/sys/mem/shmem.c index f302d04..9cbce50 100644 --- a/sys/mem/shmem.c +++ b/sys/mem/shmem.c @@ -15,124 +15,123 @@ #include "fs/node.h" #include "sys/mm.h" -static spin_t g_shm_lock = 0; -static LIST_HEAD(g_shm_region_head); -static struct slab_cache *shm_region_ref_cache = NULL; -static struct slab_cache *shm_owner_ref_cache = NULL; - -static __init void shm_init_slab_caches(void) { - shm_region_ref_cache = slab_cache_get(sizeof(struct shm_region_ref)); - shm_owner_ref_cache = slab_cache_get(sizeof(struct shm_owner_ref)); -} - -struct shm_region *shm_region_create(size_t count) { - kdebug("Allocating a SHM region of %u pages\n", count); - struct shm_region *res = kmalloc(sizeof(struct shm_region) + count * sizeof(uintptr_t)); - _assert(res); - - for (size_t i = 0; i < count; ++i) { - res->phys[i] = mm_phys_alloc_page(); - _assert(res->phys[i] != MM_NADDR); - } - - res->page_count = count; - res->ref_count = 0; - list_head_init(&res->owners); - res->lock = 0; - - return res; -} - -void shm_region_free(struct shm_region *region) { - uintptr_t irq; - spin_lock_irqsave(®ion->lock, &irq); - _assert(list_empty(®ion->owners)); - _assert(!region->ref_count); - - kdebug("Releasing %u pages\n", region->page_count); - for (size_t i = 0; i < region->page_count; ++i) { - mm_phys_free_page(region->phys[i]); - } - - spin_release_irqrestore(®ion->lock, &irq); - kfree(region); -} - -uintptr_t shm_region_map(struct shm_region *region, struct thread *thr) { - uintptr_t irq; - spin_lock_irqsave(®ion->lock, &irq); - kdebug("Mapping region of %u pages into thread %d\n", region->page_count, thr->pid); - - struct shm_region_ref *reg_ref = slab_calloc(shm_region_ref_cache); - _assert(reg_ref); - struct shm_owner_ref *own_ref = slab_calloc(shm_owner_ref_cache); - _assert(own_ref); - - uintptr_t vaddr = vmfind(thr->space, 0x80000000, 0x100000000, region->page_count); - - if (vaddr == MM_NADDR) { - slab_free(shm_region_ref_cache, reg_ref); - slab_free(shm_owner_ref_cache, own_ref); - kerror("Could not find memory for mapping\n"); - return MM_NADDR; - } - kdebug("-> %p\n", vaddr); - - reg_ref->type = SHM_TYPE_PHYS; - reg_ref->virt = vaddr; - reg_ref->map.region = region; - list_head_init(®_ref->link); - own_ref->virt = vaddr; - own_ref->owner = thr; - list_head_init(&own_ref->link); - - // Map all the pages - for (size_t i = 0; i < region->page_count; ++i) { - // TODO: access flags (prot) - _assert(mm_map_single(thr->space, - vaddr + i * MM_PAGE_SIZE, - region->phys[i], - MM_PAGE_USER | MM_PAGE_WRITE) == 0); - } - - // Add region ref to thread - list_add(®_ref->link, &thr->shm_list); - // Add owner ref to region - list_add(&own_ref->link, ®ion->owners); - - ++region->ref_count; - - spin_release_irqrestore(®ion->lock, &irq); - - return vaddr; -} - -// Slow, but kinda works -struct shm_region_ref *shm_region_find(struct thread *thr, uintptr_t addr) { - struct shm_region_ref *ref; - list_for_each_entry(ref, &thr->shm_list, link) { - switch (ref->type) { - case SHM_TYPE_PHYS: - _assert(ref->map.region); - - if (addr >= ref->virt && addr <= ref->virt + ref->map.region->page_count * MM_PAGE_SIZE) { - return ref; - } - break; - case SHM_TYPE_BLOCK: - if (addr >= ref->virt && addr <= ref->virt + ref->map.block.page_count * MM_PAGE_SIZE) { - return ref; - } - break; - } - } - return NULL; -} - +//static spin_t g_shm_lock = 0; +//static LIST_HEAD(g_shm_region_head); +//static struct slab_cache *shm_region_ref_cache = NULL; +//static struct slab_cache *shm_owner_ref_cache = NULL; +// +//static __init void shm_init_slab_caches(void) { +// shm_region_ref_cache = slab_cache_get(sizeof(struct shm_region_ref)); +// shm_owner_ref_cache = slab_cache_get(sizeof(struct shm_owner_ref)); +//} +// +//struct shm_region *shm_region_create(size_t count) { +// kdebug("Allocating a SHM region of %u pages\n", count); +// struct shm_region *res = kmalloc(sizeof(struct shm_region) + count * sizeof(uintptr_t)); +// _assert(res); +// +// for (size_t i = 0; i < count; ++i) { +// res->phys[i] = mm_phys_alloc_page(); +// _assert(res->phys[i] != MM_NADDR); +// } +// +// res->page_count = count; +// res->ref_count = 0; +// list_head_init(&res->owners); +// res->lock = 0; +// +// return res; +//} +// +//void shm_region_free(struct shm_region *region) { +// uintptr_t irq; +// spin_lock_irqsave(®ion->lock, &irq); +// _assert(list_empty(®ion->owners)); +// _assert(!region->ref_count); +// +// kdebug("Releasing %u pages\n", region->page_count); +// for (size_t i = 0; i < region->page_count; ++i) { +// mm_phys_free_page(region->phys[i]); +// } +// +// spin_release_irqrestore(®ion->lock, &irq); +// kfree(region); +//} +// +//uintptr_t shm_region_map(struct shm_region *region, struct thread *thr) { +// uintptr_t irq; +// spin_lock_irqsave(®ion->lock, &irq); +// kdebug("Mapping region of %u pages into thread %d\n", region->page_count, thr->pid); +// +// struct shm_region_ref *reg_ref = slab_calloc(shm_region_ref_cache); +// _assert(reg_ref); +// struct shm_owner_ref *own_ref = slab_calloc(shm_owner_ref_cache); +// _assert(own_ref); +// +// uintptr_t vaddr = vmfind(thr->space, 0x80000000, 0x100000000, region->page_count); +// +// if (vaddr == MM_NADDR) { +// slab_free(shm_region_ref_cache, reg_ref); +// slab_free(shm_owner_ref_cache, own_ref); +// kerror("Could not find memory for mapping\n"); +// return MM_NADDR; +// } +// kdebug("-> %p\n", vaddr); +// +// reg_ref->type = SHM_TYPE_PHYS; +// reg_ref->virt = vaddr; +// reg_ref->map.region = region; +// list_head_init(®_ref->link); +// own_ref->virt = vaddr; +// own_ref->owner = thr; +// list_head_init(&own_ref->link); +// +// // Map all the pages +// for (size_t i = 0; i < region->page_count; ++i) { +// // TODO: access flags (prot) +// _assert(mm_map_single(thr->space, +// vaddr + i * MM_PAGE_SIZE, +// region->phys[i], +// MM_PAGE_USER | MM_PAGE_WRITE) == 0); +// } +// +// // Add region ref to thread +// list_add(®_ref->link, &thr->shm_list); +// // Add owner ref to region +// list_add(&own_ref->link, ®ion->owners); +// +// ++region->ref_count; +// +// spin_release_irqrestore(®ion->lock, &irq); +// +// return vaddr; +//} +// +//// Slow, but kinda works +//struct shm_region_ref *shm_region_find(struct thread *thr, uintptr_t addr) { +// struct shm_region_ref *ref; +// list_for_each_entry(ref, &thr->shm_list, link) { +// switch (ref->type) { +// case SHM_TYPE_PHYS: +// _assert(ref->map.region); +// +// if (addr >= ref->virt && addr <= ref->virt + ref->map.region->page_count * MM_PAGE_SIZE) { +// return ref; +// } +// break; +// case SHM_TYPE_BLOCK: +// if (addr >= ref->virt && addr <= ref->virt + ref->map.block.page_count * MM_PAGE_SIZE) { +// return ref; +// } +// break; +// } +// } +// return NULL; +//} +// void *sys_mmap(void *hint, size_t length, int prot, int flags, int fd, off_t off) { uintptr_t irq; size_t page_count; - struct shm_region *shm; struct thread *thr; thr = thread_self; @@ -143,260 +142,299 @@ void *sys_mmap(void *hint, size_t length, int prot, int flags, int fd, off_t off } page_count = length / MM_PAGE_SIZE; - if (flags & MAP_ANONYMOUS) { - kdebug("Anonymous mapping requested\n"); - if ((flags & MAP_ACCESS_MASK) != MAP_PRIVATE) { - kwarn("Can't have shared anonymous mapping\n"); - return (void *) -EINVAL; - } - - shm = shm_region_create(page_count); - _assert(shm); - - spin_lock_irqsave(&g_shm_lock, &irq); - list_add(&shm->link, &g_shm_region_head); - spin_release_irqrestore(&g_shm_lock, &irq); - - uintptr_t addr = shm_region_map(shm, thr); - - kdebug("New region mapping list:\n"); - shm_thread_describe(thr); - - return addr == MM_NADDR ? (void *) -ENOMEM : (void *) addr; - } else { - kdebug("File mapping requested\n"); - - if (fd < 0 || fd >= THREAD_MAX_FDS) { - return (void *) -EBADF; - } - - struct ofile *of = thr->fds[fd]; - if (!of) { - return (void *) -EBADF; - } - - if (of->flags & OF_SOCKET) { - kwarn("%d tried to mmap a socket\n", thr->pid); - return (void *) -EINVAL; - } - - struct vnode *vn = of->file.vnode; - _assert(vn); - - if (vn->type != VN_BLK) { - kwarn("%d tried to mmap a non-block file\n", thr->pid); - return (void *) -ENOTBLK; - } - - struct blkdev *blk = vn->dev; - _assert(blk); - - if (!blk->mmap) { - kwarn("%d tried to mmap block device, but no function present\n", thr->pid); - return (void *) -ENOENT; - } - - struct shm_region_ref *reg_ref = slab_calloc(shm_region_ref_cache); - _assert(reg_ref); - void *vaddr = blk->mmap(blk, of, hint, length, flags); - - if (vaddr == NULL || vaddr == MAP_FAILED) { - slab_free(shm_region_ref_cache, reg_ref); - return MAP_FAILED; - } - - reg_ref->type = SHM_TYPE_BLOCK; - reg_ref->virt = (uintptr_t) vaddr; - reg_ref->map.block.blk = blk; - reg_ref->map.block.page_count = length / MM_PAGE_SIZE; - kinfo("Created a mapping of %u pages at %p\n", length / MM_PAGE_SIZE, vaddr); - - list_add(®_ref->link, &thr->shm_list); - - return vaddr; + if (!(flags & MAP_ANONYMOUS)) { + panic("NYI\n"); } + + if ((flags & MAP_ACCESS_MASK) != MAP_PRIVATE) { + kwarn("Can't shared anonymous mappings\n"); + return (void *) -EINVAL; + } + + uintptr_t virt_base = vmfind(thr->space, 0x100000000, 0x400000000, page_count); + _assert(virt_base != MM_NADDR); + + kdebug("mmaping %u pages at %p\n", page_count, virt_base); + + // Map pages + for (size_t i = 0; i < page_count; ++i) { + uintptr_t phys = mm_phys_alloc_page(); + _assert(phys != MM_NADDR); + struct page *page = PHYS2PAGE(phys); + _assert(page); + + page->flags |= PG_MMAPED; + // TODO: permissions + _assert(mm_map_single(thr->space, virt_base + i * MM_PAGE_SIZE, phys, MM_PAGE_USER | MM_PAGE_WRITE) == 0); + } + + return (void *) virt_base; } -int sys_munmap(void *_addr, size_t length) { - uintptr_t addr = (uintptr_t) _addr; - uintptr_t irq; +int sys_munmap(void *ptr, size_t len) { struct thread *thr; - if (addr & MM_PAGE_OFFSET_MASK) { - // Address is not page-aligned - return -EINVAL; - } - thr = thread_self; _assert(thr); - - struct shm_region_ref *reg_ref = shm_region_find(thr, addr); - switch (reg_ref->type) { - case SHM_TYPE_PHYS: { - struct shm_owner_ref *own_ref = NULL, *own_it; - if (!reg_ref) { - // No region is mapped there - return -ENOENT; - } - struct shm_region *reg = reg_ref->map.region; - _assert(reg); - - // Find region -> thread owner reference - list_for_each_entry(own_it, ®->owners, link) { - if (own_it->virt == reg_ref->virt && own_it->owner == thr) { - own_ref = own_it; - break; - } - } - _assert(own_ref); - - // Unmap pages - for (size_t i = 0; i < reg->page_count; ++i) { - _assert(mm_umap_single(thr->space, reg_ref->virt + i * MM_PAGE_SIZE, 1) != MM_NADDR); - } - - spin_lock_irqsave(®->lock, &irq); - _assert(reg->ref_count); - list_del(&own_ref->link); - --reg->ref_count; - - if (!reg->ref_count) { - spin_release_irqrestore(®->lock, &irq); - list_del(®->link); - shm_region_free(reg); - } else { - spin_release_irqrestore(®->lock, &irq); - } - - slab_free(shm_owner_ref_cache, own_ref); - } - break; - case SHM_TYPE_BLOCK: { - struct blkdev *blk = reg_ref->map.block.blk; - _assert(blk); - - kwarn("TODO: block device munmap call\n"); - - // Unmap pages - for (size_t i = 0; i < reg_ref->map.block.page_count; ++i) { - _assert(mm_umap_single(thr->space, reg_ref->virt + i * MM_PAGE_SIZE, 1) != MM_NADDR); - } - kdebug("Unmapped %u pages at %p\n", reg_ref->map.block.page_count, reg_ref->virt); - } - break; - } - list_del(®_ref->link); - slab_free(shm_region_ref_cache, reg_ref); - - kdebug("New region mapping list:\n"); - shm_thread_describe(thr); - - return 0; -} - -void shm_thread_describe(struct thread *thr) { - struct shm_region_ref *ref; - list_for_each_entry(ref, &thr->shm_list, link) { - switch (ref->type) { - case SHM_TYPE_PHYS: { - struct shm_region *reg = ref->map.region; - kdebug(" - %3u pages - %p\n", reg->page_count, ref->virt); - for (size_t i = 0; i < reg->page_count; ++i) { - kdebug(" - %p\n", reg->phys[i]); - } - } - break; - case SHM_TYPE_BLOCK: - break; - } - } -} - -void shm_space_fork(struct thread *dst, struct thread *src) { - kdebug("Forkng shared memory regions\n"); - struct shm_region_ref *ref; - list_for_each_entry(ref, &src->shm_list, link) { - switch (ref->type) { - case SHM_TYPE_PHYS: { - struct shm_region *reg = ref->map.region; - kdebug(" Fork %3u pages - %p\n", reg->page_count, ref->virt); - - shm_region_map(reg, dst); - } - break; - case SHM_TYPE_BLOCK: - break; - } - } -} - -void shm_region_release_all(struct thread *thr, int noumap) { - struct shm_region_ref *reg_ref; - struct shm_owner_ref *own_ref, *own_it; - struct shm_region *region; - uintptr_t irq; - - list_for_each_entry(reg_ref, &thr->shm_list, link) { - switch (reg_ref->type) { - case SHM_TYPE_PHYS: - region = reg_ref->map.region; - own_ref = NULL; - _assert(region); - - kdebug("Releasing %u pages at %p\n", region->page_count, reg_ref->virt); - - list_for_each_entry(own_it, ®ion->owners, link) { - if (own_it->virt == reg_ref->virt && own_it->owner == thr) { - own_ref = own_it; - break; - } - } - _assert(own_ref); - - // Unmap pages - if (!noumap) { - for (size_t i = 0; i < region->page_count; ++i) { - _assert(mm_umap_single(thr->space, reg_ref->virt + i * MM_PAGE_SIZE, 1) != MM_NADDR); - } - } - - spin_lock_irqsave(®ion->lock, &irq); - _assert(region->ref_count); - list_del(&own_ref->link); - --region->ref_count; - - if (!region->ref_count) { - spin_release_irqrestore(®ion->lock, &irq); - list_del(®ion->link); - shm_region_free(region); - } else { - spin_release_irqrestore(®ion->lock, &irq); - } - - slab_free(shm_owner_ref_cache, own_ref); - break; - - case SHM_TYPE_BLOCK: { - struct blkdev *blk = reg_ref->map.block.blk; - _assert(blk); - - kwarn("TODO: block device munmap call\n"); - - if (!noumap) { - // Unmap pages - for (size_t i = 0; i < reg_ref->map.block.page_count; ++i) { - _assert(mm_umap_single(thr->space, reg_ref->virt + i * MM_PAGE_SIZE, 1) != MM_NADDR); - } - kdebug("Unmapped %u pages at %p\n", reg_ref->map.block.page_count, reg_ref->virt); - } - } - break; - } - } - - while (!list_empty(&thr->shm_list)) { - struct shm_region_ref *ref = list_entry(thr->shm_list.next, struct shm_region_ref, link); - list_del(&ref->link); - slab_free(shm_region_ref_cache, ref); - } + kinfo("[%d] release %u pages at %p\n", thr->pid, len / 0x1000, ptr); + return -ENOMEM; } +// +// if (flags & MAP_ANONYMOUS) { +// kdebug("Anonymous mapping requested\n"); +// if ((flags & MAP_ACCESS_MASK) != MAP_PRIVATE) { +// kwarn("Can't have shared anonymous mapping\n"); +// return (void *) -EINVAL; +// } +// +// shm = shm_region_create(page_count); +// _assert(shm); +// +// spin_lock_irqsave(&g_shm_lock, &irq); +// list_add(&shm->link, &g_shm_region_head); +// spin_release_irqrestore(&g_shm_lock, &irq); +// +// uintptr_t addr = shm_region_map(shm, thr); +// +// kdebug("New region mapping list:\n"); +// shm_thread_describe(thr); +// +// return addr == MM_NADDR ? (void *) -ENOMEM : (void *) addr; +// } else { +// kdebug("File mapping requested\n"); +// +// if (fd < 0 || fd >= THREAD_MAX_FDS) { +// return (void *) -EBADF; +// } +// +// struct ofile *of = thr->fds[fd]; +// if (!of) { +// return (void *) -EBADF; +// } +// +// if (of->flags & OF_SOCKET) { +// kwarn("%d tried to mmap a socket\n", thr->pid); +// return (void *) -EINVAL; +// } +// +// struct vnode *vn = of->file.vnode; +// _assert(vn); +// +// if (vn->type != VN_BLK) { +// kwarn("%d tried to mmap a non-block file\n", thr->pid); +// return (void *) -ENOTBLK; +// } +// +// struct blkdev *blk = vn->dev; +// _assert(blk); +// +// if (!blk->mmap) { +// kwarn("%d tried to mmap block device, but no function present\n", thr->pid); +// return (void *) -ENOENT; +// } +// +// struct shm_region_ref *reg_ref = slab_calloc(shm_region_ref_cache); +// _assert(reg_ref); +// void *vaddr = blk->mmap(blk, of, hint, length, flags); +// +// if (vaddr == NULL || vaddr == MAP_FAILED) { +// slab_free(shm_region_ref_cache, reg_ref); +// return MAP_FAILED; +// } +// +// reg_ref->type = SHM_TYPE_BLOCK; +// reg_ref->virt = (uintptr_t) vaddr; +// reg_ref->map.block.blk = blk; +// reg_ref->map.block.page_count = length / MM_PAGE_SIZE; +// kinfo("Created a mapping of %u pages at %p\n", length / MM_PAGE_SIZE, vaddr); +// +// list_add(®_ref->link, &thr->shm_list); +// +// return vaddr; +// } +//} +// +//int sys_munmap(void *_addr, size_t length) { +// uintptr_t addr = (uintptr_t) _addr; +// uintptr_t irq; +// struct thread *thr; +// +// if (addr & MM_PAGE_OFFSET_MASK) { +// // Address is not page-aligned +// return -EINVAL; +// } +// +// thr = thread_self; +// _assert(thr); +// +// struct shm_region_ref *reg_ref = shm_region_find(thr, addr); +// switch (reg_ref->type) { +// case SHM_TYPE_PHYS: { +// struct shm_owner_ref *own_ref = NULL, *own_it; +// if (!reg_ref) { +// // No region is mapped there +// return -ENOENT; +// } +// struct shm_region *reg = reg_ref->map.region; +// _assert(reg); +// +// // Find region -> thread owner reference +// list_for_each_entry(own_it, ®->owners, link) { +// if (own_it->virt == reg_ref->virt && own_it->owner == thr) { +// own_ref = own_it; +// break; +// } +// } +// _assert(own_ref); +// +// // Unmap pages +// for (size_t i = 0; i < reg->page_count; ++i) { +// _assert(mm_umap_single(thr->space, reg_ref->virt + i * MM_PAGE_SIZE, 1) != MM_NADDR); +// } +// +// spin_lock_irqsave(®->lock, &irq); +// _assert(reg->ref_count); +// list_del(&own_ref->link); +// --reg->ref_count; +// +// if (!reg->ref_count) { +// spin_release_irqrestore(®->lock, &irq); +// list_del(®->link); +// shm_region_free(reg); +// } else { +// spin_release_irqrestore(®->lock, &irq); +// } +// +// slab_free(shm_owner_ref_cache, own_ref); +// } +// break; +// case SHM_TYPE_BLOCK: { +// struct blkdev *blk = reg_ref->map.block.blk; +// _assert(blk); +// +// kwarn("TODO: block device munmap call\n"); +// +// // Unmap pages +// for (size_t i = 0; i < reg_ref->map.block.page_count; ++i) { +// _assert(mm_umap_single(thr->space, reg_ref->virt + i * MM_PAGE_SIZE, 1) != MM_NADDR); +// } +// kdebug("Unmapped %u pages at %p\n", reg_ref->map.block.page_count, reg_ref->virt); +// } +// break; +// } +// list_del(®_ref->link); +// slab_free(shm_region_ref_cache, reg_ref); +// +// kdebug("New region mapping list:\n"); +// shm_thread_describe(thr); +// +// return 0; +//} +// +//void shm_thread_describe(struct thread *thr) { +// struct shm_region_ref *ref; +// list_for_each_entry(ref, &thr->shm_list, link) { +// switch (ref->type) { +// case SHM_TYPE_PHYS: { +// struct shm_region *reg = ref->map.region; +// kdebug(" - %3u pages - %p\n", reg->page_count, ref->virt); +// for (size_t i = 0; i < reg->page_count; ++i) { +// kdebug(" - %p\n", reg->phys[i]); +// } +// } +// break; +// case SHM_TYPE_BLOCK: +// break; +// } +// } +//} +// +//void shm_space_fork(struct thread *dst, struct thread *src) { +// kdebug("Forkng shared memory regions\n"); +// struct shm_region_ref *ref; +// list_for_each_entry(ref, &src->shm_list, link) { +// switch (ref->type) { +// case SHM_TYPE_PHYS: { +// struct shm_region *reg = ref->map.region; +// kdebug(" Fork %3u pages - %p\n", reg->page_count, ref->virt); +// +// shm_region_map(reg, dst); +// } +// break; +// case SHM_TYPE_BLOCK: +// break; +// } +// } +//} +// +//void shm_region_release_all(struct thread *thr, int noumap) { +// struct shm_region_ref *reg_ref; +// struct shm_owner_ref *own_ref, *own_it; +// struct shm_region *region; +// uintptr_t irq; +// +// list_for_each_entry(reg_ref, &thr->shm_list, link) { +// switch (reg_ref->type) { +// case SHM_TYPE_PHYS: +// region = reg_ref->map.region; +// own_ref = NULL; +// _assert(region); +// +// kdebug("Releasing %u pages at %p\n", region->page_count, reg_ref->virt); +// +// list_for_each_entry(own_it, ®ion->owners, link) { +// if (own_it->virt == reg_ref->virt && own_it->owner == thr) { +// own_ref = own_it; +// break; +// } +// } +// _assert(own_ref); +// +// // Unmap pages +// if (!noumap) { +// for (size_t i = 0; i < region->page_count; ++i) { +// _assert(mm_umap_single(thr->space, reg_ref->virt + i * MM_PAGE_SIZE, 1) != MM_NADDR); +// } +// } +// +// spin_lock_irqsave(®ion->lock, &irq); +// _assert(region->ref_count); +// list_del(&own_ref->link); +// --region->ref_count; +// +// if (!region->ref_count) { +// spin_release_irqrestore(®ion->lock, &irq); +// list_del(®ion->link); +// shm_region_free(region); +// } else { +// spin_release_irqrestore(®ion->lock, &irq); +// } +// +// slab_free(shm_owner_ref_cache, own_ref); +// break; +// +// case SHM_TYPE_BLOCK: { +// struct blkdev *blk = reg_ref->map.block.blk; +// _assert(blk); +// +// kwarn("TODO: block device munmap call\n"); +// +// if (!noumap) { +// // Unmap pages +// for (size_t i = 0; i < reg_ref->map.block.page_count; ++i) { +// _assert(mm_umap_single(thr->space, reg_ref->virt + i * MM_PAGE_SIZE, 1) != MM_NADDR); +// } +// kdebug("Unmapped %u pages at %p\n", reg_ref->map.block.page_count, reg_ref->virt); +// } +// } +// break; +// } +// } +// +// while (!list_empty(&thr->shm_list)) { +// struct shm_region_ref *ref = list_entry(thr->shm_list.next, struct shm_region_ref, link); +// list_del(&ref->link); +// slab_free(shm_region_ref_cache, ref); +// } +//} +//