From 8f3f600fa48387ed83e35851846f4dc4b491c366 Mon Sep 17 00:00:00 2001 From: Mark Date: Wed, 8 Apr 2020 12:45:03 +0300 Subject: [PATCH] Add slab allocator for objects < 512B --- arch/amd64/mm/map.c | 8 +- etc/make/conf.mk | 8 +- fs/node.c | 36 +++--- fs/sysfs.c | 9 +- fs/vfs_ops.c | 6 +- include/arch/amd64/mm/heap.h | 2 +- include/fs/node.h | 1 + include/sys/mem/shmem.h | 2 +- include/sys/mem/slab.h | 16 +++ sys/kernel.c | 8 -- sys/{ => mem}/shmem.c | 27 +++-- sys/mem/slab.c | 221 +++++++++++++++++++++++++++++++++++ sys/ubsan.c | 20 ++++ 13 files changed, 308 insertions(+), 56 deletions(-) create mode 100644 include/sys/mem/slab.h rename sys/{ => mem}/shmem.c (92%) create mode 100644 sys/mem/slab.c diff --git a/arch/amd64/mm/map.c b/arch/amd64/mm/map.c index 358818f..17b68ef 100644 --- a/arch/amd64/mm/map.c +++ b/arch/amd64/mm/map.c @@ -287,9 +287,7 @@ int mm_space_fork(struct thread *dst, const struct thread *src, uint32_t flags) (pdi << MM_PDI_SHIFT) | (pti << MM_PTI_SHIFT); - if (shm_region_find((struct thread *) src, src_page_virt)) { - kdebug("Skipping %p: is a shared memory region\n", src_page_virt); - } else { + if (!shm_region_find((struct thread *) src, src_page_virt)) { uintptr_t src_page_phys = src_pt[pti] & MM_PTE_MASK; uintptr_t dst_page_phys = mm_phys_alloc_page(); //amd64_phys_alloc_page(); _assert(dst_page_phys != MM_NADDR); @@ -349,9 +347,7 @@ void mm_space_release(struct thread *thr) { (pdi << MM_PDI_SHIFT) | (pti << MM_PTI_SHIFT); - if (shm_region_find(thr, page_virt)) { - kdebug("Not freeing %p: belongs to a shared region\n", page_virt); - } else { + if (!shm_region_find(thr, page_virt)) { uintptr_t page_phys = pt[pti] & MM_PTE_MASK; mm_phys_free_page(page_phys); } diff --git a/etc/make/conf.mk b/etc/make/conf.mk index 75e161b..605ebc6 100644 --- a/etc/make/conf.mk +++ b/etc/make/conf.mk @@ -19,7 +19,7 @@ CFLAGS+=-Wall \ -O$(OPTIMIZE) \ -ggdb -ifdef KERNEL_TEST_MODE +ifeq ($(KERNEL_TEST_MODE),1) CFLAGS+=-DKERNEL_TEST_MODE endif @@ -61,12 +61,14 @@ OBJS+=$(O)/sys/ubsan.o \ $(O)/sys/font/logo.o \ $(O)/sys/font/psf.o \ $(O)/sys/font/default8x16.o \ - $(O)/sys/shmem.o \ + $(O)/sys/mem/shmem.o \ + $(O)/sys/mem/slab.o \ $(O)/sys/wait.o DIRS+=$(O)/sys \ $(O)/sys/char \ $(O)/sys/block \ - $(O)/sys/font + $(O)/sys/font \ + $(O)/sys/mem OBJS+=$(O)/fs/vfs.o \ $(O)/fs/vfs_ops.o \ diff --git a/fs/node.c b/fs/node.c index abe8ea0..5c297e6 100644 --- a/fs/node.c +++ b/fs/node.c @@ -7,42 +7,32 @@ #include "sys/debug.h" #include "sys/panic.h" #include "sys/heap.h" +#include "sys/mem/slab.h" + +static struct slab_cache *vnode_cache = NULL; struct vnode *vnode_create(enum vnode_type t, const char *name) { - struct vnode *node = (struct vnode *) kmalloc(sizeof(struct vnode)); + if (!vnode_cache) { + vnode_cache = slab_cache_get(sizeof(struct vnode)); + kdebug("Initialized vnode cache\n"); + } + struct vnode *node = slab_calloc(vnode_cache); _assert(node); node->type = t; - node->flags = 0; - - node->parent = NULL; - node->first_child = NULL; - node->next_child = NULL; - - node->target = NULL; - - node->fs = NULL; - node->fs_data = NULL; - - node->op = NULL; - node->dev = NULL; - - node->uid = 0; - node->gid = 0; - node->mode = 0; - - node->open_count = 0; - if (name) { _assert(strlen(name) < NODE_MAXLEN); strcpy(node->name, name); - } else { - node->name[0] = 0; } return node; } +void vnode_destroy(struct vnode *vn) { + _assert(vnode_cache); + slab_free(vnode_cache, vn); +} + void vnode_attach(struct vnode *parent, struct vnode *child) { _assert(parent); _assert(child); diff --git a/fs/sysfs.c b/fs/sysfs.c index e70003c..88e71f4 100644 --- a/fs/sysfs.c +++ b/fs/sysfs.c @@ -10,6 +10,7 @@ #include "sys/assert.h" #include "sys/string.h" #include "sys/thread.h" +#include "sys/mem/slab.h" #include "fs/fs.h" #include "sys/debug.h" #include "sys/heap.h" @@ -230,7 +231,9 @@ static int system_mem_getter(void *ctx, char *buf, size_t lim) { struct amd64_phys_stat phys_st; struct heap_stat heap_st; struct amd64_pool_stat pool_st; + struct slab_stat slab_st; + slab_stat(&slab_st); amd64_phys_stat(&phys_st); amd64_mm_pool_stat(&pool_st); heap_stat(heap_global, &heap_st); @@ -247,12 +250,16 @@ static int system_mem_getter(void *ctx, char *buf, size_t lim) { sysfs_buf_printf(buf, lim, "HeapFree: %u kB\n", heap_st.free_size / 1024); sysfs_buf_printf(buf, lim, "HeapUsed: %u kB\n", (heap_st.total_size - heap_st.free_size) / 1024); + sysfs_buf_printf(buf, lim, "SlabObjects: %u\n", slab_st.alloc_objects); + sysfs_buf_printf(buf, lim, "SlabTotal: %u kB\n", slab_st.alloc_pages * 4); + sysfs_buf_printf(buf, lim, "SlabUsed: %u kB\n", slab_st.alloc_bytes / 1024); + return 0; } void sysfs_populate(void) { sysfs_add_config_endpoint("version", SYSFS_MODE_DEFAULT, sizeof(KERNEL_VERSION_STR) + 1, KERNEL_VERSION_STR, sysfs_config_getter, NULL); - sysfs_add_config_endpoint("mem", SYSFS_MODE_DEFAULT, 256, NULL, system_mem_getter, NULL); + sysfs_add_config_endpoint("mem", SYSFS_MODE_DEFAULT, 512, NULL, system_mem_getter, NULL); sysfs_add_config_endpoint("uptime", SYSFS_MODE_DEFAULT, 32, NULL, system_uptime_getter, NULL); diff --git a/fs/vfs_ops.c b/fs/vfs_ops.c index e4c4f16..7e07121 100644 --- a/fs/vfs_ops.c +++ b/fs/vfs_ops.c @@ -321,8 +321,7 @@ int vfs_rmdir(struct vfs_ioctx *ctx, const char *path) { } vnode_detach(node); - memset(node, 0, sizeof(struct vnode)); - kfree(node); + vnode_destroy(node); return 0; } @@ -354,8 +353,7 @@ int vfs_unlink(struct vfs_ioctx *ctx, const char *path) { } vnode_detach(node); - memset(node, 0, sizeof(struct vnode)); - kfree(node); + vnode_destroy(node); return 0; } diff --git a/include/arch/amd64/mm/heap.h b/include/arch/amd64/mm/heap.h index f809eeb..23255a0 100644 --- a/include/arch/amd64/mm/heap.h +++ b/include/arch/amd64/mm/heap.h @@ -3,7 +3,7 @@ #include "sys/heap.h" #include -#define KERNEL_HEAP (16 * 1024 * 1024) +#define KERNEL_HEAP (512 * 1024) /** * @brief Initialize heap instance of size `sz' at `phys_base' diff --git a/include/fs/node.h b/include/fs/node.h index 33258a5..dc1e6c1 100644 --- a/include/fs/node.h +++ b/include/fs/node.h @@ -88,6 +88,7 @@ struct vnode { // Node itself struct vnode *vnode_create(enum vnode_type t, const char *name); +void vnode_destroy(struct vnode *vn); // Tree manipulation void vnode_attach(struct vnode *parent, struct vnode *child); diff --git a/include/sys/mem/shmem.h b/include/sys/mem/shmem.h index 7f07a5b..538ae89 100644 --- a/include/sys/mem/shmem.h +++ b/include/sys/mem/shmem.h @@ -30,7 +30,7 @@ struct shm_owner_ref { struct shm_region_ref { enum { // -> memory pages - SHM_TYPE_PHYS, + SHM_TYPE_PHYS = 0, // -> block device mapping SHM_TYPE_BLOCK, } type; diff --git a/include/sys/mem/slab.h b/include/sys/mem/slab.h new file mode 100644 index 0000000..397c8b3 --- /dev/null +++ b/include/sys/mem/slab.h @@ -0,0 +1,16 @@ +#pragma once +#include "sys/types.h" + +struct slab_stat { + size_t alloc_bytes; + size_t alloc_pages; + size_t alloc_objects; +}; + +struct slab_cache; + +struct slab_cache *slab_cache_get(size_t obj_size); + +void slab_stat(struct slab_stat *st); +void *slab_calloc(struct slab_cache *cp); +void slab_free(struct slab_cache *cp, void *ptr); diff --git a/sys/kernel.c b/sys/kernel.c index 5f1612f..ec95a43 100644 --- a/sys/kernel.c +++ b/sys/kernel.c @@ -9,15 +9,7 @@ #include "net/net.h" #include "fs/vfs.h" -#include "sys/debug.h" - void main(void) { - char v = 0; - for (int i = 0; i < 256; ++i) { - ++v; - } - kinfo("???\n"); - pci_init(); vfs_init(); diff --git a/sys/shmem.c b/sys/mem/shmem.c similarity index 92% rename from sys/shmem.c rename to sys/mem/shmem.c index 045d4cc..ad704bd 100644 --- a/sys/shmem.c +++ b/sys/mem/shmem.c @@ -3,6 +3,7 @@ #include "sys/mem/shmem.h" #include "sys/block/blk.h" #include "sys/mem/phys.h" +#include "sys/mem/slab.h" #include "user/errno.h" #include "sys/thread.h" #include "sys/assert.h" @@ -10,11 +11,19 @@ #include "sys/debug.h" #include "fs/ofile.h" #include "sys/heap.h" +#include "sys/attr.h" #include "fs/node.h" #include "sys/mm.h" static spin_t g_shm_lock = 0; static LIST_HEAD(g_shm_region_head); +static struct slab_cache *shm_region_ref_cache = NULL; +static struct slab_cache *shm_owner_ref_cache = NULL; + +static __init void shm_init_slab_caches(void) { + shm_region_ref_cache = slab_cache_get(sizeof(struct shm_region_ref)); + shm_owner_ref_cache = slab_cache_get(sizeof(struct shm_owner_ref)); +} struct shm_region *shm_region_create(size_t count) { kdebug("Allocating a SHM region of %u pages\n", count); @@ -54,16 +63,16 @@ uintptr_t shm_region_map(struct shm_region *region, struct thread *thr) { spin_lock_irqsave(®ion->lock, &irq); kdebug("Mapping region of %u pages into thread %d\n", region->page_count, thr->pid); - struct shm_region_ref *reg_ref = kmalloc(sizeof(struct shm_region_ref)); + struct shm_region_ref *reg_ref = slab_calloc(shm_region_ref_cache); _assert(reg_ref); - struct shm_owner_ref *own_ref = kmalloc(sizeof(struct shm_owner_ref)); + struct shm_owner_ref *own_ref = slab_calloc(shm_owner_ref_cache); _assert(own_ref); uintptr_t vaddr = vmfind(thr->space, 0x80000000, 0x100000000, region->page_count); if (vaddr == MM_NADDR) { - kfree(reg_ref); - kfree(own_ref); + slab_free(shm_region_ref_cache, reg_ref); + slab_free(shm_owner_ref_cache, own_ref); kerror("Could not find memory for mapping\n"); return MM_NADDR; } @@ -187,12 +196,12 @@ void *sys_mmap(void *hint, size_t length, int prot, int flags, int fd, off_t off return (void *) -ENOENT; } - struct shm_region_ref *reg_ref = kmalloc(sizeof(struct shm_region_ref)); + struct shm_region_ref *reg_ref = slab_calloc(shm_region_ref_cache); _assert(reg_ref); void *vaddr = blk->mmap(blk, of, hint, length, flags); if (vaddr == NULL || vaddr == MAP_FAILED) { - kfree(reg_ref); + slab_free(shm_region_ref_cache, reg_ref); return MAP_FAILED; } @@ -259,7 +268,7 @@ int sys_munmap(void *_addr, size_t length) { spin_release_irqrestore(®->lock, &irq); } - kfree(own_ref); + slab_free(shm_owner_ref_cache, own_ref); } break; case SHM_TYPE_BLOCK: { @@ -277,7 +286,7 @@ int sys_munmap(void *_addr, size_t length) { break; } list_del(®_ref->link); - kfree(reg_ref); + slab_free(shm_region_ref_cache, reg_ref); kdebug("New region mapping list:\n"); shm_thread_describe(thr); @@ -364,7 +373,7 @@ void shm_region_release_all(struct thread *thr, int noumap) { spin_release_irqrestore(®ion->lock, &irq); } - kfree(own_ref); + slab_free(shm_owner_ref_cache, own_ref); break; case SHM_TYPE_BLOCK: { diff --git a/sys/mem/slab.c b/sys/mem/slab.c new file mode 100644 index 0000000..7fa2455 --- /dev/null +++ b/sys/mem/slab.c @@ -0,0 +1,221 @@ +#include "sys/mem/phys.h" +#include "sys/mem/slab.h" +#include "sys/assert.h" +#include "sys/string.h" +#include "sys/debug.h" +#include "sys/list.h" +#include "sys/attr.h" +#include "sys/mm.h" + +typedef uint32_t bufctl_t; +#define BUFCTL_END ((bufctl_t) -1) +#define slab_bufctl(slabp) \ + ((bufctl_t *) (((struct slab *) (slabp)) + 1)) + +#define CACHE_SIZE(i) \ + { 1ULL << ((i) + 4), &predefined_caches[i] } +#define CACHE_SIZE_END \ + { 0, NULL } +#define PREALLOC_COUNT 6 + +struct slab_cache { + struct list_head slabs_empty, slabs_partial, slabs_full; + size_t object_size, objects_per_slab; +}; + +struct slab { + struct list_head list; + size_t inuse; + void *base; + bufctl_t free; +}; + +struct cache_size { + size_t size; + struct slab_cache *cp; +}; + +struct slab_cache predefined_caches[PREALLOC_COUNT]; +struct cache_size predefined_cache_sizes[PREALLOC_COUNT + 1] = { + CACHE_SIZE(0), + CACHE_SIZE(1), + CACHE_SIZE(2), + CACHE_SIZE(3), + CACHE_SIZE(4), + CACHE_SIZE(5), + CACHE_SIZE_END +}; + +static void slab_init_cache(struct slab_cache *cp, size_t object_size) { + cp->object_size = object_size; + list_head_init(&cp->slabs_empty); + list_head_init(&cp->slabs_partial); + list_head_init(&cp->slabs_full); + + // Reserve space for slab descriptor + cp->objects_per_slab = (MM_PAGE_SIZE - sizeof(struct slab)) / cp->object_size; + // Reserve space for bufctl + size_t bufctl_array_size = cp->objects_per_slab * sizeof(bufctl_t); + cp->objects_per_slab -= (bufctl_array_size + cp->object_size - 1) / cp->object_size; +} + +static __init void slab_init_caches(void) { + for (size_t i = 0; i < PREALLOC_COUNT; ++i) { + kdebug("Initializing predefined cache: %u\n", 1UL << (i + 4)); + slab_init_cache(&predefined_caches[i], 1UL << (i + 4)); + } +} + +struct slab_cache *slab_cache_get(size_t size) { + struct cache_size *sizep = predefined_cache_sizes; + for (; sizep->size; ++sizep) { + if (size > sizep->size) { + continue; + } + return sizep->cp; + } + + panic("Tried to get cache of invalid size: %u\n", size); +} + +//// + +static struct slab *slab_create(struct slab_cache *cp) { + uintptr_t page_phys = mm_phys_alloc_page(); + + if (page_phys == MM_NADDR) { + return NULL; + } + + struct slab *slab = (struct slab *) MM_VIRTUALIZE(page_phys); + + slab->base = ((void *) &slab[1]) + sizeof(bufctl_t) * cp->objects_per_slab; + slab->base = (void *) (((uintptr_t) slab->base + 0x7) & ~0x7); + list_head_init(&slab->list); + slab->inuse = 0; + slab->free = 0; + + for (size_t i = 0; i < cp->objects_per_slab - 1; ++i) { + slab_bufctl(slab)[i] = i + 1; + } + slab_bufctl(slab)[cp->objects_per_slab - 1] = BUFCTL_END; + list_add(&slab->list, &cp->slabs_empty); + + kdebug("Created slab of %u x %u B\n", cp->objects_per_slab, cp->object_size); + + return slab; +} + +static void slab_destroy(struct slab_cache *cp, struct slab *slabp) { + _assert(!((uintptr_t) slabp & 0xFFF)); + kdebug("Destoyed slab of %u x %u B\n", cp->objects_per_slab, cp->object_size); + mm_phys_free_page(MM_PHYS(slabp)); +} + +static inline void *slab_alloc_from(struct slab_cache *cp, struct slab *slabp) { + void *objp; + ++slabp->inuse; + // Get object based on "free" index + objp = slabp->base + slabp->free * cp->object_size; + // Set next "free" index + slabp->free = slab_bufctl(slabp)[slabp->free]; + + // All the pages are allocated + if (slabp->free == BUFCTL_END) { + list_del_init(&slabp->list); + list_add(&slabp->list, &cp->slabs_full); + } + + memset(objp, 0, cp->object_size); + return objp; +} + +void *slab_calloc(struct slab_cache *cp) { + struct list_head *slabs_partial, *slabs_empty, *entry; + struct slab *slabp; +try_again: + + // Check if there is any partial slab to use + slabs_partial = &cp->slabs_partial; + entry = slabs_partial->next; + if (slabs_partial == entry) { + // No partial slabs, try to find an empty one + + slabs_empty = &cp->slabs_empty; + entry = slabs_empty->next; + if (slabs_empty == entry) { + // Have to allocate a new slab - no free objects are available + // After that, just try allocation again + //printf("No partial/empty slabs, allocating a new one\n"); + + if (!slab_create(cp)) { + return NULL; + } + goto try_again; + } + + list_del(entry); + list_add(entry, slabs_partial); + } + + slabp = list_entry(entry, struct slab, list); + return slab_alloc_from(cp, slabp); +} + +void slab_free(struct slab_cache *cp, void *objp) { + struct slab *slabp; + // Get page-aligned address + uintptr_t page = (uintptr_t) objp; + _assert(page & 0xFFF); + page /= MM_PAGE_SIZE; + page *= MM_PAGE_SIZE; + + // slab descriptor is on the same page objp points to + slabp = (struct slab *) page; + + unsigned int obj_number = (objp - slabp->base) / cp->object_size; + slab_bufctl(slabp)[obj_number] = slabp->free; + slabp->free = obj_number; + + size_t inuse = slabp->inuse; + + if (!--slabp->inuse) { + // Was partial or full, now empty + // Free the slab + list_del(&slabp->list); + slab_destroy(cp, slabp); + } else if (inuse == cp->objects_per_slab) { + // Was full, now partial + list_del_init(&slabp->list); + list_add(&slabp->list, &cp->slabs_partial); + } +} + +void slab_stat(struct slab_stat *st) { + st->alloc_bytes = 0; + st->alloc_objects = 0; + st->alloc_pages = 0; + + for (size_t i = 0; i < PREALLOC_COUNT; ++i) { + struct slab_cache *cp = &predefined_caches[i]; + struct slab *slab; + + list_for_each_entry(slab, &cp->slabs_full, list) { + _assert(slab->inuse == cp->objects_per_slab); + st->alloc_objects += slab->inuse; + st->alloc_bytes += slab->inuse * cp->object_size; + ++st->alloc_pages; + } + + list_for_each_entry(slab, &cp->slabs_partial, list) { + _assert(slab->inuse && slab->inuse != cp->objects_per_slab); + st->alloc_objects += slab->inuse; + st->alloc_bytes += slab->inuse * cp->object_size; + ++st->alloc_pages; + } + + list_for_each_entry(slab, &cp->slabs_empty, list) { + panic("This list should be empty, slabs are freed once they're empty\n"); + } + } +} diff --git a/sys/ubsan.c b/sys/ubsan.c index 93a42d6..31cf160 100644 --- a/sys/ubsan.c +++ b/sys/ubsan.c @@ -129,5 +129,25 @@ struct type_mismatch_info { void __ubsan_handle_type_mismatch_v1(struct type_mismatch_info *type_mismatch, uintptr_t pointer) { struct source_location *location = &type_mismatch->location; + + if (pointer == 0) { + kfatal("NULL pointer access\n"); + } else if (type_mismatch->alignment != 0 && + is_aligned(pointer, type_mismatch->alignment)) { + // Most useful on architectures with stricter memory alignment requirements, like ARM. + kfatal("Unaligned memory access\n"); + } else { + kfatal("Insufficient size:\n"); + if (type_mismatch->type_check_kind < sizeof(Type_Check_Kinds) / sizeof(Type_Check_Kinds[0])) { + kfatal("%s address %p with insufficient space for object of type %s\n", + Type_Check_Kinds[type_mismatch->type_check_kind], (void *) pointer, + type_mismatch->type->name); + } + //log("Insufficient size"); + //logf("%s address %p with insufficient space for object of type %s\n", + // Type_Check_Kinds[type_mismatch->type_check_kind], (void *)pointer, + // type_mismatch->type->name); + } + ubsan_abort(location, "type mismatch"); }