yggdrasil-old/kernel
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Add slab allocator for objects < 512B

Browse Source
This commit is contained in:
Mark
2020-04-08 12:45:03 +03:00
parent 536cb0794b
commit 8f3f600fa4
13 changed files with 308 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files
arch/amd64/mm/map.c
+2 -6
View File
@@ -287,9 +287,7 @@ int mm_space_fork(struct thread *dst, const struct thread *src, uint32_t flags)
(pdi << MM_PDI_SHIFT) |
(pti << MM_PTI_SHIFT);
if (shm_region_find((struct thread *) src, src_page_virt)) {
kdebug("Skipping %p: is a shared memory region\n", src_page_virt);
} else {
if (!shm_region_find((struct thread *) src, src_page_virt)) {
uintptr_t src_page_phys = src_pt[pti] & MM_PTE_MASK;
uintptr_t dst_page_phys = mm_phys_alloc_page(); //amd64_phys_alloc_page();
_assert(dst_page_phys != MM_NADDR);
@@ -349,9 +347,7 @@ void mm_space_release(struct thread *thr) {
(pdi << MM_PDI_SHIFT) |
(pti << MM_PTI_SHIFT);
if (shm_region_find(thr, page_virt)) {
kdebug("Not freeing %p: belongs to a shared region\n", page_virt);
} else {
if (!shm_region_find(thr, page_virt)) {
uintptr_t page_phys = pt[pti] & MM_PTE_MASK;
mm_phys_free_page(page_phys);
}
etc/make/conf.mk
+5 -3
View File
@@ -19,7 +19,7 @@ CFLAGS+=-Wall \
-O$(OPTIMIZE) \
-ggdb
ifdef KERNEL_TEST_MODE
ifeq ($(KERNEL_TEST_MODE),1)
CFLAGS+=-DKERNEL_TEST_MODE
endif
@@ -61,12 +61,14 @@ OBJS+=$(O)/sys/ubsan.o \
$(O)/sys/font/logo.o \
$(O)/sys/font/psf.o \
$(O)/sys/font/default8x16.o \
$(O)/sys/shmem.o \
$(O)/sys/mem/shmem.o \
$(O)/sys/mem/slab.o \
$(O)/sys/wait.o
DIRS+=$(O)/sys \
$(O)/sys/char \
$(O)/sys/block \
$(O)/sys/font
$(O)/sys/font \
$(O)/sys/mem
OBJS+=$(O)/fs/vfs.o \
$(O)/fs/vfs_ops.o \
fs/node.c
+13 -23
View File
@@ -7,42 +7,32 @@
#include "sys/debug.h"
#include "sys/panic.h"
#include "sys/heap.h"
#include "sys/mem/slab.h"
static struct slab_cache *vnode_cache = NULL;
struct vnode *vnode_create(enum vnode_type t, const char *name) {
struct vnode *node = (struct vnode *) kmalloc(sizeof(struct vnode));
if (!vnode_cache) {
vnode_cache = slab_cache_get(sizeof(struct vnode));
kdebug("Initialized vnode cache\n");
}
struct vnode *node = slab_calloc(vnode_cache);
_assert(node);
node->type = t;
node->flags = 0;
node->parent = NULL;
node->first_child = NULL;
node->next_child = NULL;
node->target = NULL;
node->fs = NULL;
node->fs_data = NULL;
node->op = NULL;
node->dev = NULL;
node->uid = 0;
node->gid = 0;
node->mode = 0;
node->open_count = 0;
if (name) {
_assert(strlen(name) < NODE_MAXLEN);
strcpy(node->name, name);
} else {
node->name[0] = 0;
}
return node;
}
void vnode_destroy(struct vnode *vn) {
_assert(vnode_cache);
slab_free(vnode_cache, vn);
}
void vnode_attach(struct vnode *parent, struct vnode *child) {
_assert(parent);
_assert(child);
fs/sysfs.c
+8 -1
View File
@@ -10,6 +10,7 @@
#include "sys/assert.h"
#include "sys/string.h"
#include "sys/thread.h"
#include "sys/mem/slab.h"
#include "fs/fs.h"
#include "sys/debug.h"
#include "sys/heap.h"
@@ -230,7 +231,9 @@ static int system_mem_getter(void *ctx, char *buf, size_t lim) {
struct amd64_phys_stat phys_st;
struct heap_stat heap_st;
struct amd64_pool_stat pool_st;
struct slab_stat slab_st;
slab_stat(&slab_st);
amd64_phys_stat(&phys_st);
amd64_mm_pool_stat(&pool_st);
heap_stat(heap_global, &heap_st);
@@ -247,12 +250,16 @@ static int system_mem_getter(void *ctx, char *buf, size_t lim) {
sysfs_buf_printf(buf, lim, "HeapFree: %u kB\n", heap_st.free_size / 1024);
sysfs_buf_printf(buf, lim, "HeapUsed: %u kB\n", (heap_st.total_size - heap_st.free_size) / 1024);
sysfs_buf_printf(buf, lim, "SlabObjects: %u\n", slab_st.alloc_objects);
sysfs_buf_printf(buf, lim, "SlabTotal: %u kB\n", slab_st.alloc_pages * 4);
sysfs_buf_printf(buf, lim, "SlabUsed: %u kB\n", slab_st.alloc_bytes / 1024);
return 0;
}
void sysfs_populate(void) {
sysfs_add_config_endpoint("version", SYSFS_MODE_DEFAULT, sizeof(KERNEL_VERSION_STR) + 1, KERNEL_VERSION_STR, sysfs_config_getter, NULL);
sysfs_add_config_endpoint("mem", SYSFS_MODE_DEFAULT, 256, NULL, system_mem_getter, NULL);
sysfs_add_config_endpoint("mem", SYSFS_MODE_DEFAULT, 512, NULL, system_mem_getter, NULL);
sysfs_add_config_endpoint("uptime", SYSFS_MODE_DEFAULT, 32, NULL, system_uptime_getter, NULL);
fs/vfs_ops.c
+2 -4
View File
@@ -321,8 +321,7 @@ int vfs_rmdir(struct vfs_ioctx *ctx, const char *path) {
}
vnode_detach(node);
memset(node, 0, sizeof(struct vnode));
kfree(node);
vnode_destroy(node);
return 0;
}
@@ -354,8 +353,7 @@ int vfs_unlink(struct vfs_ioctx *ctx, const char *path) {
}
vnode_detach(node);
memset(node, 0, sizeof(struct vnode));
kfree(node);
vnode_destroy(node);
return 0;
}
include/arch/amd64/mm/heap.h
+1 -1
View File
@@ -3,7 +3,7 @@
#include "sys/heap.h"
#include <stdint.h>
#define KERNEL_HEAP (16 * 1024 * 1024)
#define KERNEL_HEAP (512 * 1024)
/**
* @brief Initialize heap instance of size `sz' at `phys_base'
include/fs/node.h
+1
View File
@@ -88,6 +88,7 @@ struct vnode {
// Node itself
struct vnode *vnode_create(enum vnode_type t, const char *name);
void vnode_destroy(struct vnode *vn);
// Tree manipulation
void vnode_attach(struct vnode *parent, struct vnode *child);
include/sys/mem/shmem.h
+1 -1
View File
@@ -30,7 +30,7 @@ struct shm_owner_ref {
struct shm_region_ref {
enum {
// -> memory pages
SHM_TYPE_PHYS,
SHM_TYPE_PHYS = 0,
// -> block device mapping
SHM_TYPE_BLOCK,
} type;
include/sys/mem/slab.h
+16
View File
@@ -0,0 +1,16 @@
#pragma once
#include "sys/types.h"
struct slab_stat {
size_t alloc_bytes;
size_t alloc_pages;
size_t alloc_objects;
};
struct slab_cache;
struct slab_cache *slab_cache_get(size_t obj_size);
void slab_stat(struct slab_stat *st);
void *slab_calloc(struct slab_cache *cp);
void slab_free(struct slab_cache *cp, void *ptr);
sys/kernel.c
-8
View File
@@ -9,15 +9,7 @@
#include "net/net.h"
#include "fs/vfs.h"
#include "sys/debug.h"
void main(void) {
char v = 0;
for (int i = 0; i < 256; ++i) {
++v;
}
kinfo("???\n");
pci_init();
vfs_init();
sys/shmem.c → sys/mem/shmem.c
+18 -9
View File
@@ -3,6 +3,7 @@
#include "sys/mem/shmem.h"
#include "sys/block/blk.h"
#include "sys/mem/phys.h"
#include "sys/mem/slab.h"
#include "user/errno.h"
#include "sys/thread.h"
#include "sys/assert.h"
@@ -10,11 +11,19 @@
#include "sys/debug.h"
#include "fs/ofile.h"
#include "sys/heap.h"
#include "sys/attr.h"
#include "fs/node.h"
#include "sys/mm.h"
static spin_t g_shm_lock = 0;
static LIST_HEAD(g_shm_region_head);
static struct slab_cache *shm_region_ref_cache = NULL;
static struct slab_cache *shm_owner_ref_cache = NULL;
static __init void shm_init_slab_caches(void) {
shm_region_ref_cache = slab_cache_get(sizeof(struct shm_region_ref));
shm_owner_ref_cache = slab_cache_get(sizeof(struct shm_owner_ref));
}
struct shm_region *shm_region_create(size_t count) {
kdebug("Allocating a SHM region of %u pages\n", count);
@@ -54,16 +63,16 @@ uintptr_t shm_region_map(struct shm_region *region, struct thread *thr) {
spin_lock_irqsave(&region->lock, &irq);
kdebug("Mapping region of %u pages into thread %d\n", region->page_count, thr->pid);
struct shm_region_ref *reg_ref = kmalloc(sizeof(struct shm_region_ref));
struct shm_region_ref *reg_ref = slab_calloc(shm_region_ref_cache);
_assert(reg_ref);
struct shm_owner_ref *own_ref = kmalloc(sizeof(struct shm_owner_ref));
struct shm_owner_ref *own_ref = slab_calloc(shm_owner_ref_cache);
_assert(own_ref);
uintptr_t vaddr = vmfind(thr->space, 0x80000000, 0x100000000, region->page_count);
if (vaddr == MM_NADDR) {
kfree(reg_ref);
kfree(own_ref);
slab_free(shm_region_ref_cache, reg_ref);
slab_free(shm_owner_ref_cache, own_ref);
kerror("Could not find memory for mapping\n");
return MM_NADDR;
}
@@ -187,12 +196,12 @@ void *sys_mmap(void *hint, size_t length, int prot, int flags, int fd, off_t off
return (void *) -ENOENT;
}
struct shm_region_ref *reg_ref = kmalloc(sizeof(struct shm_region_ref));
struct shm_region_ref *reg_ref = slab_calloc(shm_region_ref_cache);
_assert(reg_ref);
void *vaddr = blk->mmap(blk, of, hint, length, flags);
if (vaddr == NULL || vaddr == MAP_FAILED) {
kfree(reg_ref);
slab_free(shm_region_ref_cache, reg_ref);
return MAP_FAILED;
}
@@ -259,7 +268,7 @@ int sys_munmap(void *_addr, size_t length) {
spin_release_irqrestore(&reg->lock, &irq);
}
kfree(own_ref);
slab_free(shm_owner_ref_cache, own_ref);
}
break;
case SHM_TYPE_BLOCK: {
@@ -277,7 +286,7 @@ int sys_munmap(void *_addr, size_t length) {
break;
}
list_del(&reg_ref->link);
kfree(reg_ref);
slab_free(shm_region_ref_cache, reg_ref);
kdebug("New region mapping list:\n");
shm_thread_describe(thr);
@@ -364,7 +373,7 @@ void shm_region_release_all(struct thread *thr, int noumap) {
spin_release_irqrestore(&region->lock, &irq);
}
kfree(own_ref);
slab_free(shm_owner_ref_cache, own_ref);
break;
case SHM_TYPE_BLOCK: {
sys/mem/slab.c
+221
View File
@@ -0,0 +1,221 @@
#include "sys/mem/phys.h"
#include "sys/mem/slab.h"
#include "sys/assert.h"
#include "sys/string.h"
#include "sys/debug.h"
#include "sys/list.h"
#include "sys/attr.h"
#include "sys/mm.h"
typedef uint32_t bufctl_t;
#define BUFCTL_END ((bufctl_t) -1)
#define slab_bufctl(slabp) \
((bufctl_t *) (((struct slab *) (slabp)) + 1))
#define CACHE_SIZE(i) \
{ 1ULL << ((i) + 4), &predefined_caches[i] }
#define CACHE_SIZE_END \
{ 0, NULL }
#define PREALLOC_COUNT 6
struct slab_cache {
struct list_head slabs_empty, slabs_partial, slabs_full;
size_t object_size, objects_per_slab;
};
struct slab {
struct list_head list;
size_t inuse;
void *base;
bufctl_t free;
};
struct cache_size {
size_t size;
struct slab_cache *cp;
};
struct slab_cache predefined_caches[PREALLOC_COUNT];
struct cache_size predefined_cache_sizes[PREALLOC_COUNT + 1] = {
CACHE_SIZE(0),
CACHE_SIZE(1),
CACHE_SIZE(2),
CACHE_SIZE(3),
CACHE_SIZE(4),
CACHE_SIZE(5),
CACHE_SIZE_END
};
static void slab_init_cache(struct slab_cache *cp, size_t object_size) {
cp->object_size = object_size;
list_head_init(&cp->slabs_empty);
list_head_init(&cp->slabs_partial);
list_head_init(&cp->slabs_full);
// Reserve space for slab descriptor
cp->objects_per_slab = (MM_PAGE_SIZE - sizeof(struct slab)) / cp->object_size;
// Reserve space for bufctl
size_t bufctl_array_size = cp->objects_per_slab * sizeof(bufctl_t);
cp->objects_per_slab -= (bufctl_array_size + cp->object_size - 1) / cp->object_size;
}
static __init void slab_init_caches(void) {
for (size_t i = 0; i < PREALLOC_COUNT; ++i) {
kdebug("Initializing predefined cache: %u\n", 1UL << (i + 4));
slab_init_cache(&predefined_caches[i], 1UL << (i + 4));
}
}
struct slab_cache *slab_cache_get(size_t size) {
struct cache_size *sizep = predefined_cache_sizes;
for (; sizep->size; ++sizep) {
if (size > sizep->size) {
continue;
}
return sizep->cp;
}
panic("Tried to get cache of invalid size: %u\n", size);
}
////
static struct slab *slab_create(struct slab_cache *cp) {
uintptr_t page_phys = mm_phys_alloc_page();
if (page_phys == MM_NADDR) {
return NULL;
}
struct slab *slab = (struct slab *) MM_VIRTUALIZE(page_phys);
slab->base = ((void *) &slab[1]) + sizeof(bufctl_t) * cp->objects_per_slab;
slab->base = (void *) (((uintptr_t) slab->base + 0x7) & ~0x7);
list_head_init(&slab->list);
slab->inuse = 0;
slab->free = 0;
for (size_t i = 0; i < cp->objects_per_slab - 1; ++i) {
slab_bufctl(slab)[i] = i + 1;
}
slab_bufctl(slab)[cp->objects_per_slab - 1] = BUFCTL_END;
list_add(&slab->list, &cp->slabs_empty);
kdebug("Created slab of %u x %u B\n", cp->objects_per_slab, cp->object_size);
return slab;
}
static void slab_destroy(struct slab_cache *cp, struct slab *slabp) {
_assert(!((uintptr_t) slabp & 0xFFF));
kdebug("Destoyed slab of %u x %u B\n", cp->objects_per_slab, cp->object_size);
mm_phys_free_page(MM_PHYS(slabp));
}
static inline void *slab_alloc_from(struct slab_cache *cp, struct slab *slabp) {
void *objp;
++slabp->inuse;
// Get object based on "free" index
objp = slabp->base + slabp->free * cp->object_size;
// Set next "free" index
slabp->free = slab_bufctl(slabp)[slabp->free];
// All the pages are allocated
if (slabp->free == BUFCTL_END) {
list_del_init(&slabp->list);
list_add(&slabp->list, &cp->slabs_full);
}
memset(objp, 0, cp->object_size);
return objp;
}
void *slab_calloc(struct slab_cache *cp) {
struct list_head *slabs_partial, *slabs_empty, *entry;
struct slab *slabp;
try_again:
// Check if there is any partial slab to use
slabs_partial = &cp->slabs_partial;
entry = slabs_partial->next;
if (slabs_partial == entry) {
// No partial slabs, try to find an empty one
slabs_empty = &cp->slabs_empty;
entry = slabs_empty->next;
if (slabs_empty == entry) {
// Have to allocate a new slab - no free objects are available
// After that, just try allocation again
//printf("No partial/empty slabs, allocating a new one\n");
if (!slab_create(cp)) {
return NULL;
}
goto try_again;
}
list_del(entry);
list_add(entry, slabs_partial);
}
slabp = list_entry(entry, struct slab, list);
return slab_alloc_from(cp, slabp);
}
void slab_free(struct slab_cache *cp, void *objp) {
struct slab *slabp;
// Get page-aligned address
uintptr_t page = (uintptr_t) objp;
_assert(page & 0xFFF);
page /= MM_PAGE_SIZE;
page *= MM_PAGE_SIZE;
// slab descriptor is on the same page objp points to
slabp = (struct slab *) page;
unsigned int obj_number = (objp - slabp->base) / cp->object_size;
slab_bufctl(slabp)[obj_number] = slabp->free;
slabp->free = obj_number;
size_t inuse = slabp->inuse;
if (!--slabp->inuse) {
// Was partial or full, now empty
// Free the slab
list_del(&slabp->list);
slab_destroy(cp, slabp);
} else if (inuse == cp->objects_per_slab) {
// Was full, now partial
list_del_init(&slabp->list);
list_add(&slabp->list, &cp->slabs_partial);
}
}
void slab_stat(struct slab_stat *st) {
st->alloc_bytes = 0;
st->alloc_objects = 0;
st->alloc_pages = 0;
for (size_t i = 0; i < PREALLOC_COUNT; ++i) {
struct slab_cache *cp = &predefined_caches[i];
struct slab *slab;
list_for_each_entry(slab, &cp->slabs_full, list) {
_assert(slab->inuse == cp->objects_per_slab);
st->alloc_objects += slab->inuse;
st->alloc_bytes += slab->inuse * cp->object_size;
++st->alloc_pages;
}
list_for_each_entry(slab, &cp->slabs_partial, list) {
_assert(slab->inuse && slab->inuse != cp->objects_per_slab);
st->alloc_objects += slab->inuse;
st->alloc_bytes += slab->inuse * cp->object_size;
++st->alloc_pages;
}
list_for_each_entry(slab, &cp->slabs_empty, list) {
panic("This list should be empty, slabs are freed once they're empty\n");
}
}
}
sys/ubsan.c
+20
View File
@@ -129,5 +129,25 @@ struct type_mismatch_info {
void __ubsan_handle_type_mismatch_v1(struct type_mismatch_info *type_mismatch,
uintptr_t pointer) {
struct source_location *location = &type_mismatch->location;
if (pointer == 0) {
kfatal("NULL pointer access\n");
} else if (type_mismatch->alignment != 0 &&
is_aligned(pointer, type_mismatch->alignment)) {
// Most useful on architectures with stricter memory alignment requirements, like ARM.
kfatal("Unaligned memory access\n");
} else {
kfatal("Insufficient size:\n");
if (type_mismatch->type_check_kind < sizeof(Type_Check_Kinds) / sizeof(Type_Check_Kinds[0])) {
kfatal("%s address %p with insufficient space for object of type %s\n",
Type_Check_Kinds[type_mismatch->type_check_kind], (void *) pointer,
type_mismatch->type->name);
}
//log("Insufficient size");
//logf("%s address %p with insufficient space for object of type %s\n",
// Type_Check_Kinds[type_mismatch->type_check_kind], (void *)pointer,
// type_mismatch->type->name);
}
ubsan_abort(location, "type mismatch");
}