Merge branch 'feat/thread' into feat/smp

feat: rerun build if linker script changes
feat: send IPIs using GIC
2021-11-30 11:07:09 +02:00 · 2021-11-10 18:37:09 +02:00 · 2021-11-10 18:36:29 +02:00 · 2021-11-10 18:35:32 +02:00 · 2021-11-10 18:17:00 +02:00
70 changed files with 1276 additions and 1243 deletions
@@ -22,9 +22,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"

 [[package]]
 name = "cortex-a"
-version = "7.0.1"
+version = "6.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5bd95fd055d118f77d4e4d527201b6ceccd13586b19b4dac1270f7081fef0f98"
+checksum = "509fc35485a2b4ddbacabe0bf2212cdfff88da93658608e5cc651afcb75b7733"
 dependencies = [
 "tock-registers",
 ]
@@ -93,7 +93,6 @@ dependencies = [
 "cfg-if",
 "cortex-a",
 "fdt-rs",
- "fs-macros",
 "kernel-macros",
 "libsys",
 "memfs",
@@ -17,12 +17,12 @@ CARGO_COMMON_OPTS=
 ifeq ($(PROFILE),release)
 CARGO_COMMON_OPTS+=--release
 endif
-ifeq ($(VERBOSE),1)
-CARGO_COMMON_OPTS+=--features verbose
-endif

 CARGO_BUILD_OPTS=$(CARGO_COMMON_OPTS) \
 				 --target=../etc/$(ARCH)-$(MACH).json
+ifeq ($(VERBOSE),1)
+CARGO_BUILD_OPTS+=--features verbose
+endif
 ifneq ($(MACH),)
 CARGO_BUILD_OPTS+=--features mach_$(MACH)
 endif
@@ -34,6 +34,7 @@ else
 ifeq ($(MACH),qemu)
 QEMU_OPTS+=-kernel $(O)/kernel.bin \
 		   -initrd $(O)/initrd.img \
+		   -smp cpus=4 \
 		   -M virt,virtualization=on \
 		   -cpu cortex-a72 \
 		   -m 512 \
@@ -68,7 +69,7 @@ endif
 all: kernel initrd

 kernel:
-	cd kernel && cargo build $(CARGO_BUILD_OPTS)
+	cd kernel && ARCH=$(ARCH) MACH=$(MACH) cargo build $(CARGO_BUILD_OPTS)
 ifeq ($(ARCH),aarch64)
 	$(LLVM_BASE)/llvm-strip -o $(O)/kernel.strip $(O)/kernel
 	$(LLVM_BASE)/llvm-size $(O)/kernel.strip
@@ -92,11 +93,8 @@ initrd:
 		--target=../etc/$(ARCH)-osdev5.json \
 		-Z build-std=core,alloc,compiler_builtins \
 		$(CARGO_COMMON_OPTS)
-	mkdir -p $(O)/rootfs/bin $(O)/rootfs/sbin $(O)/rootfs/dev $(O)/rootfs/etc $(O)/rootfs/sys
-	cp etc/initrd/passwd $(O)/rootfs/etc
-	cp etc/initrd/shadow $(O)/rootfs/etc
-	touch $(O)/rootfs/dev/.do_not_remove
-	touch $(O)/rootfs/sys/.do_not_remove
+	mkdir -p $(O)/rootfs/bin $(O)/rootfs/sbin $(O)/rootfs/dev
+	touch $(O)/rootfs/dev/.do_no_remove
 	cp target/$(ARCH)-osdev5/$(PROFILE)/init $(O)/rootfs/init
 	cp target/$(ARCH)-osdev5/$(PROFILE)/shell $(O)/rootfs/bin
 	cp target/$(ARCH)-osdev5/$(PROFILE)/fuzzy $(O)/rootfs/bin
@@ -1,2 +0,0 @@
-root:0:0:root:/:/bin/shell
-alnyan:1000:1000:alnyan:/:/bin/shell
@@ -1,2 +0,0 @@
-root:toor
-alnyan:
@@ -1,6 +1,7 @@
 #![feature(
    const_fn_trait_bound,
    const_mut_refs,
+    maybe_uninit_extra,
    maybe_uninit_uninit_array
 )]
 #![no_std]
@@ -2,7 +2,6 @@ use crate::{VnodeKind, VnodeRef, Vnode};
 use alloc::rc::Rc;
 use core::cell::RefCell;
 use core::cmp::min;
-use core::str::FromStr;
 use libsys::{
    error::Errno,
    stat::DirectoryEntry,
@@ -143,7 +142,7 @@ impl File {
                return Ok(offset);
            }

-            entries[offset] = DirectoryEntry::from_str(".").unwrap();
+            entries[offset] = DirectoryEntry::from_str(".");
            inner.pos = Self::POS_CACHE_DOT_DOT;

            offset += 1;
@@ -155,7 +154,7 @@ impl File {
                return Ok(offset);
            }

-            entries[offset] = DirectoryEntry::from_str("..").unwrap();
+            entries[offset] = DirectoryEntry::from_str("..");
            inner.pos = 0;

            offset += 1;
@@ -167,7 +166,7 @@ impl File {
        }

        let count = inner.vnode.for_each_entry(inner.pos, count, |i, e| {
-            entries[offset + i] = DirectoryEntry::from_str(e.name()).unwrap();
+            entries[offset + i] = DirectoryEntry::from_str(e.name());
        });
        inner.pos += count;
        Ok(offset + count)
@@ -1,6 +1,6 @@
 //! Virtual filesystem API and facilities
 #![warn(missing_docs)]
-#![feature(const_fn_trait_bound)]
+#![feature(destructuring_assignment, const_fn_trait_bound)]
 #![no_std]

 #[cfg(test)]
@@ -1,6 +1,6 @@
 use crate::{File, FileRef, Filesystem, Ioctx};
 use alloc::{borrow::ToOwned, boxed::Box, rc::Rc, string::String, vec::Vec};
-use core::cell::{Ref, RefCell, RefMut};
+use core::cell::{RefCell, RefMut, Ref};
 use core::fmt;
 use libsys::{
    error::Errno,
@@ -288,8 +288,10 @@ impl Vnode {
    /// vnode and will load it from disk if it's missing.
    pub fn lookup_or_load(self: &VnodeRef, name: &str) -> Result<VnodeRef, Errno> {
        if let Some(node) = self.lookup(name) {
-            Ok(node)
-        } else if let Some(ref mut data) = *self.data() {
+            return Ok(node);
+        }
+
+        if let Some(ref mut data) = *self.data() {
            let vnode = data.lookup(self.clone(), name)?;
            if let Some(fs) = self.fs() {
                vnode.set_fs(fs);
@@ -384,11 +386,13 @@ impl Vnode {

        if self.kind == VnodeKind::Directory && self.flags & Vnode::CACHE_READDIR != 0 {
            Ok(File::normal(self.clone(), File::POS_CACHE_DOT, open_flags))
-        } else if let Some(ref mut data) = *self.data() {
-            let pos = data.open(self.clone(), flags)?;
-            Ok(File::normal(self.clone(), pos, open_flags))
        } else {
-            Err(Errno::NotImplemented)
+            if let Some(ref mut data) = *self.data() {
+                let pos = data.open(self.clone(), flags)?;
+                Ok(File::normal(self.clone(), pos, open_flags))
+            } else {
+                Err(Errno::NotImplemented)
+            }
        }
    }

@@ -396,18 +400,22 @@ impl Vnode {
    pub fn close(self: &VnodeRef) -> Result<(), Errno> {
        if self.kind == VnodeKind::Directory && self.flags & Vnode::CACHE_READDIR != 0 {
            Ok(())
-        } else if let Some(ref mut data) = *self.data() {
-            data.close(self.clone())
        } else {
-            Err(Errno::NotImplemented)
+            if let Some(ref mut data) = *self.data() {
+                data.close(self.clone())
+            } else {
+                Err(Errno::NotImplemented)
+            }
        }
    }

    /// Reads data from offset `pos` into `buf`
    pub fn read(self: &VnodeRef, pos: usize, buf: &mut [u8]) -> Result<usize, Errno> {
        if self.kind == VnodeKind::Directory {
-            Err(Errno::IsADirectory)
-        } else if let Some(ref mut data) = *self.data() {
+            return Err(Errno::IsADirectory);
+        }
+
+        if let Some(ref mut data) = *self.data() {
            data.read(self.clone(), pos, buf)
        } else {
            Err(Errno::NotImplemented)
@@ -417,8 +425,10 @@ impl Vnode {
    /// Writes data from `buf` to offset `pos`
    pub fn write(self: &VnodeRef, pos: usize, buf: &[u8]) -> Result<usize, Errno> {
        if self.kind == VnodeKind::Directory {
-            Err(Errno::IsADirectory)
-        } else if let Some(ref mut data) = *self.data() {
+            return Err(Errno::IsADirectory);
+        }
+
+        if let Some(ref mut data) = *self.data() {
            data.write(self.clone(), pos, buf)
        } else {
            Err(Errno::NotImplemented)
@@ -428,8 +438,10 @@ impl Vnode {
    /// Resizes the vnode data
    pub fn truncate(self: &VnodeRef, size: usize) -> Result<(), Errno> {
        if self.kind != VnodeKind::Regular {
-            Err(Errno::IsADirectory)
-        } else if let Some(ref mut data) = *self.data() {
+            return Err(Errno::IsADirectory);
+        }
+
+        if let Some(ref mut data) = *self.data() {
            data.truncate(self.clone(), size)
        } else {
            Err(Errno::NotImplemented)
@@ -452,7 +464,7 @@ impl Vnode {
            Ok(Stat {
                blksize: 0,
                size: 0,
-                mode: props.mode,
+                mode: props.mode
            })
        } else if let Some(ref mut data) = *self.data() {
            data.stat(self.clone())
@@ -488,6 +500,7 @@ impl Vnode {
            if access.intersects(AccessMode::R_OK | AccessMode::W_OK | AccessMode::X_OK) {
                return Err(Errno::InvalidArgument);
            }
+            return Ok(());
        } else {
            if access.contains(AccessMode::F_OK) {
                return Err(Errno::InvalidArgument);
@@ -506,9 +519,9 @@ impl Vnode {

            // TODO check group
            // TODO check other
-        }

-        Ok(())
+            return Ok(());
+        }
    }
 }

@@ -0,0 +1,17 @@
+#![feature(asm)]
+#![no_std]
+#![no_main]
+
+#[macro_use]
+extern crate libusr;
+
+#[no_mangle]
+fn main() -> i32 {
+    loop {
+        trace!("Hello from userspace");
+        for _ in 0..100000 {
+            unsafe { asm!("nop"); }
+        }
+    }
+    123
+}
@@ -18,10 +18,9 @@ tock-registers = "0.7.x"
 fdt-rs = { version = "0.x.x", default-features = false }
 bitflags = "^1.3.0"
 kernel-macros = { path = "macros" }
-fs-macros = { path = "../fs/macros" }

 [target.'cfg(target_arch = "aarch64")'.dependencies]
-cortex-a = { version = "7.0.x" }
+cortex-a = { version = "6.x.x" }

 [features]
 default = ["aggressive_syscall"]
@@ -0,0 +1,12 @@
+use std::env;
+
+fn main() -> Result<(), i32> {
+    let arch = env::var("ARCH").expect("$ARCH is not set");
+    let mach = if arch == "aarch64" {
+        env::var("MACH").expect("$MACH is not set")
+    } else {
+        "none".to_owned()
+    };
+    println!("cargo:rerun-if-changed=../etc/{}-{}.ld", arch, mach);
+    Ok(())
+}
@@ -0,0 +1,16 @@
+.macro MOV_L reg, value
+    mov \reg, #((\value) & 0xFFFF)
+    movk \reg, #((\value) >> 16), lsl #16
+.endm
+
+.macro ADR_REL reg, sym
+    adrp \reg, \sym
+    add \reg, \reg, #:lo12:\sym
+.endm
+
+.macro ADR_ABS reg, sym
+    movz \reg, #:abs_g3:\sym
+    movk \reg, #:abs_g2_nc:\sym
+    movk \reg, #:abs_g1_nc:\sym
+    movk \reg, #:abs_g0_nc:\sym
+.endm
@@ -4,15 +4,16 @@ use crate::arch::{
    aarch64::reg::{CNTKCTL_EL1, CPACR_EL1},
    machine,
 };
-use core::arch::global_asm;
-use crate::config::{ConfigKey, CONFIG};
-use crate::dev::{
-    fdt::{find_prop, DeviceTree},
-    irq::IntSource,
-    Device,
+use crate::arch::{
+    aarch64::{
+        cpu,
+        smp,
+    },
 };
-use crate::fs::{devfs, sysfs};
-use crate::dev::pseudo;
+use crate::config::{ConfigKey, CONFIG};
+use crate::dev::fdt::find_prop;
+use crate::dev::{fdt::DeviceTree, irq::IntSource, Device};
+use crate::fs::devfs;
 use libsys::error::Errno;
 //use crate::debug::Level;
 use crate::mem::{
@@ -22,8 +23,8 @@ use crate::mem::{
 };
 use crate::proc;
 use cortex_a::asm::barrier::{self, dsb, isb};
-use cortex_a::registers::{SCTLR_EL1, VBAR_EL1};
-use tock_registers::interfaces::{ReadWriteable, Writeable};
+use cortex_a::registers::{MPIDR_EL1, SCTLR_EL1, VBAR_EL1};
+use tock_registers::interfaces::{ReadWriteable, Readable, Writeable};

 fn init_device_tree(fdt_base_phys: usize) -> Result<Option<DeviceTree>, Errno> {
    use fdt_rs::prelude::*;
@@ -35,6 +36,12 @@ fn init_device_tree(fdt_base_phys: usize) -> Result<Option<DeviceTree>, Errno> {
        return Ok(None);
    };

+    #[cfg(feature = "verbose")]
+    {
+        use crate::debug::Level;
+        fdt.dump(Level::Debug);
+    }
+
    let mut cfg = CONFIG.lock();

    if let Some(chosen) = fdt.node_by_path("/chosen") {
@@ -55,8 +62,7 @@ fn init_device_tree(fdt_base_phys: usize) -> Result<Option<DeviceTree>, Errno> {
    Ok(Some(fdt))
 }

-#[no_mangle]
-extern "C" fn __aa64_bsp_main(fdt_base: usize) -> ! {
+fn cpu_setup_common() {
    // Disable FP instruction trapping
    CPACR_EL1.modify(CPACR_EL1::FPEN::TrapNone);

@@ -80,6 +86,30 @@ extern "C" fn __aa64_bsp_main(fdt_base: usize) -> ! {
        isb(barrier::SY);
    }

+    unsafe {
+        cpu::init_self();
+    }
+}
+
+#[no_mangle]
+extern "C" fn __aa64_secondary_main() -> ! {
+    cpu_setup_common();
+
+    unsafe {
+        use crate::dev::irq::IntController;
+        machine::local_timer().enable().unwrap();
+        machine::intc().enable_secondary();
+        machine::intc().enable_irq(machine::IrqNumber::new(30));
+
+        proc::enter(false);
+    }
+}
+
+#[no_mangle]
+extern "C" fn __aa64_bsp_main(fdt_base: usize) -> ! {
+    // Boot CPU is MPDIR_EL1 = 0
+    cpu_setup_common();
+
    // Enable MMU
    virt::enable().expect("Failed to initialize virtual memory");

@@ -98,27 +128,24 @@ extern "C" fn __aa64_bsp_main(fdt_base: usize) -> ! {
    }

    devfs::init();
-    sysfs::init();
-
    machine::init_board().unwrap();

-    #[cfg(feature = "verbose")]
-    if let Some(fdt) = fdt {
-        use crate::debug::Level;
-        fdt.dump(Level::Debug);
+    if let Some(fdt) = &fdt {
+        unsafe {
+            smp::enable_secondary_cpus(fdt);
+        }
    }

-    devfs::add_named_char_device(&pseudo::ZERO, "zero").unwrap();
-    devfs::add_named_char_device(&pseudo::RANDOM, "random").unwrap();
-
    infoln!("Machine init finished");

    unsafe {
        machine::local_timer().enable().unwrap();
        machine::local_timer().init_irqs().unwrap();

-        proc::enter();
+        proc::enter(true);
    }
 }

+global_asm!(include_str!("macros.S"));
 global_asm!(include_str!("uboot.S"));
+global_asm!(include_str!("upper.S"));
@@ -1,54 +1,5 @@
 // vi:ft=a64asm.asm:

-.macro MOV_L reg, value
-    mov \reg, #((\value) & 0xFFFF)
-    movk \reg, #((\value) >> 16), lsl #16
-.endm
-
-.macro ADR_REL reg, sym
-    adrp \reg, \sym
-    add \reg, \reg, #:lo12:\sym
-.endm
-
-.macro ADR_ABS reg, sym
-    movz \reg, #:abs_g3:\sym
-    movk \reg, #:abs_g2_nc:\sym
-    movk \reg, #:abs_g1_nc:\sym
-    movk \reg, #:abs_g0_nc:\sym
-.endm
-
-.set PTE_BLOCK_AF,      1 << 10
-.set PTE_BLOCK_ISH,     3 << 8
-.set PTE_PRESENT,       1 << 0
-
-.set MAIR_EL1_Attr0_Normal_Inner_NC,    (4 << 0)
-.set MAIR_EL1_Attr0_Normal_Outer_NC,    (4 << 4)
-.set MAIR_EL1_Attr1_Device,             (0 << 12)
-.set MAIR_EL1_Attr1_Device_nGnRE,       (1 << 8)
-
-.set ID_AA64MMFR0_EL1_TGran4,           (0xF << 28)
-
-.set TCR_EL1_IPS_SHIFT,                 32
-
-.set TCR_EL1_TG1_4K,                    (2 << 30)
-.set TCR_EL1_SH1_Outer,                 (2 << 28)
-.set TCR_EL1_ORGN1_NC,                  (0 << 26)
-.set TCR_EL1_IRGN1_NC,                  (0 << 24)
-.set TCR_EL1_T1SZ_SHIFT,                16
-
-.set TCR_EL1_TG0_4K,                    (0 << 14)
-.set TCR_EL1_SH0_Outer,                 (2 << 12)
-.set TCR_EL1_ORGN0_NC,                  (0 << 10)
-.set TCR_EL1_IRGN0_NC,                  (0 << 8)
-.set TCR_EL1_T0SZ_SHIFT,                0
-
-.set TCR_EL1_ATTRS,                     (TCR_EL1_TG1_4K | TCR_EL1_SH1_Outer | TCR_EL1_TG0_4K | TCR_EL1_SH0_Outer | (25 << TCR_EL1_T1SZ_SHIFT) | (25 << TCR_EL1_T0SZ_SHIFT))
-
-.set SCTLR_EL1_I,                       (1 << 12)
-.set SCTLR_EL1_C,                       (1 << 2)
-.set SCTLR_EL1_M,                       (1 << 0)
-
-
 .set SCTLR_EL2_RES1,        0x30C50830

 .set SPSR_EL2_EL1h,         0x5
@@ -64,6 +15,22 @@
 _entry:
    mov x8, x0

+    // Zero .bss
+    ADR_ABS x0, __bss_start_phys
+    ADR_ABS x1, __bss_end_phys
+
+1:
+    cmp x0, x1
+    beq 2f
+
+    str xzr, [x0], #8
+
+    b 1b
+2:
+    ADR_ABS x9, __aa64_entry_upper
+    ADR_REL x10, __aa64_enter_upper
+
+_entry_common:
    // Test for EL2
    mrs x0, CurrentEL
    lsr x0, x0, #2
@@ -95,94 +62,34 @@ _entry:
    dsb sy
    isb

-    // Zero .bss
-    ADR_ABS x0, __bss_start_phys
-    ADR_ABS x1, __bss_end_phys
-1:
-    cmp x0, x1
-    beq 2f
-
-    str xzr, [x0], #8
-
-    b 1b
-2:
-
-    ADR_ABS x9, __aa64_entry_upper
-    b __aa64_enter_upper
-
-.global __aa64_enter_upper
-.type __aa64_enter_upper, %function
-__aa64_enter_upper:
-    // x8 -- FDT base
-    // x9 -- upper entry point
-
-    // Setup TTBR1_EL1
-    // TODO fix macros
-    ADR_ABS x5, KERNEL_TTBR1
-    ADR_ABS x6, KERNEL_OFFSET
-
-    // x5 = KERNEL_TTBR1 physical address
-    sub x5, x5, x6
-
-    // Fill KERNEL_TTBR1 table with upper-mapped Normal memory
-.fill_ttbr1:
-    mov x2, #256
-1:
-    sub x2, x2, #1
-
-    // x0 = (x2 << 30) | attrs...
-    lsl x1, x2, #30
-    mov x0, #(PTE_BLOCK_ISH | PTE_BLOCK_AF | PTE_PRESENT)
-    orr x0, x0, x1
-
-    str x0, [x5, x2, lsl #3]
-
-    cbnz x2, 1b
-
-.init_mmu_regs:
-    mov x0, #(MAIR_EL1_Attr0_Normal_Outer_NC | MAIR_EL1_Attr0_Normal_Inner_NC | MAIR_EL1_Attr1_Device | MAIR_EL1_Attr1_Device_nGnRE)
-    msr mair_el1, x0
-
-    // Test for 4KiB page support
-    mrs x0, ID_AA64MMFR0_EL1
-    mov x1, ID_AA64MMFR0_EL1_TGran4
-    tst x0, x1
-    bne .no_4k_gran
-
-    // x0 = PARange
-    and x0, x0, #0xF
-    lsl x0, x0, #TCR_EL1_IPS_SHIFT
-    MOV_L x1, TCR_EL1_ATTRS
-    orr x0, x0, x1
-    msr tcr_el1, x0
-
-    msr ttbr0_el1, x5
-    msr ttbr1_el1, x5
-
-    dsb ish
-    isb
-
-    mrs x0, sctlr_el1
-    orr x0, x0, #SCTLR_EL1_M
-    msr sctlr_el1, x0
-
-    mov x0, x8
-    br x9
-.no_4k_gran:
-    b .
-.size __aa64_enter_upper, . - __aa64_enter_upper
+    br x10

 .section .text._entry_upper
 __aa64_entry_upper:
    // x0 -- fdt address

-    ADR_REL x1, bsp_stack_top
+    ADR_ABS x1, bsp_stack_top
    mov sp, x1

    mov lr, xzr
    bl __aa64_bsp_main
    b .

+__aa64_entry_upper_secondary:
+    // x0 -- stack
+    mov sp, x0
+    mov lr, xzr
+    bl __aa64_secondary_main
+    b .
+
+.section .text._entry_secondary
+.global _entry_secondary
+_entry_secondary:
+    mov x8, x0
+    ADR_ABS x9, __aa64_entry_upper_secondary
+    ADR_ABS x10, __aa64_enter_upper_secondary
+    b _entry_common
+
 .section .bss
 .p2align 12
 bsp_stack_bottom:
@@ -0,0 +1,101 @@
+// vi:ft=a64asm:
+
+.set PTE_BLOCK_AF,      1 << 10
+.set PTE_BLOCK_ISH,     3 << 8
+.set PTE_PRESENT,       1 << 0
+
+.set MAIR_EL1_Attr0_Normal_Inner_NC,    (4 << 0)
+.set MAIR_EL1_Attr0_Normal_Outer_NC,    (4 << 4)
+.set MAIR_EL1_Attr1_Device,             (0 << 12)
+.set MAIR_EL1_Attr1_Device_nGnRE,       (1 << 8)
+
+.set ID_AA64MMFR0_EL1_TGran4,           (0xF << 28)
+
+.set TCR_EL1_IPS_SHIFT,                 32
+
+.set TCR_EL1_TG1_4K,                    (2 << 30)
+.set TCR_EL1_SH1_Outer,                 (2 << 28)
+.set TCR_EL1_ORGN1_NC,                  (0 << 26)
+.set TCR_EL1_IRGN1_NC,                  (0 << 24)
+.set TCR_EL1_T1SZ_SHIFT,                16
+
+.set TCR_EL1_TG0_4K,                    (0 << 14)
+.set TCR_EL1_SH0_Outer,                 (2 << 12)
+.set TCR_EL1_ORGN0_NC,                  (0 << 10)
+.set TCR_EL1_IRGN0_NC,                  (0 << 8)
+.set TCR_EL1_T0SZ_SHIFT,                0
+
+.set TCR_EL1_ATTRS,                     (TCR_EL1_TG1_4K | TCR_EL1_SH1_Outer | TCR_EL1_TG0_4K | TCR_EL1_SH0_Outer | (25 << TCR_EL1_T1SZ_SHIFT) | (25 << TCR_EL1_T0SZ_SHIFT))
+
+.set SCTLR_EL1_I,                       (1 << 12)
+.set SCTLR_EL1_C,                       (1 << 2)
+.set SCTLR_EL1_M,                       (1 << 0)
+
+.section .text._entry
+.global __aa64_enter_upper
+.type __aa64_enter_upper, %function
+__aa64_enter_upper:
+    // x8 -- FDT base
+    // x9 -- upper entry point
+
+    // Setup TTBR1_EL1
+    // TODO fix macros
+    ADR_ABS x5, KERNEL_TTBR1
+    ADR_ABS x6, KERNEL_OFFSET
+
+    // x5 = KERNEL_TTBR1 physical address
+    sub x5, x5, x6
+
+    // Fill KERNEL_TTBR1 table with upper-mapped Normal memory
+.fill_ttbr1:
+    mov x2, #256
+1:
+    sub x2, x2, #1
+
+    // x0 = (x2 << 30) | attrs...
+    lsl x1, x2, #30
+    mov x0, #(PTE_BLOCK_ISH | PTE_BLOCK_AF | PTE_PRESENT)
+    orr x0, x0, x1
+
+    str x0, [x5, x2, lsl #3]
+
+    cbnz x2, 1b
+
+__aa64_enter_upper_secondary:
+    ADR_ABS x5, KERNEL_TTBR1
+    ADR_ABS x6, KERNEL_OFFSET
+
+    // x5 = KERNEL_TTBR1 physical address
+    sub x5, x5, x6
+
+    mov x0, #(MAIR_EL1_Attr0_Normal_Outer_NC | MAIR_EL1_Attr0_Normal_Inner_NC | MAIR_EL1_Attr1_Device | MAIR_EL1_Attr1_Device_nGnRE)
+    msr mair_el1, x0
+
+    // Test for 4KiB page support
+    mrs x0, ID_AA64MMFR0_EL1
+    mov x1, ID_AA64MMFR0_EL1_TGran4
+    tst x0, x1
+    bne .no_4k_gran
+
+    // x0 = PARange
+    and x0, x0, #0xF
+    lsl x0, x0, #TCR_EL1_IPS_SHIFT
+    MOV_L x1, TCR_EL1_ATTRS
+    orr x0, x0, x1
+    msr tcr_el1, x0
+
+    msr ttbr0_el1, x5
+    msr ttbr1_el1, x5
+
+    dsb ish
+    isb
+
+    mrs x0, sctlr_el1
+    orr x0, x0, #SCTLR_EL1_M
+    msr sctlr_el1, x0
+
+    mov x0, x8
+    br x9
+.no_4k_gran:
+    b .
+.size __aa64_enter_upper, . - __aa64_enter_upper
@@ -6,7 +6,6 @@ use crate::mem::{
    phys::{self, PageUsage},
 };
 use core::mem::size_of;
-use core::arch::global_asm;

 struct Stack {
    bp: usize,
@@ -0,0 +1,72 @@
+#![allow(missing_docs)]
+
+use crate::proc::Scheduler;
+use crate::util::InitOnce;
+use core::mem::MaybeUninit;
+use core::ptr::null_mut;
+use core::sync::atomic::{AtomicUsize, Ordering};
+use cortex_a::registers::{MPIDR_EL1, TPIDR_EL1};
+use tock_registers::interfaces::{Readable, Writeable};
+
+#[repr(C)]
+pub struct Cpu {
+    counter: AtomicUsize, // 0x08
+
+    id: usize,
+    scheduler: Scheduler,
+}
+
+impl Cpu {
+    pub fn new(id: usize) -> Self {
+        Self {
+            counter: AtomicUsize::new(0),
+
+            id,
+            scheduler: Scheduler::new(),
+        }
+    }
+
+    pub fn id(&self) -> usize {
+        self.id
+    }
+
+    pub fn scheduler(&mut self) -> &Scheduler {
+        &self.scheduler
+    }
+
+    pub unsafe fn set(&mut self) {
+        TPIDR_EL1.set(self as *mut _ as u64);
+    }
+
+    pub unsafe fn get() -> &'static mut Self {
+        &mut *(TPIDR_EL1.get() as *mut Self)
+    }
+}
+
+pub unsafe fn cpus() -> impl Iterator<Item = &'static mut Cpu> {
+    CPUS[..CPU_COUNT.load(Ordering::Acquire)]
+        .iter_mut()
+        .map(|c| c.assume_init_mut())
+}
+
+pub unsafe fn by_index(idx: usize) -> &'static mut Cpu {
+    assert!(idx < CPU_COUNT.load(Ordering::Acquire));
+    CPUS[idx].assume_init_mut()
+}
+
+pub fn count() -> usize {
+    CPU_COUNT.load(Ordering::Acquire)
+}
+
+static CPU_COUNT: AtomicUsize = AtomicUsize::new(0);
+static mut CPUS: [MaybeUninit<Cpu>; 8] = MaybeUninit::uninit_array();
+
+pub unsafe fn init_self() {
+    let cpu_index = CPU_COUNT.load(Ordering::Acquire);
+    let mpidr_id = (MPIDR_EL1.get() & 0xF) as usize;
+
+    CPUS[cpu_index].write(Cpu::new(mpidr_id));
+    CPUS[cpu_index].assume_init_mut().set();
+
+    CPU_COUNT.store(cpu_index + 1, Ordering::Release);
+}
@@ -4,12 +4,11 @@ use crate::arch::machine;
 use crate::debug::Level;
 use crate::dev::irq::{IntController, IrqContext};
 use crate::mem;
-use crate::proc::{sched, Process, Thread};
+use crate::proc::{sched, Thread};
 use crate::syscall;
 use cortex_a::registers::{ESR_EL1, FAR_EL1};
-use libsys::{abi::SystemCall, signal::Signal, error::Errno};
+use libsys::{abi::SystemCall, signal::Signal};
 use tock_registers::interfaces::Readable;
-use core::arch::global_asm;

 /// Trapped SIMD/FP functionality
 pub const EC_FP_TRAP: u64 = 0b000111;
@@ -91,24 +90,28 @@ extern "C" fn __aa64_exc_sync_handler(exc: &mut ExceptionFrame) {
            let far = FAR_EL1.get() as usize;
            let iss = esr & 0x1FFFFFF;

-            if iss & (1 << 6) != 0 && far < mem::KERNEL_OFFSET && sched::is_ready() {
+            // TODO handle scenarios when sheduler is not yet initialized
+            if iss & (1 << 6) != 0 && far < mem::KERNEL_OFFSET {
                let thread = Thread::current();
                let proc = thread.owner().unwrap();
-                let asid = proc.asid();

-                let res = proc.manipulate_space(|space| {
-                    space.try_cow_copy(far)?;
-                    Process::invalidate_asid(asid);
-                    Result::<(), Errno>::Ok(())
-                });
-
-                if res.is_err() {
+                if proc
+                    .manipulate_space(|space| space.try_cow_copy(far))
+                    .is_err()
+                {
                    // Kill program
                    errorln!("Data abort from {:#x}", exc.elr_el1);
                    dump_data_abort(Level::Error, esr, far as u64);
                    proc.enter_fault_signal(thread, Signal::SegmentationFault);
                }

+                unsafe {
+                    use cortex_a::registers::TTBR0_EL1;
+                    let ttbr = TTBR0_EL1.get() as usize;
+                    let asid = (ttbr >> 48) & 0xFF;
+                    asm!("tlbi aside1, {}", in(reg) (asid << 48));
+                }
+
                return;
            }

@@ -125,7 +128,7 @@ extern "C" fn __aa64_exc_sync_handler(exc: &mut ExceptionFrame) {

            if num == SystemCall::Fork {
                match unsafe { syscall::sys_fork(exc) } {
-                    Ok(pid) => exc.x[0] = u32::from(pid) as usize,
+                    Ok(pid) => exc.x[0] = pid.value() as usize,
                    Err(err) => {
                        exc.x[0] = err.to_negative_isize() as usize;
                    }
@@ -145,14 +148,14 @@ extern "C" fn __aa64_exc_sync_handler(exc: &mut ExceptionFrame) {
        _ => {}
    }

-    if sched::is_ready() {
-        let thread = Thread::current();
-        errorln!(
-            "Unhandled exception in thread {:?}, {:?}",
-            thread.id(),
-            thread.owner().map(|e| e.id())
-        );
-    }
+    // if sched::is_ready() {
+    //     let thread = Thread::current();
+    //     errorln!(
+    //         "Unhandled exception in thread {}, {:?}",
+    //         thread.id(),
+    //         thread.owner().map(|e| e.id())
+    //     );
+    // }

    errorln!(
        "Unhandled exception at ELR={:#018x}, ESR={:#010x}",
@@ -1,7 +1,7 @@
 use crate::mem::virt::DeviceMemoryIo;
 use crate::sync::IrqSafeSpinLock;
 use tock_registers::interfaces::{Readable, Writeable};
-use tock_registers::registers::{ReadOnly, ReadWrite};
+use tock_registers::registers::{ReadOnly, WriteOnly, ReadWrite};
 use tock_registers::{register_bitfields, register_structs};

 register_bitfields! {
@@ -31,7 +31,9 @@ register_structs! {
        (0x820 => ITARGETSR: [ReadWrite<u32, ITARGETSR::Register>; 248]),
        (0xC00 => _res2),
        (0xC08 => ICFGR: [ReadWrite<u32>; 62]),
-        (0xC0C => @END),
+        (0xD00 => _res3),
+        (0xF00 => SGIR: WriteOnly<u32>),
+        (0xF04 => @END),
    }
 }

@@ -103,6 +105,14 @@ impl Gicd {
        }
    }

+    pub fn set_sgir(&self, filter: bool, mask: u32, intid: u32) {
+        let mut value = (mask << 16) | intid;
+        if filter {
+            value |= 1 << 24;
+        }
+        self.shared_regs.lock().SGIR.set(value);
+    }
+
    pub fn enable_irq(&self, irq: super::IrqNumber) {
        let irq = irq.get();

@@ -1,7 +1,7 @@
 //! ARM Generic Interrupt Controller

 use crate::dev::{
-    irq::{IntController, IntSource, IrqContext},
+    irq::{IntController, IntSource, IrqContext, IpiSender},
    Device,
 };
 use crate::mem::virt::{DeviceMemory, DeviceMemoryIo};
@@ -17,6 +17,8 @@ use gicd::Gicd;
 /// Maximum available IRQ number
 pub const MAX_IRQ: usize = 300;

+const SGI_IRQ: u32 = 2;
+
 /// Range-checked IRQ number type
 #[repr(transparent)]
 #[derive(Copy, Clone)]
@@ -61,11 +63,12 @@ impl Device for Gic {
        let gicc = Gicc::new(gicc_mmio);

        gicd.enable();
-        gicc.enable();

        self.gicd.init(gicd);
        self.gicc.init(gicc);

+        self.enable_secondary();
+
        Ok(())
    }
 }
@@ -85,9 +88,30 @@ impl IntController for Gic {
            return;
        }

+//<<<<<<< HEAD
+        if irq_number == 1 {
+            gicc.clear_irq(irq_number as u32, ic);
+            debugln!("Received IPI");
+            loop {}
+        }
+//
+//        if self.scheduler_irq.0 == irq_number {
+//            use crate::proc::sched;
+//            use cortex_a::registers::{CNTP_TVAL_EL0, CNTP_CTL_EL0};
+//            use tock_registers::interfaces::Writeable;
+//            use crate::arch::platform::cpu::Cpu;
+//            gicc.clear_irq(irq_number as u32, ic);
+//            CNTP_TVAL_EL0.set(1000000);
+//            CNTP_CTL_EL0.write(CNTP_CTL_EL0::ENABLE::SET);
+//            sched::switch(false);
+//            return;
+//        }
+//=======
        gicc.clear_irq(irq_number as u32, ic);
+//>>>>>>> feat/thread

        {
+            // TODO make timer interrupt a special case and drop table lock
            let table = self.table.lock();
            match table[irq_number] {
                None => panic!("No handler registered for irq{}", irq_number),
@@ -117,7 +141,18 @@ impl IntController for Gic {
    }
 }

+impl IpiSender for Gic {
+    fn send_to_mask(&self, exclude_self: bool, target: u32, data: u64) {
+        self.gicd.get().set_sgir(exclude_self, target, 1);
+    }
+}
+
 impl Gic {
+    ///
+    pub unsafe fn enable_secondary(&self) {
+        self.gicc.get().enable();
+    }
+
    /// Constructs an instance of GICv2.
    ///
    /// # Safety
@@ -96,4 +96,4 @@ static UART0: Uart = unsafe { Uart::new(UART0_BASE, IrqNumber::new(32)) };
 static LOCAL_TIMER: GenericTimer = GenericTimer::new(LOCAL_TIMER_IRQ);
 pub(super) static GPIO: Gpio = unsafe { Gpio::new(PIO_BASE) };
 static RTC: Rtc = unsafe { Rtc::new(RTC_BASE, RTC_IRQ) };
-static GIC: Gic = unsafe { Gic::new(GICD_BASE, GICC_BASE) };
+static GIC: Gic = unsafe { Gic::new(GICD_BASE, GICC_BASE, LOCAL_TIMER_IRQ) };
@@ -72,7 +72,13 @@ pub fn local_timer() -> &'static GenericTimer {

 /// Returns CPU's interrupt controller device
 #[inline]
-pub fn intc() -> &'static impl IntController<IrqNumber = IrqNumber> {
+pub fn intc() -> &'static Gic {
+    &GIC
+}
+
+/// Returns CPU's IPI sender device
+#[inline]
+pub fn ipi_sender() -> &'static Gic {
    &GIC
 }

@@ -2,13 +2,14 @@

 use cortex_a::registers::DAIF;
 use tock_registers::interfaces::{Readable, Writeable};
-use core::arch::asm;

 pub mod boot;
 pub mod context;
+pub mod cpu;
 pub mod exception;
 pub mod irq;
 pub mod reg;
+pub mod smp;
 pub mod timer;

 cfg_if! {
@@ -1,8 +1,6 @@
 //! CNTKCTL_EL1 register
 #![allow(missing_docs)]

-use core::arch::asm;
-
 use tock_registers::{
    interfaces::{Readable, Writeable},
    register_bitfields,
@@ -1,7 +1,6 @@
 //! CPACR_EL1 register
 #![allow(missing_docs)]

-use core::arch::asm;
 use tock_registers::{
    interfaces::{Readable, Writeable},
    register_bitfields,
@@ -0,0 +1,123 @@
+#![allow(missing_docs)]
+
+use crate::arch::{aarch64::cpu, machine};
+use crate::dev::{
+    fdt::{self, DeviceTree},
+    irq::IpiSender,
+};
+use crate::mem::{
+    self,
+    phys::{self, PageUsage},
+};
+use cortex_a::registers::MPIDR_EL1;
+use libsys::error::Errno;
+use fdt_rs::prelude::*;
+use tock_registers::interfaces::Readable;
+
+pub type NodeAddress = u32;
+
+#[derive(Clone, Copy, Debug)]
+pub enum PsciError {
+    NotSupported,
+    InvalidParameters,
+    Denied,
+    AlreadyOn,
+    OnPending,
+    InternalFailure,
+    NotPresent,
+    Disabled,
+    InvalidAddress,
+}
+
+struct Psci {
+    use_smc: bool,
+}
+
+impl Psci {
+    const PSCI_VERSION: usize = 0x84000000;
+    const PSCI_CPU_OFF: usize = 0x84000002;
+    const PSCI_CPU_ON: usize = 0xC4000003;
+
+    pub const fn new() -> Self {
+        Self { use_smc: true }
+    }
+
+    unsafe fn call(&self, x0: usize, x1: usize, x2: usize, x3: usize) -> usize {
+        if self.use_smc {
+            call_smc(x0, x1, x2, x3)
+        } else {
+            todo!()
+        }
+    }
+
+    pub unsafe fn cpu_on(
+        &self,
+        target_cpu: usize,
+        entry_point_address: usize,
+        context_id: usize,
+    ) -> Result<(), PsciError> {
+        wrap_psci_ok(self.call(
+            Self::PSCI_CPU_ON,
+            target_cpu,
+            entry_point_address,
+            context_id,
+        ))
+    }
+}
+
+const SECONDARY_STACK_PAGES: usize = 4;
+
+unsafe fn call_smc(mut x0: usize, x1: usize, x2: usize, x3: usize) -> usize {
+    asm!("smc #0", inout("x0") x0, in("x1") x1, in("x2") x2, in("x3") x3);
+    x0
+}
+
+pub unsafe fn send_ipi(exclude_self: bool, target_mask: u32, data: u64) {
+    machine::ipi_sender().send_to_mask(exclude_self, target_mask, data);
+}
+
+fn wrap_psci_ok(a: usize) -> Result<(), PsciError> {
+    const NOT_SUPPORTED: isize = -1;
+    const INVALID_PARAMETERS: isize = -2;
+    const DENIED: isize = -3;
+    const ALREADY_ON: isize = -4;
+
+    match a as isize {
+        0 => Ok(()),
+        NOT_SUPPORTED => Err(PsciError::NotSupported),
+        INVALID_PARAMETERS => Err(PsciError::InvalidParameters),
+        DENIED => Err(PsciError::Denied),
+        ALREADY_ON => Err(PsciError::AlreadyOn),
+        _ => unimplemented!(),
+    }
+}
+
+pub unsafe fn enable_secondary_cpus(dt: &DeviceTree) {
+    extern "C" {
+        fn _entry_secondary();
+    }
+
+    let cpus = dt.node_by_path("/cpus").unwrap();
+    let psci = Psci::new();
+
+    for cpu_node in cpus.children() {
+        let reg = fdt::find_prop(cpu_node, "reg").unwrap().u32(0).unwrap();
+        if reg == 0 {
+            continue;
+        }
+        infoln!("Enabling cpu{}", reg);
+        let stack_pages =
+            phys::alloc_contiguous_pages(PageUsage::Kernel, SECONDARY_STACK_PAGES).unwrap();
+        let count_old = cpu::count();
+        psci.cpu_on(
+            reg as usize,
+            _entry_secondary as usize - mem::KERNEL_OFFSET,
+            mem::virtualize(stack_pages + SECONDARY_STACK_PAGES * mem::PAGE_SIZE),
+        )
+        .unwrap();
+        while cpu::count() == count_old {
+            cortex_a::asm::wfe();
+        }
+        debugln!("Done");
+    }
+}
@@ -1,9 +1,7 @@
 //! ARM generic timer implementation

 use crate::arch::machine::{self, IrqNumber};
-use crate::proc;
 use crate::dev::{
-    pseudo,
    irq::{IntController, IntSource},
    timer::TimestampSource,
    Device,
@@ -36,9 +34,9 @@ impl IntSource for GenericTimer {
    fn handle_irq(&self) -> Result<(), Errno> {
        CNTP_TVAL_EL0.set(TIMER_TICK);
        CNTP_CTL_EL0.write(CNTP_CTL_EL0::ENABLE::SET);
-        proc::wait::tick();
-        proc::switch();
-        pseudo::RANDOM.set_state(CNTPCT_EL0.get() as u32);
+        use crate::proc::{wait, sched};
+        wait::tick();
+        sched::switch(false);
        Ok(())
    }

@@ -52,11 +50,9 @@ impl IntSource for GenericTimer {

 impl TimestampSource for GenericTimer {
    fn timestamp(&self) -> Result<Duration, Errno> {
-        let cnt = (CNTPCT_EL0.get() as u128) * 1_000_000_000u128;
-        let frq = CNTFRQ_EL0.get() as u128;
-        let secs = ((cnt / frq) / 1_000_000_000) as u64;
-        let nanos = ((cnt / frq) % 1_000_000_000) as u32;
-        Ok(Duration::new(secs, nanos))
+        let cnt = CNTPCT_EL0.get() * 1_000_000_000;
+        let frq = CNTFRQ_EL0.get();
+        Ok(Duration::from_nanos(cnt / frq))
    }
 }

@@ -74,8 +74,18 @@ __aa\bits\()_el\el\ht\()_\kind:

    EXC_SAVE_STATE
    mov x0, sp
+.if \el == 0
+    sub sp, sp, #16
+    stp fp, lr, [sp, #0]
+.else
    mov lr, xzr
+.endif
    bl __aa64_exc_\kind\()_handler
+
+.if \el == 0
+    add sp, sp, #16
+.endif
+
    EXC_RESTORE_STATE
    eret
 .endm
@@ -12,39 +12,21 @@
 //! * [errorln!]

 use crate::dev::serial::SerialDevice;
-use libsys::{debug::TraceLevel, error::Errno};
-use core::convert::TryFrom;
+use crate::sync::IrqSafeSpinLock;
+use libsys::debug::TraceLevel;
 use core::fmt;

-pub static LEVEL: Level = Level::Debug;
-
 /// Kernel logging levels
-#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)]
-#[repr(u32)]
+#[derive(Clone, Copy, PartialEq)]
 pub enum Level {
    /// Debugging information
-    Debug = 1,
+    Debug,
    /// General informational messages
-    Info = 2,
+    Info,
    /// Non-critical warnings
-    Warn = 3,
+    Warn,
    /// Critical errors
-    Error = 4,
-}
-
-impl TryFrom<u32> for Level {
-    type Error = Errno;
-
-    #[inline(always)]
-    fn try_from(l: u32) -> Result<Level, Errno> {
-        match l {
-            1 => Ok(Level::Debug),
-            2 => Ok(Level::Info),
-            3 => Ok(Level::Warn),
-            4 => Ok(Level::Error),
-            _ => Err(Errno::InvalidArgument)
-        }
-    }
+    Error,
 }

 impl From<TraceLevel> for Level {
@@ -133,15 +115,15 @@ macro_rules! errorln {
 }

 #[doc(hidden)]
-pub fn _debug(level: Level, args: fmt::Arguments) {
+pub fn _debug(_level: Level, args: fmt::Arguments) {
+    static LOCK: IrqSafeSpinLock<()> = IrqSafeSpinLock::new(());
    use crate::arch::machine;
    use fmt::Write;

-    if level >= LEVEL {
-        SerialOutput {
-            inner: machine::console(),
-        }
-        .write_fmt(args)
-        .ok();
+    let _lock = LOCK.lock();
+    SerialOutput {
+        inner: machine::console(),
    }
+    .write_fmt(args)
+    .ok();
 }
@@ -3,7 +3,9 @@ use crate::debug::Level;
 use fdt_rs::prelude::*;
 use fdt_rs::{
    base::DevTree,
-    index::{DevTreeIndex, DevTreeIndexNode, DevTreeIndexProp},
+    index::{
+        iters::DevTreeIndexCompatibleNodeIter, DevTreeIndex, DevTreeIndexNode, DevTreeIndexProp,
+    },
 };
 use libsys::{error::Errno, path::path_component_left};

@@ -44,7 +46,7 @@ fn dump_node(level: Level, node: &INode, depth: usize) {
        print!(level, "{:?} = ", name);

        match name {
-            "compatible" => print!(level, "{:?}", prop.str().unwrap()),
+            "compatible" | "enable-method" => print!(level, "{:?}", prop.str().unwrap()),
            "#address-cells" | "#size-cells" => print!(level, "{}", prop.u32(0).unwrap()),
            "reg" => {
                print!(level, "<");
@@ -115,6 +117,20 @@ impl DeviceTree {

    /// Loads a device tree from physical `base` address and
    /// creates an index for it
+    pub fn compatible<'a, 's>(&'a self, compat: &'s str) -> DevTreeIndexCompatibleNodeIter<'s, 'a, 'a, 'a> {
+        self.index.compatible_nodes(compat)
+    }
+
+    pub fn initrd(&self) -> Option<(usize, usize)> {
+        let chosen = self.node_by_path("/chosen")?;
+        let initrd_start = find_prop(chosen.clone(), "linux,initrd-start")?
+            .u32(0)
+            .ok()?;
+        let initrd_end = find_prop(chosen, "linux,initrd-end")?.u32(0).ok()?;
+
+        Some((initrd_start as usize, initrd_end as usize))
+    }
+
    pub fn from_phys(base: usize) -> Result<DeviceTree, Errno> {
        // TODO virtualize address
        let tree = unsafe { DevTree::from_raw_pointer(base as *const _) }
@@ -1,4 +1,5 @@
 //! Interrupt controller and handler interfaces
+use crate::arch::platform::smp::NodeAddress;
 use crate::dev::Device;
 use core::marker::PhantomData;
 use libsys::error::Errno;
@@ -27,6 +28,12 @@ pub trait IntController: Device {
    fn handle_pending_irqs<'irq_context>(&'irq_context self, ic: &IrqContext<'irq_context>);
 }

+/// Inter-processor interrupt delivery method
+pub trait IpiSender: Device {
+    /// Raise an IPI for the target CPU mask, optionally excluding source CPU
+    fn send_to_mask(&self, except_self: bool, target: u32, data: u64);
+}
+
 /// Interface for peripherals capable of emitting IRQs
 pub trait IntSource: Device {
    /// Handles pending IRQs, if any, of this [IntSource].
@@ -11,7 +11,6 @@ pub mod rtc;
 pub mod sd;
 pub mod serial;
 pub mod timer;
-pub mod pseudo;
 pub mod tty;

 /// Generic device trait
@@ -1,102 +0,0 @@
-use crate::arch::machine::{self, IrqNumber};
-use crate::dev::{
-    irq::{IntController, IntSource},
-    serial::SerialDevice,
-    tty::{CharRing, TtyDevice},
-    Device,
-};
-use crate::mem::virt::DeviceMemoryIo;
-use crate::sync::IrqSafeSpinLock;
-use crate::util::InitOnce;
-use libsys::{error::Errno, ioctl::IoctlCmd};
-use core::sync::atomic::{AtomicU32, Ordering};
-use tock_registers::{
-    interfaces::{ReadWriteable, Readable, Writeable},
-    register_bitfields, register_structs,
-    registers::{ReadOnly, ReadWrite, WriteOnly},
-};
-use vfs::CharDevice;
-
-pub struct Random {
-    state: AtomicU32
-}
-pub struct Zero;
-
-impl Device for Random {
-    fn name(&self) -> &'static str {
-        "Pseudo-random device"
-    }
-
-    unsafe fn enable(&self) -> Result<(), Errno> {
-        Ok(())
-    }
-}
-
-impl CharDevice for Random {
-    fn read(&self, _blocking: bool, data: &mut [u8]) -> Result<usize, Errno> {
-        for byte in data.iter_mut() {
-            *byte = self.read_single() as u8;
-        }
-        Ok(data.len())
-    }
-
-    fn write(&self, _blocking: bool, _data: &[u8]) -> Result<usize, Errno> {
-        Ok(0)
-    }
-
-    fn is_ready(&self, _write: bool) -> Result<bool, Errno> {
-        Ok(true)
-    }
-
-    fn ioctl(&self, _cmd: IoctlCmd, _ptr: usize, _lim: usize) -> Result<usize, Errno> {
-        Err(Errno::InvalidArgument)
-    }
-}
-
-
-impl Device for Zero {
-    fn name(&self) -> &'static str {
-        "Zero device"
-    }
-
-    unsafe fn enable(&self) -> Result<(), Errno> {
-        Ok(())
-    }
-}
-
-impl CharDevice for Zero {
-    fn read(&self, _blocking: bool, data: &mut [u8]) -> Result<usize, Errno> {
-        data.fill(0);
-        Ok(data.len())
-    }
-
-    fn write(&self, _blocking: bool, _data: &[u8]) -> Result<usize, Errno> {
-        Ok(0)
-    }
-
-    fn is_ready(&self, _write: bool) -> Result<bool, Errno> {
-        Ok(true)
-    }
-
-    fn ioctl(&self, _cmd: IoctlCmd, _ptr: usize, _lim: usize) -> Result<usize, Errno> {
-        Err(Errno::InvalidArgument)
-    }
-}
-
-impl Random {
-    pub fn set_state(&self, state: u32) {
-        self.state.store(state, Ordering::Release);
-    }
-
-    pub fn read_single(&self) -> u32 {
-        let mut x = self.state.load(Ordering::Acquire);
-        x ^= x << 13;
-        x ^= x >> 7;
-        x ^= x << 17;
-        self.state.store(x, Ordering::Release);
-        x
-    }
-}
-
-pub static RANDOM: Random = Random { state: AtomicU32::new(0) };
-pub static ZERO: Zero = Zero;
@@ -131,7 +131,6 @@ impl IntSource for Pl011 {
    fn handle_irq(&self) -> Result<(), Errno> {
        let inner = self.inner.get().lock();
        inner.regs.ICR.write(ICR::ALL::CLEAR);
-
        let byte = inner.regs.DR.get();
        drop(inner);

@@ -60,7 +60,7 @@ pub trait TtyDevice<const N: usize>: SerialDevice {
            },
            IoctlCmd::TtySetPgrp => {
                let src = arg::struct_ref::<u32>(ptr)?;
-                self.ring().inner.lock().fg_pgid = Some(Pid::try_from(*src)?);
+                self.ring().inner.lock().fg_pgid = Some(unsafe { Pid::from_raw(*src) });
                Ok(0)
            },
            _ => Err(Errno::InvalidArgument)
@@ -26,7 +26,7 @@ pub fn root() -> &'static VnodeRef {
    DEVFS_ROOT.get()
 }

-pub fn add_named_char_device(dev: &'static dyn CharDevice, name: &str) -> Result<(), Errno> {
+fn _add_char_device(dev: &'static dyn CharDevice, name: &str) -> Result<(), Errno> {
    infoln!("Add char device: {}", name);

    let node = Vnode::new(name, VnodeKind::Char, Vnode::CACHE_STAT);
@@ -56,5 +56,5 @@ pub fn add_char_device(dev: &'static dyn CharDevice, kind: CharDeviceType) -> Re

    let name = core::str::from_utf8(&buf[..=prefix.len()]).map_err(|_| Errno::InvalidArgument)?;

-    add_named_char_device(dev, name)
+    _add_char_device(dev, name)
 }
@@ -8,7 +8,6 @@ use vfs::VnodeRef;
 use memfs::BlockAllocator;

 pub mod devfs;
-pub mod sysfs;

 /// Allocator implementation for memfs
 #[derive(Clone, Copy)]
@@ -33,9 +32,9 @@ unsafe impl BlockAllocator for MemfsBlockAlloc {
 pub fn create_filesystem(options: &MountOptions) -> Result<VnodeRef, Errno> {
    let fs_name = options.fs.unwrap();

-    match fs_name {
-        "devfs" => Ok(devfs::root().clone()),
-        "sysfs" => Ok(sysfs::root().clone()),
-        _ => todo!()
+    if fs_name == "devfs" {
+        Ok(devfs::root().clone())
+    } else {
+        todo!();
    }
 }
@@ -1,169 +0,0 @@
-use crate::util::InitOnce;
-use alloc::boxed::Box;
-use core::sync::atomic::{AtomicUsize, Ordering};
-use fs_macros::auto_inode;
-use libsys::{
-    error::Errno,
-    stat::{FileMode, OpenFlags, Stat},
-};
-use vfs::{CharDevice, CharDeviceWrapper, Vnode, VnodeImpl, VnodeKind, VnodeRef};
-use core::fmt::{self, Write};
-use core::str::FromStr;
-use crate::debug::{self, Level};
-
-struct NodeData<
-    R: Fn(&mut [u8]) -> Result<usize, Errno>,
-    W: Fn(&[u8]) -> Result<usize, Errno>,
-> {
-    read_func: R,
-    write_func: W,
-}
-
-struct BufferWriter<'a> {
-    dst: &'a mut [u8],
-    pos: usize
-}
-
-impl<'a> fmt::Write for BufferWriter<'a> {
-    fn write_str(&mut self, s: &str) -> fmt::Result {
-        for byte in s.bytes() {
-            if self.pos == self.dst.len() {
-                todo!();
-            }
-            self.dst[self.pos] = byte;
-            self.pos += 1;
-        }
-        Ok(())
-    }
-}
-
-impl<'a> BufferWriter<'a> {
-    pub const fn new(dst: &'a mut [u8]) -> Self {
-        Self { dst, pos: 0 }
-    }
-
-    pub const fn count(&self) -> usize {
-        self.pos
-    }
-}
-
-#[auto_inode]
-impl<
-        R: Fn(&mut [u8]) -> Result<usize, Errno>,
-        W: Fn(&[u8]) -> Result<usize, Errno>,
-    > VnodeImpl for NodeData<R, W>
-{
-    fn open(&mut self, _node: VnodeRef, _mode: OpenFlags) -> Result<usize, Errno> {
-        Ok(0)
-    }
-
-    fn close(&mut self, _node: VnodeRef) -> Result<(), Errno> {
-        Ok(())
-    }
-
-    fn read(&mut self, _node: VnodeRef, pos: usize, data: &mut [u8]) -> Result<usize, Errno> {
-        if pos != 0 {
-            // TODO handle this
-            Ok(0)
-        } else {
-            (self.read_func)(data)
-        }
-    }
-
-    fn write(&mut self, _node: VnodeRef, pos: usize, data: &[u8]) -> Result<usize, Errno> {
-        if pos != 0 {
-            todo!();
-        }
-        (self.write_func)(data)
-    }
-}
-
-impl<
-        R: Fn(&mut [u8]) -> Result<usize, Errno>,
-        W: Fn(&[u8]) -> Result<usize, Errno>,
-    > NodeData<R, W>
-{
-    pub const fn new(read_func: R, write_func: W) -> Self {
-        Self {
-            read_func,
-            write_func,
-        }
-    }
-}
-
-static SYSFS_ROOT: InitOnce<VnodeRef> = InitOnce::new();
-static TEST_COUNTER: AtomicUsize = AtomicUsize::new(0);
-
-// TODO subdirs
-fn add_generic_node<R, W>(parent: Option<VnodeRef>, name: &str, mode: FileMode, read: R, write: W)
-where
-    R: Fn(&mut [u8]) -> Result<usize, Errno> + 'static,
-    W: Fn(&[u8]) -> Result<usize, Errno> + 'static,
-{
-    let node = Vnode::new(name, VnodeKind::Regular, Vnode::CACHE_STAT);
-    node.props_mut().mode = mode | FileMode::S_IFREG;
-    node.set_data(Box::new(NodeData::new(read, write)));
-
-    if let Some(parent) = parent {
-        parent.attach(node);
-    } else {
-        SYSFS_ROOT.get().attach(node);
-    }
-}
-
-pub fn add_read_write_node<R, W>(parent: Option<VnodeRef>, name: &str, read: R, write: W)
-where
-    R: Fn(&mut [u8]) -> Result<usize, Errno> + 'static,
-    W: Fn(&[u8]) -> Result<usize, Errno> + 'static,
-{
-    add_generic_node(parent, name, FileMode::from_bits(0o600).unwrap(), read, write)
-}
-
-pub fn add_read_node<R>(parent: Option<VnodeRef>, name: &str, read: R) where R: Fn(&mut [u8]) -> Result<usize, Errno> + 'static {
-    add_generic_node(parent, name, FileMode::from_bits(0o400).unwrap(), read, |_| Err(Errno::ReadOnly))
-}
-
-pub fn add_directory(parent: Option<VnodeRef>, name: &str) -> Result<VnodeRef, Errno> {
-    let node = Vnode::new(name, VnodeKind::Directory, Vnode::CACHE_READDIR | Vnode::CACHE_STAT);
-    node.props_mut().mode = FileMode::from_bits(0o500).unwrap() | FileMode::S_IFDIR;
-
-    if let Some(parent) = parent {
-        parent.attach(node.clone());
-    } else {
-        SYSFS_ROOT.get().attach(node.clone());
-    }
-
-    Ok(node)
-}
-
-pub fn root() -> &'static VnodeRef {
-    SYSFS_ROOT.get()
-}
-
-pub fn init() {
-    let node = Vnode::new("", VnodeKind::Directory, Vnode::CACHE_READDIR | Vnode::CACHE_STAT);
-    node.props_mut().mode = FileMode::default_dir();
-    SYSFS_ROOT.init(node);
-
-    let debug_dir = add_directory(None, "debug").unwrap();
-
-    add_read_write_node(Some(debug_dir.clone()), "level", |buf| {
-        let mut writer = BufferWriter::new(buf);
-        write!(&mut writer, "{}\n", debug::LEVEL as u32).map_err(|_| Errno::InvalidArgument)?;
-        Ok(writer.count())
-    }, |buf| {
-        let s = core::str::from_utf8(buf).map_err(|_| Errno::InvalidArgument)?;
-        let value = u32::from_str(s).map_err(|_| Errno::InvalidArgument).and_then(Level::try_from)?;
-        todo!()
-    });
-
-    add_read_node(None, "uptime", |buf| {
-        use crate::arch::machine;
-        use crate::dev::timer::TimestampSource;
-
-        let mut writer = BufferWriter::new(buf);
-        let time = machine::local_timer().timestamp()?;
-        write!(&mut writer, "{} {}\n", time.as_secs(), time.subsec_nanos()).map_err(|_| Errno::InvalidArgument)?;
-        Ok(writer.count())
-    });
-}
@@ -1,15 +1,19 @@
 //! osdve5 crate (lol)
 #![feature(
+    asm,
+    global_asm,
    const_for,
    const_mut_refs,
+    const_raw_ptr_deref,
    const_fn_fn_ptr_basics,
    const_fn_trait_bound,
    const_trait_impl,
+    const_panic,
    panic_info_message,
    alloc_error_handler,
    linked_list_cursors,
    const_btree_new,
-    asm_const,
+    maybe_uninit_uninit_array
 )]
 #![no_std]
 #![no_main]
@@ -37,15 +41,23 @@ pub mod sync;
 pub mod syscall;
 pub mod util;

-use core::arch::asm;
-
 #[panic_handler]
 fn panic_handler(pi: &core::panic::PanicInfo) -> ! {
    unsafe {
        asm!("msr daifset, #2");
+        use crate::arch::platform::cpu::{self, Cpu};
+
+        crate::arch::platform::smp::send_ipi(true, (1 << cpu::count()) - 1, 0);
    }

-    errorln!("Panic: {:?}", pi);
+    use cortex_a::registers::MPIDR_EL1;
+    use tock_registers::interfaces::Readable;
+
+    errorln!("Panic on node{}: {:?}", MPIDR_EL1.get() & 0xF, pi);
    // TODO
-    loop {}
+    loop {
+        unsafe {
+            asm!("wfe");
+        }
+    }
 }
@@ -6,7 +6,6 @@ use crate::mem::{
 };
 use core::ops::{Index, IndexMut};
 use libsys::{error::Errno, mem::memset};
-use core::arch::asm;

 /// Transparent wrapper structure representing a single
 /// translation table entry
@@ -2,8 +2,14 @@

 use crate::init;
 use crate::sync::IrqSafeSpinLock;
-use alloc::collections::BTreeMap;
-use libsys::proc::{Tid, Pid};
+use crate::mem;
+use alloc::{
+    boxed::Box,
+    collections::{BTreeMap},
+};
+use core::sync::atomic::{AtomicUsize, Ordering};
+use crate::arch::platform::cpu::{self, Cpu};
+use libsys::proc::Pid;

 pub mod elf;
 pub mod thread;
@@ -18,30 +24,91 @@ pub mod wait;

 pub mod sched;
 pub use sched::Scheduler;
-pub(self) use sched::SCHED;
+//pub(self) use sched::SCHED;

-/// Performs a task switch.
-///
-/// See [Scheduler::switch]
-pub fn switch() {
-    SCHED.switch(false);
+//<<<<<<< HEAD
+// <<<<<<< HEAD
+// macro_rules! spawn {
+//     (fn ($dst_arg:ident : usize) $body:block, $src_arg:expr) => {{
+//         #[inline(never)]
+//         extern "C" fn __inner_func($dst_arg : usize) -> ! {
+//             let __res = $body;
+//             {
+//                 #![allow(unreachable_code)]
+//                 SCHED.current_process().exit(__res);
+//                 panic!();
+//             }
+//         }
+//
+//         let __proc = $crate::proc::Process::new_kernel(__inner_func, $src_arg).unwrap();
+//         $crate::proc::SCHED.enqueue(__proc.id());
+//     }};
+//
+//     (fn () $body:block) => (spawn!(fn (_arg: usize) $body, 0usize))
+// }
+
+///// Performs a task switch.
+/////
+///// See [Scheduler::switch]
+//pub fn switch() {
+//    SCHED.switch(false);
+//}
+
+// ///
+// pub fn process(id: Pid) -> ProcessRef {
+//     PROCESSES.lock().get(&id).unwrap().clone()
+// }
+
+macro_rules! spawn {
+    (fn ($dst_arg:ident : usize) $body:block, $src_arg:expr) => {{
+        #[inline(never)]
+        extern "C" fn __inner_func($dst_arg : usize) -> ! {
+            let __res = $body;
+            {
+                todo!();
+                // #![allow(unreachable_code)]
+                // SCHED.current_process().exit(__res);
+                panic!();
+            }
+        }
+
+        let __proc = $crate::proc::Process::new_kernel(__inner_func, $src_arg).unwrap();
+        $crate::proc::sched::enqueue(__proc.id());
+    }};
+
+    (fn () $body:block) => (spawn!(fn (_arg: usize) $body, 0usize))
 }

+// /// Global list of all processes in the system
+// // =======
+// /// Performs a task switch.
+// ///
+// /// See [Scheduler::switch]
+// pub fn switch() {
+//     SCHED.switch(false);
+// }
+
+// >>>>>>> feat/thread
 pub(self) static PROCESSES: IrqSafeSpinLock<BTreeMap<Pid, ProcessRef>> =
    IrqSafeSpinLock::new(BTreeMap::new());

-pub(self) static THREADS: IrqSafeSpinLock<BTreeMap<Tid, ThreadRef>> =
+pub(self) static THREADS: IrqSafeSpinLock<BTreeMap<u32, ThreadRef>> =
    IrqSafeSpinLock::new(BTreeMap::new());

-/// Sets up initial process and enters it.
-///
-/// See [Scheduler::enter]
-///
-/// # Safety
-///
-/// Unsafe: May only be called once.
-pub unsafe fn enter() -> ! {
-    SCHED.init();
-    Process::new_kernel(init::init_fn, 0).unwrap().enqueue();
-    SCHED.enter();
+pub unsafe fn enter(is_bsp: bool) -> ! {
+    static COUNTER: AtomicUsize = AtomicUsize::new(0);
+
+    let sched = Cpu::get().scheduler();
+    sched.init();
+
+    COUNTER.fetch_add(1, Ordering::Release);
+    while COUNTER.load(Ordering::Acquire) != cpu::count() {
+        cortex_a::asm::nop();
+    }
+
+    if is_bsp {
+        Process::new_kernel(init::init_fn, 0).unwrap().enqueue();
+    }
+
+    sched.enter();
 }
@@ -6,9 +6,9 @@ use crate::mem::{
    virt::{MapAttributes, Space},
 };
 use crate::proc::{
-    wait::Wait, Context, ProcessIo, Thread, ThreadRef, ThreadState, PROCESSES, SCHED, Tid,
+    wait::Wait, Context, ProcessIo, Thread, ThreadRef, ThreadState, PROCESSES, sched,
 };
-use crate::sync::{IrqSafeSpinLock};
+use crate::sync::IrqSafeSpinLock;
 use alloc::{rc::Rc, vec::Vec};
 use core::sync::atomic::{AtomicU32, Ordering};
 use libsys::{
@@ -18,7 +18,6 @@ use libsys::{
    signal::Signal,
    ProgramArgs,
 };
-use core::arch::asm;

 /// Wrapper type for a process struct reference
 pub type ProcessRef = Rc<Process>;
@@ -40,7 +39,7 @@ struct ProcessInner {
    ppid: Option<Pid>,
    sid: Pid,
    exit: Option<ExitCode>,
-    threads: Vec<Tid>,
+    threads: Vec<u32>,
 }

 /// Structure describing an operating system process
@@ -138,7 +137,7 @@ impl Process {
    pub fn enqueue(&self) {
        let inner = self.inner.lock();
        for &tid in inner.threads.iter() {
-            SCHED.enqueue(tid);
+            sched::enqueue(tid);
        }
    }

@@ -147,73 +146,116 @@ impl Process {
        PROCESSES.lock().get(&pid).cloned()
    }

-    fn find1(a: u32) -> Option<usize> {
-        for i in 0..32 {
-            if a & (1 << i) != 0 {
-                return Some(i);
-            }
-        }
-        None
-    }
-
-    /// Handles all pending signals (when returning from aborted syscall)
-    pub fn handle_pending_signals(&self) {
-        let mut lock = self.inner.lock();
-        let ttbr0 = lock.space.as_mut().unwrap().address_phys() | ((lock.id.asid() as usize) << 48);
-        let main_thread = Thread::get(lock.threads[0]).unwrap();
-        drop(lock);
-
-        loop {
-            let state = self.signal_state.load(Ordering::Acquire);
-            if let Some(signal) = Self::find1(state).map(|e| Signal::try_from(e as u32).unwrap()) {
-                self.signal_state.fetch_and(!(1 << (signal as u32)), Ordering::Release);
-                main_thread.clone().enter_signal(signal, ttbr0);
-            } else {
-                break;
-            }
-        }
-    }
+// <<<<<<< HEAD
+    // /// Schedules an initial thread for execution
+    // ///
+    // /// # Safety
+    // ///
+    // /// Unsafe: only allowed to be called once, repeated calls
+    // ///         will generate undefined behavior
+    // pub unsafe fn enter(cpu: u32, proc: ProcessRef) -> ! {
+    //     // FIXME use some global lock to guarantee atomicity of thread entry?
+    //     proc.inner.lock().state = State::Running;
+    //     proc.cpu.store(cpu, Ordering::SeqCst);
+    //     let ctx = proc.ctx.get();

+    //     // I don't think this is bad: process can't be dropped fully unless
+    //     // it's been reaped (and this function won't run for such process)
+    //     // drop(proc);
+    //     (&mut *ctx).enter()
+    // }
+// =======
    /// Sets a pending signal for a process
    pub fn set_signal(&self, signal: Signal) {
-        let mut lock = self.inner.lock();
-        let ttbr0 = lock.space.as_mut().unwrap().address_phys() | ((lock.id.asid() as usize) << 48);
-        let main_thread = Thread::get(lock.threads[0]).unwrap();
-        drop(lock);
+        todo!();
+        // let mut lock = self.inner.lock();
+        // let ttbr0 = lock.space.as_mut().unwrap().address_phys() | ((lock.id.asid() as usize) << 48);
+        // let main_thread = Thread::get(lock.threads[0]).unwrap();
+        // drop(lock);

-        // TODO check that `signal` is not a fault signal
-        //      it is illegal to call this function with
-        //      fault signals
-
-        match main_thread.state() {
-            ThreadState::Running => {
-                main_thread.enter_signal(signal, ttbr0);
-            }
-            ThreadState::Waiting => {
-                self.signal_state.fetch_or(1 << (signal as u32), Ordering::Release);
-                main_thread.interrupt_wait(true);
-            }
-            ThreadState::Ready => {
-                main_thread.clone().setup_signal(signal, ttbr0);
-                main_thread.interrupt_wait(false);
-            }
-            ThreadState::Finished => {
-                // TODO report error back
-                todo!()
-            }
-        }
+        // // TODO check that `signal` is not a fault signal
+        // //      it is illegal to call this function with
+        // //      fault signals
+        // match main_thread.state() {
+        //     ThreadState::Running => {
+        //         main_thread.enter_signal(signal, ttbr0);
+        //     }
+        //     ThreadState::Waiting => {
+        //         main_thread.clone().setup_signal(signal, ttbr0);
+        //         main_thread.interrupt_wait(true);
+        //     }
+        //     ThreadState::Ready => {
+        //         main_thread.clone().setup_signal(signal, ttbr0);
+        //         main_thread.interrupt_wait(false);
+        //     }
+        //     ThreadState::Finished => {
+        //         // TODO report error back
+        //         todo!()
+        //     }
+        // }
    }

    /// Immediately delivers a signal to requested thread
    pub fn enter_fault_signal(&self, thread: ThreadRef, signal: Signal) {
-        let mut lock = self.inner.lock();
-        let ttbr0 = lock.space.as_mut().unwrap().address_phys() | ((lock.id.asid() as usize) << 48);
-        drop(lock);
-        thread.enter_signal(signal, ttbr0);
+        todo!();
+        // let mut lock = self.inner.lock();
+        // let ttbr0 = lock.space.as_mut().unwrap().address_phys() | ((lock.id.asid() as usize) << 48);
+        // thread.enter_signal(signal, ttbr0);
    }

+    // /// Schedules a next thread for execution
+    // ///
+    // /// # Safety
+    // ///
+    // /// Unsafe:
+    // ///
+    // /// * Does not ensure src and dst threads are not the same thread
+    // /// * Does not ensure src is actually current context
+    // pub unsafe fn switch(cpu: u32, src: ProcessRef, dst: ProcessRef, discard: bool) {
+    //     {
+    //         let mut src_lock = src.inner.lock();
+    //         let mut dst_lock = dst.inner.lock();
+
+    //         if !discard {
+    //             assert_eq!(src_lock.state, State::Running);
+    //             src_lock.state = State::Ready;
+    //         }
+    //         assert!(dst_lock.state == State::Ready || dst_lock.state == State::Waiting);
+    //         dst_lock.state = State::Running;
+
+    //         src.cpu.store(Self::CPU_NONE, Ordering::SeqCst);
+    //         dst.cpu.store(cpu, Ordering::SeqCst);
+    //     }
+
+    //     let src_ctx = src.ctx.get();
+    //     let dst_ctx = dst.ctx.get();
+
+    //     // See "drop" note in Process::enter()
+    //     // drop(src);
+    //     // drop(dst);
+
+    //     (&mut *src_ctx).switch(&mut *dst_ctx);
+    // }
+
+    // /// Suspends current process with a "waiting" status
+    // pub fn enter_wait(&self) {
+    //     let drop = {
+    //         let mut lock = self.inner.lock();
+    //         let drop = lock.state == State::Running;
+    //         lock.state = State::Waiting;
+    //         sched::dequeue(lock.id);
+    //         // SCHED.dequeue(lock.id);
+    //         drop
+    //     };
+    //     if drop {
+    //         sched::switch(true);
+    //         // todo!();
+    //         // SCHED.switch(true);
+    //     }
+    // }
+
    /// Crates a new thread in the process
-    pub fn new_user_thread(&self, entry: usize, stack: usize, arg: usize) -> Result<Tid, Errno> {
+    pub fn new_user_thread(&self, entry: usize, stack: usize, arg: usize) -> Result<u32, Errno> {
        let mut lock = self.inner.lock();

        let space_phys = lock.space.as_mut().unwrap().address_phys();
@@ -222,11 +264,32 @@ impl Process {
        let thread = Thread::new_user(lock.id, entry, stack, arg, ttbr0)?;
        let tid = thread.id();
        lock.threads.push(tid);
-        SCHED.enqueue(tid);
+        sched::enqueue(tid);

        Ok(tid)
    }

+    // /// Creates a new kernel process
+    // pub fn new_kernel(entry: extern "C" fn(usize) -> !, arg: usize) -> Result<ProcessRef, Errno> {
+    //     let id = Pid::new_kernel();
+    //     let res = Rc::new(Self {
+    //         ctx: UnsafeCell::new(Context::kernel(entry as usize, arg)),
+    //         io: IrqSafeSpinLock::new(ProcessIo::new()),
+    //         exit_wait: Wait::new(),
+    //         inner: IrqSafeSpinLock::new(ProcessInner {
+    //             id,
+    //             exit: None,
+    //             space: None,
+    //             wait_flag: false,
+    //             state: State::Ready,
+    //         }),
+    //         cpu: AtomicU32::new(Self::CPU_NONE),
+    //     });
+    //     debugln!("New kernel process: {}", id);
+    //     assert!(PROCESSES.lock().insert(id, res.clone()).is_none());
+    //     Ok(res)
+    // }
+
    /// Creates a "fork" of the process, cloning its address space and
    /// resources
    pub fn fork(&self, frame: &mut ExceptionFrame) -> Result<Pid, Errno> {
@@ -261,7 +324,8 @@ impl Process {
        debugln!("Process {:?} forked into {:?}", src_inner.id, dst_id);
        assert!(PROCESSES.lock().insert(dst_id, dst).is_none());

-        SCHED.enqueue(tid);
+        sched::enqueue(tid);
+        // SCHED.enqueue(dst_id);

        Ok(dst_id)
    }
@@ -278,18 +342,15 @@ impl Process {
        lock.state = ProcessState::Finished;

        for &tid in lock.threads.iter() {
-            let thread = Thread::get(tid).unwrap();
-            if thread.state() == ThreadState::Waiting {
-                todo!()
-            }
-            thread.terminate(status);
-            SCHED.dequeue(tid);
+            Thread::get(tid).unwrap().terminate(status);
+            sched::dequeue(tid);
+            // SCHED.dequeue(tid);
        }

        if let Some(space) = lock.space.take() {
            unsafe {
                Space::release(space);
-                Process::invalidate_asid((lock.id.asid() as usize) << 48);
+                asm!("tlbi aside1, {}", in(reg) ((lock.id.asid() as usize) << 48));
            }
        }

@@ -302,7 +363,7 @@ impl Process {
        self.exit_wait.wakeup_all();

        if is_running {
-            SCHED.switch(true);
+            sched::switch(true);
            panic!("This code should never run");
        }
    }
@@ -326,8 +387,9 @@ impl Process {
            lock.threads.retain(|&e| e != tid);

            thread.terminate(status);
-            SCHED.dequeue(tid);
-            debugln!("Thread {:?} terminated", tid);
+            todo!();
+            // SCHED.dequeue(tid);
+            debugln!("Thread {} terminated", tid);

            switch
        };
@@ -335,7 +397,8 @@ impl Process {
        if switch {
            // TODO retain thread ID in process "finished" list and
            //      drop it when process finishes
-            SCHED.switch(true);
+            // SCHED.switch(true);
+            todo!();
            panic!("This code should not run");
        } else {
            // Can drop this thread: it's not running
@@ -443,7 +506,7 @@ impl Process {
        let mut data_offset = 0usize;
        for arg in argv.iter() {
            // XXX this is really unsafe and I am not really sure ABI will stay like this XXX
-            Self::write_paged(space, base + offset, base + data_offset)?;
+            Self::write_paged(space, base + offset + 0, base + data_offset)?;
            Self::write_paged(space, base + offset + 8, arg.len())?;
            offset += 16;
            data_offset += arg.len();
@@ -461,36 +524,50 @@ impl Process {
        Ok(base + offset)
    }

-    pub fn asid(&self) -> usize {
-        (self.id().asid() as usize) << 48
-    }
-
-    pub fn invalidate_tlb(&self) {
-        Process::invalidate_asid(self.asid());
-    }
-
-    #[inline]
-    pub fn invalidate_asid(asid: usize) {
-        unsafe {
-            asm!("tlbi aside1, {}", in(reg) asid);
-        }
-    }
-
-    /// Loads a new program into current process address space
-    pub fn execve<F: FnOnce(&mut Space) -> Result<usize, Errno>>(
-        loader: F,
-        argv: &[&str],
-    ) -> Result<(), Errno> {
-        unsafe {
-            // Run with interrupts disabled
-            asm!("msr daifset, #2");
-        }
+   /// Loads a new program into current process address space
+   pub fn execve<F: FnOnce(&mut Space) -> Result<usize, Errno>>(
+       loader: F,
+       argv: &[&str],
+   ) -> Result<(), Errno> {
+       unsafe {
+           // Run with interrupts disabled
+           asm!("msr daifset, #2");
+       }

+// <<<<<<< HEAD
+//         let proc = sched::current_process();
+//         let mut lock = proc.inner.lock();
+//         if lock.id.is_kernel() {
+//             let mut proc_lock = PROCESSES.lock();
+//             let old_pid = lock.id;
+//             assert!(
+//                 proc_lock.remove(&old_pid).is_some(),
+//                 "Failed to downgrade kernel process (remove kernel pid)"
+//             );
+//             lock.id = Pid::new_user();
+//             debugln!(
+//                 "Process downgrades from kernel to user: {} -> {}",
+//                 old_pid,
+//                 lock.id
+//             );
+//             assert!(proc_lock.insert(lock.id, proc.clone()).is_none());
+//             unsafe {
+//                 use crate::arch::platform::cpu::Cpu;
+//                 Cpu::get().scheduler().hack_current_pid(lock.id);
+//             }
+//         } else {
+//             // Invalidate user ASID
+//             let input = (lock.id.asid() as usize) << 48;
+//             unsafe {
+//                 asm!("tlbi aside1, {}", in(reg) input);
+//             }
+// =======
        let proc = Process::current();
        let mut process_lock = proc.inner.lock();

        if process_lock.threads.len() != 1 {
            todo!();
+// >>>>>>> feat/thread
        }

        let thread = Thread::get(process_lock.threads[0]).unwrap();
@@ -533,6 +610,7 @@ impl Process {
        let entry = loader(new_space)?;
        let arg = Self::store_arguments(new_space, argv)?;

+        debugln!("Will now enter at {:#x}", entry);
        // TODO drop old address space
        process_lock.space = Some(new_space);

@@ -540,7 +618,7 @@ impl Process {
            // TODO drop old context
            let ctx = thread.ctx.get();
            let asid = (process_lock.id.asid() as usize) << 48;
-            Process::invalidate_asid(asid);
+            asm!("tlbi aside1, {}", in(reg) asid);

            ctx.write(Context::user(
                entry,
@@ -1,15 +1,17 @@
 //!
 use crate::proc::{Thread, ThreadRef, THREADS};
-use crate::sync::IrqSafeSpinLock;
 use crate::util::InitOnce;
-use libsys::proc::Tid;
 use alloc::{collections::VecDeque, rc::Rc};
-use core::arch::asm;
+use crate::sync::{IrqSafeSpinLock, IrqSafeSpinLockGuard};
+use crate::arch::platform::cpu::{self, Cpu};
+use cortex_a::registers::{MPIDR_EL1, DAIF};
+use core::ops::Deref;
+use tock_registers::interfaces::Readable;

 struct SchedulerInner {
-    queue: VecDeque<Tid>,
-    idle: Option<Tid>,
-    current: Option<Tid>,
+    queue: VecDeque<u32>,
+    idle: Option<u32>,
+    current: Option<u32>,
 }

 /// Process scheduler state and queues
@@ -25,15 +27,31 @@ impl SchedulerInner {
            current: None,
        };

-        let idle = Thread::new_kernel(None, idle_fn, 0).unwrap().id();
-        assert_eq!(idle, Tid::IDLE);
-        this.idle = Some(idle);
+        this.idle = Some(Thread::new_kernel(None, idle_fn, 0).unwrap().id());

        this
    }
 }

 impl Scheduler {
+    ///
+    pub const fn new() -> Self {
+        Self {
+            inner: InitOnce::new()
+        }
+    }
+
+    ///
+    pub fn queue_size(&self) -> usize {
+        let lock = self.inner.get().lock();
+        let c = if lock.current.is_some() {
+            1
+        } else {
+            0
+        };
+        lock.queue.len() + c
+    }
+
    /// Initializes inner data structure:
    ///
    /// * idle thread
@@ -43,12 +61,12 @@ impl Scheduler {
    }

    /// Schedules a thread for execution
-    pub fn enqueue(&self, tid: Tid) {
+    pub fn enqueue(&self, tid: u32) {
        self.inner.get().lock().queue.push_back(tid);
    }

    /// Removes given `tid` from execution queue
-    pub fn dequeue(&self, tid: Tid) {
+    pub fn dequeue(&self, tid: u32) {
        self.inner.get().lock().queue.retain(|&p| p != tid)
    }

@@ -71,7 +89,7 @@ impl Scheduler {
        };

        asm!("msr daifset, #2");
-        Thread::enter(thread)
+        Thread::enter((MPIDR_EL1.get() & 0xF) as u32, thread)
    }

    /// This hack is required to be called from execve() when downgrading current
@@ -80,7 +98,7 @@ impl Scheduler {
    /// # Safety
    ///
    /// Unsafe: only allowed to be called from Process::execve()
-    pub unsafe fn hack_current_tid(&self, old: Tid, new: Tid) {
+    pub unsafe fn hack_current_tid(&self, old: u32, new: u32) {
        let mut lock = self.inner.get().lock();
        match lock.current {
            Some(t) if t == old => {
@@ -92,14 +110,20 @@ impl Scheduler {

    /// Switches to the next task scheduled for execution. If there're
    /// none present in the queue, switches to the idle task.
-    pub fn switch(&self, discard: bool) {
+    pub fn switch(&self, discard: bool, sched_lock: IrqSafeSpinLockGuard<()>) {
        let (from, to) = {
            let mut inner = self.inner.get().lock();
            let current = inner.current.unwrap();

-            if !discard && current != Tid::IDLE {
+//<<<<<<< HEAD
+            if !discard && current != inner.idle.unwrap() {
+//=======
+//            if !discard && current != 0 {
+//>>>>>>> feat/thread
                // Put the process into the back of the queue
-                inner.queue.push_back(current);
+                if !enqueue_somewhere_else((MPIDR_EL1.get() & 0xF) as usize, current, &sched_lock) {
+                    inner.queue.push_back(current);
+                }
            }

            let next = if inner.queue.is_empty() {
@@ -122,8 +146,13 @@ impl Scheduler {

        if !Rc::ptr_eq(&from, &to) {
            unsafe {
+//<<<<<<< HEAD
+                drop(sched_lock);
+//                Process::switch((MPIDR_EL1.get() & 0xF) as u32, from, to, discard);
+//=======
                asm!("msr daifset, #2");
-                Thread::switch(from, to, discard);
+                Thread::switch((MPIDR_EL1.get() & 0xF) as u32, from, to, discard);
+//>>>>>>> feat/thread
            }
        }
    }
@@ -143,10 +172,16 @@ impl Scheduler {
    // }
 }

-/// Returns `true` if the scheduler has been initialized
-pub fn is_ready() -> bool {
-    SCHED.inner.is_initialized()
-}
+// <<<<<<< HEAD
+// // pub fn is_ready() -> bool {
+// //     SCHED.inner.is_initialized()
+// // }
+// =======
+// /// Returns `true` if the scheduler has been initialized
+// pub fn is_ready() -> bool {
+//     SCHED.inner.is_initialized()
+// }
+// >>>>>>> feat/thread

 #[inline(never)]
 extern "C" fn idle_fn(_a: usize) -> ! {
@@ -155,8 +190,94 @@ extern "C" fn idle_fn(_a: usize) -> ! {
    }
 }

+pub fn current_thread() -> ThreadRef {
+    let guard = SCHED_LOCK.lock();
+    unsafe { Cpu::get().scheduler().current_thread() }
+}
+
+/// Performs a task switch.
+///
+/// See [Scheduler::switch]
+pub fn switch(discard: bool) {
+    assert!(DAIF.matches_all(DAIF::I::SET));
+    let guard = SCHED_LOCK.lock();
+    unsafe { Cpu::get().scheduler().switch(discard, guard); }
+}
+
+///
+pub fn enqueue_to(cpu: usize, tid: u32) {
+    todo!()
+    //let _lock = SCHED_LOCK.lock();
+    //debugln!("Queue {} to cpu{}", pid, cpu);
+    //unsafe {
+    //    cpu::by_index(cpu).scheduler().enqueue(pid)
+    //}
+}
+
+///
+pub fn enqueue(tid: u32) {
+    let _lock = SCHED_LOCK.lock();
+    let mut min_idx = 0;
+    let mut min_cnt = usize::MAX;
+    for (i, cpu) in unsafe { cpu::cpus() }.enumerate() {
+        let size = cpu.scheduler().queue_size();
+        if size < min_cnt {
+            min_cnt = size;
+            min_idx = i;
+        }
+    }
+
+    // debugln!("Queue {} to cpu{}", pid, min_idx);
+    unsafe {
+        cpu::by_index(min_idx).scheduler().enqueue(tid)
+    }
+}
+
+///
+pub fn enqueue_somewhere_else(ignore: usize, tid: u32, _guard: &IrqSafeSpinLockGuard<()>) -> bool {
+    let mut min_idx = 0;
+    //let mut min_cnt = usize::MAX;
+    static mut LAST: usize = 0;
+    //for (i, cpu) in unsafe { cpu::cpus() }.enumerate() {
+    //for (i, cpu) in wacky_cpu_iterate() {
+    //    if i == ignore {
+    //        continue;
+    //    }
+    //    let size = cpu.scheduler().queue_size();
+    //    if size < min_cnt {
+    //        min_cnt = size;
+    //        min_idx = i;
+    //    }
+    //}
+    unsafe {
+        LAST = (LAST + 1) % cpu::count();
+        min_idx = LAST;
+    }
+
+    if min_idx == ignore {
+        false
+    } else {
+        unsafe {
+            cpu::by_index(min_idx).scheduler().enqueue(tid)
+        }
+        true
+    }
+}
+
+///
+pub fn dequeue(tid: u32) {
+    // TODO process can be rescheduled to other CPU between scheduler locks
+    let lock = SCHED_LOCK.lock();
+    let cpu_id = Thread::get(tid).unwrap().cpu();
+    unsafe {
+        cpu::by_index(cpu_id as usize).scheduler().dequeue(tid);
+    }
+}
+
+static SCHED_LOCK: IrqSafeSpinLock<()> = IrqSafeSpinLock::new(());
+
 // TODO maybe move this into a per-CPU struct
-/// Global scheduler struct
-pub static SCHED: Scheduler = Scheduler {
-    inner: InitOnce::new(),
-};
+// /// Global scheduler struct
+// pub static SCHED: Scheduler = Scheduler {
+//     inner: InitOnce::new(),
+// };
@@ -3,7 +3,7 @@
 use crate::arch::aarch64::exception::ExceptionFrame;
 use crate::proc::{
    wait::{Wait, WaitStatus},
-    Process, ProcessRef, SCHED, THREADS,
+    Process, ProcessRef, sched, THREADS,
 };
 use crate::sync::IrqSafeSpinLock;
 use crate::util::InitOnce;
@@ -12,7 +12,7 @@ use core::cell::UnsafeCell;
 use core::sync::atomic::{AtomicU32, Ordering};
 use libsys::{
    error::Errno,
-    proc::{ExitCode, Pid, Tid},
+    proc::{ExitCode, Pid},
    signal::Signal,
 };

@@ -35,7 +35,7 @@ pub enum State {
 }

 struct ThreadInner {
-    id: Tid,
+    id: u32,
    state: State,
    owner: Option<Pid>,
    pending_wait: Option<&'static Wait>,
@@ -52,23 +52,21 @@ pub struct Thread {
    pub(super) ctx: UnsafeCell<Context>,
    signal_ctx: UnsafeCell<Context>,
    signal_pending: AtomicU32,
+    cpu: AtomicU32,
 }

 impl Thread {
+    const CPU_NONE: u32 = u32::MAX;
+
    /// Returns currently active thread [Rc]-reference
    #[inline]
    pub fn current() -> ThreadRef {
-        SCHED.current_thread()
-    }
-
-    /// Returns `true` if the thread is currently executing a signal handler context
-    pub fn is_handling_signal(&self) -> bool {
-        self.signal_pending.load(Ordering::Acquire) != 0
+        sched::current_thread()
    }

    /// Returns a reference to thread `tid`, if it exists
    #[inline]
-    pub fn get(tid: Tid) -> Option<ThreadRef> {
+    pub fn get(tid: u32) -> Option<ThreadRef> {
        THREADS.lock().get(&tid).cloned()
    }

@@ -92,6 +90,7 @@ impl Thread {
        let id = new_tid();

        let res = Rc::new(Self {
+            cpu: AtomicU32::new(Self::CPU_NONE),
            ctx: UnsafeCell::new(Context::kernel(entry as usize, arg)),
            signal_ctx: UnsafeCell::new(Context::empty()),
            signal_pending: AtomicU32::new(0),
@@ -123,6 +122,7 @@ impl Thread {
        let id = new_tid();

        let res = Rc::new(Self {
+            cpu: AtomicU32::new(Self::CPU_NONE),
            ctx: UnsafeCell::new(Context::user(entry, arg, ttbr0, stack)),
            signal_ctx: UnsafeCell::new(Context::empty()),
            signal_pending: AtomicU32::new(0),
@@ -152,6 +152,7 @@ impl Thread {
        let id = new_tid();

        let res = Rc::new(Self {
+            cpu: AtomicU32::new(Self::CPU_NONE),
            ctx: UnsafeCell::new(Context::fork(frame, ttbr0)),
            signal_ctx: UnsafeCell::new(Context::empty()),
            signal_pending: AtomicU32::new(0),
@@ -174,7 +175,7 @@ impl Thread {

    /// Returns the thread ID
    #[inline]
-    pub fn id(&self) -> Tid {
+    pub fn id(&self) -> u32 {
        self.inner.lock().id
    }

@@ -184,12 +185,18 @@ impl Thread {
    ///
    /// Unsafe: only allowed to be called once, repeated calls
    ///         will generate undefined behavior
-    pub unsafe fn enter(thread: ThreadRef) -> ! {
+    pub unsafe fn enter(cpu: u32, thread: ThreadRef) -> ! {
        // FIXME use some global lock to guarantee atomicity of thread entry?
        thread.inner.lock().state = State::Running;
+        thread.cpu.store(cpu, Ordering::SeqCst);
        thread.current_context().enter()
    }

+    ///
+    pub fn cpu(&self) -> u32 {
+        self.cpu.load(Ordering::SeqCst)
+    }
+
    /// Schedules a next thread for execution
    ///
    /// # Safety
@@ -198,7 +205,7 @@ impl Thread {
    ///
    /// * Does not ensure src and dst threads are not the same thread
    /// * Does not ensure src is actually current context
-    pub unsafe fn switch(src: ThreadRef, dst: ThreadRef, discard: bool) {
+    pub unsafe fn switch(cpu: u32, src: ThreadRef, dst: ThreadRef, discard: bool) {
        {
            let mut src_lock = src.inner.lock();
            let mut dst_lock = dst.inner.lock();
@@ -209,6 +216,9 @@ impl Thread {
            }
            // assert!(dst_lock.state == State::Ready || dst_lock.state == State::Waiting);
            dst_lock.state = State::Running;
+
+            src.cpu.store(Self::CPU_NONE, Ordering::SeqCst);
+            dst.cpu.store(cpu, Ordering::SeqCst);
        }

        let src_ctx = src.current_context();
@@ -232,17 +242,18 @@ impl Thread {
            let mut lock = self.inner.lock();
            let drop = lock.state == State::Running;
            lock.state = State::Waiting;
-            SCHED.dequeue(lock.id);
+            sched::dequeue(lock.id);
+            // SCHED.dequeue(lock.id);
            drop
        };
        if drop {
-            SCHED.switch(true);
+            sched::switch(true);
+            // SCHED.switch(true);
        }
    }

    /// Changes process wait condition status
    pub fn setup_wait(&self, wait: *const Wait) {
-        #![allow(clippy::not_unsafe_ptr_arg_deref)]
        let mut lock = self.inner.lock();
        // FIXME this is not cool
        lock.pending_wait = Some(unsafe { &*wait });
@@ -250,7 +261,7 @@ impl Thread {
    }

    /// Suspends current thread until thread `tid` terminates
-    pub fn waittid(tid: Tid) -> Result<(), Errno> {
+    pub fn waittid(tid: u32) -> Result<(), Errno> {
        loop {
            let thread = THREADS
                .lock()
@@ -340,7 +351,7 @@ impl Thread {
        let signal_ctx = unsafe { &mut *self.signal_ctx.get() };

        debugln!(
-            "Signal entry: tid={:?}, pc={:#x}, sp={:#x}, ttbr0={:#x}",
+            "Signal entry: tid={}, pc={:#x}, sp={:#x}, ttbr0={:#x}",
            lock.id,
            lock.signal_entry,
            lock.signal_stack,
@@ -404,8 +415,9 @@ impl Drop for Thread {
 }

 /// Allocates a new thread ID
-pub fn new_tid() -> Tid {
-    static LAST: AtomicU32 = AtomicU32::new(0);
+pub fn new_tid() -> u32 {
+    static LAST: AtomicU32 = AtomicU32::new(1);
    let id = LAST.fetch_add(1, Ordering::Relaxed);
-    Tid::from(id)
+    assert!(id < 256, "Out of user TIDs");
+    id
 }
@@ -2,16 +2,16 @@

 use crate::arch::machine;
 use crate::dev::timer::TimestampSource;
-use crate::proc::{sched::SCHED, Thread, ThreadRef};
+use crate::proc::{self, sched, Thread, ThreadRef};
 use crate::sync::IrqSafeSpinLock;
 use alloc::collections::LinkedList;
 use core::time::Duration;
-use libsys::{error::Errno, proc::Tid, stat::FdSet};
+use libsys::{error::Errno, stat::FdSet};

 /// Wait channel structure. Contains a queue of processes
 /// waiting for some event to happen.
 pub struct Wait {
-    queue: IrqSafeSpinLock<LinkedList<Tid>>,
+    queue: IrqSafeSpinLock<LinkedList<u32>>,
    #[allow(dead_code)]
    name: &'static str
 }
@@ -28,7 +28,7 @@ pub enum WaitStatus {
 }

 struct Timeout {
-    tid: Tid,
+    tid: u32,
    deadline: Duration,
 }

@@ -47,7 +47,12 @@ pub fn tick() {
        if time > item.deadline {
            let tid = item.tid;
            cursor.remove_current();
-            SCHED.enqueue(tid);
+            todo!();
+//<<<<<<< HEAD
+//            sched::enqueue(pid);
+//=======
+//            SCHED.enqueue(tid);
+//>>>>>>> feat/thread
        } else {
            cursor.move_next();
        }
@@ -83,12 +88,8 @@ pub fn select(
    }
    let read = rfds.as_deref().map(FdSet::clone);
    let write = wfds.as_deref().map(FdSet::clone);
-    if let Some(rfds) = &mut rfds {
-        rfds.reset();
-    }
-    if let Some(wfds) = &mut wfds {
-        wfds.reset();
-    }
+    rfds.as_deref_mut().map(FdSet::reset);
+    wfds.as_deref_mut().map(FdSet::reset);

    let deadline = timeout.map(|v| v + machine::local_timer().timestamp().unwrap());
    let proc = thread.owner().unwrap();
@@ -133,7 +134,7 @@ impl Wait {
    }

    /// Interrupt wait pending on the channel
-    pub fn abort(&self, tid: Tid, enqueue: bool) {
+    pub fn abort(&self, tid: u32, enqueue: bool) {
        let mut queue = self.queue.lock();
        let mut tick_lock = TICK_LIST.lock();
        let mut cursor = tick_lock.cursor_front_mut();
@@ -153,7 +154,8 @@ impl Wait {
                let thread = Thread::get(tid).unwrap();
                thread.set_wait_status(WaitStatus::Interrupted);
                if enqueue {
-                    SCHED.enqueue(tid);
+                    sched::enqueue(tid);
+                    // SCHED.enqueue(tid);
                }
                break;
            } else {
@@ -182,7 +184,7 @@ impl Wait {
                drop(tick_lock);

                Thread::get(tid).unwrap().set_wait_status(WaitStatus::Done);
-                SCHED.enqueue(tid);
+                sched::enqueue(tid);
            }

            limit -= 1;
@@ -4,12 +4,14 @@ use crate::arch::platform::{irq_mask_save, irq_restore};
 use core::cell::UnsafeCell;
 use core::fmt;
 use core::ops::{Deref, DerefMut};
-use core::sync::atomic::{AtomicBool, Ordering};
+use core::sync::atomic::{AtomicUsize, Ordering};
+use cortex_a::registers::MPIDR_EL1;
+use tock_registers::interfaces::Readable;

 /// Lock structure ensuring IRQs are disabled when inner value is accessed
 pub struct IrqSafeSpinLock<T> {
    value: UnsafeCell<T>,
-    state: AtomicBool,
+    state: AtomicUsize,
 }

 /// Guard-structure wrapping a reference to value owned by [IrqSafeSpinLock].
@@ -25,28 +27,25 @@ impl<T> IrqSafeSpinLock<T> {
    pub const fn new(value: T) -> Self {
        Self {
            value: UnsafeCell::new(value),
-            state: AtomicBool::new(false),
+            state: AtomicUsize::new(usize::MAX),
        }
    }

-    #[inline(always)]
-    fn try_lock(&self) -> Result<bool, bool> {
-        self.state
-            .compare_exchange_weak(false, true, Ordering::Acquire, Ordering::Relaxed)
-    }
-
-    #[inline(always)]
-    unsafe fn force_release(&self) {
-        self.state.store(false, Ordering::Release);
-        cortex_a::asm::sev();
-    }
-
    /// Returns [IrqSafeSpinLockGuard] for this lock
    #[inline]
    pub fn lock(&self) -> IrqSafeSpinLockGuard<T> {
        let irq_state = unsafe { irq_mask_save() };
+        let id = MPIDR_EL1.get() & 0xF;

-        while self.try_lock().is_err() {
+        while let Err(e) = self.state.compare_exchange_weak(
+            usize::MAX,
+            id as usize,
+            Ordering::Acquire,
+            Ordering::Relaxed,
+        ) {
+            // if e == id as usize {
+            //     break;
+            // }
            cortex_a::asm::wfe();
        }

@@ -55,11 +54,17 @@ impl<T> IrqSafeSpinLock<T> {
            irq_state,
        }
    }
+
+    pub unsafe fn force_release(&self) {
+        self.state.store(usize::MAX, Ordering::Release);
+        cortex_a::asm::sev();
+    }
 }

 impl<T> Deref for IrqSafeSpinLockGuard<'_, T> {
    type Target = T;

+    #[inline(always)]
    fn deref(&self) -> &Self::Target {
        unsafe { &*self.lock.value.get() }
    }
@@ -4,7 +4,6 @@ use crate::mem;
 use core::alloc::Layout;
 use libsys::error::Errno;
 use crate::proc::Process;
-use core::arch::asm;

 // TODO _mut() versions checking whether pages are actually writable

@@ -123,7 +122,6 @@ pub fn validate_ptr(base: usize, len: usize, write: bool) -> Result<(), Errno> {
    }

    let process = Process::current();
-    let asid = process.asid();

    for i in (base / mem::PAGE_SIZE)..((base + len + mem::PAGE_SIZE - 1) / mem::PAGE_SIZE) {
        if !is_el0_accessible(i * mem::PAGE_SIZE, write) {
@@ -131,9 +129,7 @@ pub fn validate_ptr(base: usize, len: usize, write: bool) -> Result<(), Errno> {
            // a write access
            let res = if write {
                process.manipulate_space(|space| {
-                    space.try_cow_copy(i * mem::PAGE_SIZE)?;
-                    Process::invalidate_asid(asid);
-                    Ok(())
+                    space.try_cow_copy(i * mem::PAGE_SIZE)
                })
            } else {
                Err(Errno::DoesNotExist)
@@ -1,11 +1,11 @@
 //! System call implementation

 use crate::arch::{machine, platform::exception::ExceptionFrame};
+use crate::mem::{virt::MapAttributes, phys::PageUsage};
 use crate::debug::Level;
 use crate::dev::timer::TimestampSource;
 use crate::fs::create_filesystem;
-use crate::mem::{phys::PageUsage, virt::MapAttributes};
-use crate::proc::{self, elf, wait, Process, ProcessIo, Thread};
+use crate::proc::{self, sched, elf, wait, Process, ProcessIo, Thread};
 use core::mem::size_of;
 use core::ops::DerefMut;
 use core::time::Duration;
@@ -14,7 +14,7 @@ use libsys::{
    debug::TraceLevel,
    error::Errno,
    ioctl::IoctlCmd,
-    proc::{ExitCode, MemoryAccess, Pid, Tid},
+    proc::{ExitCode, Pid, MemoryAccess},
    signal::{Signal, SignalDestination},
    stat::{
        AccessMode, DirectoryEntry, FdSet, FileDescriptor, FileMode, GroupId, MountOptions,
@@ -56,7 +56,8 @@ fn find_at_node<T: DerefMut<Target = ProcessIo>>(
    }
 }

-fn _syscall(num: SystemCall, args: &[usize]) -> Result<usize, Errno> {
+/// Main system call dispatcher function
+pub fn syscall(num: SystemCall, args: &[usize]) -> Result<usize, Errno> {
    match num {
        // I/O
        SystemCall::Read => {
@@ -209,8 +210,7 @@ fn _syscall(num: SystemCall, args: &[usize]) -> Result<usize, Errno> {
            let acc = MemoryAccess::from_bits(args[2] as u32).ok_or(Errno::InvalidArgument)?;
            let _flags = MemoryAccess::from_bits(args[3] as u32).ok_or(Errno::InvalidArgument)?;

-            let mut attrs =
-                MapAttributes::NOT_GLOBAL | MapAttributes::SH_OUTER | MapAttributes::PXN;
+            let mut attrs = MapAttributes::NOT_GLOBAL | MapAttributes::SH_OUTER | MapAttributes::PXN;
            if !acc.contains(MemoryAccess::READ) {
                return Err(Errno::NotImplemented);
            }
@@ -244,7 +244,9 @@ fn _syscall(num: SystemCall, args: &[usize]) -> Result<usize, Errno> {
            }

            let proc = Process::current();
-            proc.manipulate_space(move |space| space.free(addr, len / 4096))?;
+            proc.manipulate_space(move |space| {
+                space.free(addr, len / 4096)
+            })?;
            Ok(0)
        }

@@ -256,7 +258,7 @@ fn _syscall(num: SystemCall, args: &[usize]) -> Result<usize, Errno> {

            Process::current()
                .new_user_thread(entry, stack, arg)
-                .map(|e| u32::from(e) as usize)
+                .map(|e| e as usize)
        }
        SystemCall::Exec => {
            let filename = arg::str_ref(args[0], args[1])?;
@@ -291,7 +293,7 @@ fn _syscall(num: SystemCall, args: &[usize]) -> Result<usize, Errno> {
        }
        SystemCall::WaitPid => {
            // TODO special "pid" values
-            let pid = Pid::try_from(args[0] as u32)?;
+            let pid = unsafe { Pid::from_raw(args[0] as u32) };
            let status = arg::struct_mut::<i32>(args[1])?;

            match Process::waitpid(pid) {
@@ -303,15 +305,15 @@ fn _syscall(num: SystemCall, args: &[usize]) -> Result<usize, Errno> {
            }
        }
        SystemCall::WaitTid => {
-            let tid = Tid::from(args[0] as u32);
+            let tid = args[0] as u32;

            match Thread::waittid(tid) {
                Ok(_) => Ok(0),
                _ => todo!(),
            }
        }
-        SystemCall::GetPid => Ok(u32::from(Process::current().id()) as usize),
-        SystemCall::GetTid => Ok(u32::from(Thread::current().id()) as usize),
+        SystemCall::GetPid => Ok(Process::current().id().value() as usize),
+        SystemCall::GetTid => Ok(Thread::current().id() as usize),
        SystemCall::Sleep => {
            let rem_buf = arg::option_buf_ref(args[1], size_of::<u64>() * 2)?;
            let mut rem = Duration::new(0, 0);
@@ -346,38 +348,40 @@ fn _syscall(num: SystemCall, args: &[usize]) -> Result<usize, Errno> {
            Ok(0)
        }
        SystemCall::Yield => {
-            proc::switch();
+            sched::switch(false);
            Ok(0)
        }
        SystemCall::GetSid => {
            // TODO handle kernel processes here?
-            let pid = Pid::to_option(args[0] as u32);
+            let pid = args[0] as u32;
            let current = Process::current();
-            let proc = if let Some(pid) = pid {
+            let proc = if pid == 0 {
+                current
+            } else {
+                let pid = unsafe { Pid::from_raw(pid) };
                let proc = Process::get(pid).ok_or(Errno::DoesNotExist)?;
                if proc.sid() != current.sid() {
                    return Err(Errno::PermissionDenied);
                }
                proc
-            } else {
-                current
            };

-            Ok(u32::from(proc.sid()) as usize)
+            Ok(proc.sid().value() as usize)
        }
        SystemCall::GetPgid => {
            // TODO handle kernel processes here?
-            let pid = Pid::to_option(args[0] as u32);
+            let pid = args[0] as u32;
            let current = Process::current();
-            let proc = if let Some(pid) = pid {
-                Process::get(pid).ok_or(Errno::DoesNotExist)?
-            } else {
+            let proc = if pid == 0 {
                current
+            } else {
+                let pid = unsafe { Pid::from_raw(pid) };
+                Process::get(pid).ok_or(Errno::DoesNotExist)?
            };

-            Ok(u32::from(proc.pgid()) as usize)
+            Ok(proc.pgid().value() as usize)
        }
-        SystemCall::GetPpid => Ok(u32::from(Process::current().ppid().unwrap()) as usize),
+        SystemCall::GetPpid => Ok(Process::current().ppid().unwrap().value() as usize),
        SystemCall::SetSid => {
            let proc = Process::current();
            let mut io = proc.io.lock();
@@ -388,26 +392,22 @@ fn _syscall(num: SystemCall, args: &[usize]) -> Result<usize, Errno> {

            let id = proc.id();
            proc.set_sid(id);
-            Ok(u32::from(id) as usize)
+            Ok(id.value() as usize)
        }
        SystemCall::SetPgid => {
-            let pid = Pid::to_option(args[0] as u32);
-            let pgid = Pid::to_option(args[1] as u32);
+            let pid = args[0] as u32;
+            let pgid = args[1] as u32;

            let current = Process::current();
-            let proc = if let Some(_pid) = pid {
-                todo!()
-            } else {
-                current
-            };
+            let proc = if pid == 0 { current } else { todo!() };

-            if let Some(_pgid) = pgid {
-                todo!();
-            } else {
+            if pgid == 0 {
                proc.set_pgid(proc.id());
+            } else {
+                todo!();
            }

-            Ok(u32::from(proc.pgid()) as usize)
+            Ok(proc.pgid().value() as usize)
        }

        // System
@@ -440,7 +440,7 @@ fn _syscall(num: SystemCall, args: &[usize]) -> Result<usize, Errno> {
            let buf = arg::str_ref(args[1], args[2])?;
            let thread = Thread::current();
            let proc = thread.owner().unwrap();
-            println!(level, "[trace {:?}:{:?}] {}", proc.id(), thread.id(), buf);
+            println!(level, "[trace {:?}:{}] {}", proc.id(), thread.id(), buf);
            Ok(args[1])
        }

@@ -448,14 +448,3 @@ fn _syscall(num: SystemCall, args: &[usize]) -> Result<usize, Errno> {
        SystemCall::Fork => unreachable!(),
    }
 }
-
-/// Main system call dispatcher function
-pub fn syscall(num: SystemCall, args: &[usize]) -> Result<usize, Errno> {
-    let thread = Thread::current();
-    let process = thread.owner().unwrap();
-    let result = _syscall(num, args);
-    if !thread.is_handling_signal() {
-        process.handle_pending_signals();
-    }
-    result
-}
@@ -1,10 +1,9 @@
 use crate::abi::SystemCall;
-use core::arch::asm;
 use crate::{
    debug::TraceLevel,
    error::Errno,
    ioctl::IoctlCmd,
-    proc::{ExitCode, MemoryAccess, MemoryMap, Pid, Tid},
+    proc::{ExitCode, MemoryAccess, MemoryMap, Pid},
    signal::{Signal, SignalDestination},
    stat::{
        AccessMode, DirectoryEntry, FdSet, FileDescriptor, FileMode, GroupId, MountOptions,
@@ -76,6 +75,9 @@ macro_rules! argp {
 //     ($a:expr) => ($a as *const core::ffi::c_void as usize)
 // }

+/// # Safety
+///
+/// System call
 #[inline(always)]
 pub fn sys_exit(code: ExitCode) -> ! {
    unsafe {
@@ -84,11 +86,17 @@ pub fn sys_exit(code: ExitCode) -> ! {
    unreachable!();
 }

+/// # Safety
+///
+/// System call
 #[inline(always)]
 pub fn sys_close(fd: FileDescriptor) -> Result<(), Errno> {
    Errno::from_syscall_unit(unsafe { syscall!(SystemCall::Close, argn!(u32::from(fd))) })
 }

+/// # Safety
+///
+/// System call
 #[inline(always)]
 pub fn sys_ex_nanosleep(ns: u64, rem: &mut [u64; 2]) -> Result<(), Errno> {
    Errno::from_syscall_unit(unsafe {
@@ -96,6 +104,9 @@ pub fn sys_ex_nanosleep(ns: u64, rem: &mut [u64; 2]) -> Result<(), Errno> {
    })
 }

+/// # Safety
+///
+/// System call
 #[inline(always)]
 pub fn sys_ex_debug_trace(level: TraceLevel, msg: &[u8]) -> Result<(), Errno> {
    Errno::from_syscall_unit(unsafe {
@@ -108,6 +119,9 @@ pub fn sys_ex_debug_trace(level: TraceLevel, msg: &[u8]) -> Result<(), Errno> {
    })
 }

+/// # Safety
+///
+/// System call
 #[inline(always)]
 pub fn sys_openat(
    at: Option<FileDescriptor>,
@@ -128,6 +142,9 @@ pub fn sys_openat(
    .map(|e| FileDescriptor::from(e as u32))
 }

+/// # Safety
+///
+/// System call
 #[inline(always)]
 pub fn sys_read(fd: FileDescriptor, data: &mut [u8]) -> Result<usize, Errno> {
    Errno::from_syscall(unsafe {
@@ -152,6 +169,9 @@ pub fn sys_write(fd: FileDescriptor, data: &[u8]) -> Result<usize, Errno> {
    })
 }

+/// # Safety
+///
+/// System call
 #[inline(always)]
 pub fn sys_fstatat(
    at: Option<FileDescriptor>,
@@ -176,15 +196,18 @@ pub fn sys_fstatat(
 /// System call
 #[inline(always)]
 pub unsafe fn sys_fork() -> Result<Option<Pid>, Errno> {
-    Errno::from_syscall(syscall!(SystemCall::Fork)).and_then(|res| {
+    Errno::from_syscall(syscall!(SystemCall::Fork)).map(|res| {
        if res != 0 {
-            Pid::try_from(res as u32).map(Some)
+            Some(unsafe { Pid::from_raw(res as u32) })
        } else {
-            Ok(None)
+            None
        }
    })
 }

+/// # Safety
+///
+/// System call
 #[inline(always)]
 pub fn sys_execve(pathname: &str, argv: &[&str]) -> Result<(), Errno> {
    Errno::from_syscall_unit(unsafe {
@@ -198,17 +221,23 @@ pub fn sys_execve(pathname: &str, argv: &[&str]) -> Result<(), Errno> {
    })
 }

+/// # Safety
+///
+/// System call
 #[inline(always)]
 pub fn sys_waitpid(pid: Pid, status: &mut i32) -> Result<(), Errno> {
    Errno::from_syscall_unit(unsafe {
        syscall!(
            SystemCall::WaitPid,
-            argn!(u32::from(pid)),
+            argn!(pid.value()),
            argp!(status as *mut i32)
        )
    })
 }

+/// # Safety
+///
+/// System call
 #[inline(always)]
 pub fn sys_ioctl(
    fd: FileDescriptor,
@@ -260,10 +289,10 @@ pub fn sys_ex_kill(pid: SignalDestination, signum: Signal) -> Result<(), Errno>
 }

 #[inline(always)]
-pub fn sys_ex_clone(entry: usize, stack: usize, arg: usize) -> Result<Tid, Errno> {
+pub fn sys_ex_clone(entry: usize, stack: usize, arg: usize) -> Result<usize, Errno> {
    Errno::from_syscall(unsafe {
        syscall!(SystemCall::Clone, argn!(entry), argn!(stack), argn!(arg))
-    }).map(|e| Tid::from(e as u32))
+    })
 }

 #[inline(always)]
@@ -275,8 +304,8 @@ pub fn sys_ex_thread_exit(status: ExitCode) -> ! {
 }

 #[inline(always)]
-pub fn sys_ex_thread_wait(tid: Tid) -> Result<ExitCode, Errno> {
-    Errno::from_syscall(unsafe { syscall!(SystemCall::WaitTid, argn!(u32::from(tid))) })
+pub fn sys_ex_thread_wait(tid: u32) -> Result<ExitCode, Errno> {
+    Errno::from_syscall(unsafe { syscall!(SystemCall::WaitTid, argn!(tid)) })
        .map(|_| ExitCode::from(0))
 }

@@ -327,31 +356,27 @@ pub fn sys_faccessat(
 }

 #[inline(always)]
-pub fn sys_ex_gettid() -> Tid {
-    Tid::from(unsafe { syscall!(SystemCall::GetTid) as u32 })
+pub fn sys_ex_gettid() -> u32 {
+    unsafe { syscall!(SystemCall::GetTid) as u32 }
 }

 #[inline(always)]
 pub fn sys_getpid() -> Pid {
-    Pid::try_from(unsafe { syscall!(SystemCall::GetPid) as u32 }).unwrap()
+    unsafe { Pid::from_raw(syscall!(SystemCall::GetPid) as u32) }
 }

 #[inline(always)]
-pub fn sys_getpgid(pid: Option<Pid>) -> Result<Pid, Errno> {
-    Errno::from_syscall(unsafe { syscall!(SystemCall::GetPgid, argn!(Pid::from_option(pid))) })
-        .and_then(|e| Pid::try_from(e as u32))
+pub fn sys_getpgid(pid: Pid) -> Result<Pid, Errno> {
+    Errno::from_syscall(unsafe { syscall!(SystemCall::GetPgid, argn!(pid.value())) })
+        .map(|e| unsafe { Pid::from_raw(e as u32) })
 }

 #[inline(always)]
-pub fn sys_setpgid(pid: Option<Pid>, pgid: Option<Pid>) -> Result<Pid, Errno> {
+pub fn sys_setpgid(pid: Pid, pgid: Pid) -> Result<Pid, Errno> {
    Errno::from_syscall(unsafe {
-        syscall!(
-            SystemCall::SetPgid,
-            argn!(Pid::from_option(pid)),
-            argn!(Pid::from_option(pgid))
-        )
+        syscall!(SystemCall::SetPgid, argn!(pid.value()), argn!(pgid.value()))
    })
-    .and_then(|e| Pid::try_from(e as u32))
+    .map(|e| unsafe { Pid::from_raw(e as u32) })
 }

 #[inline(always)]
@@ -379,7 +404,7 @@ pub fn sys_getgid() -> GroupId {
 #[inline(always)]
 pub fn sys_setsid() -> Result<Pid, Errno> {
    Errno::from_syscall(unsafe { syscall!(SystemCall::SetSid) })
-        .and_then(|e| Pid::try_from(e as u32))
+        .map(|e| unsafe { Pid::from_raw(e as u32) })
 }

 #[inline(always)]
@@ -445,10 +470,6 @@ pub fn sys_mmap(
    })
 }

-
-/// # Safety
-///
-/// System call
 #[inline(always)]
 pub unsafe fn sys_munmap(addr: usize, len: usize) -> Result<(), Errno> {
    Errno::from_syscall_unit(syscall!(SystemCall::UnmapMemory, argn!(addr), argn!(len)))
@@ -1,3 +1,4 @@
+#![feature(asm, const_panic)]
 #![no_std]

 #[macro_use]
@@ -23,55 +24,6 @@ pub struct ProgramArgs {
    pub size: usize
 }

-// TODO utils
-use core::fmt;
-
-#[derive(Clone, Copy)]
-pub struct FixedStr<const N: usize> {
-    len: usize,
-    data: [u8; N],
-}
-
-impl<const N: usize> FixedStr<N> {
-    pub const fn empty() -> Self {
-        Self {
-            len: 0,
-            data: [0; N]
-        }
-    }
-
-    pub fn copy_from_str(&mut self, src: &str) {
-        if src.len() > self.data.len() {
-            panic!("copy_from_str: src len > data len");
-        }
-        self.len = src.len();
-        self.data[..self.len].copy_from_slice(src.as_bytes());
-    }
-
-    pub fn as_str(&self) -> &str {
-        unsafe {
-            core::str::from_utf8_unchecked(&self.data[..self.len])
-        }
-    }
-}
-
-impl<const N: usize> fmt::Debug for FixedStr<N> {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "\"")?;
-        fmt::Display::fmt(self, f)?;
-        write!(f, "\"")
-    }
-}
-
-impl<const N: usize> fmt::Display for FixedStr<N> {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        for &byte in &self.data[..self.len] {
-            write!(f, "{}", byte as char)?;
-        }
-        Ok(())
-    }
-}
-
 #[cfg(feature = "user")]
 pub mod calls;
 #[cfg(feature = "user")]
@@ -12,11 +12,6 @@ pub struct ExitCode(i32);
 #[repr(transparent)]
 pub struct Pid(u32);

-/// Wrapper type for thread ID
-#[derive(Clone, Copy, PartialOrd, Ord, PartialEq, Eq)]
-#[repr(transparent)]
-pub struct Tid(u32);
-
 #[derive(Clone, Copy, PartialOrd, Ord, PartialEq, Eq, Debug)]
 #[repr(transparent)]
 pub struct Pgid(u32);
@@ -31,8 +26,8 @@ bitflags! {

 bitflags! {
    pub struct MemoryMap: u32 {
-        const BACKEND = 0x3;
-        const ANONYMOUS = 1;
+        const BACKEND = 0x3 << 0;
+        const ANONYMOUS = 1 << 0;

        const SHARING = 0x3 << 2;
        const PRIVATE = 1 << 2;
@@ -58,15 +53,14 @@ impl From<ExitCode> for i32 {
 }

 impl Pid {
+    /// Kernel idle process always has PID of zero
+    pub const IDLE: Self = Self(Self::KERNEL_BIT);
+
    const KERNEL_BIT: u32 = 1 << 31;
-    const USER_MAX: u32 = 256;

    /// Constructs an instance of user-space PID
    pub const fn user(id: u32) -> Self {
-        assert!(id < Self::USER_MAX, "PID is too high");
-        if id == 0 {
-            panic!("User PID cannot be zero");
-        }
+        assert!(id < 256, "PID is too high");
        Self(id)
    }

@@ -89,20 +83,18 @@ impl Pid {
        self.0 as u8
    }

-    pub fn from_option(m: Option<Self>) -> u32 {
-        if let Some(pid) = m {
-            u32::from(pid)
-        } else {
-            0
-        }
+    /// Returns bit value of this pid
+    pub const fn value(self) -> u32 {
+        self.0
    }

-    pub fn to_option(m: u32) -> Option<Self> {
-        if m != 0 {
-            Some(Self::try_from(m).unwrap())
-        } else {
-            None
-        }
+    /// Constructs [Pid] from raw [u32] value
+    ///
+    /// # Safety
+    ///
+    /// Unsafe: does not check `num`
+    pub const unsafe fn from_raw(num: u32) -> Self {
+        Self(num)
    }
 }

@@ -117,26 +109,6 @@ impl fmt::Debug for Pid {
    }
 }

-impl TryFrom<u32> for Pid {
-    type Error = Errno;
-
-    fn try_from(raw: u32) -> Result<Pid, Errno> {
-        if raw & Self::KERNEL_BIT != 0 {
-            Ok(Pid::kernel(raw & !Self::KERNEL_BIT))
-        } else if raw != 0 && raw < Self::USER_MAX {
-            Ok(Pid::user(raw))
-        } else {
-            Err(Errno::InvalidArgument)
-        }
-    }
-}
-
-impl From<Pid> for u32 {
-    fn from(pid: Pid) -> u32 {
-        pid.0
-    }
-}
-
 impl TryFrom<Pid> for Pgid {
    type Error = Errno;

@@ -160,25 +132,3 @@ impl From<Pgid> for u32 {
        p.0
    }
 }
-
-impl Tid {
-    pub const IDLE: Tid = Tid(0);
-}
-
-impl fmt::Debug for Tid {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "Tid(#{})", self.0)
-    }
-}
-
-impl From<u32> for Tid {
-    fn from(p: u32) -> Tid {
-        Self(p)
-    }
-}
-
-impl From<Tid> for u32 {
-    fn from(p: Tid) -> u32 {
-        p.0
-    }
-}
@@ -37,7 +37,7 @@ impl From<isize> for SignalDestination {
 impl From<SignalDestination> for isize {
    fn from(p: SignalDestination) -> isize {
        match p {
-            SignalDestination::Process(pid) => u32::from(pid) as isize,
+            SignalDestination::Process(pid) => pid.value() as isize,
            SignalDestination::Group(pgid) => -(u32::from(pgid) as isize),
            SignalDestination::This => 0,
            SignalDestination::All => -1
@@ -1,6 +1,5 @@
 // TODO split up this file
 use crate::error::Errno;
-use core::str::FromStr;
 use core::fmt;

 const AT_FDCWD: i32 = -2;
@@ -144,20 +143,16 @@ impl DirectoryEntry {
        Self { name: [0; 64] }
    }

-    pub fn as_str(&self) -> &str {
-        let zero = self.name.iter().position(|&c| c == 0).unwrap();
-        core::str::from_utf8(&self.name[..zero]).unwrap()
-    }
-}
-
-impl FromStr for DirectoryEntry {
-    type Err = Errno;
-
-    fn from_str(i: &str) -> Result<Self, Errno> {
+    pub fn from_str(i: &str) -> DirectoryEntry {
        let mut res = DirectoryEntry { name: [0; 64] };
        let bytes = i.as_bytes();
        res.name[..bytes.len()].copy_from_slice(bytes);
-        Ok(res)
+        res
+    }
+
+    pub fn as_str(&self) -> &str {
+        let zero = self.name.iter().position(|&c| c == 0).unwrap();
+        core::str::from_utf8(&self.name[..zero]).unwrap()
    }
 }

@@ -9,6 +9,3 @@ edition = "2021"
 libsys = { path = "../libsys", features = ["user"] }
 lazy_static = { version = "^1.4.0", features = ["spin_no_std"] }
 memoffset = "^0.6.4"
-
-[features]
-verbose = []
@@ -82,7 +82,6 @@ impl Zone {
            MemoryAccess::READ | MemoryAccess::WRITE,
            MemoryMap::ANONYMOUS | MemoryMap::PRIVATE,
        )?;
-        #[cfg(feature = "verbose")]
        trace_debug!("Zone::alloc({}) => {:#x}", size, pages);

        let zone_ptr = pages as *mut Zone;
@@ -102,9 +101,8 @@ impl Zone {
    }

    unsafe fn free(zone: *mut Self) {
-        #[cfg(feature = "verbose")]
        trace_debug!("Zone::free({:p})", zone);
-        sys_munmap(zone as usize, (*zone).size + size_of::<Zone>())
+        sys_munmap(zone as usize, (&*zone).size + size_of::<Zone>())
            .expect("Failed to unmap heap pages");
    }

@@ -167,7 +165,7 @@ unsafe fn alloc_from(list: &mut ZoneList, zone_size: usize, size: usize) -> *mut
            if !ptr.is_null() {
                return ptr;
            }
-            zone = (*zone).next;
+            zone = (&mut *zone).next;
        }

        let zone = match Zone::alloc(zone_size) {
@@ -177,7 +175,7 @@ unsafe fn alloc_from(list: &mut ZoneList, zone_size: usize, size: usize) -> *mut
                return null_mut();
            }
        };
-        list.add(&mut (*zone).list);
+        list.add(&mut (&mut *zone).list);
    }
 }

@@ -185,7 +183,6 @@ unsafe impl GlobalAlloc for Allocator {
    unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
        assert!(layout.align() < 16);
        let size = (layout.size() + 15) & !15;
-        #[cfg(feature = "verbose")]
        trace_debug!("alloc({:?})", layout);
        if size <= SMALL_ZONE_ELEM {
            alloc_from(SMALL_ZONE_LIST.assume_init_mut(), SMALL_ZONE_SIZE, size)
@@ -199,7 +196,6 @@ unsafe impl GlobalAlloc for Allocator {
    }

    unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {
-        #[cfg(feature = "verbose")]
        trace_debug!("free({:p}, {:?})", ptr, layout);
        assert!(!ptr.is_null());
        let mut block = ptr.sub(size_of::<Block>()) as *mut Block;
@@ -236,7 +232,7 @@ unsafe impl GlobalAlloc for Allocator {
        if !next.is_null() && next_ref.flags & BLOCK_ALLOC == 0 {
            next_ref.flags = 0;
            if !next_ref.next.is_null() {
-                (*next_ref.next).prev = block;
+                (&mut *(next_ref.next)).prev = block;
            }
            block_ref.next = next_ref.next;
            block_ref.size += (next_ref.size as usize + size_of::<Block>()) as u32;
@@ -245,15 +241,15 @@ unsafe impl GlobalAlloc for Allocator {
        if block_ref.prev.is_null() && block_ref.next.is_null() {
            let zone = (block as usize - size_of::<Zone>()) as *mut Zone;
            assert_eq!((zone as usize) & 0xFFF, 0);
-            (*zone).list.del();
+            (&mut *zone).list.del();
            Zone::free(zone);
        }
    }
 }

 #[alloc_error_handler]
-fn alloc_error_handler(layout: Layout) -> ! {
-    panic!("Allocation failed: {:?}", layout);
+fn alloc_error_handler(_layout: Layout) -> ! {
+    loop {}
 }

 #[global_allocator]
@@ -1,14 +1,6 @@
-use crate::trace;
+use libsys::{debug::TraceLevel, ProgramArgs};
 use alloc::vec::Vec;
-use libsys::{
-    debug::TraceLevel,
-    ProgramArgs,
-};
-
-mod passwd;
-pub use passwd::UserInfo;
-mod shadow;
-pub use shadow::UserShadow;
+use crate::trace;

 static mut PROGRAM_ARGS: Vec<&'static str> = Vec::new();

@@ -18,13 +10,12 @@ pub fn args() -> &'static [&'static str] {

 pub(crate) unsafe fn setup_env(arg: &ProgramArgs) {
    for i in 0..arg.argc {
-        let base = core::ptr::read((arg.argv + i * 16) as *const *const u8);
+        let base = core::ptr::read((arg.argv + i * 16 + 0) as *const *const u8);
        let len = core::ptr::read((arg.argv + i * 16 + 8) as *const usize);

        let string = core::str::from_utf8(core::slice::from_raw_parts(base, len)).unwrap();
        PROGRAM_ARGS.push(string);
    }

-    #[cfg(feature = "verbose")]
    trace!(TraceLevel::Debug, "args = {:?}", PROGRAM_ARGS);
 }
@@ -1,99 +0,0 @@
-use crate::io::{Read, read_line};
-use core::str::FromStr;
-use core::fmt;
-use crate::trace_debug;
-use crate::file::File;
-use libsys::{FixedStr, stat::{UserId, GroupId}};
-
-#[derive(Debug, Clone, Copy)]
-pub struct UserInfo {
-    name: FixedStr<32>,
-    uid: UserId,
-    gid: GroupId,
-    home: FixedStr<64>,
-    shell: FixedStr<64>,
-}
-
-impl UserInfo {
-    pub fn name(&self) -> &str {
-        self.name.as_str()
-    }
-
-    pub fn home(&self) -> &str {
-        self.home.as_str()
-    }
-
-    pub fn shell(&self) -> &str {
-        self.shell.as_str()
-    }
-
-    pub fn uid(&self) -> UserId {
-        self.uid
-    }
-
-    pub fn gid(&self) -> GroupId {
-        self.gid
-    }
-
-    pub fn find<F: Fn(&Self) -> bool>(pred: F) -> Result<Self, ()> {
-        let mut file = File::open("/etc/passwd").map_err(|_| ())?;
-        let mut buf = [0; 128];
-        loop {
-            let line = read_line(&mut file, &mut buf).map_err(|_| ())?;
-            if let Some(line) = line {
-                let ent = UserInfo::from_str(line)?;
-                if pred(&ent) {
-                    return Ok(ent);
-                }
-            } else {
-                break;
-            }
-        }
-        Err(())
-    }
-
-    pub fn by_name(name: &str) -> Result<Self, ()> {
-        Self::find(|ent| ent.name() == name)
-    }
-}
-
-impl FromStr for UserInfo {
-    type Err = ();
-
-    fn from_str(s: &str) -> Result<Self, ()> {
-        let mut iter = s.split(":");
-
-        let name = iter.next().ok_or(())?;
-        let uid = iter
-            .next()
-            .ok_or(())
-            .and_then(|e| u32::from_str(e).map_err(|_| ()))
-            .map(UserId::from)?;
-        let gid = iter
-            .next()
-            .ok_or(())
-            .and_then(|e| u32::from_str(e).map_err(|_| ()))
-            .map(GroupId::from)?;
-        let comment = iter.next().ok_or(())?;
-        let home = iter.next().ok_or(())?;
-        let shell = iter.next().ok_or(())?;
-
-        if iter.next().is_some() {
-            return Err(());
-        }
-
-        let mut res = Self {
-            uid,
-            gid,
-            name: FixedStr::empty(),
-            home: FixedStr::empty(),
-            shell: FixedStr::empty(),
-        };
-
-        res.name.copy_from_str(&name);
-        res.home.copy_from_str(&home);
-        res.shell.copy_from_str(&shell);
-
-        Ok(res)
-    }
-}
@@ -1,67 +0,0 @@
-use crate::file::File;
-use crate::io::{Read, read_line};
-use core::str::FromStr;
-use libsys::FixedStr;
-
-#[derive(Debug, Clone, Copy)]
-pub struct UserShadow {
-    name: FixedStr<32>,
-    password: FixedStr<64>,
-}
-
-impl UserShadow {
-    pub fn name(&self) -> &str {
-        self.name.as_str()
-    }
-
-    pub fn password(&self) -> &str {
-        self.password.as_str()
-    }
-
-
-    pub fn find<F: Fn(&Self) -> bool>(pred: F) -> Result<Self, ()> {
-        let mut file = File::open("/etc/shadow").map_err(|_| ())?;
-        let mut buf = [0; 128];
-        loop {
-            let line = read_line(&mut file, &mut buf).map_err(|_| ())?;
-            if let Some(line) = line {
-                let ent = UserShadow::from_str(line)?;
-                if pred(&ent) {
-                    return Ok(ent);
-                }
-            } else {
-                break;
-            }
-        }
-        Err(())
-    }
-
-    pub fn by_name(name: &str) -> Result<Self, ()> {
-        Self::find(|ent| ent.name() == name)
-    }
-}
-
-impl FromStr for UserShadow {
-    type Err = ();
-
-    fn from_str(s: &str) -> Result<Self, ()> {
-        let mut iter = s.split(':');
-
-        let name = iter.next().ok_or(())?;
-        let password = iter.next().ok_or(())?;
-
-        if iter.next().is_some() {
-            return Err(());
-        }
-
-        let mut res = Self {
-            name: FixedStr::empty(),
-            password: FixedStr::empty(),
-        };
-
-        res.name.copy_from_str(name);
-        res.password.copy_from_str(password);
-
-        Ok(res)
-    }
-}
@@ -42,27 +42,3 @@ pub fn stat(pathname: &str) -> Result<Stat, Error> {
    sys_fstatat(None, pathname, &mut buf, 0).unwrap();
    Ok(buf)
 }
-
-// TODO use BufRead instead once it's implemented
-pub(crate) fn read_line<'a, F: Read>(f: &mut F, buf: &'a mut [u8]) -> Result<Option<&'a str>, ()> {
-    let mut pos = 0;
-    loop {
-        if pos == buf.len() {
-            return Err(());
-        }
-
-        let count = f.read(&mut buf[pos..=pos]).map_err(|_| ())?;
-        if count == 0 {
-            if pos == 0 {
-                return Ok(None);
-            }
-            break;
-        }
-        if buf[pos] == b'\n' {
-            break;
-        }
-
-        pos += 1;
-    }
-    core::str::from_utf8(&buf[..pos]).map_err(|_| ()).map(Some)
-}
@@ -8,8 +8,7 @@ macro_rules! print {

 #[macro_export]
 macro_rules! println {
-    ($($args:tt)+) => (print!("{}\n", format_args!($($args)+)));
-    () => (print!("\n"));
+    ($($args:tt)+) => (print!("{}\n", format_args!($($args)+)))
 }

 #[macro_export]
@@ -19,8 +18,7 @@ macro_rules! eprint {

 #[macro_export]
 macro_rules! eprintln {
-    ($($args:tt)+) => (eprint!("{}\n", format_args!($($args)+)));
-    () => (eprint!("\n"));
+    ($($args:tt)+) => (eprint!("{}\n", format_args!($($args)+)))
 }

 pub fn _print<T: Write>(out: fn() -> T, args: fmt::Arguments) {
@@ -1,4 +1,4 @@
-#![feature(alloc_error_handler)]
+#![feature(asm, alloc_error_handler)]
 #![no_std]

 #[macro_use]
@@ -6,9 +6,8 @@ use core::fmt;
 use core::mem::MaybeUninit;
 use libsys::{
    calls::{sys_ex_clone, sys_ex_gettid, sys_ex_signal, sys_ex_thread_exit, sys_ex_thread_wait},
-    proc::{ExitCode, Tid},
+    proc::ExitCode,
 };
-use core::arch::asm;

 struct NativeData<F, T>
 where
@@ -23,19 +22,19 @@ where

 #[derive(Clone)]
 pub struct Thread {
-    id: Tid,
+    id: u32,
 }

 pub type ThreadResult<T> = Result<T, Box<dyn Any + Send + Sync>>;
 pub type ThreadPacket<T> = Arc<UnsafeCell<MaybeUninit<ThreadResult<T>>>>;

 pub struct JoinHandle<T> {
-    native: Tid,
+    native: u32,
    result: ThreadPacket<T>,
 }

 impl Thread {
-    pub const fn id(&self) -> Tid {
+    pub const fn id(&self) -> u32 {
        self.id
    }
 }
@@ -61,7 +60,7 @@ impl<T> JoinHandle<T> {
 }

 unsafe fn init_common(signal_stack_pointer: *mut u8) {
-    let tid = u32::from(sys_ex_gettid()) as u64;
+    let tid = sys_ex_gettid() as u64;
    asm!("msr tpidr_el0, {:x}", in(reg) tid);

    // thread::current() should be valid at this point
@@ -87,7 +86,8 @@ pub fn current() -> Thread {
    unsafe {
        asm!("mrs {:x}, tpidr_el0", out(reg) id);
    }
-    Thread { id: Tid::from(id as u32) }
+
+    Thread { id: id as u32 }
 }

 pub fn spawn<F, T>(f: F) -> JoinHandle<T>
@@ -136,7 +136,7 @@ where
            result: result.clone(),
        }));

-        sys_ex_clone(thread_entry::<F, T> as usize, stack, data as usize).unwrap()
+        sys_ex_clone(thread_entry::<F, T> as usize, stack, data as usize).unwrap() as u32
    };

    JoinHandle { native, result }
@@ -37,6 +37,3 @@ path = "src/sbin/login.rs"
 libusr = { path = "../libusr" }
 libsys = { path = "../libsys" }
 lazy_static = { version = "*", features = ["spin_no_std"] }
-
-[features]
-verbose = ["libusr/verbose"]
@@ -30,7 +30,7 @@ fn main() -> i32 {

    if args.len() == 1 {
        if let Err(e) = do_cat(io::stdin()) {
-            eprintln!(".: {:?}", e);
+            eprintln!("{}: {:?}", ".", e);
            res = -1;
        }
    } else {
@@ -1,3 +1,4 @@
+#![feature(asm)]
 #![no_std]
 #![no_main]

@@ -8,7 +9,6 @@
 extern crate libusr;

 use libusr::sys::{abi::SystemCall, stat::Stat};
-use core::arch::asm;

 static mut STATE: u64 = 0;

@@ -27,7 +27,7 @@ fn line_print(off: usize, line: &[u8]) {
            print!(".");
        }
    }
-    println!();
+    println!("");
 }

 fn do_hexd<F: Read>(mut fd: F) -> Result<(), io::Error> {
@@ -53,7 +53,7 @@ fn main() -> i32 {

    if args.len() == 1 {
        if let Err(e) = do_hexd(io::stdin()) {
-            eprintln!(".: {:?}", e);
+            eprintln!("{}: {:?}", ".", e);
            res = -1;
        }
    } else {
@@ -57,7 +57,7 @@ fn main() -> i32 {

    if args.len() == 1 {
        if let Err(e) = list_directory(".") {
-            eprintln!(".: {:?}", e);
+            eprintln!("{}: {:?}", ".", e);
            res = -1;
        }
    } else {
@@ -9,7 +9,7 @@ use alloc::{borrow::ToOwned, vec::Vec};
 use libusr::io::{self, Read};
 use libusr::signal::{self, SignalHandler};
 use libusr::sys::{
-    sys_chdir, sys_execve, sys_exit, sys_faccessat, sys_fork, sys_getpgid, sys_setpgid,
+    proc::Pid, sys_chdir, sys_execve, sys_exit, sys_faccessat, sys_fork, sys_getpgid, sys_setpgid,
    sys_waitpid, AccessMode, Errno, ExitCode, FileDescriptor, Signal,
 };

@@ -22,11 +22,13 @@ fn cmd_cd(args: &[&str]) -> ExitCode {
    if args.len() != 2 {
        eprintln!("Usage: cd DIR");
        ExitCode::from(-1)
-    } else if let Err(err) = sys_chdir(args[1]) {
-        eprintln!("{}: {:?}", args[1], err);
-        ExitCode::from(-1)
    } else {
-        ExitCode::from(0)
+        if let Err(err) = sys_chdir(args[1]) {
+            eprintln!("{}: {:?}", args[1], err);
+            ExitCode::from(-1)
+        } else {
+            ExitCode::from(0)
+        }
    }
 }

@@ -65,11 +67,11 @@ fn execute(line: &str) -> Result<ExitCode, Errno> {
    if let Some(pid) = unsafe { sys_fork()? } {
        let mut status = 0;
        sys_waitpid(pid, &mut status)?;
-        let pgid = sys_getpgid(None).unwrap();
+        let pgid = sys_getpgid(unsafe { Pid::from_raw(0) }).unwrap();
        io::tcsetpgrp(FileDescriptor::STDIN, pgid).unwrap();
        Ok(ExitCode::from(status))
    } else {
-        let pgid = sys_setpgid(None, None).unwrap();
+        let pgid = sys_setpgid(unsafe { Pid::from_raw(0) }, unsafe { Pid::from_raw(0) }).unwrap();
        io::tcsetpgrp(FileDescriptor::STDIN, pgid).unwrap();
        sys_execve(&filename, &args).unwrap();
        sys_exit(ExitCode::from(-1));
@@ -82,7 +84,7 @@ fn main() -> i32 {
    let mut stdin = io::stdin();

    signal::set_handler(Signal::Interrupt, SignalHandler::Ignore);
-    let pgid = sys_setpgid(None, None).unwrap();
+    let pgid = sys_setpgid(unsafe { Pid::from_raw(0) }, unsafe { Pid::from_raw(0) }).unwrap();
    io::tcsetpgrp(FileDescriptor::STDIN, pgid).unwrap();

    loop {
@@ -1,11 +1,10 @@
+#![feature(asm)]
 #![no_std]
 #![no_main]

 #[macro_use]
 extern crate libusr;

-use core::arch::asm;
-
 use libusr::sys::{stat::MountOptions, sys_execve, sys_fork, sys_mount, sys_waitpid};

 #[no_mangle]
@@ -18,14 +17,6 @@ fn main() -> i32 {
        },
    )
    .expect("Failed to mount devfs");
-    sys_mount(
-        "/sys",
-        &MountOptions {
-            device: None,
-            fs: Some("sysfs"),
-        },
-    )
-    .expect("Failed to mount sysfs");

    if let Some(pid) = unsafe { sys_fork().unwrap() } {
        let mut status = 0;
@@ -39,6 +30,6 @@ fn main() -> i32 {
        }
    } else {
        sys_execve("/sbin/login", &["/sbin/login", "/dev/ttyS0"]).unwrap();
-        unreachable!();
+        loop {}
    }
 }
@@ -11,11 +11,11 @@ use libsys::{
    },
    error::Errno,
    ioctl::IoctlCmd,
+    proc::Pid,
    stat::{FileDescriptor, FileMode, GroupId, OpenFlags, UserId},
    termios::{Termios, TermiosLflag},
 };
-use libusr::{env::{self, UserInfo, UserShadow}, io};
-use core::str::FromStr;
+use libusr::{env, io};

 struct HiddenInput {
    fd: FileDescriptor,
@@ -74,28 +74,23 @@ fn readline(fd: FileDescriptor, buf: &mut [u8]) -> Result<&str, Errno> {
    }
 }

-fn login(uid: UserId, gid: GroupId, shell: &str) -> Result<(), Errno> {
+fn login_as(uid: UserId, gid: GroupId, shell: &str) -> Result<(), Errno> {
    if let Some(pid) = unsafe { sys_fork() }? {
        let mut status = 0;
        sys_waitpid(pid, &mut status).ok();
-        let pgid = sys_getpgid(None).unwrap();
+        let pgid = sys_getpgid(unsafe { Pid::from_raw(0) }).unwrap();
        io::tcsetpgrp(FileDescriptor::STDIN, pgid).unwrap();
        Ok(())
    } else {
        sys_setuid(uid).expect("setuid failed");
        sys_setgid(gid).expect("setgid failed");
-        let pgid = sys_setpgid(None, None).unwrap();
+        let pgid = sys_setpgid(unsafe { Pid::from_raw(0) }, unsafe { Pid::from_raw(0) }).unwrap();
        io::tcsetpgrp(FileDescriptor::STDIN, pgid).unwrap();
        sys_execve(shell, &[shell]).expect("execve() failed");
        panic!();
    }
 }

-fn login_as(name: &str) -> Result<(), Errno> {
-    let ent = UserInfo::by_name(name).map_err(|_| Errno::DoesNotExist)?;
-    login(ent.uid(), ent.gid(), ent.shell())
-}
-
 // TODO baud rate and misc port settings
 #[no_mangle]
 fn main() -> i32 {
@@ -138,26 +133,15 @@ fn main() -> i32 {
    loop {
        print!("login: ");
        let username = readline(FileDescriptor::STDIN, &mut user_buf).expect("Login read failed");
-
-        let shadow = match UserShadow::by_name(username) {
-            Ok(e) => e,
-            Err(_) => continue
-        };
-
-        if !shadow.password().is_empty() {
-            print!("password: ");
-            let password = {
-                let mut input = HiddenInput::open(FileDescriptor::STDIN).unwrap();
-                input.readline(&mut password_buf)
-            }
-            .expect("Password read failed");
-
-            if password != shadow.password() {
-                eprintln!("Incorrect password");
-                continue;
-            }
+        print!("password: ");
+        let password = {
+            let mut input = HiddenInput::open(FileDescriptor::STDIN).unwrap();
+            input.readline(&mut password_buf)
        }
+        .expect("Password read failed");

-        login_as(username);
+        if username == "root" && password == "toor" {
+            login_as(UserId::from(0), GroupId::from(0), "/bin/shell").unwrap();
+        }
    }
 }
Author	SHA1	Message	Date
alnyan	5128286220	Merge branch 'feat/thread' into feat/smp	2021-11-30 11:07:09 +02:00
alnyan	b61cb052ec	feat: rerun build if linker script changes	2021-11-10 18:37:09 +02:00
alnyan	3e6b7a71e6	feat: send IPIs using GIC	2021-11-10 18:36:29 +02:00
alnyan	f242948f82	xxx: doesn't really work	2021-11-10 18:35:32 +02:00
alnyan	0ee19cde96	merge: smp	2021-11-10 18:17:00 +02:00