From 64233db3df05899620d4e6639d27e9d847377345 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Fri, 21 Jul 2023 17:09:43 +0300 Subject: [PATCH] fs: implement in-memory writable filesystem --- Cargo.toml | 1 + lib/memfs/Cargo.toml | 15 ++ lib/memfs/src/block.rs | 260 +++++++++++++++++++ lib/memfs/src/bvec.rs | 459 ++++++++++++++++++++++++++++++++++ lib/memfs/src/dir.rs | 32 +++ lib/memfs/src/file.rs | 37 +++ lib/memfs/src/lib.rs | 287 +++++++++++++++++++++ lib/memfs/src/tar.rs | 136 ++++++++++ lib/memfs/test/dir1/test1.txt | 1 + lib/memfs/test/test1.txt | 1 + lib/memfs/test/test_image.tar | Bin 0 -> 10240 bytes src/fs/mod.rs | 31 ++- src/fs/tar.rs | 310 ----------------------- src/main.rs | 5 +- 14 files changed, 1261 insertions(+), 314 deletions(-) create mode 100644 lib/memfs/Cargo.toml create mode 100644 lib/memfs/src/block.rs create mode 100644 lib/memfs/src/bvec.rs create mode 100644 lib/memfs/src/dir.rs create mode 100644 lib/memfs/src/file.rs create mode 100644 lib/memfs/src/lib.rs create mode 100644 lib/memfs/src/tar.rs create mode 100644 lib/memfs/test/dir1/test1.txt create mode 100644 lib/memfs/test/test1.txt create mode 100644 lib/memfs/test/test_image.tar delete mode 100644 src/fs/tar.rs diff --git a/Cargo.toml b/Cargo.toml index 062ebf14..7ae70ef8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,7 @@ edition = "2021" [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } vfs = { path = "lib/vfs" } +memfs = { path = "lib/memfs" } aarch64-cpu = "9.3.1" atomic_enum = "0.2.0" diff --git a/lib/memfs/Cargo.toml b/lib/memfs/Cargo.toml new file mode 100644 index 00000000..4369a818 --- /dev/null +++ b/lib/memfs/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "memfs" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } +vfs = { path = "../vfs" } +static_assertions = "1.1.0" + +[features] +default = [] +test-io = [] diff --git a/lib/memfs/src/block.rs b/lib/memfs/src/block.rs new file mode 100644 index 00000000..ec68ef93 --- /dev/null +++ b/lib/memfs/src/block.rs @@ -0,0 +1,260 @@ +//! Block management interfaces and structures +use core::{ + marker::PhantomData, + mem::{size_of, MaybeUninit}, + ops::{Deref, DerefMut}, + ptr::NonNull, +}; + +use yggdrasil_abi::error::Error; + +/// Number of bytes in a block +pub const SIZE: usize = 4096; +/// Maximum number of indirection pointers a block can hold +pub const ENTRY_COUNT: usize = SIZE / size_of::(); + +/// Interface for a block allocator +/// +/// # Safety +/// +/// This trait is unsafe to implement because it has to provide and accept raw data pointers of +/// exactly [SIZE]. +pub unsafe trait BlockAllocator: 'static { + /// Allocates a contiguous block of size [SIZE] + fn alloc() -> Result, Error>; + + /// Dealocates a block. + /// + /// # Safety + /// + /// Unsafe: accepts arbitrary data pointers. + unsafe fn dealloc(block: NonNull); +} + +#[repr(transparent)] +struct BlockRaw<'a, A: BlockAllocator> { + inner: Option<&'a mut [u8; SIZE]>, + _pd: PhantomData, +} + +/// Block containing file data +#[repr(transparent)] +pub struct BlockData<'a, A: BlockAllocator> { + inner: BlockRaw<'a, A>, +} + +/// Block containing indirection pointers to other blocks +#[repr(transparent)] +pub struct BlockIndirect<'a, A: BlockAllocator> { + inner: BlockRaw<'a, A>, +} + +impl<'a, A: BlockAllocator> BlockRaw<'a, A> { + const fn null() -> Self { + Self { + inner: None, + _pd: PhantomData, + } + } + + fn new() -> Result { + let ptr = A::alloc()?; + unsafe { Ok(Self::from_raw(ptr)) } + } + + unsafe fn from_raw(ptr: NonNull) -> Self { + Self { + inner: Some(&mut *(ptr.as_ptr() as *mut _)), + _pd: PhantomData, + } + } + + unsafe fn as_uninit_indirect_mut( + &mut self, + ) -> &'a mut [MaybeUninit>; ENTRY_COUNT] { + unsafe { &mut *(self.inner.as_ref().unwrap().as_ptr() as *mut _) } + } + + #[inline] + fn is_null(&self) -> bool { + self.inner.is_none() + } +} + +impl Drop for BlockRaw<'_, A> { + fn drop(&mut self) { + if let Some(inner) = self.inner.take() { + unsafe { + A::dealloc(NonNull::new_unchecked(inner as *mut _)); + } + } + } +} + +// Data block +impl<'a, A: BlockAllocator> BlockData<'a, A> { + /// Dummy entry representing a missing block + pub const fn null() -> Self { + Self { + inner: BlockRaw::null(), + } + } + + /// Allocates a new block for data + pub fn new() -> Result { + Ok(Self { + inner: BlockRaw::new()?, + }) + } + + /// Replaces self with a null block and drops any data that might've been allocated + pub fn set_null(&mut self) { + self.inner = BlockRaw::null(); + } + + /// Returns `true` if the block this structure refers to has not yet been allocated + #[inline] + pub fn is_null(&self) -> bool { + self.inner.is_null() + } +} + +impl Deref for BlockData<'_, A> { + type Target = [u8; SIZE]; + + fn deref(&self) -> &Self::Target { + self.inner.inner.as_ref().unwrap() + } +} + +impl DerefMut for BlockData<'_, A> { + fn deref_mut(&mut self) -> &mut Self::Target { + self.inner.inner.as_mut().unwrap() + } +} + +// Indirect block +impl<'a, A: BlockAllocator> BlockIndirect<'a, A> { + /// Dummy entry representing a missing block + pub const fn null() -> Self { + Self { + inner: BlockRaw::null(), + } + } + + /// Allocates a new indirection block + pub fn new() -> Result { + let mut inner = BlockRaw::new()?; + for item in unsafe { inner.as_uninit_indirect_mut() } { + item.write(BlockData::null()); + } + Ok(Self { inner }) + } + + /// Returns `true` if the block this structure refers to has not yet been allocated + #[inline] + pub fn is_null(&self) -> bool { + self.inner.is_null() + } +} + +impl<'a, A: BlockAllocator> Deref for BlockIndirect<'a, A> { + type Target = [BlockData<'a, A>; ENTRY_COUNT]; + + fn deref(&self) -> &Self::Target { + unsafe { &*(self.inner.inner.as_ref().unwrap().as_ptr() as *const _) } + } +} + +impl<'a, A: BlockAllocator> DerefMut for BlockIndirect<'a, A> { + fn deref_mut(&mut self) -> &mut Self::Target { + unsafe { &mut *(self.inner.inner.as_mut().unwrap().as_mut_ptr() as *mut _) } + } +} + +impl<'a, A: BlockAllocator> Drop for BlockIndirect<'a, A> { + fn drop(&mut self) { + if self.is_null() { + return; + } + + for item in self.iter_mut() { + item.set_null(); + } + } +} + +#[cfg(test)] +mod tests { + use core::sync::atomic::Ordering; + use std::vec::Vec; + + use crate::block::{BlockData, BlockIndirect}; + + #[test] + fn block_indirect_allocation() { + test_allocator_with_counter!(A_COUNTER, A); + + const N: usize = 7; + const M: usize = 13; + + assert_eq!(A_COUNTER.load(Ordering::Acquire), 0); + + { + let mut indirect = Vec::new(); + + // Allocate indirect blocks + { + for _ in 0..N { + indirect.push(BlockIndirect::::new().unwrap()); + } + } + + assert_eq!(A_COUNTER.load(Ordering::Acquire), N); + + // Allocate L1 indirection blocks + { + for l1_block in indirect.iter_mut() { + for i in 0..M { + let l0_block = BlockData::new().unwrap(); + + l1_block[i] = l0_block; + } + } + } + + // N * M data blocks and N indirection blocks + assert_eq!(A_COUNTER.load(Ordering::Acquire), N * M + N); + + // Drop 1 indirect block for test + indirect.pop(); + + assert_eq!(A_COUNTER.load(Ordering::Acquire), (N - 1) * M + (N - 1)); + } + + assert_eq!(A_COUNTER.load(Ordering::Acquire), 0); + } + + #[test] + fn block_allocation() { + test_allocator_with_counter!(A_COUNTER, A); + + const N: usize = 13; + { + assert_eq!(A_COUNTER.load(Ordering::Acquire), 0); + + { + let mut s = Vec::new(); + + for _ in 0..N { + let mut block = BlockData::::new().unwrap(); + block.fill(1); + s.push(block); + } + assert_eq!(A_COUNTER.load(Ordering::Acquire), N); + } + + assert_eq!(A_COUNTER.load(Ordering::Acquire), 0); + } + } +} diff --git a/lib/memfs/src/bvec.rs b/lib/memfs/src/bvec.rs new file mode 100644 index 00000000..f731115b --- /dev/null +++ b/lib/memfs/src/bvec.rs @@ -0,0 +1,459 @@ +//! Block vector management structures +use core::{ + cmp::Ordering, + mem::MaybeUninit, + ops::{Index, IndexMut}, +}; + +use yggdrasil_abi::error::Error; + +use crate::block::{self, BlockAllocator, BlockData, BlockIndirect}; + +// 16.125M total +const L0_BLOCKS: usize = 32; // 128K in L0 +const L1_BLOCKS: usize = 8; // 16M in L1 + +/// Block vector for efficient in-memory files +pub struct BVec<'a, A: BlockAllocator> { + capacity: usize, + size: usize, + l0: [BlockData<'a, A>; L0_BLOCKS], + l1: [BlockIndirect<'a, A>; L1_BLOCKS], +} + +impl<'a, A: BlockAllocator> BVec<'a, A> { + /// Creates an empty block vector. + /// + /// # Note + /// + /// The function is guaranteed to make no allocations before the vector is actually written to. + pub fn new() -> Self { + let mut l0 = MaybeUninit::uninit_array(); + let mut l1 = MaybeUninit::uninit_array(); + + for it in l0.iter_mut() { + it.write(BlockData::null()); + } + + for it in l1.iter_mut() { + it.write(BlockIndirect::null()); + } + + Self { + capacity: 0, + size: 0, + l0: unsafe { MaybeUninit::array_assume_init(l0) }, + l1: unsafe { MaybeUninit::array_assume_init(l1) }, + } + } + + /// Returns the size of the data inside this vector + #[inline] + pub const fn size(&self) -> usize { + self.size + } + + fn grow_l1(&mut self, old_l1_cap: usize, new_l1_cap: usize) -> Result<(), Error> { + for i in old_l1_cap..new_l1_cap { + assert!(self.l1[i].is_null()); + self.l1[i] = BlockIndirect::new()?; + } + Ok(()) + } + + fn shrink_l1(&mut self, old_l1_cap: usize, new_l1_cap: usize) { + debug_assert!(new_l1_cap <= old_l1_cap); + for i in new_l1_cap..old_l1_cap { + assert!(!self.l1[i].is_null()); + self.l1[i] = BlockIndirect::null(); + } + } + + #[inline] + fn caps(cap: usize) -> (usize, usize) { + let l0_cap = core::cmp::min(cap, L0_BLOCKS); + let l1_cap = if cap > L0_BLOCKS { + core::cmp::min( + (cap - L0_BLOCKS + block::ENTRY_COUNT - 1) / block::ENTRY_COUNT, + L1_BLOCKS, + ) + } else { + 0 + }; + (l0_cap, l1_cap) + } + + /// Resizes the vector to hold exactly `new_capacity` data blocks + pub fn resize(&mut self, new_capacity: usize) -> Result<(), Error> { + // TODO handle L2 capacity + match new_capacity.cmp(&self.capacity) { + Ordering::Less => { + let (_, new_l1_cap) = Self::caps(new_capacity); + let (_, old_l1_cap) = Self::caps(self.capacity); + + // Shrink data blocks + for index in new_capacity..self.capacity { + let block = &mut self[index]; + assert!(!block.is_null()); + block.set_null(); + } + + // Shrink L1 blocks + self.shrink_l1(old_l1_cap, new_l1_cap); + } + + Ordering::Greater => { + let (_, new_l1_cap) = Self::caps(new_capacity); + let (_, old_l1_cap) = Self::caps(self.capacity); + + // Allocate L1 indirection blocks + assert!(new_l1_cap >= old_l1_cap); + if new_l1_cap > old_l1_cap { + self.grow_l1(old_l1_cap, new_l1_cap)?; + } + + // Grow data blocks + for index in self.capacity..new_capacity { + let block = unsafe { self.index_unchecked_mut(index) }; + assert!(block.is_null()); + *block = BlockData::new()?; + } + } + + Ordering::Equal => (), + } + + self.capacity = new_capacity; + Ok(()) + } + + fn ensure_write_capacity(&mut self, pos: usize, need_to_write: usize) -> Result<(), Error> { + let current_capacity = self.capacity * block::SIZE; + let need_capacity = + (core::cmp::max(pos + need_to_write, self.size) + block::SIZE - 1) / block::SIZE; + + if need_capacity > current_capacity { + self.resize(need_capacity) + } else { + Ok(()) + } + } + + /// Writes data to the vector, growing it if needed + pub fn write(&mut self, pos: u64, data: &[u8]) -> Result { + let mut pos = pos as usize; + // if pos > self.size { + // return Err(Error::InvalidFile); + // } + + let mut rem = data.len(); + let mut doff = 0usize; + + self.ensure_write_capacity(pos, rem)?; + + if pos + rem > self.size { + self.size = pos + rem; + } + + while rem > 0 { + let index = pos / block::SIZE; + let offset = pos % block::SIZE; + let count = core::cmp::min(rem, block::SIZE - offset); + + let block = &mut self[index]; + + let dst = &mut block[offset..offset + count]; + let src = &data[doff..doff + count]; + dst.copy_from_slice(src); + + doff += count; + pos += count; + rem -= count; + } + + Ok(doff) + } + + /// Reads data from the vector + pub fn read(&self, pos: u64, data: &mut [u8]) -> Result { + let mut pos = pos as usize; + if pos > self.size { + return Err(Error::InvalidFile); + } + + let mut rem = core::cmp::min(self.size - pos, data.len()); + let mut doff = 0usize; + + while rem > 0 { + let index = pos / block::SIZE; + let offset = pos % block::SIZE; + let count = core::cmp::min(block::SIZE - offset, rem); + + let block = &self[index]; + + let src = &block[offset..offset + count]; + let dst = &mut data[doff..doff + count]; + + dst.copy_from_slice(src); + + doff += count; + pos += count; + rem -= count; + } + + Ok(doff) + } + + unsafe fn index_unchecked(&self, mut index: usize) -> &BlockData<'a, A> { + if index < L0_BLOCKS { + return &self.l0[index]; + } + index -= L0_BLOCKS; + if index < L1_BLOCKS * block::ENTRY_COUNT { + let l1i = index / block::ENTRY_COUNT; + let l0i = index % block::ENTRY_COUNT; + + let l1r = &self.l1[l1i]; + assert!(!l1r.is_null()); + + return &l1r[l0i]; + } + + todo!(); + } + + unsafe fn index_unchecked_mut(&mut self, mut index: usize) -> &mut BlockData<'a, A> { + if index < L0_BLOCKS { + return &mut self.l0[index]; + } + index -= L0_BLOCKS; + if index < L1_BLOCKS * block::ENTRY_COUNT { + let l1i = index / block::ENTRY_COUNT; + let l0i = index % block::ENTRY_COUNT; + + let l1r = &mut self.l1[l1i]; + assert!(!l1r.is_null()); + + return &mut l1r[l0i]; + } + + todo!() + } +} + +impl<'a, A: BlockAllocator> Index for BVec<'a, A> { + type Output = BlockData<'a, A>; + + fn index(&self, index: usize) -> &Self::Output { + if index > self.capacity { + panic!( + "Block index out of bounds: capacity={}, index={}", + self.capacity, index + ); + } + + unsafe { self.index_unchecked(index) } + } +} + +impl<'a, A: BlockAllocator> IndexMut for BVec<'a, A> { + fn index_mut(&mut self, index: usize) -> &mut Self::Output { + if index > self.capacity { + panic!( + "Block index out of bounds: capacity={}, index={}", + self.capacity, index + ); + } + + unsafe { self.index_unchecked_mut(index) } + } +} + +impl<'a, A: BlockAllocator> TryFrom<&[u8]> for BVec<'a, A> { + type Error = Error; + + fn try_from(value: &[u8]) -> Result { + let mut res = Self::new(); + res.write(0, value)?; + assert_eq!(res.size(), value.len()); + Ok(res) + } +} + +#[cfg(test)] +mod bvec_allocation { + use core::sync::atomic::Ordering; + + use crate::{ + block, + bvec::{BVec, L0_BLOCKS, L1_BLOCKS}, + }; + + #[test] + fn bvec_grow_shrink() { + test_allocator_with_counter!(A_COUNTER, A); + + assert_eq!(A_COUNTER.load(Ordering::Acquire), 0); + + { + let mut bvec = BVec::::new(); + + assert_eq!( + A_COUNTER.load(Ordering::Acquire), + 0, + "BVec should not allocate on creation" + ); + + const N: usize = 123; + bvec.resize(N).unwrap(); + + // N data blocks (12 in L0 + 111 in L1) + assert_eq!(A_COUNTER.load(Ordering::Acquire), N + 1); + + // Test the index interface + for i in 0..N { + assert!(!bvec[i].is_null(), "Index {} must be allocated", i); + } + + // Test the data structure + for i in 0..L0_BLOCKS { + assert!(!bvec.l0[i].is_null()); + } + assert!(!bvec.l1[0].is_null()); + for i in L0_BLOCKS..N { + let l1i = (i - L0_BLOCKS) / block::ENTRY_COUNT; + let l0i = (i - L0_BLOCKS) % block::ENTRY_COUNT; + + let l1r = &bvec.l1[l1i]; + assert!(!l1r.is_null()); + assert!(!l1r[l0i].is_null()); + } + + for i in 1..L1_BLOCKS { + assert!(bvec.l1[i].is_null()); + } + + // Shrink to 100 blocks, test if L1 is still allocated + const M: usize = 100; + bvec.resize(M).unwrap(); + + assert_eq!(A_COUNTER.load(Ordering::Acquire), M + 1); + + // Test the index interface + for i in 0..M { + assert!(!bvec[i].is_null(), "Index {} must be allocated", i); + } + + // Test the data structure + for i in 0..L0_BLOCKS { + assert!(!bvec.l0[i].is_null()); + } + assert!(!bvec.l1[0].is_null()); + for i in L0_BLOCKS..M { + let l1i = (i - L0_BLOCKS) / block::ENTRY_COUNT; + let l0i = (i - L0_BLOCKS) % block::ENTRY_COUNT; + + let l1r = &bvec.l1[l1i]; + assert!(!l1r.is_null()); + assert!(!l1r[l0i].is_null()); + } + for i in M..N { + let l1i = (i - L0_BLOCKS) / block::ENTRY_COUNT; + let l0i = (i - L0_BLOCKS) % block::ENTRY_COUNT; + + let l1r = &bvec.l1[l1i]; + assert!(!l1r.is_null()); + assert!(l1r[l0i].is_null()); + } + + for i in 1..L1_BLOCKS { + assert!(bvec.l1[i].is_null()); + } + + // Shrink to 13 blocks, test if L1 got deallocated + const O: usize = 13; + bvec.resize(O).unwrap(); + + assert_eq!(A_COUNTER.load(Ordering::Acquire), O); + } + + assert_eq!(A_COUNTER.load(Ordering::Acquire), 0); + } +} + +#[cfg(all(test, feature = "test-io"))] +mod bvec_io { + use crate::{block, bvec::L0_BLOCKS}; + + use super::BVec; + + #[test] + fn test_bvec_write() { + test_allocator_with_counter!(A_COUNTER, A); + + { + let data = [1, 2, 3, 4, 5]; + let mut bvec = BVec::::new(); + + // Write at 0 + assert_eq!(bvec.write(0, &data).unwrap(), data.len()); + assert_eq!(bvec.capacity, 1); + assert_eq!(bvec.size(), data.len()); + + assert_eq!(&bvec[0][..bvec.size()], &data[..]); + + // Write at 3 + assert_eq!(bvec.write(3, &data).unwrap(), data.len()); + assert_eq!(bvec.capacity, 1); + assert_eq!(bvec.size(), 3 + data.len()); + + assert_eq!(&bvec[0][..bvec.size()], &[1, 2, 3, 1, 2, 3, 4, 5]); + } + + { + let data = [5, 4, 3, 2, 1]; + let mut bvec = BVec::::new(); + + // Write at the end of L0-region + assert_eq!( + bvec.write((L0_BLOCKS * block::SIZE) as u64, &data).unwrap(), + data.len() + ); + // L0_BLOCKS + 1 L1 data block + assert_eq!(bvec.capacity, L0_BLOCKS + 1); + assert_eq!(bvec.size(), L0_BLOCKS * block::SIZE + data.len()); + + assert_eq!(&bvec[L0_BLOCKS][..data.len()], &data[..]); + + // Write at zero + assert_eq!(bvec.write(0, &data).unwrap(), data.len()); + assert_eq!(bvec.capacity, L0_BLOCKS + 1); + assert_eq!(bvec.size(), L0_BLOCKS * block::SIZE + data.len()); + + assert_eq!(&bvec[0][..data.len()], &data[..]); + + // Test write crossing L0 block boundary + assert_eq!( + bvec.write((block::SIZE - 3) as u64, &data).unwrap(), + data.len() + ); + assert_eq!(bvec.capacity, L0_BLOCKS + 1); + assert_eq!(bvec.size(), L0_BLOCKS * block::SIZE + data.len()); + + assert_eq!(&bvec[0][block::SIZE - 3..], &[5, 4, 3]); + assert_eq!(&bvec[1][..2], &[2, 1]); + + // Test write crossing L0-L1 boundary + assert_eq!( + bvec.write((L0_BLOCKS * block::SIZE) as u64 - 2, &data) + .unwrap(), + data.len() + ); + assert_eq!(bvec.capacity, L0_BLOCKS + 1); + assert_eq!(bvec.size(), L0_BLOCKS * block::SIZE + data.len()); + + assert_eq!(&bvec[L0_BLOCKS - 1][block::SIZE - 2..], &[5, 4]); + assert_eq!(&bvec[L0_BLOCKS][..data.len()], &[3, 2, 1, 2, 1]); + } + } +} diff --git a/lib/memfs/src/dir.rs b/lib/memfs/src/dir.rs new file mode 100644 index 00000000..77f4f796 --- /dev/null +++ b/lib/memfs/src/dir.rs @@ -0,0 +1,32 @@ +use core::marker::PhantomData; + +use alloc::boxed::Box; + +use vfs::{Vnode, VnodeImpl, VnodeKind, VnodeRef}; +use yggdrasil_abi::error::Error; + +use crate::{block::BlockAllocator, bvec::BVec, file::FileNode}; + +pub(crate) struct DirectoryNode { + _pd: PhantomData, +} + +impl VnodeImpl for DirectoryNode { + fn create(&mut self, _at: &VnodeRef, name: &str, kind: VnodeKind) -> Result { + let child = Vnode::new(name, kind); + match kind { + VnodeKind::Directory => child.set_data(Box::new(Self { _pd: PhantomData })), + VnodeKind::Regular => child.set_data(Box::new(FileNode { + data: BVec::::new(), + })), + _ => todo!(), + } + Ok(child) + } +} + +impl DirectoryNode { + pub fn new() -> Self { + Self { _pd: PhantomData } + } +} diff --git a/lib/memfs/src/file.rs b/lib/memfs/src/file.rs new file mode 100644 index 00000000..57fcf54c --- /dev/null +++ b/lib/memfs/src/file.rs @@ -0,0 +1,37 @@ +use vfs::{VnodeImpl, VnodeRef}; +use yggdrasil_abi::{ + error::Error, + io::{FileMode, OpenOptions}, +}; + +use crate::{block::BlockAllocator, bvec::BVec}; + +pub(crate) struct FileNode { + pub(crate) data: BVec<'static, A>, +} + +impl VnodeImpl for FileNode { + fn open(&mut self, _node: &VnodeRef, opts: OpenOptions, _mode: FileMode) -> Result { + if opts.contains(OpenOptions::APPEND) { + Ok(self.data.size() as u64) + } else { + Ok(0) + } + } + + fn close(&mut self, _node: &VnodeRef) -> Result<(), Error> { + Ok(()) + } + + fn read(&mut self, _node: &VnodeRef, pos: u64, data: &mut [u8]) -> Result { + self.data.read(pos, data) + } + + fn write(&mut self, _node: &VnodeRef, pos: u64, data: &[u8]) -> Result { + self.data.write(pos, data) + } + + fn size(&mut self, _node: &VnodeRef) -> Result { + Ok(self.data.size() as u64) + } +} diff --git a/lib/memfs/src/lib.rs b/lib/memfs/src/lib.rs new file mode 100644 index 00000000..db5ef084 --- /dev/null +++ b/lib/memfs/src/lib.rs @@ -0,0 +1,287 @@ +//! In-memory filesystem driver +#![no_std] +#![warn(missing_docs)] +#![allow(clippy::new_without_default)] +#![feature( + const_mut_refs, + maybe_uninit_uninit_array, + const_maybe_uninit_uninit_array, + maybe_uninit_array_assume_init +)] + +use core::{ + any::Any, + cell::{Ref, RefCell}, + marker::PhantomData, +}; + +use alloc::{boxed::Box, rc::Rc}; +use block::BlockAllocator; +use vfs::{BlockDevice, Filesystem, Vnode, VnodeKind, VnodeRef}; +use yggdrasil_abi::{error::Error, path}; + +use crate::{bvec::BVec, dir::DirectoryNode, file::FileNode, tar::TarIterator}; + +#[cfg(test)] +extern crate std; + +extern crate alloc; + +#[cfg(test)] +macro_rules! test_allocator_with_counter { + ($counter:ident, $allocator:ident) => { + static $counter: core::sync::atomic::AtomicUsize = core::sync::atomic::AtomicUsize::new(0); + + struct $allocator; + + unsafe impl $crate::block::BlockAllocator for $allocator { + fn alloc() -> Result, yggdrasil_abi::error::Error> { + let b = std::boxed::Box::into_raw(std::boxed::Box::new([0; $crate::block::SIZE])); + $counter.fetch_add(1, core::sync::atomic::Ordering::Release); + Ok(unsafe { core::ptr::NonNull::new_unchecked(b as _) }) + } + + unsafe fn dealloc(block: core::ptr::NonNull) { + $counter.fetch_sub(1, core::sync::atomic::Ordering::Release); + drop(std::boxed::Box::from_raw( + block.as_ptr() as *mut [u8; $crate::block::SIZE] + )); + } + } + }; +} + +pub mod block; +pub mod bvec; + +mod dir; +mod file; +mod tar; + +/// In-memory read/write filesystem +pub struct MemoryFilesystem { + root: RefCell>, + _pd: PhantomData, +} + +impl Filesystem for MemoryFilesystem { + fn dev(self: Rc) -> Option<&'static dyn BlockDevice> { + todo!() + } + + fn data(&self) -> Option> { + todo!() + } + + fn root(self: Rc) -> Result { + Ok(self.root.borrow().clone().unwrap()) + } +} + +impl MemoryFilesystem { + fn make_path( + self: &Rc, + at: &VnodeRef, + path: &str, + kind: VnodeKind, + create: bool, + ) -> Result { + if path.is_empty() { + return Ok(at.clone()); + } + let (element, rest) = path::split_left(path); + assert!(!element.is_empty()); + + let node = at.lookup(element); + let node = match node { + Some(node) => node, + None => { + if !create { + return Err(Error::DoesNotExist); + } + + let node = self.create_node_initial(element, kind); + at.add_child(node.clone()); + + node + } + }; + + if rest.is_empty() { + Ok(node) + } else { + assert!(node.is_directory()); + self.make_path(&node, rest, kind, create) + } + } + + fn create_node_initial(self: &Rc, name: &str, kind: VnodeKind) -> VnodeRef { + assert!(!name.is_empty()); + assert!(!name.contains('/')); + + let node = Vnode::new(name, kind); + node.set_fs(self.clone()); + + match kind { + VnodeKind::Directory => node.set_data(Box::new(DirectoryNode::::new())), + VnodeKind::Regular => {} + _ => todo!(), + } + + node + } + + fn from_slice_internal(self: &Rc, tar_data: &'static [u8]) -> Result { + let root = Vnode::new("", VnodeKind::Directory); + root.set_fs(self.clone()); + root.set_data(Box::new(DirectoryNode::::new())); + + // 1. Create paths in tar + for item in TarIterator::new(tar_data) { + let Ok((hdr, _)) = item else { + return Err(Error::InvalidArgument); + }; + + let path = hdr.name.as_str()?.trim_matches('/'); + let (dirname, filename) = path::split_right(path); + let parent = self.make_path(&root, dirname, VnodeKind::Directory, true)?; + let node = self.create_node_initial(filename, hdr.node_kind()); + + parent.add_child(node); + } + + // 2. Associate files with their data + for item in TarIterator::new(tar_data) { + let Ok((hdr, data)) = item else { + panic!("Unreachable"); + }; + if hdr.node_kind() == VnodeKind::Regular { + let data = data.unwrap(); + let path = hdr.name.as_str()?.trim_matches('/'); + let node = self.make_path(&root, path, VnodeKind::Directory, false)?; + + let bvec = BVec::::try_from(data)?; + assert_eq!(bvec.size(), data.len()); + node.set_data(Box::new(FileNode { data: bvec })); + } + } + + Ok(root) + } + + /// Constructs a filesystem tree from a tar image in memory + pub fn from_slice(tar_data: &'static [u8]) -> Result, Error> { + let fs = Rc::new(Self { + root: RefCell::new(None), + _pd: PhantomData, + }); + let root = fs.from_slice_internal(tar_data)?; + fs.root.replace(Some(root)); + + Ok(fs) + } + + /// Constructs an empty memory filesystem + pub fn empty() -> Rc { + let fs = Rc::new(Self { + root: RefCell::new(None), + _pd: PhantomData, + }); + let root = Vnode::new("", VnodeKind::Directory); + root.set_data(Box::new(DirectoryNode::::new())); + root.set_fs(fs.clone()); + fs.root.replace(Some(root)); + fs + } +} + +#[cfg(test)] +mod tests { + use std::rc::Rc; + use vfs::{Filesystem, IoContext, Read, Seek, SeekFrom, VnodeKind, Write}; + use yggdrasil_abi::io::{FileMode, OpenOptions}; + + use crate::MemoryFilesystem; + + #[test] + fn test_memfs_construction() { + fn check_file(ioctx: &IoContext, path: &str, expected_data: &str) { + let node = ioctx.find(None, path, false, false).unwrap(); + + assert_eq!(node.kind(), VnodeKind::Regular); + assert_eq!(node.size().unwrap(), expected_data.len() as u64); + + let file = node.open(OpenOptions::READ, FileMode::empty()).unwrap(); + let mut buf = [0; 512]; + + assert_eq!( + file.borrow_mut().read(&mut buf).unwrap(), + expected_data.len() + ); + + assert_eq!(&buf[..expected_data.len()], expected_data.as_bytes()); + } + + static TEST_IMAGE: &[u8] = include_bytes!("../test/test_image.tar"); + test_allocator_with_counter!(A_COUNTER, A); + + let fs = MemoryFilesystem::::from_slice(TEST_IMAGE).unwrap(); + let root = fs.root().unwrap(); + + let ioctx = IoContext::new(root.clone()); + + assert!(Rc::ptr_eq( + &root, + &ioctx.find(None, "/", false, false).unwrap() + )); + + check_file(&ioctx, "/test1.txt", include_str!("../test/test1.txt")); + } + + #[test] + fn test_memfs_create_and_write() { + test_allocator_with_counter!(A_COUNTER, A); + + let fs = MemoryFilesystem::::empty(); + let root = fs.root().unwrap(); + + let ioctx = IoContext::new(root.clone()); + + // Create, write, seek and read file + { + // TODO CREATE option handling + root.create("test1.txt", VnodeKind::Regular).unwrap(); + + let file = ioctx + .open( + None, + "/test1.txt", + OpenOptions::WRITE | OpenOptions::READ, + FileMode::empty(), + ) + .unwrap(); + + let write_data = [1, 2, 3, 4]; + let mut read_data = [0; 512]; + + let mut file = file.borrow_mut(); + assert_eq!(file.write(&write_data).unwrap(), write_data.len()); + assert_eq!(file.seek(SeekFrom::Start(0)).unwrap(), 0); + assert_eq!(file.read(&mut read_data).unwrap(), write_data.len()); + assert_eq!(&read_data[..write_data.len()], &write_data[..]); + } + + // Create a directory + { + // TODO read directory + root.create("dir1", VnodeKind::Directory).unwrap(); + + let dir1 = ioctx.find(None, "/dir1", false, false).unwrap(); + let node = dir1.create("file1.txt", VnodeKind::Regular).unwrap(); + assert!(Rc::ptr_eq( + &ioctx.find(None, "/dir1/file1.txt", false, false).unwrap(), + &node + )); + } + } +} diff --git a/lib/memfs/src/tar.rs b/lib/memfs/src/tar.rs new file mode 100644 index 00000000..a6e7afe0 --- /dev/null +++ b/lib/memfs/src/tar.rs @@ -0,0 +1,136 @@ +use vfs::VnodeKind; +use yggdrasil_abi::error::Error; + +#[repr(C)] +pub(crate) struct OctalField { + data: [u8; N], +} + +#[repr(C)] +pub(crate) struct TarString { + data: [u8; N], +} + +pub(crate) struct TarIterator<'a> { + data: &'a [u8], + offset: usize, + zero_blocks: usize, +} + +#[repr(packed)] +pub(crate) struct TarEntry { + pub name: TarString<100>, + _mode: OctalField<8>, + _uid: OctalField<8>, + _gid: OctalField<8>, + pub size: OctalField<12>, + _mtime: OctalField<12>, + _checksum: OctalField<8>, + type_: u8, + _link_name: TarString<100>, + _magic: [u8; 8], + _user: TarString<32>, + _group: TarString<32>, + _dev_major: OctalField<8>, + _dev_minor: OctalField<8>, + _prefix: TarString<155>, + __pad: [u8; 12], +} + +impl<'a> TarIterator<'a> { + pub const fn new(data: &'a [u8]) -> Self { + Self { + data, + offset: 0, + zero_blocks: 0, + } + } +} + +impl<'a> Iterator for TarIterator<'a> { + type Item = Result<(&'a TarEntry, Option<&'a [u8]>), Error>; + + fn next(&mut self) -> Option { + loop { + if self.offset + 512 > self.data.len() { + break None; + } + + let hdr_ptr = &self.data[self.offset..]; + let hdr = unsafe { &*(hdr_ptr.as_ptr() as *const TarEntry) }; + + if hdr.is_empty() { + if self.zero_blocks == 1 { + self.offset = self.data.len(); + return None; + } + self.zero_blocks += 1; + continue; + } + + let (data, size_aligned) = match hdr.type_ { + 0 | b'0' => { + let size = usize::from(&hdr.size); + + if self.offset + 512 + size > self.data.len() { + return Some(Err(Error::InvalidArgument)); + } + + let data = &self.data[self.offset + 512..self.offset + 512 + size]; + let size_aligned = (size + 511) & !511; + + (Some(data), size_aligned) + } + // Directory + b'5' => (None, 0), + _ => todo!("Unknown node kind: {}", hdr.type_), + }; + self.offset += size_aligned + 512; + + break Some(Ok((hdr, data))); + } + } +} + +impl From<&OctalField> for usize { + fn from(value: &OctalField) -> Self { + let mut acc = 0; + for i in 0..N { + if value.data[i] == 0 { + break; + } + acc <<= 3; + acc |= (value.data[i] - b'0') as usize; + } + acc + } +} + +impl TarString { + pub fn as_str(&self) -> Result<&str, Error> { + core::str::from_utf8(&self.data[..self.len()]).map_err(|_| Error::InvalidArgument) + } + + pub fn len(&self) -> usize { + for i in 0..N { + if self.data[i] == 0 { + return i; + } + } + N + } +} + +impl TarEntry { + pub fn is_empty(&self) -> bool { + self.name.data[0] == 0 + } + + pub fn node_kind(&self) -> VnodeKind { + match self.type_ { + 0 | b'0' => VnodeKind::Regular, + b'5' => VnodeKind::Directory, + _ => todo!(), + } + } +} diff --git a/lib/memfs/test/dir1/test1.txt b/lib/memfs/test/dir1/test1.txt new file mode 100644 index 00000000..ac439756 --- /dev/null +++ b/lib/memfs/test/dir1/test1.txt @@ -0,0 +1 @@ +This is another test file diff --git a/lib/memfs/test/test1.txt b/lib/memfs/test/test1.txt new file mode 100644 index 00000000..9f4b6d8b --- /dev/null +++ b/lib/memfs/test/test1.txt @@ -0,0 +1 @@ +This is a test file diff --git a/lib/memfs/test/test_image.tar b/lib/memfs/test/test_image.tar new file mode 100644 index 0000000000000000000000000000000000000000..15dd01d95793af66dc35379801bbcd144887e5cc GIT binary patch literal 10240 zcmeIz;R=E<6o%pb?kaW#bvAeOM!Eq(BOwfQLEU|>C@QFaEu!8b3^}-s=j_9#8{ER= z9Wsqd)l8GCRBq=~O2fDiv<#|+R7RZhr5LMuk9A&sH^zFf*3_H+Q1{Pwd{=+|>bHD1J_kX$WMms%qzm4m5u<0t; zRkvwp_e{L~raxlK?12CR2q1s}0tg_0 g00IagfB*srAb = OneTimeInit::new(); +/// Implementation of [memfs::block::BlockAllocator] for the kernel +pub struct FileBlockAllocator; + +unsafe impl BlockAllocator for FileBlockAllocator { + fn alloc() -> Result, Error> { + // TODO make this a static assertion + assert_eq!(block::SIZE, 4096); + let page = phys::alloc_page(PageUsage::Used)?; + Ok(unsafe { NonNull::new_unchecked(page.virtualize() as *mut _) }) + } + + unsafe fn dealloc(block: NonNull) { + let page = block.as_ptr() as usize; + assert!(page > mem::KERNEL_VIRT_OFFSET); + todo!("Release physical memory"); + } +} + /// Constructs an instance of a filesystem for given set of [MountOptions] pub fn create_filesystem(options: &MountOptions) -> Result { let fs_name = options.filesystem.unwrap(); diff --git a/src/fs/tar.rs b/src/fs/tar.rs deleted file mode 100644 index c1d2358f..00000000 --- a/src/fs/tar.rs +++ /dev/null @@ -1,310 +0,0 @@ -//! In-memory filesystem implementation -use abi::{error::Error, io::FileMode}; -use alloc::{boxed::Box, rc::Rc}; -use vfs::{Filesystem, Vnode, VnodeImpl, VnodeKind, VnodeRef}; -use yggdrasil_abi::io::OpenOptions; - -use crate::util::OneTimeInit; - -#[repr(C)] -struct OctalField { - data: [u8; N], -} - -#[repr(C)] -struct TarString { - data: [u8; N], -} - -struct TarIterator<'a> { - data: &'a [u8], - offset: usize, - zero_blocks: usize, -} - -#[repr(packed)] -struct TarEntry { - name: TarString<100>, - _mode: OctalField<8>, - _uid: OctalField<8>, - _gid: OctalField<8>, - size: OctalField<12>, - _mtime: OctalField<12>, - _checksum: OctalField<8>, - type_: u8, - _link_name: TarString<100>, - _magic: [u8; 8], - _user: TarString<32>, - _group: TarString<32>, - _dev_major: OctalField<8>, - _dev_minor: OctalField<8>, - _prefix: TarString<155>, - __pad: [u8; 12], -} - -impl<'a> TarIterator<'a> { - pub const fn new(data: &'a [u8]) -> Self { - Self { - data, - offset: 0, - zero_blocks: 0, - } - } -} - -impl<'a> Iterator for TarIterator<'a> { - type Item = Result<(&'a TarEntry, Option<&'a [u8]>), Error>; - - fn next(&mut self) -> Option { - loop { - if self.offset + 512 > self.data.len() { - break None; - } - - let hdr_ptr = &self.data[self.offset..]; - let hdr = unsafe { &*(hdr_ptr.as_ptr() as *const TarEntry) }; - - if hdr.is_empty() { - if self.zero_blocks == 1 { - self.offset = self.data.len(); - return None; - } - self.zero_blocks += 1; - continue; - } - - let (data, size_aligned) = match hdr.type_ { - 0 | b'0' => { - let size = usize::from(&hdr.size); - - if self.offset + 512 + size > self.data.len() { - return Some(Err(Error::InvalidArgument)); - } - - let data = &self.data[self.offset + 512..self.offset + 512 + size]; - let size_aligned = (size + 511) & !511; - - (Some(data), size_aligned) - } - // Directory - b'5' => (None, 0), - _ => todo!("Unknown node kind: {}", hdr.type_), - }; - self.offset += size_aligned + 512; - - break Some(Ok((hdr, data))); - } - } -} - -impl From<&OctalField> for usize { - fn from(value: &OctalField) -> Self { - let mut acc = 0; - for i in 0..N { - if value.data[i] == 0 { - break; - } - acc <<= 3; - acc |= (value.data[i] - b'0') as usize; - } - acc - } -} - -impl TarString { - pub fn as_str(&self) -> Result<&str, Error> { - core::str::from_utf8(&self.data[..self.len()]).map_err(|_| Error::InvalidArgument) - } - - pub fn len(&self) -> usize { - for i in 0..N { - if self.data[i] == 0 { - return i; - } - } - N - } -} - -impl TarEntry { - pub fn is_empty(&self) -> bool { - self.name.data[0] == 0 - } - - pub fn node_kind(&self) -> VnodeKind { - match self.type_ { - 0 | b'0' => VnodeKind::Regular, - b'5' => VnodeKind::Directory, - _ => todo!(), - } - } -} - -/// tar-image based in-memory filesystem -pub struct TarFilesystem { - root: OneTimeInit, -} - -impl Filesystem for TarFilesystem { - fn dev(self: Rc) -> Option<&'static dyn vfs::BlockDevice> { - todo!() - } - - fn root(self: Rc) -> Result { - self.root.try_get().cloned().ok_or(Error::DoesNotExist) - } - - fn data(&self) -> Option> { - todo!() - } -} - -struct DirInode; -struct RegularInode { - data: &'static [u8], -} - -impl VnodeImpl for DirInode { - fn create(&mut self, _at: &VnodeRef, name: &str, kind: VnodeKind) -> Result { - let child = Vnode::new(name, kind); - match kind { - VnodeKind::Directory => child.set_data(Box::new(DirInode)), - VnodeKind::Regular => (), - _ => todo!(), - } - Ok(child) - } -} - -impl VnodeImpl for RegularInode { - fn open(&mut self, _node: &VnodeRef, opts: OpenOptions, _mode: FileMode) -> Result { - if opts.contains(OpenOptions::WRITE) { - panic!("TODO: tarfs write"); - } - - Ok(0) - } - - fn close(&mut self, _node: &VnodeRef) -> Result<(), Error> { - Ok(()) - } - - fn read(&mut self, _node: &VnodeRef, pos: u64, data: &mut [u8]) -> Result { - let pos = pos as usize; - if pos > self.data.len() { - return Err(Error::InvalidFile); - } - let rem = core::cmp::min(self.data.len() - pos, data.len()); - data[..rem].copy_from_slice(&self.data[pos..pos + rem]); - Ok(rem) - } - - fn size(&mut self, _node: &VnodeRef) -> Result { - Ok(self.data.len() as u64) - } -} - -impl TarFilesystem { - fn make_path( - self: &Rc, - at: &VnodeRef, - path: &str, - kind: VnodeKind, - create: bool, - ) -> Result { - debugln!("make_path {:?}", path); - if path.is_empty() { - return Ok(at.clone()); - } - let (element, rest) = abi::path::split_left(path); - assert!(!element.is_empty()); - - let node = at.lookup(element); - let node = match node { - Some(node) => node, - None => { - if !create { - debugln!("path {:?} does not exist", path); - return Err(Error::DoesNotExist); - } - - infoln!("Create {:?}", element); - let node = self.create_node_initial(element, kind); - at.add_child(node.clone()); - - node - } - }; - - if rest.is_empty() { - Ok(node) - } else { - assert!(node.is_directory()); - self.make_path(&node, rest, kind, create) - } - } - - fn create_node_initial(self: &Rc, name: &str, kind: VnodeKind) -> VnodeRef { - assert!(!name.is_empty()); - assert!(!name.contains('/')); - - let node = Vnode::new(name, kind); - node.set_fs(self.clone()); - - match kind { - VnodeKind::Directory => node.set_data(Box::new(DirInode)), - VnodeKind::Regular => {} - _ => todo!(), - } - - node - } - - fn from_slice_internal(self: &Rc, tar_data: &'static [u8]) -> Result { - let root = Vnode::new("", VnodeKind::Directory); - root.set_fs(self.clone()); - root.set_data(Box::new(DirInode)); - - // 1. Create paths in tar - for item in TarIterator::new(tar_data) { - let Ok((hdr, _)) = item else { - warnln!("Tar image is truncated"); - return Err(Error::InvalidArgument); - }; - - let path = hdr.name.as_str()?.trim_matches('/'); - infoln!("path = {:?}", path); - let (dirname, filename) = abi::path::split_right(path); - let parent = self.make_path(&root, dirname, VnodeKind::Directory, true)?; - let node = self.create_node_initial(filename, hdr.node_kind()); - - parent.add_child(node); - } - - // 2. Associate files with their data - for item in TarIterator::new(tar_data) { - let Ok((hdr, data)) = item else { - panic!("Unreachable"); - }; - if hdr.node_kind() == VnodeKind::Regular { - let data = data.unwrap(); - let path = hdr.name.as_str()?.trim_matches('/'); - let node = self.make_path(&root, path, VnodeKind::Directory, false)?; - node.set_data(Box::new(RegularInode { data })); - } - } - - Ok(root) - } - - /// Constructs a filesystem tree from a tar image in memory - pub fn from_slice(tar_data: &'static [u8]) -> Result, Error> { - let fs = Rc::new(TarFilesystem { - root: OneTimeInit::new(), - }); - let root = fs.from_slice_internal(tar_data)?; - fs.root.init(root); - - Ok(fs) - } -} diff --git a/src/main.rs b/src/main.rs index 91f56fd7..9ed9318e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -19,7 +19,8 @@ use abi::{ error::Error, io::{FileMode, OpenOptions}, }; -use fs::{tar::TarFilesystem, INITRD_DATA}; +use fs::{FileBlockAllocator, INITRD_DATA}; +use memfs::MemoryFilesystem; use task::process::Process; use vfs::{Filesystem, IoContext, VnodeRef}; @@ -42,7 +43,7 @@ pub mod util; fn setup_root() -> Result { let initrd_data = INITRD_DATA.get(); - let fs = TarFilesystem::from_slice(initrd_data.data)?; + let fs = MemoryFilesystem::::from_slice(initrd_data.data).unwrap(); fs.root() }