shell: reimplement shell

userspace/Cargo.lock

@@ -1023,9 +1023,9 @@ dependencies = [
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.89"
+version = "1.0.93"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f139b0662de085916d1fb67d2b4169d1addddda1919e696f3252b740b629986e"
+checksum = "60946a68e5f9d28b0dc1c21bb8a97ee7d018a8b322fa57838ba31cc878e22d99"
 dependencies = [
  "unicode-ident",
 ]
@@ -1330,11 +1330,8 @@ version = "0.1.0"
 dependencies = [
  "clap",
  "cross",
- "libc",
  "nom",
  "thiserror",
- "yggdrasil-abi",
- "yggdrasil-rt",
 ]
 
 [[package]]
@@ -1451,9 +1448,9 @@ checksum = "9ac8fb7895b4afa060ad731a32860db8755da3449a47e796d5ecf758db2671d4"
 
 [[package]]
 name = "syn"
-version = "2.0.86"
+version = "2.0.96"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e89275301d38033efb81a6e60e3497e734dfcc62571f2854bf4b16690398824c"
+checksum = "d5d0adab1ae378d7f53bdebc67a39f1f151407ef230f0ce2883572f5d8985c80"
 dependencies = [
  "proc-macro2",
  "quote",

userspace/etc/profile

@@ -1 +1 @@
-set PATH /bin:/sbin
+export PATH=/bin:/sbin

userspace/shell/.gitignore

@@ -0,0 +1 @@
+/target

userspace/shell/Cargo.lock

@@ -0,0 +1,88 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 4
+
+[[package]]
+name = "memchr"
+version = "2.7.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
+
+[[package]]
+name = "minimal-lexical"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
+
+[[package]]
+name = "nom"
+version = "7.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
+dependencies = [
+ "memchr",
+ "minimal-lexical",
+]
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.93"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "60946a68e5f9d28b0dc1c21bb8a97ee7d018a8b322fa57838ba31cc878e22d99"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.38"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "shell"
+version = "0.1.0"
+dependencies = [
+ "nom",
+ "thiserror",
+]
+
+[[package]]
+name = "syn"
+version = "2.0.96"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d5d0adab1ae378d7f53bdebc67a39f1f151407ef230f0ce2883572f5d8985c80"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "thiserror"
+version = "2.0.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d452f284b73e6d76dd36758a0c8684b1d5be31f92b89d07fd5822175732206fc"
+dependencies = [
+ "thiserror-impl",
+]
+
+[[package]]
+name = "thiserror-impl"
+version = "2.0.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "26afc1baea8a989337eeb52b6e72a039780ce45c3edfcc9c5b9d112feeb173c2"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83"

userspace/shell/Cargo.toml

@@ -2,7 +2,6 @@
 name = "shell"
 version = "0.1.0"
 edition = "2021"
-authors = ["Mark Poliakov <mark@alnyan.me>"]
 
 [dependencies]
 cross.workspace = true
@@ -12,12 +11,5 @@ thiserror.workspace = true
 
 nom = "7.1.3"
 
-[target.'cfg(target_os = "yggdrasil")'.dependencies]
-yggdrasil-rt = { path = "../../lib/runtime" }
-yggdrasil-abi = { path = "../../lib/abi" }
-
-[target.'cfg(unix)'.dependencies]
-libc = "*"
-
 [lints]
 workspace = true

userspace/shell/example.qs

@@ -0,0 +1,28 @@
+let x = 123
+let y = [1, 2, $x]
+
+fn f(a, b) {
+    echo ${a}
+    cat "/some/path/$b.txt"
+}
+
+fn g(xs) {
+    for item in $xs {
+        echo "* $item"
+    }
+}
+
+fn h() {
+    let counter = 0
+    for line in io:lines() {
+        echo "$counter $line"
+        counter = $counter + 1
+    }
+}
+
+a \
+    b \
+    c
+
+f $x "filename"
+g $y | h >output.txt

userspace/shell/src/builtin.rs

@@ -0,0 +1,96 @@
+use std::{collections::BTreeMap, env, io::Write, process::ExitCode, sync::RwLock};
+
+use crate::exec::{InheritStderr, InheritStdout, Input, Outcome, Output};
+
+pub type Builtin = fn(Io, Vec<String>, Envs) -> Outcome;
+
+pub struct Io {
+    pub stdin: Input,
+    pub stdout: Output<InheritStdout>,
+    pub stderr: Output<InheritStderr>,
+}
+pub struct Envs(Vec<(String, String)>);
+
+impl From<Vec<(String, String)>> for Envs {
+    fn from(value: Vec<(String, String)>) -> Self {
+        Self(value)
+    }
+}
+
+static BUILTINS: RwLock<BTreeMap<String, Builtin>> = RwLock::new(BTreeMap::new());
+
+pub fn get(program: &str) -> Option<Builtin> {
+    let builtins = BUILTINS.read().unwrap();
+    builtins.get(program).copied()
+}
+
+pub fn register(name: &str, function: Builtin) {
+    BUILTINS.write().unwrap().insert(name.into(), function);
+}
+
+fn b_echo(mut io: Io, args: Vec<String>, _env: Envs) -> Outcome {
+    for (i, arg) in args.iter().enumerate() {
+        if i != 0 {
+            write!(io.stdout, " ").ok();
+        }
+        write!(io.stdout, "{arg}").ok();
+    }
+    writeln!(io.stdout).ok();
+    Outcome::ok()
+}
+
+fn b_cd(mut io: Io, args: Vec<String>, _env: Envs) -> Outcome {
+    if args.len() != 1 {
+        writeln!(io.stderr, "`cd` requires one argument").ok();
+        return Outcome::err();
+    }
+    let path = &args[0];
+    match env::set_current_dir(path) {
+        Ok(()) => Outcome::ok(),
+        Err(error) => {
+            writeln!(io.stderr, "{path}: {error}").ok();
+            Outcome::err()
+        }
+    }
+}
+
+fn b_pwd(mut io: Io, _args: Vec<String>, _env: Envs) -> Outcome {
+    match env::current_dir() {
+        Ok(path) => {
+            writeln!(io.stdout, "{}", path.display()).ok();
+            Outcome::ok()
+        }
+        Err(error) => {
+            writeln!(io.stderr, "pwd: {error}").ok();
+            Outcome::err()
+        }
+    }
+}
+
+fn b_exit(mut io: Io, args: Vec<String>, _env: Envs) -> Outcome {
+    let code = match args.len() {
+        0 => ExitCode::SUCCESS,
+        1 => todo!(),
+        _ => {
+            writeln!(io.stderr, "Usage: exit [<code>]").ok();
+            return Outcome::err();
+        }
+    };
+
+    Outcome::ExitShell(code)
+}
+
+fn b_export(_io: Io, _args: Vec<String>, env: Envs) -> Outcome {
+    for (key, value) in env.0 {
+        env::set_var(key, value);
+    }
+    Outcome::ok()
+}
+
+pub fn register_default() {
+    register("echo", b_echo);
+    register("cd", b_cd);
+    register("pwd", b_pwd);
+    register("exit", b_exit);
+    register("export", b_export);
+}

userspace/shell/src/builtins.rs

@@ -1,147 +0,0 @@
-use std::{
-    collections::HashMap,
-    env,
-    fmt::{self, Write as FmtWrite},
-    fs::File,
-    io::Write as IoWrite,
-    path::{Path, PathBuf},
-    process::ExitCode,
-};
-
-use crate::{Error, Outcome};
-
-pub struct Io<'a> {
-    pub stdout: Output<'a>,
-    pub stderr: Output<'a>,
-}
-
-pub enum Output<'a> {
-    File(&'a mut File),
-    Default(&'a mut dyn IoWrite),
-}
-
-impl FmtWrite for Output<'_> {
-    fn write_str(&mut self, s: &str) -> fmt::Result {
-        match self {
-            Self::File(file) => file.write_all(s.as_bytes()).map_err(|_| fmt::Error),
-            Self::Default(file) => file.write_all(s.as_bytes()).map_err(|_| fmt::Error),
-        }
-    }
-}
-
-pub type BuiltinCommand = fn(&[String], &mut HashMap<String, String>, Io) -> Result<Outcome, Error>;
-
-static BUILTINS: &[(&str, BuiltinCommand)] = &[
-    ("echo", b_echo),
-    ("set", b_set),
-    #[cfg(target_os = "yggdrasil")]
-    ("cd", b_cd),
-    #[cfg(target_os = "yggdrasil")]
-    ("pwd", b_pwd),
-    ("which", b_which),
-    ("exit", b_exit),
-];
-
-pub fn get_builtin(name: &str) -> Option<BuiltinCommand> {
-    BUILTINS
-        .iter()
-        .find_map(|&(key, value)| if key == name { Some(value) } else { None })
-}
-
-fn b_which(
-    args: &[String],
-    _envs: &mut HashMap<String, String>,
-    mut io: Io,
-) -> Result<Outcome, Error> {
-    fn find_in_path(path: &str, program: &str) -> Option<String> {
-        for entry in path.split(':') {
-            let full_path = PathBuf::from(entry).join(program);
-
-            if full_path.exists() {
-                return Some(full_path.to_str().unwrap().to_owned());
-            }
-        }
-        None
-    }
-
-    if args.len() != 1 {
-        writeln!(io.stderr, "which usage: which PROGRAM").ok();
-        return Ok(Outcome::Exited(1));
-    }
-
-    let program = args[0].as_str();
-
-    let resolution = if program.starts_with('/') || program.starts_with('.') {
-        if Path::new(program).exists() {
-            Some(program.to_owned())
-        } else {
-            None
-        }
-    } else if let Ok(path) = env::var("PATH") {
-        find_in_path(&path, program)
-    } else {
-        None
-    };
-
-    match resolution {
-        Some(path) => {
-            writeln!(io.stdout, "{}: {}", program, path).ok();
-            Ok(Outcome::Exited(0))
-        }
-        _ => Ok(Outcome::Exited(1)),
-    }
-}
-
-fn b_set(args: &[String], envs: &mut HashMap<String, String>, mut io: Io) -> Result<Outcome, Error> {
-    if args.len() != 2 {
-        writeln!(io.stderr, "set usage: set VAR VALUE").ok();
-        return Ok(Outcome::Exited(1));
-    }
-    envs.insert(args[0].clone(), args[1].clone());
-    Ok(Outcome::ok())
-}
-
-fn b_echo(args: &[String], _envs: &mut HashMap<String, String>, mut io: Io) -> Result<Outcome, Error> {
-    for (i, arg) in args.iter().enumerate() {
-        if i != 0 {
-            write!(io.stdout, " ").ok();
-        }
-        write!(io.stdout, "{}", arg).ok();
-    }
-    writeln!(io.stdout).ok();
-    Ok(Outcome::ok())
-}
-
-fn b_exit(args: &[String], _envs: &mut HashMap<String, String>, mut io: Io) -> Result<Outcome, Error> {
-    match args.len() {
-        0 => Ok(Outcome::ExitShell(ExitCode::SUCCESS)),
-        _ => {
-            writeln!(io.stderr, "Usage: exit [CODE]").ok();
-            Ok(Outcome::Exited(1))
-        }
-    }
-}
-
-#[cfg(target_os = "yggdrasil")]
-fn b_cd(args: &[String], _envs: &mut HashMap<String, String>, _io: Io) -> Result<Outcome, Error> {
-    let path = if args.is_empty() {
-        "/"
-    } else {
-        args[0].as_str()
-    };
-    yggdrasil_rt::io::set_current_directory(path).map_err(Error::RtError)?;
-    Ok(Outcome::Exited(0))
-}
-
-#[cfg(target_os = "yggdrasil")]
-fn b_pwd(args: &[String], _envs: &mut HashMap<String, String>, mut io: Io) -> Result<Outcome, Error> {
-    if !args.is_empty() {
-        writeln!(io.stderr, "Usage: pwd").ok();
-        return Ok(Outcome::Exited(1));
-    }
-
-    let pwd = yggdrasil_rt::io::current_directory_string().map_err(Error::RtError)?;
-    writeln!(io.stdout, "{pwd}").ok();
-
-    Ok(Outcome::Exited(0))
-}

userspace/shell/src/env.rs

@@ -0,0 +1,236 @@
+use std::{
+    collections::{hash_map::Entry, HashMap},
+    fmt,
+    path::PathBuf,
+};
+
+use crate::syntax::{
+    lex::{Fragment, Word},
+    parse::{ICommand, IPipelineElement, IRedirects},
+};
+
+pub trait Expand {
+    type Output;
+
+    fn expand(&self, env: &Env) -> Result<Self::Output, Error>;
+}
+
+#[derive(Clone)]
+pub enum Variable {
+    String(String),
+    Array(Vec<Variable>),
+}
+
+pub struct Env {
+    vars: HashMap<String, Variable>,
+}
+
+#[derive(Debug, thiserror::Error)]
+pub enum Error {
+    #[error("Undefined variable {0:?}")]
+    UndefinedVariable(String),
+}
+
+impl Env {
+    pub fn new() -> Self {
+        Self {
+            vars: HashMap::new(),
+        }
+    }
+
+    pub fn setup_builtin(&mut self) {
+        let bin_name = std::env::args().next().unwrap_or_default();
+        self.vars.insert("SHELL".into(), Variable::String(bin_name));
+    }
+
+    pub fn lookup(&self, name: &str) -> Option<Variable> {
+        self.vars
+            .get(name)
+            .cloned()
+            .or_else(|| std::env::var(name).ok().map(Variable::String))
+    }
+
+    pub fn put_var(&mut self, name: &str, value: Variable) -> bool {
+        match self.vars.entry(name.into()) {
+            Entry::Vacant(entry) => {
+                entry.insert(value);
+                true
+            }
+            Entry::Occupied(_) => false,
+        }
+    }
+}
+
+impl fmt::Display for Variable {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::String(text) => f.write_str(text),
+            Self::Array(items) => {
+                for (i, item) in items.iter().enumerate() {
+                    if i != 0 {
+                        f.write_str(" ")?;
+                    }
+                    fmt::Display::fmt(item, f)?;
+                }
+                Ok(())
+            }
+        }
+    }
+}
+
+impl From<&str> for Variable {
+    fn from(value: &str) -> Self {
+        Self::String(value.into())
+    }
+}
+impl From<String> for Variable {
+    fn from(value: String) -> Self {
+        Self::String(value)
+    }
+}
+
+impl<T: Expand> Expand for Option<T> {
+    type Output = Option<T::Output>;
+
+    fn expand(&self, env: &Env) -> Result<Self::Output, Error> {
+        self.as_ref().map(|e| e.expand(env)).transpose()
+    }
+}
+
+impl<T: Expand> Expand for Vec<T> {
+    type Output = Vec<T::Output>;
+
+    fn expand(&self, env: &Env) -> Result<Self::Output, Error> {
+        self.iter().map(|e| e.expand(env)).collect()
+    }
+}
+
+impl<T: Expand, U: Expand> Expand for (T, U) {
+    type Output = (T::Output, U::Output);
+
+    fn expand(&self, env: &Env) -> Result<Self::Output, Error> {
+        Ok((self.0.expand(env)?, self.1.expand(env)?))
+    }
+}
+
+pub struct Command {
+    pub pipeline: Vec<PipelineElement>,
+    pub redirects: Redirects,
+}
+
+pub struct PipelineElement {
+    pub envs: Vec<(String, Option<String>)>,
+    pub words: Vec<String>,
+}
+pub struct Redirects {
+    pub stdin: Option<PathBuf>,
+    pub stdout: Option<PathBuf>,
+    pub stderr: Option<PathBuf>,
+}
+
+impl Expand for ICommand<'_> {
+    type Output = Command;
+
+    fn expand(&self, env: &Env) -> Result<Self::Output, Error> {
+        let pipeline = self.pipeline.expand(env)?;
+        let redirects = self.redirects.expand(env)?;
+        Ok(Command {
+            pipeline,
+            redirects,
+        })
+    }
+}
+
+impl Expand for IRedirects<'_> {
+    type Output = Redirects;
+
+    fn expand(&self, env: &Env) -> Result<Self::Output, Error> {
+        let stdin = self.stdin.expand(env)?.map(PathBuf::from);
+        let stdout = self.stdout.expand(env)?.map(PathBuf::from);
+        let stderr = self.stderr.expand(env)?.map(PathBuf::from);
+        Ok(Redirects {
+            stdin,
+            stdout,
+            stderr,
+        })
+    }
+}
+
+impl Expand for IPipelineElement<'_> {
+    type Output = PipelineElement;
+
+    fn expand(&self, env: &Env) -> Result<Self::Output, Error> {
+        let words = self.words.expand(env)?;
+        let envs = self.envs.expand(env)?;
+        Ok(PipelineElement { envs, words })
+    }
+}
+
+impl Expand for Word<'_> {
+    type Output = String;
+
+    fn expand(&self, env: &Env) -> Result<Self::Output, Error> {
+        let mut output = String::new();
+        for fragment in self.0.iter() {
+            output.push_str(&fragment.expand(env)?);
+        }
+        Ok(output)
+    }
+}
+
+impl Expand for Fragment<'_> {
+    type Output = String;
+
+    fn expand(&self, env: &Env) -> Result<Self::Output, Error> {
+        match *self {
+            Self::Literal(text) => Ok(text.into()),
+            Self::Variable(name) => {
+                let text = env.lookup(name).map(|v| v.to_string()).unwrap_or_default();
+                Ok(text)
+            }
+            Self::QuotedLiteral(text) => Ok(text.into()),
+        }
+    }
+}
+
+impl fmt::Display for Command {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        for (i, element) in self.pipeline.iter().enumerate() {
+            if i != 0 {
+                f.write_str(" | ")?;
+            }
+            fmt::Display::fmt(element, f)?;
+        }
+        if !self.pipeline.is_empty() {
+            f.write_str(" ")?;
+        }
+        fmt::Display::fmt(&self.redirects, f)
+    }
+}
+
+impl fmt::Display for Redirects {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        if let Some(path) = self.stdin.as_ref() {
+            write!(f, "<{}", path.display())?;
+        }
+        if let Some(path) = self.stdout.as_ref() {
+            if self.stdin.is_some() {
+                f.write_str(" ")?;
+            }
+            write!(f, ">{}", path.display())?;
+        }
+        Ok(())
+    }
+}
+
+impl fmt::Display for PipelineElement {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        for (i, word) in self.words.iter().enumerate() {
+            if i != 0 {
+                f.write_str(" ")?;
+            }
+            f.write_str(word)?;
+        }
+        Ok(())
+    }
+}

userspace/shell/src/exec.rs

@@ -0,0 +1,297 @@
+use std::{
+    fmt,
+    fs::File,
+    io::{self, BufRead, BufReader, Stderr, Stdout, Write},
+    marker::PhantomData,
+    os::fd::{FromRawFd, IntoRawFd},
+    pipe::{PipeReader, PipeWriter},
+    process::{self, Child, ExitCode, ExitStatus, Stdio},
+    thread::{self, JoinHandle},
+};
+
+use crate::{builtin, env::Command, Error};
+
+pub enum Outcome {
+    Process(ExitStatus),
+    Builtin(ExitCode),
+    ExitShell(ExitCode),
+}
+
+pub enum Handle {
+    Process(Child),
+    Thread(JoinHandle<Outcome>),
+}
+
+pub trait InheritOutput: fmt::Debug {
+    type Stream: Write;
+
+    fn inherited() -> Self::Stream;
+}
+#[derive(Debug)]
+pub struct InheritStdout;
+#[derive(Debug)]
+pub struct InheritStderr;
+
+impl InheritOutput for InheritStdout {
+    type Stream = Stdout;
+    fn inherited() -> Self::Stream {
+        io::stdout()
+    }
+}
+impl InheritOutput for InheritStderr {
+    type Stream = Stderr;
+    fn inherited() -> Self::Stream {
+        io::stderr()
+    }
+}
+
+impl io::Read for Input {
+    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
+        match self {
+            Self::Pipe(pipe) => pipe.read(buf),
+            Self::File(file) => file.read(buf),
+            Self::Inherit => io::stdin().read(buf),
+        }
+    }
+}
+impl From<Input> for Stdio {
+    fn from(value: Input) -> Self {
+        match value {
+            Input::Inherit => Stdio::inherit(),
+            Input::File(file) => Stdio::from(file.into_inner()),
+            Input::Pipe(pipe) => unsafe { Stdio::from_raw_fd(pipe.into_inner().into_raw_fd()) },
+        }
+    }
+}
+impl Input {
+    pub fn read_line(&mut self, line: &mut String) -> Result<usize, io::Error> {
+        match self {
+            Self::Pipe(pipe) => pipe.read_line(line),
+            Self::File(file) => file.read_line(line),
+            Self::Inherit => io::stdin().read_line(line),
+        }
+    }
+}
+
+impl<I: InheritOutput> io::Write for Output<I> {
+    fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
+        match self {
+            Self::Pipe(pipe) => pipe.write(buf),
+            Self::File(file) => file.write(buf),
+            Self::Inherit(_) => I::inherited().write(buf),
+        }
+    }
+
+    fn flush(&mut self) -> io::Result<()> {
+        match self {
+            Self::Pipe(pipe) => pipe.flush(),
+            Self::File(file) => file.flush(),
+            Self::Inherit(_) => I::inherited().flush(),
+        }
+    }
+}
+impl<I: InheritOutput> From<Output<I>> for Stdio {
+    fn from(value: Output<I>) -> Self {
+        match value {
+            Output::Inherit(_) => Stdio::inherit(),
+            Output::File(file) => Stdio::from(file),
+            Output::Pipe(pipe) => unsafe { Stdio::from_raw_fd(pipe.into_raw_fd()) },
+        }
+    }
+}
+impl<I: InheritOutput> Output<I> {
+    pub fn try_clone(&self) -> Result<Self, Error> {
+        match self {
+            Self::File(file) => Ok(Self::File(file.try_clone()?)),
+            Self::Pipe(pipe) => Ok(Self::Pipe(pipe.try_clone()?)),
+            Self::Inherit(_) => Ok(Self::Inherit(PhantomData)),
+        }
+    }
+}
+
+#[derive(Debug)]
+pub enum Input {
+    Pipe(BufReader<PipeReader>),
+    File(BufReader<File>),
+    Inherit,
+}
+
+pub enum Output<I: InheritOutput> {
+    Pipe(PipeWriter),
+    File(File),
+    Inherit(PhantomData<I>),
+}
+
+pub struct Execution {
+    pub program: String,
+    pub arguments: Vec<String>,
+    pub envs: Vec<(String, String)>,
+    pub stdin: Input,
+    pub stdout: Output<InheritStdout>,
+    pub stderr: Output<InheritStderr>,
+}
+
+impl Outcome {
+    pub fn is_success(&self) -> bool {
+        match self {
+            Self::Process(status) => status.success(),
+            Self::Builtin(status) => *status == ExitCode::SUCCESS,
+            Self::ExitShell(_code) => true,
+        }
+    }
+
+    pub fn err() -> Self {
+        Self::Builtin(ExitCode::FAILURE)
+    }
+
+    pub fn ok() -> Self {
+        Self::Builtin(ExitCode::SUCCESS)
+    }
+}
+
+// TODO move pipelines into process groups
+fn spawn_command(execution: Execution) -> Result<Child, Error> {
+    let mut command = process::Command::new(execution.program);
+
+    command
+        .args(execution.arguments)
+        .stdin(execution.stdin)
+        .stdout(execution.stdout)
+        .stderr(execution.stderr);
+
+    for (key, value) in execution.envs {
+        command.env(key, value);
+    }
+
+    let child = command.spawn()?;
+
+    Ok(child)
+}
+
+pub fn exec_pipeline<I: IntoIterator<Item = Execution>>(
+    pipeline: I,
+) -> Result<Vec<Handle>, Error> {
+    let mut handles = vec![];
+    for element in pipeline.into_iter() {
+        let handle = if let Some(builtin) = builtin::get(&element.program) {
+            let io = builtin::Io {
+                stdin: element.stdin,
+                stdout: element.stdout,
+                stderr: element.stderr,
+            };
+
+            Handle::Thread(thread::spawn(move || {
+                builtin(io, element.arguments, element.envs.into())
+            }))
+        } else {
+            let child = spawn_command(element)?;
+
+            Handle::Process(child)
+        };
+
+        handles.push(handle);
+    }
+    Ok(handles)
+}
+
+pub fn wait_for_pipeline(handles: Vec<Handle>) -> Result<(Outcome, Option<ExitCode>), Error> {
+    let mut outcomes = vec![];
+
+    for element in handles {
+        match element {
+            Handle::Thread(thread) => {
+                let result = match thread.join() {
+                    Ok(result) => result,
+                    Err(_error) => {
+                        eprintln!("Builtin thread panicked");
+                        Outcome::err()
+                    }
+                };
+                outcomes.push(result);
+            }
+            Handle::Process(mut child) => {
+                let result = child.wait()?;
+                outcomes.push(Outcome::Process(result));
+            }
+        }
+    }
+
+    // Find first non-successful outcome
+    let mut exit = None;
+    let mut error = None;
+
+    for outcome in outcomes {
+        if let Outcome::ExitShell(code) = outcome {
+            exit = Some(code);
+        } else if error.is_none() && !outcome.is_success() {
+            error = Some(outcome);
+        }
+    }
+
+    let error = match error {
+        Some(exit) => exit,
+        None => Outcome::ok(),
+    };
+
+    Ok((error, exit))
+}
+
+pub fn eval(command: Command) -> Result<(Outcome, Option<ExitCode>), Error> {
+    // Set up pipeline I/O
+    let mut stdins = vec![];
+    let mut stdouts = vec![];
+
+    let stdin = match command.redirects.stdin.as_ref() {
+        Some(path) => {
+            let file = File::open(path)?;
+            Input::File(BufReader::new(file))
+        }
+        None => Input::Inherit,
+    };
+    let stdout = match command.redirects.stdout.as_ref() {
+        Some(path) => Output::File(File::create(path)?),
+        None => Output::Inherit(PhantomData),
+    };
+    let stderr = match command.redirects.stderr.as_ref() {
+        Some(path) => Output::File(File::create(path)?),
+        None => Output::Inherit(PhantomData),
+    };
+
+    stdins.push(stdin);
+    for _ in 1..command.pipeline.len() {
+        let (read, write) = std::pipe::pipe()?;
+        stdins.push(Input::Pipe(BufReader::new(read)));
+        stdouts.push(Output::Pipe(write));
+    }
+    stdouts.push(stdout);
+
+    assert_eq!(stdins.len(), stdouts.len());
+    assert_eq!(stdins.len(), command.pipeline.len());
+
+    let io = Iterator::zip(stdins.drain(..), stdouts.drain(..));
+    let mut pipeline = vec![];
+
+    for (command, (stdin, stdout)) in command.pipeline.iter().zip(io) {
+        let (program, arguments) = command.words.split_first().unwrap();
+        let stderr = stderr.try_clone()?;
+        let envs = command
+            .envs
+            .iter()
+            .map(|(a, b)| (a.clone(), b.clone().unwrap_or_default()))
+            .collect();
+        let execution = Execution {
+            program: program.into(),
+            arguments: arguments.to_vec(),
+            envs,
+            stdin,
+            stdout,
+            stderr,
+        };
+        pipeline.push(execution);
+    }
+
+    let handles = exec_pipeline(pipeline)?;
+    let (status, exit) = wait_for_pipeline(handles)?;
+
+    Ok((status, exit))
+}

userspace/shell/src/main.rs

@@ -1,81 +1,59 @@
-#![cfg_attr(not(unix), feature(yggdrasil_os, rustc_private))]
+#![feature(
+    if_let_guard,
+    iter_chain,
+    anonymous_pipe,
+    trait_alias,
+    exitcode_exit_method
+)]
+#![cfg_attr(target_os = "yggdrasil", feature(yggdrasil_os))]
+#![allow(clippy::new_without_default, clippy::should_implement_trait)]
 
 use std::{
-    collections::HashMap,
-    env,
     fs::File,
-    io::{self, stderr, stdin, stdout, BufRead, BufReader, Stdin, Write},
+    io::{self, stdin, stdout, BufRead, BufReader, Write},
-    os::fd::{FromRawFd, IntoRawFd},
+    process::ExitCode,
-    path::Path,
-    process::{self, Child, ExitCode, Stdio},
 };
 
 use clap::Parser;
-use parser::{Command, CommandOutput};
+use env::{Env, Expand};
+use exec::Outcome;
+use syntax::parse::parse_interactive;
 
-mod builtins;
+pub mod builtin;
-mod parser;
+pub mod env;
-mod readline;
+pub mod exec;
-mod sys;
+pub mod readline;
+pub mod syntax;
 
 #[derive(Debug, thiserror::Error)]
 pub enum Error {
     #[error("{0}")]
-    IoError(#[from] io::Error),
+    Io(#[from] io::Error),
-    #[cfg(any(target_os = "yggdrasil", rust_analyzer))]
-    #[error("{0:?}")]
-    RtError(yggdrasil_rt::Error),
 }
 
-#[derive(Parser)]
+#[derive(Debug, Parser)]
-pub struct Args {
+pub struct ShellArgs {
     #[arg(short)]
     login: bool,
     script: Option<String>,
     args: Vec<String>,
 }
 
-pub enum Outcome {
+pub enum ShellInput {
-    Exited(i32),
-    Killed(i32),
-    ExitShell(ExitCode),
-}
-
-pub struct PipelineElement<'a> {
-    pub command: &'a str,
-    pub args: &'a [String],
-    pub input: Stdio,
-    pub output: Stdio,
-}
-
-pub struct Pipeline<'a> {
-    pub elements: Vec<PipelineElement<'a>>,
-    pub env: &'a HashMap<String, String>,
-}
-
-pub struct SpawnedPipeline {
-    pub children: Vec<Child>,
-}
-
-pub enum Input {
-    Interactive(Stdin),
     File(BufReader<File>),
+    Interactive,
 }
 
-impl Outcome {
+impl ShellInput {
-    pub const fn ok() -> Self {
+    pub fn read_line(&mut self, line: &mut String) -> Result<usize, Error> {
-        Self::Exited(0)
-    }
-}
-
-impl Input {
-    pub fn getline(&mut self, buf: &mut [u8]) -> Result<usize, Error> {
         match self {
-            Self::Interactive(input) => {
+            Self::File(file) => Ok(file.read_line(line)?),
+            Self::Interactive => {
                 let mut stdout = stdout();
+                let mut stdin = stdin();
 
-                readline::readline(input, &mut stdout, buf, |stdout| {
+                readline::readline(&mut stdin, &mut stdout, line, |stdout| {
-                    let cwd = env::current_dir();
+                    let cwd = std::env::current_dir();
                     let cwd = match cwd {
                         Ok(cwd) => format!("{}", cwd.display()),
                         Err(_) => "???".into(),
@@ -84,254 +62,103 @@ impl Input {
                     stdout.write_all(prompt.as_bytes()).ok();
                 })
             }
-            Self::File(input) => {
-                let mut string = String::new();
-                input.read_line(&mut string)?;
-                buf[..string.len()].copy_from_slice(string.as_bytes());
-                Ok(string.len())
-            }
         }
     }
-
-    pub fn is_interactive(&self) -> bool {
-        matches!(self, Self::Interactive(_))
-    }
 }
 
-// TODO group pipeline commands into a single process group
+fn run(mut input: ShellInput, env: &Env) -> Result<(), Error> {
-pub fn exec(
+    let mut line = String::new();
-    interactive: bool,
+    loop {
-    command: &Command,
+        line.clear();
-    env: &mut HashMap<String, String>,
-) -> Result<Outcome, Error> {
-    // Pipeline "a | b | c" execution:
-    //
-    // 1. a.stdin = STDIN, a.stdout = pipe0
-    // 2. b.stdin = pipe0, b.stdout = pipe1
-    // 3. c.stdin = pipe1, c.stdout = STDOUT
-    //
-    // Pipe count: command count - 1
-
-    if command.pipeline.is_empty() {
-        return Ok(Outcome::ok());
-    }
-
-    let stdin = if let Some(path) = command.stdin.as_ref() {
-        Some(File::open(path)?)
-    } else {
-        None
-    };
-    let mut stdout = match command.stdout.as_ref() {
-        Some(CommandOutput::Path(path)) => Some(File::create(path)?),
-        Some(CommandOutput::Fd(_fd)) => None,
-        None => None,
-    };
-
-    if command.pipeline.len() == 1 {
-        let command = &command.pipeline[0];
-        let (cmd, args) = command.0.split_first().unwrap();
-
-        if let Some(builtin) = builtins::get_builtin(cmd) {
-            let mut default_stdout = io::stdout();
-            let mut default_stderr = io::stderr();
-
-            let stdout = match stdout.as_mut() {
-                Some(file) => builtins::Output::File(file),
-                None => builtins::Output::Default(&mut default_stdout)
-            };
-            let stderr = builtins::Output::Default(&mut default_stderr);
-            let io = builtins::Io {
-                stdout,
-                stderr,
-            };
-            return builtin(args, env, io);
-        }
-    }
-
-    let mut inputs = vec![];
-    let mut outputs = vec![];
-
-    if let Some(stdin) = stdin {
-        inputs.push(unsafe { Stdio::from_raw_fd(stdin.into_raw_fd()) });
-    } else {
-        inputs.push(Stdio::inherit());
-    }
-    for _ in 1..command.pipeline.len() {
-        let pipe = sys::create_pipe()?;
-
-        let read_fd = pipe.read.into_raw_fd();
-        let write_fd = pipe.write.into_raw_fd();
-
-        let input = unsafe { Stdio::from_raw_fd(read_fd) };
-        let output = unsafe { Stdio::from_raw_fd(write_fd) };
-
-        inputs.push(input);
-        outputs.push(output);
-    }
-    if let Some(stdout) = stdout {
-        outputs.push(unsafe { Stdio::from_raw_fd(stdout.into_raw_fd()) });
-    } else {
-        outputs.push(Stdio::inherit());
-    }
-
-    assert_eq!(inputs.len(), outputs.len());
-    assert_eq!(inputs.len(), command.pipeline.len());
-
-    let mut elements = vec![];
-    let ios = inputs.drain(..).zip(outputs.drain(..));
-
-    for (command, (input, output)) in command.pipeline.iter().zip(ios) {
-        let (cmd, args) = command.0.split_first().unwrap();
-
-        let element = PipelineElement {
-            command: cmd,
-            args,
-            input,
-            output,
-        };
-
-        elements.push(element);
-    }
-
-    let pipeline = Pipeline { elements, env };
-    let pipeline = sys::spawn_pipeline(interactive, pipeline)?;
-
-    let status = sys::wait_for_pipeline(interactive, pipeline)?;
-
-    Ok(status)
-}
-
-fn run(mut input: Input, vars: &mut HashMap<String, String>) -> Result<ExitCode, Error> {
-    // let mut line = String::new();
-    let mut line = [0; 4096];
-
-    if input.is_interactive() {
-        sys::init_signal_handler();
-    }
-
-    let code = loop {
-        // line.clear();
-
-        let len = input.getline(&mut line)?;
 
+        let len = input.read_line(&mut line)?;
         if len == 0 {
-            break ExitCode::SUCCESS;
+            break Ok(());
         }
 
-        let Ok(line) = std::str::from_utf8(&line[..len]) else {
-            continue;
-        };
         let line = line.trim();
-        if line.starts_with('#') || line.is_empty() {
+        if line.is_empty() || line.starts_with('#') {
             continue;
         }
-        let cmd = match parser::parse_line(vars, line) {
+
-            Ok(cmd) => cmd,
+        let command = match parse_interactive(line) {
-            Err(error) if input.is_interactive() => {
+            Ok(c) => c,
-                eprintln!("stdin: {error}");
+            Err(e) => {
+                eprintln!("Syntax error: {e}");
                 continue;
             }
-            Err(error) => {
-                eprintln!("Command error: {error}");
-                return Ok(ExitCode::FAILURE);
-            }
         };
-
+        let command = match command.expand(env) {
-        let q_code = match exec(input.is_interactive(), &cmd, vars) {
+            Ok(c) => c,
-            Ok(Outcome::ExitShell(code)) => {
-                break code;
-            }
-            Ok(Outcome::Killed(signal)) => {
-                if input.is_interactive() {
-                    eprintln!("Killed: {}", signal);
-                }
-                signal + 128
-            }
-            Ok(Outcome::Exited(code)) => code % 256,
             Err(e) => {
-                eprintln!("{}: {}", line, e);
+                eprintln!("{e}");
-                127
+                continue;
             }
         };
 
-        vars.insert("?".to_owned(), q_code.to_string());
+        let (outcome, exit) = match exec::eval(command) {
-    };
+            Ok(res) => res,
+            Err(error) => {
+                eprintln!("{error}");
+                continue;
+            }
+        };
 
-    Ok(code)
+        if let Some(exit) = exit {
-}
+            exit.exit_process();
-
-fn run_file<P: AsRef<Path>>(path: P, env: &mut HashMap<String, String>) -> Result<ExitCode, Error> {
-    let input = BufReader::new(File::open(path)?);
-    run(Input::File(input), env)
-}
-
-fn run_stdin(env: &mut HashMap<String, String>) -> Result<ExitCode, Error> {
-    run(Input::Interactive(stdin()), env)
-}
-
-// Sets up builtin variables
-fn setup_env(vars: &mut HashMap<String, String>, script: &Option<String>, args: &[String]) {
-    let pid = process::id();
-    let bin_name = env::args().next();
-    if let Some(bin_name) = bin_name.as_ref() {
-        vars.insert("SHELL".into(), bin_name.clone());
-    } else {
-        vars.remove("SHELL");
-    }
-
-    vars.insert("#".into(), format!("{}", args.len()));
-
-    if let Some(script) = script {
-        vars.insert("0".into(), script.clone());
-    } else if let Some(bin_name) = bin_name {
-        vars.insert("0".into(), bin_name);
-    } else {
-        vars.remove("0");
-    }
-
-    let mut args_string = String::new();
-    for (i, arg) in args.iter().enumerate() {
-        if i != 0 {
-            args_string.push(' ');
         }
-        args_string.push_str(arg);
 
-        vars.insert(format!("{}", i + 1), arg.clone());
+        match outcome {
+            Outcome::ExitShell(_) => unreachable!(),
+            Outcome::Process(status) => {
+                if !status.success() {
+                    if let Some(code) = status.code() {
+                        eprintln!("Exit code: {code}");
+                    }
+                }
+            }
+            Outcome::Builtin(code) => {
+                if code != ExitCode::SUCCESS {
+                    eprintln!("Builtin command failed");
+                }
+            }
+        }
     }
-    vars.insert("*".into(), args_string.clone());
+}
-    // TODO array types
-    vars.insert("@".into(), args_string);
 
-    vars.insert("$".into(), format!("{pid}"));
+fn run_wrapper(args: ShellArgs, env: &Env) -> Result<(), Error> {
-
+    match args.script {
-    // Insert PATH to current process env
+        Some(script) => {
-    if let Some(path) = vars.get("PATH") {
+            let script = BufReader::new(File::open(script)?);
-        env::set_var("PATH", path);
+            run(ShellInput::File(script), env)
+        }
+        None => run(ShellInput::Interactive, env),
     }
 }
 
 fn main() -> ExitCode {
-    let args = Args::parse();
+    const PROFILE_PATH: &str = "/etc/profile";
-    let mut vars = HashMap::new();
 
-    for (key, value) in env::vars() {
+    let args = ShellArgs::parse();
-        vars.insert(key, value);
+    let mut env = Env::new();
-    }
 
-    setup_env(&mut vars, &args.script, &args.args);
+    env.setup_builtin();
+    builtin::register_default();
 
     if args.login {
-        run_file("/etc/profile", &mut vars).ok();
+        if let Ok(profile_script) = File::open(PROFILE_PATH) {
+            let profile_script = BufReader::new(profile_script);
+            match run(ShellInput::File(profile_script), &env) {
+                Ok(()) => (),
+                Err(error) => {
+                    eprintln!("{PROFILE_PATH}: {error}");
+                }
+            }
+        }
     }
 
-    let result = if let Some(script) = &args.script {
+    match run_wrapper(args, &env) {
-        run_file(script, &mut vars)
+        Ok(()) => ExitCode::SUCCESS,
-    } else {
-        run_stdin(&mut vars)
-    };
-
-    match result {
-        Ok(_) => ExitCode::SUCCESS,
         Err(error) => {
             eprintln!("{error}");
             ExitCode::FAILURE

userspace/shell/src/parser.rs

@@ -1,534 +0,0 @@
-use std::{collections::HashMap, path::PathBuf};
-
-use nom::{
-    branch::alt,
-    bytes::complete::{is_a, is_not, tag, take_while1},
-    character::complete::{char, space0, u8 as num_u8},
-    combinator::{map, recognize, verify},
-    multi::{fold_many0, many0, many1, separated_list1},
-    sequence::{delimited, pair, preceded, separated_pair, terminated},
-    IResult,
-};
-
-#[derive(Debug, thiserror::Error)]
-pub enum Error {
-    #[error("{0}")]
-    Lex(#[from] nom::Err<nom::error::Error<String>>),
-}
-
-pub struct Command {
-    pub pipeline: Vec<CommandPipelineElement>,
-    pub stdin: Option<PathBuf>,
-    pub stdout: Option<CommandOutput>,
-    pub stderr: Option<CommandOutput>
-}
-
-pub struct CommandPipelineElement(pub Vec<String>);
-
-pub enum CommandOutput {
-    Fd(u8),
-    Path(PathBuf),
-}
-
-trait Expand {
-    type Output;
-
-    fn expand(self, env: &HashMap<String, String>) -> Self::Output;
-}
-
-#[derive(Clone, Debug, PartialEq)]
-enum QuoteFragment<'a> {
-    Text(&'a str),
-    Var(&'a str),
-}
-
-#[derive(Clone, Debug, PartialEq)]
-enum WordToken<'a> {
-    Text(&'a str),
-    Var(&'a str),
-    Quote(Vec<QuoteFragment<'a>>),
-}
-
-#[derive(Clone, Debug, PartialEq)]
-struct Word<'a>(Vec<WordToken<'a>>);
-
-#[derive(Clone, Debug, PartialEq)]
-struct ParsedPipelineElement<'a>(Vec<Word<'a>>);
-
-#[derive(Clone, Debug, PartialEq)]
-struct ParsedPipeline<'a>(Vec<ParsedPipelineElement<'a>>);
-
-#[derive(Clone, Debug, PartialEq)]
-struct ParsedCommand<'a> {
-    pipeline: ParsedPipeline<'a>,
-    redirects: Redirects<'a>,
-}
-
-#[derive(Clone, Debug, PartialEq)]
-struct Redirects<'a> {
-    stdin: Option<Word<'a>>,
-    stdout: Option<OutputTarget<'a>>,
-    stderr: Option<OutputTarget<'a>>,
-}
-
-#[derive(Clone, Debug, PartialEq)]
-enum OutputTarget<'a> {
-    Path(Word<'a>),
-    Fd(u8),
-}
-
-#[derive(Clone, Debug, PartialEq)]
-struct OutputRedirect<'a> {
-    fd: u8,
-    target: OutputTarget<'a>,
-}
-
-#[derive(Clone, Debug, PartialEq)]
-enum Redirect<'a> {
-    Input(Word<'a>),
-    Output(OutputRedirect<'a>),
-}
-
-fn is_ident_tail(c: char) -> bool {
-    c.is_alphanumeric() || c == '_'
-}
-
-fn is_punctuation(c: char) -> bool {
-    [
-        ';', '|', '{', '}', '(', ')', ',', '<', '>', '$', '"', '\'', '\\',
-    ]
-    .contains(&c)
-}
-
-fn is_filename(c: char) -> bool {
-    !c.is_whitespace() && !is_punctuation(c)
-}
-
-fn lex_name(i: &str) -> IResult<&str, &str> {
-    alt((recognize(is_a("#*@?!$-")), take_while1(is_ident_tail)))(i)
-}
-
-fn lex_var_braced(i: &str) -> IResult<&str, &str> {
-    // ${ABCD}
-    delimited(tag("${"), lex_name, tag("}"))(i)
-}
-
-fn lex_var_unbraced(i: &str) -> IResult<&str, &str> {
-    // $ABCD $# $* $@ $? $! $$ $-
-    preceded(tag("$"), lex_name)(i)
-}
-
-fn lex_var(i: &str) -> IResult<&str, &str> {
-    alt((lex_var_braced, lex_var_unbraced))(i)
-}
-
-fn lex_filename(i: &str) -> IResult<&str, &str> {
-    take_while1(is_filename)(i)
-}
-
-fn lex_quoted_literal(i: &str) -> IResult<&str, QuoteFragment> {
-    let not_quote_slash = is_not("$\"\\");
-    map(
-        verify(not_quote_slash, |s: &str| !s.is_empty()),
-        QuoteFragment::Text,
-    )(i)
-}
-
-fn lex_quoted_var(i: &str) -> IResult<&str, QuoteFragment> {
-    map(lex_var, QuoteFragment::Var)(i)
-}
-
-fn lex_quoted(i: &str) -> IResult<&str, WordToken> {
-    // "abcdef $abcdef"
-    map(
-        delimited(
-            char('"'),
-            many0(alt((lex_quoted_var, lex_quoted_literal))),
-            char('"'),
-        ),
-        WordToken::Quote,
-    )(i)
-}
-
-fn lex_word_token(i: &str) -> IResult<&str, WordToken> {
-    alt((
-        lex_quoted,
-        map(lex_var, WordToken::Var),
-        map(lex_filename, WordToken::Text),
-    ))(i)
-}
-
-fn lex_word(i: &str) -> IResult<&str, Word> {
-    map(many1(lex_word_token), Word)(i)
-}
-
-fn lex_pipeline_element(i: &str) -> IResult<&str, ParsedPipelineElement> {
-    map(
-        preceded(space0, many1(terminated(lex_word, space0))),
-        ParsedPipelineElement,
-    )(i)
-}
-
-fn lex_pipeline(i: &str) -> IResult<&str, ParsedPipeline> {
-    map(
-        separated_list1(preceded(space0, char('|')), lex_pipeline_element),
-        ParsedPipeline,
-    )(i)
-}
-
-fn lex_output_target(i: &str) -> IResult<&str, OutputTarget> {
-    // abcdef $a
-    // &2
-    preceded(space0, alt((
-        map(preceded(char('&'), num_u8), OutputTarget::Fd),
-        map(lex_word, OutputTarget::Path),
-    )))(i)
-}
-
-fn lex_output_redirect(i: &str) -> IResult<&str, OutputRedirect> {
-    // >$a
-    // 2>&1
-    alt((
-        map(
-            separated_pair(num_u8, char('>'), lex_output_target),
-            |(fd, target)| OutputRedirect { fd, target },
-        ),
-        map(preceded(char('>'), lex_output_target), |target| {
-            OutputRedirect { fd: 1, target }
-        }),
-    ))(i)
-}
-
-fn lex_input_redirect(i: &str) -> IResult<&str, Word> {
-    preceded(char('<'), lex_word)(i)
-}
-
-fn lex_redirect(i: &str) -> IResult<&str, Redirect> {
-    alt((
-        map(lex_input_redirect, Redirect::Input),
-        map(lex_output_redirect, Redirect::Output),
-    ))(i)
-}
-
-fn lex_redirects(i: &str) -> IResult<&str, Redirects> {
-    fold_many0(
-        preceded(space0, lex_redirect),
-        || Redirects {
-            stdin: None,
-            stdout: None,
-            stderr: None,
-        },
-        |mut acc, redirect| match redirect {
-            Redirect::Input(path) => {
-                acc.stdin = Some(path);
-                acc
-            }
-            Redirect::Output(redirect) if redirect.fd == 1 => {
-                acc.stdout = Some(redirect.target);
-                acc
-            }
-            Redirect::Output(redirect) if redirect.fd == 2 => {
-                acc.stderr = Some(redirect.target);
-                acc
-            }
-            _ => acc,
-        },
-    )(i)
-}
-
-fn lex_command(i: &str) -> IResult<&str, ParsedCommand> {
-    map(
-        terminated(pair(lex_pipeline, lex_redirects), space0),
-        |(pipeline, redirects)| ParsedCommand {
-            pipeline,
-            redirects,
-        },
-    )(i)
-}
-
-impl Expand for ParsedCommand<'_> {
-    type Output = Command;
-
-    fn expand(self, env: &HashMap<String, String>) -> Self::Output {
-        let pipeline = self.pipeline.expand(env);
-        let Redirects { stdin, stdout, stderr } = self.redirects;
-        let stdin = stdin.map(|e| e.expand(env)).map(PathBuf::from);
-        let stdout = stdout.map(|e| e.expand(env));
-        let stderr = stderr.map(|e| e.expand(env));
-        Command {
-            pipeline,
-            stdin,
-            stdout,
-            stderr
-        }
-    }
-}
-
-impl Expand for OutputTarget<'_> {
-    type Output = CommandOutput;
-
-    fn expand(self, env: &HashMap<String, String>) -> Self::Output {
-        match self {
-            Self::Fd(fd) => CommandOutput::Fd(fd),
-            Self::Path(path) => CommandOutput::Path(path.expand(env).into())
-        }
-    }
-}
-
-impl Expand for Word<'_> {
-    type Output = String;
-
-    fn expand(self, env: &HashMap<String, String>) -> Self::Output {
-        let mut word = String::new();
-        for token in &self.0 {
-            match token {
-                &WordToken::Var(var) => {
-                    let val = env.get(var).map_or("", |s| s.as_str());
-                    word.push_str(val);
-                }
-                &WordToken::Text(text) => {
-                    word.push_str(text);
-                }
-                WordToken::Quote(frags) => {
-                    for fragment in frags {
-                        match *fragment {
-                            QuoteFragment::Var(var) => {
-                                let val = env.get(var).map_or("", |s| s.as_str());
-                                word.push_str(val);
-                            }
-                            QuoteFragment::Text(text) => {
-                                word.push_str(text);
-                            }
-                        }
-                    }
-                }
-            }
-        }
-        word
-    }
-}
-
-impl Expand for ParsedPipeline<'_> {
-    type Output = Vec<CommandPipelineElement>;
-
-    fn expand(self, env: &HashMap<String, String>) -> Self::Output {
-        self.0.into_iter().map(|e| e.expand(env)).collect()
-    }
-}
-
-impl Expand for ParsedPipelineElement<'_> {
-    type Output = CommandPipelineElement;
-
-    fn expand(self, env: &HashMap<String, String>) -> Self::Output {
-        CommandPipelineElement(self.0.into_iter().map(|e| e.expand(env)).collect::<Vec<_>>())
-    }
-}
-
-pub fn parse_line(env: &HashMap<String, String>, input: &str) -> Result<Command, Error> {
-    let (rest, command) = lex_command(input).map_err(|e| e.map_input(ToOwned::to_owned))?;
-    if !rest.is_empty() {
-        todo!("Trailing characters: {rest:?}")
-    }
-    let command = command.expand(env);
-    Ok(command)
-}
-
-#[cfg(test)]
-mod tests {
-    use std::collections::HashMap;
-
-    use crate::parser::{
-        lex_filename, lex_quoted, lex_word, lex_word_token, OutputTarget, ParsedPipeline,
-        ParsedPipelineElement, Redirects,
-    };
-
-    use super::{
-        lex_pipeline, lex_pipeline_element, lex_redirects, lex_var, QuoteFragment,
-        Word, WordToken, Expand
-    };
-
-    #[test]
-    fn test_lex_var() {
-        let tests = [
-            ("$A1_", "A1_"),
-            ("$1", "1"),
-            ("${A1_}", "A1_"),
-            ("${1}", "1"),
-            ("$#", "#"),
-        ];
-
-        for (input, expect) in tests {
-            let (rest, output) = lex_var(input).unwrap();
-            assert!(rest.is_empty());
-            assert_eq!(output, expect);
-        }
-    }
-
-    #[test]
-    fn test_lex_filename() {
-        let tests = [
-            ("abcdef", "abcdef"),
-            ("abcdef1", "abcdef1"),
-            ("1", "1"),
-            ("_", "_"),
-            ("[", "["),
-            ("/a/b/c", "/a/b/c"),
-        ];
-
-        for (input, expect) in tests {
-            let (rest, output) = lex_filename(input).unwrap();
-            assert!(rest.is_empty());
-            assert_eq!(output, expect);
-        }
-    }
-
-    #[test]
-    fn test_lex_word_token() {
-        let tests = [("a", WordToken::Text("a")), ("$b", WordToken::Var("b"))];
-
-        for (input, expect) in tests {
-            let (rest, output) = lex_word_token(input).unwrap();
-            assert!(rest.is_empty());
-            assert_eq!(output, expect);
-        }
-    }
-
-    #[test]
-    fn test_lex_quoted() {
-        let tests = [
-            (
-                "\"abcdef ghijkl\"",
-                WordToken::Quote(vec![QuoteFragment::Text("abcdef ghijkl")]),
-            ),
-            (
-                "\"abcdef$ghijkl 123\"",
-                WordToken::Quote(vec![
-                    QuoteFragment::Text("abcdef"),
-                    QuoteFragment::Var("ghijkl"),
-                    QuoteFragment::Text(" 123"),
-                ]),
-            ),
-        ];
-
-        for (input, expect) in tests {
-            let (rest, output) = lex_quoted(input).unwrap();
-            assert!(rest.is_empty());
-            assert_eq!(output, expect);
-        }
-    }
-
-    #[test]
-    fn test_lex_word() {
-        let tests = [
-            (
-                "a$a$b",
-                Word(vec![
-                    WordToken::Text("a"),
-                    WordToken::Var("a"),
-                    WordToken::Var("b"),
-                ]),
-            ),
-            (
-                "a$1\"b$c d\"e${f}",
-                Word(vec![
-                    WordToken::Text("a"),
-                    WordToken::Var("1"),
-                    WordToken::Quote(vec![
-                        QuoteFragment::Text("b"),
-                        QuoteFragment::Var("c"),
-                        QuoteFragment::Text(" d"),
-                    ]),
-                    WordToken::Text("e"),
-                    WordToken::Var("f"),
-                ]),
-            ),
-        ];
-
-        for (input, expect) in tests {
-            let (rest, output) = lex_word(input).unwrap();
-            assert_eq!(rest, "");
-            assert_eq!(output, expect);
-        }
-    }
-
-    #[test]
-    fn test_lex_pipeline_element() {
-        let input = "a 1 $c d\"e $f g\" | ...";
-        let (rest, output) = lex_pipeline_element(input).unwrap();
-        assert_eq!(rest, "| ...");
-        assert_eq!(
-            output,
-            ParsedPipelineElement(vec![
-                Word(vec![WordToken::Text("a")]),
-                Word(vec![WordToken::Text("1")]),
-                Word(vec![WordToken::Var("c")]),
-                Word(vec![
-                    WordToken::Text("d"),
-                    WordToken::Quote(vec![
-                        QuoteFragment::Text("e "),
-                        QuoteFragment::Var("f"),
-                        QuoteFragment::Text(" g")
-                    ])
-                ]),
-            ])
-        );
-    }
-
-    #[test]
-    fn test_lex_pipeline() {
-        let input = "a b $c | d $e f g | h 1";
-        let (rest, output) = lex_pipeline(input).unwrap();
-        assert_eq!(rest, "");
-        assert_eq!(
-            output,
-            ParsedPipeline(vec![
-                ParsedPipelineElement(vec![
-                    Word(vec![WordToken::Text("a")]),
-                    Word(vec![WordToken::Text("b")]),
-                    Word(vec![WordToken::Var("c")]),
-                ]),
-                ParsedPipelineElement(vec![
-                    Word(vec![WordToken::Text("d")]),
-                    Word(vec![WordToken::Var("e")]),
-                    Word(vec![WordToken::Text("f")]),
-                    Word(vec![WordToken::Text("g")]),
-                ]),
-                ParsedPipelineElement(vec![
-                    Word(vec![WordToken::Text("h")]),
-                    Word(vec![WordToken::Text("1")]),
-                ]),
-            ])
-        );
-    }
-
-    #[test]
-    fn test_lex_redirects() {
-        let input = "2>$c >&2 <\"$d\"";
-        let (rest, output) = lex_redirects(input).unwrap();
-        assert_eq!(rest, "");
-        assert_eq!(
-            output,
-            Redirects {
-                stdin: Some(Word(vec![WordToken::Quote(vec![QuoteFragment::Var("d")])])),
-                stdout: Some(OutputTarget::Fd(2)),
-                stderr: Some(OutputTarget::Path(Word(vec![WordToken::Var("c")]))),
-            }
-        );
-    }
-
-    #[test]
-    fn test_expand_word() {
-        let word = Word(vec![
-            WordToken::Text("a"),
-            WordToken::Var("b"),
-            WordToken::Quote(vec![
-                QuoteFragment::Text(" my_text "),
-                QuoteFragment::Var("c"),
-            ]),
-        ]);
-        let env = HashMap::from_iter([("b".to_owned(), "my_var".to_owned())]);
-
-        let result = word.expand(&env);
-        assert_eq!(result, "amy_var my_text ");
-    }
-}

userspace/shell/src/readline.rs

@@ -9,11 +9,11 @@ enum Outcome {
     Data(usize),
 }
 
-fn readline_inner(stdin: &mut RawStdin, stdout: &mut Stdout, buffer: &mut [u8]) -> Result<Outcome, Error> {
+fn readline_inner(stdin: &mut RawStdin, stdout: &mut Stdout, buffer: &mut String) -> Result<Outcome, Error> {
     let mut pos = 0;
     let mut ch = [0];
 
-    while pos < buffer.len() {
+    loop {
         let len = stdin.read(&mut ch)?;
         if len == 0 {
             break;
@@ -38,6 +38,7 @@ fn readline_inner(stdin: &mut RawStdin, stdout: &mut Stdout, buffer: &mut [u8])
                 if pos != 0 {
                     stdout.write_all(b"\x1B[D \x1B[D").ok();
                     stdout.flush().ok();
+                    buffer.pop();
                     pos -= 1;
                 }
             }
@@ -45,7 +46,7 @@ fn readline_inner(stdin: &mut RawStdin, stdout: &mut Stdout, buffer: &mut [u8])
                 stdout.write_all(&[ch]).ok();
                 stdout.flush().ok();
 
-                buffer[pos] = ch;
+                buffer.push(ch as char);
                 pos += 1;
             }
             _ => (),
@@ -66,7 +67,7 @@ fn readline_inner(stdin: &mut RawStdin, stdout: &mut Stdout, buffer: &mut [u8])
 pub fn readline<P: Fn(&mut Stdout)>(
     stdin: &mut Stdin,
     stdout: &mut Stdout,
-    buffer: &mut [u8],
+    buffer: &mut String,
     prompt: P
 ) -> Result<usize, Error> {
     let mut stdin = RawStdin::new(stdin)?;

userspace/shell/src/syntax/lex.rs

@@ -0,0 +1,410 @@
+use std::str::FromStr;
+
+use nom::{
+    branch::alt,
+    bytes::complete::{is_a, is_not, tag},
+    character::complete::{alphanumeric1, char, space0},
+    combinator::{map, recognize, value, verify},
+    multi::{fold_many1, many0, many1_count},
+    sequence::{delimited, pair, preceded, separated_pair},
+    IResult,
+};
+
+#[derive(Debug, Clone, PartialEq)]
+pub enum Fragment<'a> {
+    Literal(&'a str),
+    QuotedLiteral(&'a str),
+    Variable(&'a str),
+}
+#[derive(Debug, Clone, PartialEq)]
+pub struct Word<'a>(pub Vec<Fragment<'a>>);
+
+#[derive(Debug)]
+pub struct TokenStream<'a> {
+    input: &'a str,
+    buffer: Option<Token<'a>>,
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub enum Redirect<'a> {
+    Output(OutputRedirect<'a>),
+    Input(Word<'a>),
+}
+#[derive(Debug, Clone, PartialEq)]
+pub enum OutputRedirect<'a> {
+    Err(Word<'a>),
+    Out(Word<'a>),
+    Both(Word<'a>),
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub enum Keyword {
+    If,
+    While,
+    For,
+    Match,
+    Let,
+}
+#[derive(Debug, Clone, PartialEq)]
+pub enum Operator {
+    Eq,
+    Gt,
+    Lt,
+    Or,
+    Assign,
+}
+#[derive(Debug, Clone, PartialEq)]
+pub enum Punctuation {
+    LBrace,
+    RBrace,
+    LParen,
+    RParen,
+    LBracket,
+    RBracket,
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub enum Token<'a> {
+    Word(Word<'a>),
+    Redirect(Redirect<'a>),
+    Keyword(Keyword),
+    Punctuation(Punctuation),
+    Operator(Operator),
+}
+
+type NomError<'a> = nom::Err<nom::error::Error<&'a str>>;
+impl<'a> TokenStream<'a> {
+    pub fn new(input: &'a str) -> Self {
+        Self {
+            input,
+            buffer: None,
+        }
+    }
+
+    pub fn is_eof(&self) -> bool {
+        self.input.is_empty() && self.buffer.is_none()
+    }
+
+    fn read(&mut self) -> Result<Option<Token<'a>>, NomError<'a>> {
+        if self.input.is_empty() {
+            self.buffer = None;
+            Ok(None)
+        } else {
+            let (rest, token) = lex_token(self.input)?;
+            self.input = rest;
+            self.buffer = Some(token.clone());
+            Ok(Some(token))
+        }
+    }
+
+    pub fn next(&mut self) -> Result<Option<Token<'a>>, NomError<'a>> {
+        let token = self.peek()?;
+        self.read()?;
+        Ok(token)
+    }
+    pub fn peek(&mut self) -> Result<Option<Token<'a>>, NomError<'a>> {
+        if let Some(buffer) = self.buffer.clone() {
+            return Ok(Some(buffer));
+        }
+        self.read()
+    }
+}
+
+impl Word<'_> {
+    pub fn as_literal(&self) -> Option<&str> {
+        if self.0.len() != 1 {
+            return None;
+        }
+        let Fragment::Literal(lit) = self.0[0] else {
+            return None;
+        };
+        Some(lit)
+    }
+}
+
+impl FromStr for Keyword {
+    type Err = ();
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s {
+            "if" => Ok(Self::If),
+            "while" => Ok(Self::While),
+            "for" => Ok(Self::For),
+            "match" => Ok(Self::Match),
+            "let" => Ok(Self::Let),
+            _ => Err(()),
+        }
+    }
+}
+
+fn lex_identifier(i: &str) -> IResult<&str, &str> {
+    recognize(many1_count(alt((alphanumeric1, is_a("-_")))))(i)
+}
+fn lex_filename(i: &str) -> IResult<&str, &str> {
+    recognize(many1_count(alt((alphanumeric1, is_a("./-_:")))))(i)
+}
+
+fn lex_braced_var(i: &str) -> IResult<&str, &str> {
+    // ${ABCD}
+    delimited(tag("${"), lex_identifier, char('}'))(i)
+}
+fn lex_unbraced_var(i: &str) -> IResult<&str, &str> {
+    // $ABCD
+    preceded(char('$'), lex_identifier)(i)
+}
+fn lex_var(i: &str) -> IResult<&str, &str> {
+    alt((lex_braced_var, lex_unbraced_var))(i)
+}
+
+fn lex_dquoted_literal(i: &str) -> IResult<&str, &str> {
+    let is_not_var_slash_quote = is_not("\\\"$");
+    verify(is_not_var_slash_quote, |s: &str| !s.is_empty())(i)
+}
+fn lex_dquoted(i: &str) -> IResult<&str, Vec<Fragment>> {
+    delimited(
+        char('"'),
+        many0(alt((
+            map(lex_var, Fragment::Variable),
+            map(lex_dquoted_literal, Fragment::QuotedLiteral),
+        ))),
+        char('"'),
+    )(i)
+}
+
+fn lex_squoted_text(i: &str) -> IResult<&str, &str> {
+    let is_not_slash_quote = is_not("\\'");
+    recognize(many0(is_not_slash_quote))(i)
+}
+fn lex_squoted(i: &str) -> IResult<&str, &str> {
+    delimited(char('\''), lex_squoted_text, char('\''))(i)
+}
+
+fn lex_unquoted_fragment(i: &str) -> IResult<&str, Fragment> {
+    alt((
+        map(lex_var, Fragment::Variable),
+        map(lex_filename, Fragment::Literal),
+    ))(i)
+}
+
+fn lex_word(i: &str) -> IResult<&str, Word> {
+    fold_many1(
+        alt((
+            lex_dquoted,
+            map(lex_squoted, |s| vec![Fragment::QuotedLiteral(s)]),
+            map(lex_unquoted_fragment, |s| vec![s]),
+        )),
+        || Word(vec![]),
+        |mut acc, items| {
+            acc.0.extend(items);
+            acc
+        },
+    )(i)
+}
+
+fn lex_explicit_output_redirect(i: &str) -> IResult<&str, OutputRedirect> {
+    // out>abcdef
+    // err>abcdef
+    // out+err>abcdef
+    // oe>abcdef
+    // eo>abcdef
+
+    #[derive(Debug, Clone)]
+    enum Source {
+        Out,
+        Err,
+        Both,
+    }
+
+    map(
+        separated_pair(
+            alt((
+                value(Source::Out, tag("out")),
+                value(Source::Err, tag("err")),
+                value(Source::Both, tag("oe")),
+                value(Source::Both, tag("eo")),
+                value(Source::Out, char('o')),
+                value(Source::Err, char('e')),
+            )),
+            char('>'),
+            lex_word,
+        ),
+        |(source, path)| match source {
+            Source::Out => OutputRedirect::Out(path),
+            Source::Err => OutputRedirect::Err(path),
+            Source::Both => OutputRedirect::Both(path),
+        },
+    )(i)
+}
+fn lex_implicit_output_redirect(i: &str) -> IResult<&str, OutputRedirect> {
+    // >abcdef
+    map(
+        preceded(pair(char('>'), space0), lex_word),
+        OutputRedirect::Out,
+    )(i)
+}
+
+fn lex_output_redirect(i: &str) -> IResult<&str, OutputRedirect> {
+    alt((lex_implicit_output_redirect, lex_explicit_output_redirect))(i)
+}
+fn lex_input_redirect(i: &str) -> IResult<&str, Word> {
+    // <abcdef
+    preceded(pair(char('<'), space0), lex_word)(i)
+}
+fn lex_redirect(i: &str) -> IResult<&str, Redirect> {
+    alt((
+        map(lex_input_redirect, Redirect::Input),
+        map(lex_output_redirect, Redirect::Output),
+    ))(i)
+}
+
+fn lex_maybe_keyword(i: &str) -> IResult<&str, Token> {
+    // TODO this will recognize quoted text as a keyword
+    map(lex_word, |word| {
+        if let Some(kw) = word.as_literal().and_then(|s| Keyword::from_str(s).ok()) {
+            return Token::Keyword(kw);
+        }
+        Token::Word(word)
+    })(i)
+}
+
+fn lex_punctuation(i: &str) -> IResult<&str, Punctuation> {
+    alt((
+        value(Punctuation::LBrace, char('{')),
+        value(Punctuation::RBrace, char('}')),
+        value(Punctuation::LParen, char('(')),
+        value(Punctuation::RParen, char(')')),
+        value(Punctuation::LBracket, char('[')),
+        value(Punctuation::RBracket, char(']')),
+    ))(i)
+}
+
+fn lex_operator(i: &str) -> IResult<&str, Operator> {
+    alt((
+        value(Operator::Eq, tag("==")),
+        value(Operator::Assign, char('=')),
+        value(Operator::Or, char('|')),
+    ))(i)
+}
+
+pub fn lex_token(i: &str) -> IResult<&str, Token> {
+    preceded(
+        space0,
+        alt((
+            map(lex_punctuation, Token::Punctuation),
+            map(lex_redirect, Token::Redirect),
+            map(lex_operator, Token::Operator),
+            lex_maybe_keyword,
+        )),
+    )(i)
+}
+
+pub fn lex_tokens(i: &str) -> IResult<&str, Vec<Token>> {
+    many0(lex_token)(i)
+}
+
+#[cfg(test)]
+mod tests {
+    use std::fmt;
+
+    use nom::IResult;
+
+    use super::{
+        lex_filename, lex_tokens, Fragment, Keyword, Operator, OutputRedirect, Redirect, Token,
+        Word,
+    };
+
+    #[track_caller]
+    fn run_tests<
+        'a,
+        T: PartialEq + fmt::Debug,
+        I: IntoIterator<Item = (&'a str, T, &'a str)>,
+        F: Fn(&'a str) -> IResult<&'a str, T>,
+    >(
+        it: I,
+        parser: F,
+    ) {
+        let location = std::panic::Location::caller();
+
+        for (i, (input, expect, expect_rest)) in it.into_iter().enumerate() {
+            let (rest, output) = match parser(input) {
+                Ok(ok) => ok,
+                Err(error) => {
+                    eprintln!("Test #{i} in {location:?} failed:");
+                    eprintln!("* Input: {input:?}");
+                    eprintln!("* Parser returned error: {error}");
+                    panic!();
+                }
+            };
+
+            if rest != expect_rest {
+                eprintln!("Test #{i} in {location:?} failed:");
+                eprintln!("* Input: {input:?}");
+                if expect_rest.is_empty() {
+                    eprintln!("* Unexpected trailing characters: {rest:?}");
+                } else {
+                    eprintln!("* Expected trailing characters: {expect_rest:?}");
+                    eprintln!("* Actual trailing characters: {rest:?}");
+                }
+                panic!();
+            }
+
+            if output != expect {
+                eprintln!("Test #{i} in {location:?} failed:");
+                eprintln!("* Input: {input:?}");
+                eprintln!("* Expected output: {expect:?}");
+                eprintln!("* Actual output: {output:?}");
+                panic!();
+            }
+        }
+    }
+
+    #[test]
+    fn test_lex_filename() {
+        run_tests(
+            [
+                ("./abc123_a-a/file>other", "./abc123_a-a/file", ">other"),
+                ("/a/b/c d e f g", "/a/b/c", " d e f g"),
+            ],
+            lex_filename,
+        )
+    }
+
+    #[test]
+    fn test_lex_tokens() {
+        run_tests(
+            [
+                (
+                    " if  /a/b/c\" $a b c\"$d efg",
+                    vec![
+                        Token::Keyword(Keyword::If),
+                        Token::Word(Word(vec![
+                            Fragment::Literal("/a/b/c"),
+                            Fragment::Literal(" "),
+                            Fragment::Variable("a"),
+                            Fragment::Literal(" b c"),
+                            Fragment::Variable("d"),
+                        ])),
+                        Token::Word(Word(vec![Fragment::Literal("efg")])),
+                    ],
+                    "",
+                ),
+                (
+                    "\t>$d\"filename\"",
+                    vec![Token::Redirect(Redirect::Output(OutputRedirect::Out(
+                        Word(vec![Fragment::Variable("d"), Fragment::Literal("filename")]),
+                    )))],
+                    "",
+                ),
+                (
+                    "| abc",
+                    vec![
+                        Token::Operator(Operator::Or),
+                        Token::Word(Word(vec![Fragment::Literal("abc")])),
+                    ],
+                    "",
+                ),
+            ],
+            lex_tokens,
+        )
+    }
+}

userspace/shell/src/syntax/mod.rs

@@ -0,0 +1,2 @@
+pub mod lex;
+pub mod parse;

userspace/shell/src/syntax/parse.rs

@@ -0,0 +1,137 @@
+use crate::syntax::lex::{Operator, Redirect, Token, TokenStream};
+
+use super::lex::{OutputRedirect, Word};
+
+#[derive(Debug, thiserror::Error)]
+pub enum Error {
+    #[error("{0}")]
+    Lex(nom::Err<nom::error::Error<String>>),
+    #[error("Unexpected token `{0}`")]
+    UnexpectedToken(String),
+    #[error("Empty command")]
+    EmptyPipelineCommand,
+}
+
+#[derive(Debug, PartialEq, Clone)]
+pub struct ICommand<'a> {
+    pub pipeline: Vec<IPipelineElement<'a>>,
+    pub redirects: IRedirects<'a>,
+}
+
+#[derive(Debug, PartialEq, Clone)]
+pub struct IRedirects<'a> {
+    pub stdin: Option<Word<'a>>,
+    pub stdout: Option<Word<'a>>,
+    pub stderr: Option<Word<'a>>,
+}
+
+#[derive(Debug, PartialEq, Clone)]
+pub struct IPipelineElement<'a> {
+    pub envs: Vec<(Word<'a>, Option<Word<'a>>)>,
+    pub words: Vec<Word<'a>>,
+}
+
+pub fn parse_pipeline_element<'a>(ts: &mut TokenStream<'a>) -> Result<IPipelineElement<'a>, Error> {
+    let mut words = vec![];
+    let mut envs = vec![];
+
+    while let Some(token) = ts.peek()? {
+        let Token::Word(word) = token else {
+            break;
+        };
+        ts.next()?;
+        if let Some(Token::Operator(Operator::Assign)) = ts.peek()? {
+            ts.next()?;
+            let value = if let Some(Token::Word(word)) = ts.peek()? {
+                ts.next()?;
+                Some(word)
+            } else {
+                None
+            };
+
+            envs.push((word, value));
+        } else {
+            words.push(word);
+        }
+    }
+
+    Ok(IPipelineElement { words, envs })
+}
+
+pub fn parse_pipeline<'a>(ts: &mut TokenStream<'a>) -> Result<Vec<IPipelineElement<'a>>, Error> {
+    let mut elements = vec![];
+    let mut expect_command = false;
+    while !ts.is_eof() {
+        let element = parse_pipeline_element(ts)?;
+        let is_empty = element.words.is_empty();
+        if !is_empty {
+            expect_command = false;
+            elements.push(element);
+        } else {
+            return Err(Error::EmptyPipelineCommand);
+        }
+
+        // maybe followed by eof, redirect or pipe
+        let token = ts.peek()?;
+        match token {
+            Some(Token::Operator(Operator::Or)) => {
+                expect_command = true;
+                ts.next()?;
+                continue;
+            }
+            Some(Token::Redirect(_)) => break,
+            // parse_pipeline_element() should've consumed all of these
+            Some(Token::Word(_)) => unreachable!(),
+            None => break,
+            _ => todo!(),
+        }
+    }
+    if expect_command {
+        return Err(Error::EmptyPipelineCommand);
+    }
+    Ok(elements)
+}
+
+pub fn parse_redirects<'a>(ts: &mut TokenStream<'a>) -> Result<IRedirects<'a>, Error> {
+    let mut result = IRedirects {
+        stdin: None,
+        stdout: None,
+        stderr: None,
+    };
+    while let Some(token) = ts.next()? {
+        match token {
+            Token::Redirect(Redirect::Output(redirect)) => match redirect {
+                OutputRedirect::Out(path) => result.stdout = Some(path),
+                OutputRedirect::Err(path) => result.stderr = Some(path),
+                OutputRedirect::Both(path) => {
+                    result.stdout = Some(path.clone());
+                    result.stderr = Some(path);
+                }
+            },
+            Token::Redirect(Redirect::Input(path)) => result.stdin = Some(path),
+            // parse_pipeline() should've consumed all of these
+            _ => unreachable!(),
+        }
+    }
+    Ok(result)
+}
+
+pub fn parse_interactive(line: &str) -> Result<ICommand, Error> {
+    let mut ts = TokenStream::new(line);
+
+    // pipeline itself
+    let pipeline = parse_pipeline(&mut ts)?;
+    // maybe followed by redirects
+    let redirects = parse_redirects(&mut ts)?;
+
+    Ok(ICommand {
+        pipeline,
+        redirects,
+    })
+}
+
+impl<S: Into<String>> From<nom::Err<nom::error::Error<S>>> for Error {
+    fn from(value: nom::Err<nom::error::Error<S>>) -> Self {
+        Self::Lex(value.map_input(Into::into))
+    }
+}

userspace/shell/src/sys/mod.rs

@@ -1,11 +0,0 @@
-#[cfg(any(unix, rust_analyzer))]
-pub mod unix;
-#[cfg(any(unix, rust_analyzer))]
-pub use unix as imp;
-
-#[cfg(target_os = "yggdrasil")]
-pub mod yggdrasil;
-#[cfg(target_os = "yggdrasil")]
-pub use yggdrasil as imp;
-
-pub use imp::*;

userspace/shell/src/sys/unix.rs

@@ -1,72 +0,0 @@
-use std::{
-    io,
-    os::{
-        fd::{FromRawFd, OwnedFd},
-        unix::process::ExitStatusExt,
-    },
-    process::{Command, ExitStatus},
-};
-
-use crate::{Outcome, Pipeline, SpawnedPipeline};
-
-pub struct Pipe {
-    pub read: OwnedFd,
-    pub write: OwnedFd,
-}
-
-pub fn create_pipe() -> Result<Pipe, io::Error> {
-    let mut fds = [0; 2];
-    let (read, write) = unsafe {
-        if libc::pipe(fds.as_mut_ptr()) != 0 {
-            return Err(io::Error::last_os_error());
-        }
-
-        (OwnedFd::from_raw_fd(fds[0]), OwnedFd::from_raw_fd(fds[1]))
-    };
-
-    Ok(Pipe { read, write })
-}
-
-impl From<ExitStatus> for Outcome {
-    fn from(value: ExitStatus) -> Self {
-        if let Some(code) = value.code() {
-            Self::Exited(code)
-        } else if let Some(sig) = value.signal() {
-            Self::Killed(sig)
-        } else {
-            todo!()
-        }
-    }
-}
-
-pub fn init_signal_handler() {}
-
-pub fn spawn_pipeline(
-    _interactive: bool,
-    pipeline: Pipeline<'_>,
-) -> Result<SpawnedPipeline, io::Error> {
-    let mut children = vec![];
-    for element in pipeline.elements {
-        let mut command = Command::new(element.command);
-        command
-            .args(element.args)
-            .envs(pipeline.env.iter())
-            .stdin(element.input)
-            .stdout(element.output);
-
-        children.push(command.spawn()?);
-    }
-
-    Ok(SpawnedPipeline { children })
-}
-
-pub fn wait_for_pipeline(
-    _interactive: bool,
-    mut pipeline: SpawnedPipeline,
-) -> Result<Outcome, io::Error> {
-    for mut child in pipeline.children.drain(..) {
-        child.wait()?;
-    }
-
-    Ok(Outcome::ok())
-}

userspace/shell/src/sys/yggdrasil.rs

@@ -1,105 +0,0 @@
-use std::os::{
-    fd::AsRawFd,
-    yggdrasil::{
-        rt::io::device,
-        io::pipe,
-        process::{self, CommandExt, ExitStatusExt, ProcessGroupId},
-        signal::{set_signal_handler, Signal, SignalHandler},
-    },
-};
-use std::{
-    io::{self, stdin},
-    os::fd::OwnedFd,
-    process::{Command, ExitStatus},
-};
-
-use crate::{Outcome, Pipeline, SpawnedPipeline};
-
-pub struct Pipe {
-    pub read: OwnedFd,
-    pub write: OwnedFd,
-}
-
-fn set_terminal_group(group_id: ProcessGroupId) -> Result<(), io::Error> {
-    let mut buffer = [0; 64];
-    device::device_request::<device::SetTerminalGroup>(stdin().as_raw_fd(), &mut buffer, &group_id)?;
-    Ok(())
-}
-
-pub fn spawn_pipeline(
-    interactive: bool,
-    pipeline: Pipeline<'_>,
-) -> Result<SpawnedPipeline, io::Error> {
-    let group_id = process::create_process_group();
-
-    if interactive {
-        set_terminal_group(group_id)?;
-    }
-
-    let mut children = vec![];
-    for element in pipeline.elements {
-        let mut command = Command::new(element.command);
-        command
-            .args(element.args)
-            .envs(pipeline.env.iter())
-            .stdin(element.input)
-            .stdout(element.output)
-            .process_group(group_id);
-
-        children.push(command.spawn()?);
-    }
-
-    Ok(SpawnedPipeline { children })
-}
-
-pub fn wait_for_pipeline(
-    interactive: bool,
-    mut pipeline: SpawnedPipeline,
-) -> Result<Outcome, io::Error> {
-    let self_group_id = process::group_id();
-
-    for mut child in pipeline.children.drain(..) {
-        let status = child.wait()?;
-
-        if !status.success() {
-            if interactive {
-                set_terminal_group(self_group_id).ok();
-            }
-            return Ok(Outcome::from(status));
-        }
-    }
-
-    if interactive {
-        set_terminal_group(self_group_id).ok();
-    }
-
-    Ok(Outcome::ok())
-}
-
-pub fn create_pipe() -> Result<Pipe, io::Error> {
-    let (read, write) = pipe::create_pipe_pair(false, false)?;
-    Ok(Pipe { read, write })
-}
-
-fn signal_handler(_sig: Signal) {
-    println!();
-}
-
-pub fn init_signal_handler() {
-    let _ = set_signal_handler(Signal::Interrupted, SignalHandler::Function(signal_handler));
-}
-
-impl From<ExitStatus> for Outcome {
-    fn from(value: ExitStatus) -> Self {
-        if let Some(code) = value.code() {
-            Self::Exited(code)
-        } else if let Some(sig) = value.signal() {
-            match sig {
-                Ok(sig) => Self::Killed(sig as _),
-                Err(sig) => Self::Killed(sig as _),
-            }
-        } else {
-            todo!()
-        }
-    }
-}

userspace/sysutils/src/ls.rs

@@ -2,13 +2,7 @@
 #![feature(let_chains, decl_macro)]
 
 use std::{
-    cmp::Ordering,
+    cmp::Ordering, ffi::OsString, fmt, fs::{read_dir, FileType, Metadata}, io, path::{Path, PathBuf}, process::ExitCode, time::SystemTime
-    fmt,
-    fs::{read_dir, FileType, Metadata},
-    io,
-    path::{Path, PathBuf},
-    process::ExitCode,
-    time::SystemTime,
 };
 
 #[cfg(unix)]
@@ -29,6 +23,8 @@ pub struct Args {
     inodes: bool,
     #[arg(short, long)]
     human_readable: bool,
+    #[arg(short)]
+    all: bool,
 
     paths: Vec<PathBuf>,
 }
@@ -325,7 +321,17 @@ fn sort_dirs_first(a: &Entry, b: &Entry) -> Ordering {
     by_type.then(by_name)
 }
 
-fn list_directory(path: &Path) -> io::Result<Vec<Entry>> {
+fn include(filename: &OsString, all: bool) -> bool {
+    if all {
+        return true;
+    }
+    let Some(filename) = filename.to_str() else {
+        return true;
+    };
+    !filename.starts_with(".")
+}
+
+fn list_directory(path: &Path, all: bool) -> io::Result<Vec<Entry>> {
     let mut entries = vec![];
     for entry in read_dir(path)? {
         let Ok(entry) = entry else {
@@ -334,6 +340,11 @@ fn list_directory(path: &Path) -> io::Result<Vec<Entry>> {
         };
 
         let os_filename = entry.file_name();
+
+        if !include(&os_filename, all) {
+            continue;
+        }
+
         let ty = entry.file_type().ok();
         let attrs = entry.path().symlink_metadata().ok();
 
@@ -363,7 +374,7 @@ fn list_directory(path: &Path) -> io::Result<Vec<Entry>> {
 
 fn list(opts: &Args, path: &Path) -> io::Result<()> {
     if path.is_dir() {
-        let entries = list_directory(path)?;
+        let entries = list_directory(path, opts.all)?;
 
         for entry in entries {
             println!("{}", entry.display_with(opts));