shell: better parser, kernel: better fd inheritance in spawn

kernel/libk/src/vfs/file/mod.rs

@@ -609,11 +609,13 @@ impl FileSet {
     /// Removes and closes a [FileRef] from the struct
     pub fn close_file(&mut self, fd: RawFd) -> Result<(), Error> {
         // Do nothing, file will be dropped and closed
-        if self.map.remove(&fd).is_some() {
+        // TODO call File's close() and return its status
-            Ok(())
+        let _ = self.take_file(fd)?;
-        } else {
+        Ok(())
-            Err(Error::InvalidFile)
+    }
-        }
+
+    pub fn take_file(&mut self, fd: RawFd) -> Result<FileRef, Error> {
+        self.map.remove(&fd).ok_or(Error::InvalidFile)
     }
 
     /// Removes all [FileRef]s from the struct which do not pass the `predicate` check

kernel/src/syscall/imp/sys_process.rs

@@ -100,7 +100,7 @@ pub(crate) fn spawn_process(options: &SpawnOptions<'_>) -> Result<ProcessId, Err
     let thread = Thread::current();
     let process = thread.process();
 
-    run_with_io(&process, |mut io| {
+    let result = run_with_io(&process, |mut io| {
         let attach_debugger = options
             .optional
             .iter()
@@ -139,9 +139,14 @@ pub(crate) fn spawn_process(options: &SpawnOptions<'_>) -> Result<ProcessId, Err
 
         for opt in options.optional {
             match opt {
-                &SpawnOption::InheritFile { source, child } => {
+                &SpawnOption::MoveFile { source, child } => {
+                    if let Ok(src_file) = io.files.take_file(source) {
+                        child_io.files.set_file(child, src_file)?;
+                    }
+                }
+                &SpawnOption::CopyFile { source, child } => {
                     if let Ok(src_file) = io.files.file(source) {
-                        child_io.files.set_file(child, src_file.clone())?;
+                        child_io.files.set_file(child, src_file.send()?)?;
                     }
                 }
                 &SpawnOption::SetProcessGroup(pgroup) => {
@@ -184,7 +189,13 @@ pub(crate) fn spawn_process(options: &SpawnOptions<'_>) -> Result<ProcessId, Err
         }
 
         Ok(pid as _)
-    })
+    });
+
+    //if let Err(error) = result {
+    // log::error!("spawn({options:#?}) -> {result:?}");
+    //}
+
+    result
 }
 
 pub(crate) fn wait_process(

lib/abi/src/process/mod.rs

@@ -49,13 +49,21 @@ pub enum ProcessOption {
 /// Defines an optional argument for controlling process creation
 #[derive(Debug)]
 pub enum SpawnOption {
-    /// Indicates a new process should inherit a file descriptor from its creator
+    MoveFile {
-    InheritFile {
-        /// FD on the creator side
         source: RawFd,
-        /// What FD number should be used in the child
         child: RawFd,
     },
+    CopyFile {
+        source: RawFd,
+        child: RawFd,
+    },
+    // /// Indicates a new process should inherit a file descriptor from its creator
+    // InheritFile {
+    //     /// FD on the creator side
+    //     source: RawFd,
+    //     /// What FD number should be used in the child
+    //     child: RawFd,
+    // },
     /// The new process should be placed in the specified group
     SetProcessGroup(ProcessGroupId),
     /// Gain terminal control for the given FD

userspace/shell/Cargo.toml

@@ -16,3 +16,6 @@ yggdrasil-abi = { path = "../../lib/abi" }
 
 [target.'cfg(unix)'.dependencies]
 libc = "*"
+
+[lints]
+workspace = true

userspace/shell/src/builtins.rs

@@ -12,7 +12,9 @@ pub type BuiltinCommand = fn(&[String], &mut HashMap<String, String>) -> Result<
 static BUILTINS: &[(&str, BuiltinCommand)] = &[
     ("echo", b_echo),
     ("set", b_set),
+    #[cfg(target_os = "yggdrasil")]
     ("cd", b_cd),
+    #[cfg(target_os = "yggdrasil")]
     ("pwd", b_pwd),
     ("which", b_which),
     ("exit", b_exit),
@@ -94,6 +96,7 @@ fn b_exit(args: &[String], _envs: &mut HashMap<String, String>) -> Result<Outcom
     }
 }
 
+#[cfg(target_os = "yggdrasil")]
 fn b_cd(args: &[String], _envs: &mut HashMap<String, String>) -> Result<Outcome, Error> {
     let path = if args.is_empty() {
         "/"
@@ -104,6 +107,7 @@ fn b_cd(args: &[String], _envs: &mut HashMap<String, String>) -> Result<Outcome,
     Ok(Outcome::Exited(0))
 }
 
+#[cfg(target_os = "yggdrasil")]
 fn b_pwd(args: &[String], _envs: &mut HashMap<String, String>) -> Result<Outcome, Error> {
     if !args.is_empty() {
         eprintln!("Usage: pwd");

userspace/shell/src/main.rs

@@ -5,12 +5,13 @@ use std::{
     env,
     fs::File,
     io::{self, stdin, stdout, BufRead, BufReader, Stdin, Write},
-    os::fd::{FromRawFd, IntoRawFd, OwnedFd},
+    os::fd::{FromRawFd, IntoRawFd},
     path::Path,
     process::{Child, ExitCode, Stdio},
 };
 
 use clap::Parser;
+use parser::Command;
 
 mod builtins;
 mod parser;
@@ -20,8 +21,9 @@ mod sys;
 pub enum Error {
     #[error("{0}")]
     IoError(#[from] io::Error),
+    #[cfg(any(target_os = "yggdrasil", rust_analyzer))]
     #[error("{0:?}")]
-    RtError(yggdrasil_rt::Error)
+    RtError(yggdrasil_rt::Error),
 }
 
 #[derive(Parser)]
@@ -87,7 +89,7 @@ impl Input {
 // TODO group pipeline commands into a single process group
 pub fn exec(
     interactive: bool,
-    pipeline: &[parser::Command],
+    command: &Command,
     env: &mut HashMap<String, String>,
 ) -> Result<Outcome, Error> {
     // Pipeline "a | b | c" execution:
@@ -98,12 +100,23 @@ pub fn exec(
     //
     // Pipe count: command count - 1
 
-    if pipeline.is_empty() {
+    if command.commands.is_empty() {
         return Ok(Outcome::ok());
     }
 
-    if pipeline.len() == 1 {
+    let stdin = if let Some(path) = command.stdin.as_ref() {
-        let command = &pipeline[0];
+        Some(File::open(path)?)
+    } else {
+        None
+    };
+    let stdout = if let Some(path) = command.stdout.as_ref() {
+        Some(File::create(path)?)
+    } else {
+        None
+    };
+
+    if command.commands.len() == 1 {
+        let command = &command.commands[0];
         let (cmd, args) = command.words.split_first().unwrap();
 
         if let Some(builtin) = builtins::get_builtin(cmd) {
@@ -113,16 +126,17 @@ pub fn exec(
 
     let mut inputs = vec![];
     let mut outputs = vec![];
-    let mut pipe_fds = vec![];
 
-    inputs.push(Stdio::inherit());
+    if let Some(stdin) = stdin {
-    for _ in 1..pipeline.len() {
+        inputs.push(unsafe { Stdio::from_raw_fd(stdin.into_raw_fd()) });
+    } else {
+        inputs.push(Stdio::inherit());
+    }
+    for _ in 1..command.commands.len() {
         let pipe = sys::create_pipe()?;
 
         let read_fd = pipe.read.into_raw_fd();
         let write_fd = pipe.write.into_raw_fd();
-        pipe_fds.push(unsafe { OwnedFd::from_raw_fd(read_fd) });
-        pipe_fds.push(unsafe { OwnedFd::from_raw_fd(write_fd) });
 
         let input = unsafe { Stdio::from_raw_fd(read_fd) };
         let output = unsafe { Stdio::from_raw_fd(write_fd) };
@@ -130,15 +144,19 @@ pub fn exec(
         inputs.push(input);
         outputs.push(output);
     }
-    outputs.push(Stdio::inherit());
+    if let Some(stdout) = stdout {
+        outputs.push(unsafe { Stdio::from_raw_fd(stdout.into_raw_fd()) });
+    } else {
+        outputs.push(Stdio::inherit());
+    }
 
     assert_eq!(inputs.len(), outputs.len());
-    assert_eq!(inputs.len(), pipeline.len());
+    assert_eq!(inputs.len(), command.commands.len());
 
     let mut elements = vec![];
     let ios = inputs.drain(..).zip(outputs.drain(..));
 
-    for (command, (input, output)) in pipeline.iter().zip(ios) {
+    for (command, (input, output)) in command.commands.iter().zip(ios) {
         let (cmd, args) = command.words.split_first().unwrap();
 
         let element = PipelineElement {
@@ -154,8 +172,6 @@ pub fn exec(
     let pipeline = Pipeline { elements, env };
     let pipeline = sys::spawn_pipeline(interactive, pipeline)?;
 
-    drop(pipe_fds);
-
     let status = sys::wait_for_pipeline(interactive, pipeline)?;
 
     Ok(status)
@@ -182,7 +198,17 @@ fn run(mut input: Input, vars: &mut HashMap<String, String>) -> io::Result<ExitC
             Some((line, _)) => line.trim(),
             None => line,
         };
-        let cmd = parser::parse_line(vars, line).unwrap();
+        let cmd = match parser::parse_line(vars, line) {
+            Ok(cmd) => cmd,
+            Err(error) if input.is_interactive() => {
+                eprintln!("stdin: {error}");
+                continue;
+            }
+            Err(error) => {
+                eprintln!("Command error: {error}");
+                return Ok(ExitCode::FAILURE);
+            }
+        };
 
         let q_code = match exec(input.is_interactive(), &cmd, vars) {
             Ok(Outcome::ExitShell(code)) => {

userspace/shell/src/parser.rs

@@ -1,157 +1,143 @@
-use std::collections::HashMap;
+use std::{collections::HashMap, mem, path::PathBuf};
+
+use nom::{
+    branch::alt,
+    bytes::complete::{is_a, tag},
+    character::complete::{alphanumeric1, space0, u8 as num_u8},
+    combinator::{map, recognize, value},
+    multi::many1,
+    sequence::{preceded, terminated},
+    IResult, Parser,
+};
 
 #[derive(Debug, PartialEq)]
-pub struct Command {
+pub struct PipelineElement {
     pub words: Vec<String>,
 }
 
+#[derive(Debug, thiserror::Error)]
+pub enum Error {
+    #[error("{0}")]
+    Lex(#[from] nom::Err<nom::error::Error<String>>),
+}
+
 #[derive(Debug, PartialEq)]
-pub enum Token {
+pub struct Command {
-    Word(String),
+    pub commands: Vec<PipelineElement>,
+    pub stdin: Option<PathBuf>,
+    pub stdout: Option<PathBuf>,
+    pub stderr: Option<PathBuf>,
+}
+
+#[derive(Debug, Clone)]
+enum Token<'a> {
+    Word(&'a str),
     Pipe,
+    Output(u8),
+    Input,
 }
 
-fn lex_skip_whitespace(mut input: &[u8]) -> &[u8] {
+fn lex_word(i: &str) -> IResult<&str, Token> {
-    while input.first().map(u8::is_ascii_whitespace).unwrap_or(false) {
+    map(
-        input = &input[1..];
+        recognize(many1(alt((alphanumeric1, is_a("_-+=%!@/.[]:"))))),
-    }
+        Token::Word,
-    input
+    )
+    .parse(i)
 }
 
-pub fn lex_word(mut input: &[u8]) -> Result<(Token, &[u8]), &[u8]> {
+fn lex_output(i: &str) -> IResult<&str, Token> {
-    let mut buffer = String::new();
+    alt((
-    while !input.is_empty() {
+        map(terminated(num_u8, tag(">")), Token::Output),
-        if input[0].is_ascii_whitespace() || input[0] == b'"' {
+        value(Token::Output(1), tag(">")),
-            break;
+    ))
-        }
+    .parse(i)
-
-        buffer.push(input[0] as char);
-        input = &input[1..];
-    }
-
-    Ok((Token::Word(buffer), input))
 }
 
-pub fn lex_token(mut input: &[u8]) -> Result<(Option<Token>, &[u8]), &[u8]> {
+fn lex_input(i: &str) -> IResult<&str, Token> {
-    input = lex_skip_whitespace(input);
+    value(Token::Input, tag("<")).parse(i)
-
-    let Some(&ch) = input.first() else {
-        return Ok((None, &[]));
-    };
-
-    match ch {
-        b'|' => Ok((Some(Token::Pipe), &input[1..])),
-        b'"' => todo!(),
-        _ => lex_word(input).map(|(x, y)| (Some(x), y)),
-    }
 }
 
-pub fn lex_line(mut input: &[u8]) -> Result<Vec<Token>, &[u8]> {
+fn lex_pipe(i: &str) -> IResult<&str, Token> {
-    let mut res = Vec::new();
+    value(Token::Pipe, tag("|")).parse(i)
-    while let (Some(token), output) = lex_token(input)? {
-        res.push(token);
-        input = output;
-    }
-    Ok(res)
 }
 
-pub fn collect_pipeline(tokens: &[Token]) -> Vec<Command> {
+fn lex_token(i: &str) -> IResult<&str, Token> {
-    let mut pipeline = Vec::new();
+    preceded(space0, alt((lex_output, lex_word, lex_input, lex_pipe))).parse(i)
-    let mut current = None;
+}
 
-    for token in tokens {
+fn parse_command(_env: &HashMap<String, String>, input: &[Token]) -> Result<Command, Error> {
+    let mut elements = vec![];
+    let mut stdin = None;
+    let mut stdout = None;
+    let mut stderr = None;
+
+    let mut current = vec![];
+
+    let mut it = input.into_iter();
+    while let Some(token) = it.next() {
         match token {
-            Token::Word(word) => {
+            &Token::Word(word) => {
-                let current = current.get_or_insert_with(|| Command { words: vec![] });
+                current.push(word.into());
-                current.words.push(word.clone());
             }
             Token::Pipe => {
-                if let Some(current) = current.take() {
+                if current.is_empty() {
-                    pipeline.push(current);
+                    todo!();
                 }
+
+                elements.push(PipelineElement {
+                    words: mem::replace(&mut current, vec![]),
+                });
+            }
+            Token::Output(1) => {
+                // TODO ok_or
+                let path = it.next().unwrap();
+                let Token::Word(word) = path else {
+                    todo!();
+                };
+                stdout = Some(PathBuf::from(word));
+            }
+            Token::Output(2) => {
+                // TODO ok_or
+                let path = it.next().unwrap();
+                let Token::Word(word) = path else {
+                    todo!();
+                };
+                stderr = Some(PathBuf::from(word));
+            }
+            Token::Input => {
+                // TODO ok_or
+                let path = it.next().unwrap();
+                let Token::Word(word) = path else {
+                    todo!();
+                };
+                stdin = Some(PathBuf::from(word));
+            }
+            Token::Output(_) => {
+                todo!();
             }
         }
     }
 
-    if let Some(current) = current {
+    if !current.is_empty() {
-        pipeline.push(current);
+        elements.push(PipelineElement { words: current });
     }
 
-    pipeline
+    Ok(Command {
+        commands: elements,
+        stdin,
+        stdout,
+        stderr,
+    })
 }
 
-pub fn parse_line(_env: &HashMap<String, String>, input: &str) -> Result<Vec<Command>, ()> {
+pub fn parse_line(env: &HashMap<String, String>, input: &str) -> Result<Command, Error> {
-    let tokens = lex_line(input.as_bytes()).map_err(|_| ())?;
+    let mut input = input;
-    let pipeline = collect_pipeline(&tokens);
+    let mut tokens = vec![];
-    Ok(pipeline)
+    while !input.is_empty() {
-}
+        let (tail, token) = lex_token(input).map_err(|error| error.map_input(String::from))?;
-
+        tokens.push(token);
-#[cfg(test)]
+        input = tail;
-mod tests {
+    }
-    use crate::parser::{lex_line, lex_skip_whitespace, lex_token, Command, Token};
+
-
+    parse_command(env, &tokens)
-    use super::collect_pipeline;
-
-    #[test]
-    fn collect() {
-        let tokens = lex_line(b"abc def | ghi jkl | mno pqr").unwrap();
-        let pipeline = collect_pipeline(&tokens);
-
-        assert_eq!(
-            &pipeline,
-            &[
-                Command {
-                    words: vec!["abc".to_owned(), "def".to_owned()]
-                },
-                Command {
-                    words: vec!["ghi".to_owned(), "jkl".to_owned()]
-                },
-                Command {
-                    words: vec!["mno".to_owned(), "pqr".to_owned()]
-                },
-            ]
-        );
-    }
-
-    #[test]
-    fn skip_whitespace() {
-        let w = b"   \t\na";
-        assert_eq!(lex_skip_whitespace(w), b"a");
-
-        let w = b"";
-        assert_eq!(lex_skip_whitespace(w), b"");
-
-        let w = b"a";
-        assert_eq!(lex_skip_whitespace(w), b"a");
-    }
-
-    #[test]
-    fn line_tokens() {
-        let w = b"abc def";
-        assert_eq!(
-            lex_line(w).unwrap(),
-            vec![Token::Word("abc".to_owned()), Token::Word("def".to_owned())]
-        );
-
-        let w = b"abc def | ghi jkl";
-        assert_eq!(
-            lex_line(w).unwrap(),
-            vec![
-                Token::Word("abc".to_owned()),
-                Token::Word("def".to_owned()),
-                Token::Pipe,
-                Token::Word("ghi".to_owned()),
-                Token::Word("jkl".to_owned()),
-            ]
-        );
-    }
-
-    #[test]
-    fn token() {
-        let w = b"abc def";
-        let (t0, w) = lex_token(w).unwrap();
-        assert_eq!(t0, Some(Token::Word("abc".to_owned())));
-        let (t1, w) = lex_token(w).unwrap();
-        assert_eq!(t1, Some(Token::Word("def".to_owned())));
-    }
 }

userspace/shell/src/sys/mod.rs

@@ -1,6 +1,6 @@
-#[cfg(unix)]
+#[cfg(any(unix, rust_analyzer))]
 pub mod unix;
-#[cfg(unix)]
+#[cfg(any(unix, rust_analyzer))]
 pub use unix as imp;
 
 #[cfg(target_os = "yggdrasil")]

userspace/shell/src/sys/unix.rs

@@ -8,7 +8,7 @@ use std::{
     process::{Child, Command, ExitStatus, Stdio},
 };
 
-use crate::Outcome;
+use crate::{Outcome, Pipeline, SpawnedPipeline};
 
 pub struct Pipe {
     pub read: OwnedFd,
@@ -63,3 +63,53 @@ impl From<ExitStatus> for Outcome {
         }
     }
 }
+
+pub fn init_signal_handler() {}
+
+pub fn spawn_pipeline(
+    interactive: bool,
+    pipeline: Pipeline<'_>,
+) -> Result<SpawnedPipeline, io::Error> {
+    let mut children = vec![];
+    for element in pipeline.elements {
+        let mut command = Command::new(element.command);
+        command
+            .args(element.args)
+            .envs(pipeline.env.iter())
+            .stdin(element.input)
+            .stdout(element.output);
+
+        children.push(command.spawn()?);
+    }
+
+    Ok(SpawnedPipeline { children })
+}
+
+pub fn wait_for_pipeline(
+    interactive: bool,
+    mut pipeline: SpawnedPipeline,
+) -> Result<Outcome, io::Error> {
+    for mut child in pipeline.children.drain(..) {
+        child.wait()?;
+    }
+
+    Ok(Outcome::ok())
+    // let self_group_id = process::group_id();
+
+    // for mut child in pipeline.children.drain(..) {
+    //     let status = child.wait()?;
+
+    //     if !status.success() {
+    //         if interactive {
+    //             set_terminal_group(self_group_id).ok();
+    //         }
+    //         return Ok(Outcome::from(status));
+    //     }
+    // }
+
+    // if interactive {
+    //     set_terminal_group(self_group_id).ok();
+    // }
+
+    // Ok(Outcome::ok())
+}

userspace/term/src/main.rs

@@ -4,7 +4,7 @@ use std::{
     fs::File,
     io::{Read, Write},
     os::{
-        fd::{AsRawFd, FromRawFd, RawFd},
+        fd::{self, AsRawFd, FromRawFd, IntoRawFd, RawFd},
         yggdrasil::{
             self,
             io::{
@@ -262,14 +262,19 @@ impl Terminal<'_> {
         poll.add(conn_fd)?;
         poll.add(pty_master_fd)?;
 
-        let pty_slave_fd = pty_slave.as_raw_fd();
+        let pty_slave_stdin = pty_slave.into_raw_fd();
+        let pty_slave_stdout = fd::clone_fd(pty_slave_stdin)?;
+        let pty_slave_stderr = fd::clone_fd(pty_slave_stdin)?;
+
+        debug_trace!("stdin = {pty_slave_stdin:?}, stdout = {pty_slave_stdout:?}, stderr = {pty_slave_stderr:?}");
+
         let group_id = yggdrasil::process::create_process_group();
         let shell = unsafe {
             Command::new("/bin/sh")
                 .arg("-l")
-                .stdin(Stdio::from_raw_fd(pty_slave_fd))
+                .stdin(Stdio::from_raw_fd(pty_slave_stdin))
-                .stdout(Stdio::from_raw_fd(pty_slave_fd))
+                .stdout(Stdio::from_raw_fd(pty_slave_stdout))
-                .stderr(Stdio::from_raw_fd(pty_slave_fd))
+                .stderr(Stdio::from_raw_fd(pty_slave_stderr))
                 .process_group(group_id)
                 .gain_terminal(0)
                 .spawn()?