feat(env): listFiles operation in the pi-env daemon

Lists a directory tree like fd --hidden --exclude .git without following symbolic links, with the ignore crate of fd and ripgrep for git's ignore rules. Records travel as payload (kind byte, relative / path, NUL); the listing stops at maxEntries, maxBytes or timeoutMs with truncated, and a cancel stops it between entries. hello reports features: ["listFiles"] so clients can tell older daemons apart.
2026-11-05 23:37:25 +00:00 · 2026-10-12 15:13:14 +00:00 · 2026-10-12 15:13:14 +00:00 · aa499c2426
commit aa499c2426
parent abfe62928b
5 changed files with 487 additions and 5 deletions
--- a/packages/env/daemon/Cargo.lock
+++ b/packages/env/daemon/Cargo.lock
@ -2,12 +2,31 @@
 # It is not intended for manual editing.
 version = 4

+[[package]]
+name = "aho-corasick"
+version = "1.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "bitflags"
 version = "2.13.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06"

+[[package]]
+name = "bstr"
+version = "1.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6bb31b46c14244e20ee9984b11bf5c992b91fb6939fea616e3512c8baecdbe5f"
+dependencies = [
+ "memchr",
+ "serde_core",
+]
+
 [[package]]
 name = "cfg-if"
 version = "1.0.5"
@ -20,6 +39,31 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48"

+[[package]]
+name = "crossbeam-deque"
+version = "0.8.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a"
+dependencies = [
+ "crossbeam-epoch",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-epoch"
+version = "0.9.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d"
+dependencies = [
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-utils"
+version = "0.8.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6"
+
 [[package]]
 name = "encoding_rs"
 version = "0.8.42"
@ -43,6 +87,35 @@ dependencies = [
 "libc",
 ]

+[[package]]
+name = "globset"
+version = "0.4.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "07c34a9410465b45bd9787443bc7370f37735bad04b0f0cd57ff1a3186c98988"
+dependencies = [
+ "aho-corasick",
+ "bstr",
+ "log",
+ "regex-automata",
+ "regex-syntax",
+]
+
+[[package]]
+name = "ignore"
+version = "0.4.33"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "00b69833ed729dc5aa7d19541d96d6cf8e9137194207a04916d658e43168402f"
+dependencies = [
+ "crossbeam-deque",
+ "globset",
+ "log",
+ "memchr",
+ "regex-automata",
+ "same-file",
+ "walkdir",
+ "winapi-util",
+]
+
 [[package]]
 name = "inotify"
 version = "0.11.5"
@ -157,6 +230,7 @@ name = "pi-env"
 version = "0.1.0"
 dependencies = [
 "encoding_rs",
+ "ignore",
 "libc",
 "notify",
 "serde_json",
@ -181,6 +255,23 @@ dependencies = [
 "proc-macro2",
 ]

+[[package]]
+name = "regex-automata"
+version = "0.4.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-syntax",
+]
+
+[[package]]
+name = "regex-syntax"
+version = "0.8.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
+
 [[package]]
 name = "rustversion"
 version = "1.0.23"
--- a/packages/env/daemon/Cargo.toml
+++ b/packages/env/daemon/Cargo.toml
@ -7,6 +7,7 @@ publish = false

 [dependencies]
 encoding_rs = "=0.8.42"
+ignore = "=0.4.33"
 notify = "=8.2.0"
 serde_json = "=1.0.151"

--- a/packages/env/daemon/src/list.rs
+++ b/packages/env/daemon/src/list.rs
@ -0,0 +1,368 @@
+//! `listFiles`: the files and directories below a directory, as `fd --hidden --exclude .git` lists them (without
+//! following symbolic links), skipping what git ignores. Uses the `ignore` crate, which fd and ripgrep use.
+
+use crate::errors::Failure;
+use crate::frame::MAX_PAYLOAD;
+use ignore::WalkBuilder;
+use serde_json::{Value, json};
+use std::ffi::OsString;
+use std::fs;
+use std::path::{Component, Path};
+use std::time::{Duration, Instant};
+
+const DEFAULT_MAX_ENTRIES: u64 = 50_000;
+const DEFAULT_MAX_BYTES: u64 = 4 * 1024 * 1024;
+const DEFAULT_TIMEOUT_MS: u64 = 10_000;
+
+pub struct Options {
+    /// Include names starting with `.` (and, on Windows, hidden files).
+    pub hidden: bool,
+    /// Skip what git ignores inside git work trees: `.gitignore` files, `.git/info/exclude`, the global excludes file.
+    pub gitignore: bool,
+    /// Names skipped with everything below them, in addition to `.git`.
+    pub exclude: Vec<OsString>,
+    pub max_entries: u64,
+    /// Bytes of the listing payload: per entry one kind byte, the path, and a NUL.
+    pub max_bytes: u64,
+    pub timeout: Duration,
+}
+
+impl Options {
+    pub fn from_json(json: &Value) -> Result<Options, Failure> {
+        let count = |key: &str, default: u64| -> Result<u64, Failure> {
+            match &json[key] {
+                Value::Null => Ok(default),
+                value => value
+                    .as_u64()
+                    .ok_or_else(|| Failure::new("EINVAL", format!("invalid field {key}"))),
+            }
+        };
+        let exclude = match &json["exclude"] {
+            Value::Null => Vec::new(),
+            Value::Array(names) => names
+                .iter()
+                .map(|name| name.as_str().map(OsString::from))
+                .collect::<Option<Vec<_>>>()
+                .ok_or_else(|| Failure::new("EINVAL", "invalid field exclude"))?,
+            _ => return Err(Failure::new("EINVAL", "invalid field exclude")),
+        };
+        Ok(Options {
+            hidden: json["hidden"].as_bool().unwrap_or(true),
+            gitignore: json["gitignore"].as_bool().unwrap_or(true),
+            exclude,
+            max_entries: count("maxEntries", DEFAULT_MAX_ENTRIES)?,
+            max_bytes: count("maxBytes", DEFAULT_MAX_BYTES)?.min(MAX_PAYLOAD as u64),
+            timeout: Duration::from_millis(count("timeoutMs", DEFAULT_TIMEOUT_MS)?),
+        })
+    }
+}
+
+pub struct Listing {
+    /// Records of a kind byte (`f`, `d`, `l`), the path relative to the root with `/` separators, and a NUL.
+    pub payload: Vec<u8>,
+    pub count: u64,
+    pub truncated: bool,
+}
+
+/// The path of `path` below `root`, its components joined with `/`; names that are not UTF-8 get replacement
+/// characters, as `readdir` reports them.
+fn relative(root: &Path, path: &Path) -> Option<String> {
+    let rest = path.strip_prefix(root).ok()?;
+    let mut text = String::new();
+    for component in rest.components() {
+        let Component::Normal(name) = component else {
+            return None;
+        };
+        if !text.is_empty() {
+            text.push('/');
+        }
+        text.push_str(&name.to_string_lossy());
+    }
+    (!text.is_empty()).then_some(text)
+}
+
+/// List `root` depth first, siblings in byte order of their names. Stops with `truncated` at the first entry beyond
+/// a bound; fails with `aborted` once `aborted()` is true. Entries that cannot be read are skipped.
+pub fn list(root: &str, options: &Options, aborted: &dyn Fn() -> bool) -> Result<Listing, Failure> {
+    let metadata = fs::metadata(root).map_err(|error| Failure::io(&error, "scandir", root))?;
+    if !metadata.is_dir() {
+        return Err(Failure::new(
+            "ENOTDIR",
+            format!("ENOTDIR: not a directory, scandir '{root}'"),
+        )
+        .path(root));
+    }
+    let root_path = Path::new(root);
+    let exclude = options.exclude.clone();
+    let walk = WalkBuilder::new(root_path)
+        .hidden(!options.hidden)
+        .ignore(false)
+        .parents(options.gitignore)
+        .git_ignore(options.gitignore)
+        .git_exclude(options.gitignore)
+        .git_global(options.gitignore)
+        .require_git(true)
+        .follow_links(false)
+        // The global excludes file is found relative to the listed directory, not the daemon's.
+        .current_dir(root_path)
+        .sort_by_file_name(|a, b| a.cmp(b))
+        .filter_entry(move |entry| {
+            let name = entry.file_name();
+            name != ".git" && !exclude.iter().any(|excluded| excluded == name)
+        })
+        .build();
+    let started = Instant::now();
+    let mut listing = Listing {
+        payload: Vec::new(),
+        count: 0,
+        truncated: false,
+    };
+    for entry in walk {
+        if aborted() {
+            return Err(Failure::new("aborted", "aborted"));
+        }
+        let Ok(entry) = entry else { continue };
+        if entry.depth() == 0 {
+            continue;
+        }
+        let Some(file_type) = entry.file_type() else {
+            continue;
+        };
+        let kind = if file_type.is_symlink() {
+            b'l'
+        } else if file_type.is_dir() {
+            b'd'
+        } else if file_type.is_file() {
+            b'f'
+        } else {
+            continue;
+        };
+        let Some(path) = relative(root_path, entry.path()) else {
+            continue;
+        };
+        let size = path.len() as u64 + 2;
+        if listing.count >= options.max_entries
+            || listing.payload.len() as u64 + size > options.max_bytes
+            || started.elapsed() > options.timeout
+        {
+            listing.truncated = true;
+            break;
+        }
+        listing.payload.push(kind);
+        listing.payload.extend_from_slice(path.as_bytes());
+        listing.payload.push(0);
+        listing.count += 1;
+    }
+    Ok(listing)
+}
+
+pub fn run(json: &Value, aborted: &dyn Fn() -> bool) -> Result<(Value, Vec<u8>), Failure> {
+    let root = json["path"]
+        .as_str()
+        .ok_or_else(|| Failure::new("EINVAL", "missing field path"))?;
+    let listing = list(root, &Options::from_json(json)?, aborted)?;
+    Ok((
+        json!({ "count": listing.count, "truncated": listing.truncated }),
+        listing.payload,
+    ))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::sync::atomic::{AtomicU64, Ordering};
+
+    struct Tree(std::path::PathBuf);
+
+    impl Tree {
+        fn new(files: &[&str]) -> Tree {
+            static NEXT: AtomicU64 = AtomicU64::new(0);
+            let root = std::env::temp_dir().join(format!(
+                "pi-env-list-{}-{}",
+                std::process::id(),
+                NEXT.fetch_add(1, Ordering::Relaxed)
+            ));
+            let tree = Tree(root);
+            for file in files {
+                tree.write(file, "");
+            }
+            tree
+        }
+
+        fn write(&self, path: &str, content: &str) {
+            let path = self.0.join(path);
+            if path.to_string_lossy().ends_with('/') {
+                fs::create_dir_all(&path).unwrap();
+                return;
+            }
+            fs::create_dir_all(path.parent().unwrap()).unwrap();
+            fs::write(path, content).unwrap();
+        }
+
+        fn list(&self, options: Options) -> (Vec<String>, bool) {
+            let listing = list(&self.0.to_string_lossy(), &options, &|| false).unwrap();
+            let entries: Vec<String> = listing
+                .payload
+                .split(|byte| *byte == 0)
+                .filter(|record| !record.is_empty())
+                .map(|record| {
+                    format!(
+                        "{} {}",
+                        record[0] as char,
+                        String::from_utf8_lossy(&record[1..])
+                    )
+                })
+                .collect();
+            assert_eq!(entries.len() as u64, listing.count);
+            (entries, listing.truncated)
+        }
+    }
+
+    impl Drop for Tree {
+        fn drop(&mut self) {
+            let _ = fs::remove_dir_all(&self.0);
+        }
+    }
+
+    fn options() -> Options {
+        Options::from_json(&json!({})).unwrap()
+    }
+
+    #[test]
+    fn lists_files_and_directories_depth_first_in_name_order() {
+        let tree = Tree::new(&[
+            "b.txt",
+            "a/z.txt",
+            "a/b/c.txt",
+            "empty/",
+            ".hidden/x",
+            "A.txt",
+        ]);
+        let (entries, truncated) = tree.list(options());
+        assert_eq!(
+            entries,
+            [
+                "d .hidden",
+                "f .hidden/x",
+                "f A.txt",
+                "d a",
+                "d a/b",
+                "f a/b/c.txt",
+                "f a/z.txt",
+                "f b.txt",
+                "d empty"
+            ]
+        );
+        assert!(!truncated);
+        let (entries, _) = tree.list(Options {
+            hidden: false,
+            ..options()
+        });
+        assert!(!entries.iter().any(|entry| entry.contains(".hidden")));
+    }
+
+    #[test]
+    fn skips_what_git_ignores_only_in_work_trees() {
+        let tree = Tree::new(&[
+            "keep.txt",
+            "out/x.js",
+            "sub/keep.txt",
+            "sub/drop.log",
+            "node_modules/m/i.js",
+        ]);
+        tree.write(".gitignore", "out/\n*.log\n");
+        tree.write("sub/.gitignore", "!keep.txt\n");
+        // Not a work tree yet: everything is listed.
+        let (entries, _) = tree.list(options());
+        assert!(entries.contains(&"f out/x.js".to_string()));
+        tree.write(".git/HEAD", "ref: refs/heads/main\n");
+        tree.write(".git/info/exclude", "keep.txt\n");
+        let (entries, _) = tree.list(Options {
+            exclude: vec!["node_modules".into()],
+            ..options()
+        });
+        assert_eq!(
+            entries,
+            [
+                "f .gitignore",
+                "d sub",
+                "f sub/.gitignore",
+                "f sub/keep.txt"
+            ]
+        );
+        let (entries, _) = tree.list(Options {
+            gitignore: false,
+            ..options()
+        });
+        assert!(entries.contains(&"f out/x.js".to_string()));
+        assert!(!entries.iter().any(|entry| entry.contains(".git/")));
+        // A subdirectory of a work tree applies the rules of its parents.
+        let sub = list(&tree.0.join("sub").to_string_lossy(), &options(), &|| false).unwrap();
+        assert_eq!(sub.payload, b"f.gitignore\0fkeep.txt\0");
+    }
+
+    #[cfg(unix)]
+    #[test]
+    fn lists_symbolic_links_without_following_them() {
+        let tree = Tree::new(&["dir/inner.txt"]);
+        std::os::unix::fs::symlink(tree.0.join("dir"), tree.0.join("link")).unwrap();
+        std::os::unix::fs::symlink("missing", tree.0.join("dangling")).unwrap();
+        std::os::unix::fs::symlink(&tree.0, tree.0.join("dir/loop")).unwrap();
+        let (entries, _) = tree.list(options());
+        assert_eq!(
+            entries,
+            [
+                "l dangling",
+                "d dir",
+                "f dir/inner.txt",
+                "l dir/loop",
+                "l link"
+            ]
+        );
+    }
+
+    #[test]
+    fn stops_at_bounds_with_truncated() {
+        let tree = Tree::new(&["a", "b", "c"]);
+        let (entries, truncated) = tree.list(Options {
+            max_entries: 2,
+            ..options()
+        });
+        assert_eq!((entries.len(), truncated), (2, true));
+        let (entries, truncated) = tree.list(Options {
+            max_entries: 3,
+            ..options()
+        });
+        assert_eq!((entries.len(), truncated), (3, false));
+        let (entries, truncated) = tree.list(Options {
+            max_bytes: 6,
+            ..options()
+        });
+        assert_eq!(
+            (entries, truncated),
+            (vec!["f a".to_string(), "f b".to_string()], true)
+        );
+        let (entries, truncated) = tree.list(Options {
+            timeout: Duration::ZERO,
+            ..options()
+        });
+        assert!(truncated && entries.is_empty());
+    }
+
+    #[test]
+    fn fails_on_missing_roots_files_and_aborts() {
+        let tree = Tree::new(&["file"]);
+        let missing = list(
+            &tree.0.join("missing").to_string_lossy(),
+            &options(),
+            &|| false,
+        );
+        assert_eq!(missing.err().unwrap().code, "ENOENT");
+        let file = list(&tree.0.join("file").to_string_lossy(), &options(), &|| {
+            false
+        });
+        assert_eq!(file.err().unwrap().code, "ENOTDIR");
+        let aborted = list(&tree.0.to_string_lossy(), &options(), &|| true);
+        assert_eq!(aborted.err().unwrap().code, "aborted");
+    }
+}
--- a/packages/env/daemon/src/main.rs
+++ b/packages/env/daemon/src/main.rs
@ -6,6 +6,7 @@ mod errors;
 mod exec;
 mod frame;
 mod fs;
+mod list;
 mod output;
 mod pty;
 mod scan;
@ -29,6 +30,8 @@ use std::thread;
 use std::time::{Duration, SystemTime, UNIX_EPOCH};

 const PROTOCOL: u64 = 1;
+/// Operations added after protocol 1's first release; clients check `hello.features` before using them.
+const FEATURES: &[&str] = &["listFiles"];
 /// The npm package version this daemon ships with (build.rs).
 const VERSION: &str = env!("PI_ENV_VERSION");
 const PING_INTERVAL: Duration = Duration::from_secs(5);
@ -153,6 +156,7 @@ impl Server {
                "cwd": sys::cwd(),
                "driveCwds": sys::drive_cwds(),
                "pid": std::process::id(),
+                "features": FEATURES,
            }))),
            "lstat" => plain(fs::lstat(field(json, "path")?)),
            "realpath" => plain(fs::realpath(field(json, "path")?)),
@ -239,6 +243,7 @@ impl Server {
                plain(Ok(json!({ "handle": handle })))
            }
            "readdir" => plain(self.read_dir(json)),
+            "listFiles" => list::run(json, &|| control.aborted.load(Ordering::SeqCst)),
            "close" => {
                self.handles
                    .lock()
--- a/packages/env/docs/protocol.md
+++ b/packages/env/docs/protocol.md
@ -47,7 +47,8 @@ contents and output travel as raw payload bytes.
 A cancel names one request. The daemon registers each request before running it, so a cancel sent right after its
 request is never lost. `exec` and `pty` kill the command's process group and finish with `aborted`; `{ mode: "kill" }` kills
 without aborting, so the command finishes with the killed process's status, as `cleanup()` does. `scanLines` stops
-between chunks with `aborted`. `watch` stops and finishes with `{}`. Other operations are short and finish normally.
+between chunks and `listFiles` between entries, with `aborted`. `watch` stops and finishes with `{}`. Other operations
+are short and finish normally.

 ## Scheduling

@ -67,10 +68,13 @@ handles belong to the daemon that opened them, so requests on them fail instead

 ## Operations

-`hello { protocol }` → `{ protocol, version, os, arch, home, tmpdir, cwd, driveCwds, pid }`. The first request.
-`version` is the npm package version the daemon shipped with; `os` and `arch` are Rust's `std::env::consts` values;
-`tmpdir` follows the remote Node's `os.tmpdir()` rules; `cwd` and `driveCwds` (Windows' `=C:` variables) are what
-Node's `path.resolve` falls back to for drive-relative paths.
+`hello { protocol }` → `{ protocol, version, os, arch, home, tmpdir, cwd, driveCwds, pid, features }`. The first
+request. `version` is the npm package version the daemon shipped with; `os` and `arch` are Rust's `std::env::consts`
+values (`linux`, `android`, `macos`, `windows`; `x86_64`, `aarch64`), which the client's `nodePlatform` maps to Node's
+names; `tmpdir` follows the remote Node's `os.tmpdir()` rules; `cwd` and `driveCwds` (Windows' `=C:` variables) are
+what Node's `path.resolve` falls back to for drive-relative paths. `features` lists operations added to protocol 1
+after its first release (`listFiles`); older daemons omit the field and refuse those operations as unknown, so clients
+check it first.

 File operations take `{ path }` (and the listed fields) and return `{}` unless noted:

@ -94,6 +98,19 @@ File operations take `{ path }` (and the listed fields) and return `{}` unless n
 | `readdir` | `handle`, `max` | `{ entries, done }` |
 | `close` | `handle` | |

+`listFiles { path, hidden?, gitignore?, exclude?, maxEntries?, maxBytes?, timeoutMs? }` lists the tree below the
+directory `path` (feature `listFiles`): files, directories and symbolic links, which are not followed; other kinds of
+files are left out, and entries that cannot be read are skipped. The payload holds one record per entry: a kind byte
+(`f` file, `d` directory, `l` symbolic link), the path relative to `path` with `/` separators as UTF-8 (names that are
+not UTF-8 get replacement characters), and a NUL byte. Records come depth first, siblings in byte order of their
+names. The result is `{ count, truncated }`. Entries named `.git` and the names in `exclude` are skipped with
+everything below them. `hidden` (default true) includes names starting with `.` (and Windows' hidden files).
+`gitignore` (default true) skips, inside git work trees, what the `.gitignore` files of `path`, its subdirectories and
+its parents, `.git/info/exclude` and the global excludes file ignore, using the `ignore` crate of `fd` and ripgrep. The
+listing stops with `truncated: true` at the first entry beyond `maxEntries` (default 50 000), beyond `maxBytes` of
+payload (default 4 MiB, at most the frame limit less 64 KiB), or after `timeoutMs` (default 10 000). A missing `path`
+fails with `ENOENT`, a file with `ENOTDIR`.
+
 `info` is `{ name, kind, size, mtimeSec, mtimeNsec, dev, ino }` with `kind` one of `file`, `directory`, `symlink`, `other`.
 `readdir` entries are `{ name, raw?, info }` or `{ name, raw?, error }` when the entry's `lstat` failed. Names are
 decoded as UTF-8 with replacement characters, and lstat'ed under that name, as Node does; `raw` holds the original