diff --git a/packages/env/daemon/Cargo.lock b/packages/env/daemon/Cargo.lock index 9c70d27f3..5d05477b4 100644 --- a/packages/env/daemon/Cargo.lock +++ b/packages/env/daemon/Cargo.lock @@ -2,12 +2,31 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + [[package]] name = "bitflags" version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" +[[package]] +name = "bstr" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bb31b46c14244e20ee9984b11bf5c992b91fb6939fea616e3512c8baecdbe5f" +dependencies = [ + "memchr", + "serde_core", +] + [[package]] name = "cfg-if" version = "1.0.5" @@ -20,6 +39,31 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48" +[[package]] +name = "crossbeam-deque" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" + [[package]] name = "encoding_rs" version = "0.8.42" @@ -43,6 +87,35 @@ dependencies = [ "libc", ] +[[package]] +name = "globset" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07c34a9410465b45bd9787443bc7370f37735bad04b0f0cd57ff1a3186c98988" +dependencies = [ + "aho-corasick", + "bstr", + "log", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "ignore" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b69833ed729dc5aa7d19541d96d6cf8e9137194207a04916d658e43168402f" +dependencies = [ + "crossbeam-deque", + "globset", + "log", + "memchr", + "regex-automata", + "same-file", + "walkdir", + "winapi-util", +] + [[package]] name = "inotify" version = "0.11.5" @@ -157,6 +230,7 @@ name = "pi-env" version = "0.1.0" dependencies = [ "encoding_rs", + "ignore", "libc", "notify", "serde_json", @@ -181,6 +255,23 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + [[package]] name = "rustversion" version = "1.0.23" diff --git a/packages/env/daemon/Cargo.toml b/packages/env/daemon/Cargo.toml index d7372d768..e362aaa3d 100644 --- a/packages/env/daemon/Cargo.toml +++ b/packages/env/daemon/Cargo.toml @@ -7,6 +7,7 @@ publish = false [dependencies] encoding_rs = "=0.8.42" +ignore = "=0.4.33" notify = "=8.2.0" serde_json = "=1.0.151" diff --git a/packages/env/daemon/src/list.rs b/packages/env/daemon/src/list.rs new file mode 100644 index 000000000..c990ada96 --- /dev/null +++ b/packages/env/daemon/src/list.rs @@ -0,0 +1,368 @@ +//! `listFiles`: the files and directories below a directory, as `fd --hidden --exclude .git` lists them (without +//! following symbolic links), skipping what git ignores. Uses the `ignore` crate, which fd and ripgrep use. + +use crate::errors::Failure; +use crate::frame::MAX_PAYLOAD; +use ignore::WalkBuilder; +use serde_json::{Value, json}; +use std::ffi::OsString; +use std::fs; +use std::path::{Component, Path}; +use std::time::{Duration, Instant}; + +const DEFAULT_MAX_ENTRIES: u64 = 50_000; +const DEFAULT_MAX_BYTES: u64 = 4 * 1024 * 1024; +const DEFAULT_TIMEOUT_MS: u64 = 10_000; + +pub struct Options { + /// Include names starting with `.` (and, on Windows, hidden files). + pub hidden: bool, + /// Skip what git ignores inside git work trees: `.gitignore` files, `.git/info/exclude`, the global excludes file. + pub gitignore: bool, + /// Names skipped with everything below them, in addition to `.git`. + pub exclude: Vec, + pub max_entries: u64, + /// Bytes of the listing payload: per entry one kind byte, the path, and a NUL. + pub max_bytes: u64, + pub timeout: Duration, +} + +impl Options { + pub fn from_json(json: &Value) -> Result { + let count = |key: &str, default: u64| -> Result { + match &json[key] { + Value::Null => Ok(default), + value => value + .as_u64() + .ok_or_else(|| Failure::new("EINVAL", format!("invalid field {key}"))), + } + }; + let exclude = match &json["exclude"] { + Value::Null => Vec::new(), + Value::Array(names) => names + .iter() + .map(|name| name.as_str().map(OsString::from)) + .collect::>>() + .ok_or_else(|| Failure::new("EINVAL", "invalid field exclude"))?, + _ => return Err(Failure::new("EINVAL", "invalid field exclude")), + }; + Ok(Options { + hidden: json["hidden"].as_bool().unwrap_or(true), + gitignore: json["gitignore"].as_bool().unwrap_or(true), + exclude, + max_entries: count("maxEntries", DEFAULT_MAX_ENTRIES)?, + max_bytes: count("maxBytes", DEFAULT_MAX_BYTES)?.min(MAX_PAYLOAD as u64), + timeout: Duration::from_millis(count("timeoutMs", DEFAULT_TIMEOUT_MS)?), + }) + } +} + +pub struct Listing { + /// Records of a kind byte (`f`, `d`, `l`), the path relative to the root with `/` separators, and a NUL. + pub payload: Vec, + pub count: u64, + pub truncated: bool, +} + +/// The path of `path` below `root`, its components joined with `/`; names that are not UTF-8 get replacement +/// characters, as `readdir` reports them. +fn relative(root: &Path, path: &Path) -> Option { + let rest = path.strip_prefix(root).ok()?; + let mut text = String::new(); + for component in rest.components() { + let Component::Normal(name) = component else { + return None; + }; + if !text.is_empty() { + text.push('/'); + } + text.push_str(&name.to_string_lossy()); + } + (!text.is_empty()).then_some(text) +} + +/// List `root` depth first, siblings in byte order of their names. Stops with `truncated` at the first entry beyond +/// a bound; fails with `aborted` once `aborted()` is true. Entries that cannot be read are skipped. +pub fn list(root: &str, options: &Options, aborted: &dyn Fn() -> bool) -> Result { + let metadata = fs::metadata(root).map_err(|error| Failure::io(&error, "scandir", root))?; + if !metadata.is_dir() { + return Err(Failure::new( + "ENOTDIR", + format!("ENOTDIR: not a directory, scandir '{root}'"), + ) + .path(root)); + } + let root_path = Path::new(root); + let exclude = options.exclude.clone(); + let walk = WalkBuilder::new(root_path) + .hidden(!options.hidden) + .ignore(false) + .parents(options.gitignore) + .git_ignore(options.gitignore) + .git_exclude(options.gitignore) + .git_global(options.gitignore) + .require_git(true) + .follow_links(false) + // The global excludes file is found relative to the listed directory, not the daemon's. + .current_dir(root_path) + .sort_by_file_name(|a, b| a.cmp(b)) + .filter_entry(move |entry| { + let name = entry.file_name(); + name != ".git" && !exclude.iter().any(|excluded| excluded == name) + }) + .build(); + let started = Instant::now(); + let mut listing = Listing { + payload: Vec::new(), + count: 0, + truncated: false, + }; + for entry in walk { + if aborted() { + return Err(Failure::new("aborted", "aborted")); + } + let Ok(entry) = entry else { continue }; + if entry.depth() == 0 { + continue; + } + let Some(file_type) = entry.file_type() else { + continue; + }; + let kind = if file_type.is_symlink() { + b'l' + } else if file_type.is_dir() { + b'd' + } else if file_type.is_file() { + b'f' + } else { + continue; + }; + let Some(path) = relative(root_path, entry.path()) else { + continue; + }; + let size = path.len() as u64 + 2; + if listing.count >= options.max_entries + || listing.payload.len() as u64 + size > options.max_bytes + || started.elapsed() > options.timeout + { + listing.truncated = true; + break; + } + listing.payload.push(kind); + listing.payload.extend_from_slice(path.as_bytes()); + listing.payload.push(0); + listing.count += 1; + } + Ok(listing) +} + +pub fn run(json: &Value, aborted: &dyn Fn() -> bool) -> Result<(Value, Vec), Failure> { + let root = json["path"] + .as_str() + .ok_or_else(|| Failure::new("EINVAL", "missing field path"))?; + let listing = list(root, &Options::from_json(json)?, aborted)?; + Ok(( + json!({ "count": listing.count, "truncated": listing.truncated }), + listing.payload, + )) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::{AtomicU64, Ordering}; + + struct Tree(std::path::PathBuf); + + impl Tree { + fn new(files: &[&str]) -> Tree { + static NEXT: AtomicU64 = AtomicU64::new(0); + let root = std::env::temp_dir().join(format!( + "pi-env-list-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + let tree = Tree(root); + for file in files { + tree.write(file, ""); + } + tree + } + + fn write(&self, path: &str, content: &str) { + let path = self.0.join(path); + if path.to_string_lossy().ends_with('/') { + fs::create_dir_all(&path).unwrap(); + return; + } + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, content).unwrap(); + } + + fn list(&self, options: Options) -> (Vec, bool) { + let listing = list(&self.0.to_string_lossy(), &options, &|| false).unwrap(); + let entries: Vec = listing + .payload + .split(|byte| *byte == 0) + .filter(|record| !record.is_empty()) + .map(|record| { + format!( + "{} {}", + record[0] as char, + String::from_utf8_lossy(&record[1..]) + ) + }) + .collect(); + assert_eq!(entries.len() as u64, listing.count); + (entries, listing.truncated) + } + } + + impl Drop for Tree { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } + } + + fn options() -> Options { + Options::from_json(&json!({})).unwrap() + } + + #[test] + fn lists_files_and_directories_depth_first_in_name_order() { + let tree = Tree::new(&[ + "b.txt", + "a/z.txt", + "a/b/c.txt", + "empty/", + ".hidden/x", + "A.txt", + ]); + let (entries, truncated) = tree.list(options()); + assert_eq!( + entries, + [ + "d .hidden", + "f .hidden/x", + "f A.txt", + "d a", + "d a/b", + "f a/b/c.txt", + "f a/z.txt", + "f b.txt", + "d empty" + ] + ); + assert!(!truncated); + let (entries, _) = tree.list(Options { + hidden: false, + ..options() + }); + assert!(!entries.iter().any(|entry| entry.contains(".hidden"))); + } + + #[test] + fn skips_what_git_ignores_only_in_work_trees() { + let tree = Tree::new(&[ + "keep.txt", + "out/x.js", + "sub/keep.txt", + "sub/drop.log", + "node_modules/m/i.js", + ]); + tree.write(".gitignore", "out/\n*.log\n"); + tree.write("sub/.gitignore", "!keep.txt\n"); + // Not a work tree yet: everything is listed. + let (entries, _) = tree.list(options()); + assert!(entries.contains(&"f out/x.js".to_string())); + tree.write(".git/HEAD", "ref: refs/heads/main\n"); + tree.write(".git/info/exclude", "keep.txt\n"); + let (entries, _) = tree.list(Options { + exclude: vec!["node_modules".into()], + ..options() + }); + assert_eq!( + entries, + [ + "f .gitignore", + "d sub", + "f sub/.gitignore", + "f sub/keep.txt" + ] + ); + let (entries, _) = tree.list(Options { + gitignore: false, + ..options() + }); + assert!(entries.contains(&"f out/x.js".to_string())); + assert!(!entries.iter().any(|entry| entry.contains(".git/"))); + // A subdirectory of a work tree applies the rules of its parents. + let sub = list(&tree.0.join("sub").to_string_lossy(), &options(), &|| false).unwrap(); + assert_eq!(sub.payload, b"f.gitignore\0fkeep.txt\0"); + } + + #[cfg(unix)] + #[test] + fn lists_symbolic_links_without_following_them() { + let tree = Tree::new(&["dir/inner.txt"]); + std::os::unix::fs::symlink(tree.0.join("dir"), tree.0.join("link")).unwrap(); + std::os::unix::fs::symlink("missing", tree.0.join("dangling")).unwrap(); + std::os::unix::fs::symlink(&tree.0, tree.0.join("dir/loop")).unwrap(); + let (entries, _) = tree.list(options()); + assert_eq!( + entries, + [ + "l dangling", + "d dir", + "f dir/inner.txt", + "l dir/loop", + "l link" + ] + ); + } + + #[test] + fn stops_at_bounds_with_truncated() { + let tree = Tree::new(&["a", "b", "c"]); + let (entries, truncated) = tree.list(Options { + max_entries: 2, + ..options() + }); + assert_eq!((entries.len(), truncated), (2, true)); + let (entries, truncated) = tree.list(Options { + max_entries: 3, + ..options() + }); + assert_eq!((entries.len(), truncated), (3, false)); + let (entries, truncated) = tree.list(Options { + max_bytes: 6, + ..options() + }); + assert_eq!( + (entries, truncated), + (vec!["f a".to_string(), "f b".to_string()], true) + ); + let (entries, truncated) = tree.list(Options { + timeout: Duration::ZERO, + ..options() + }); + assert!(truncated && entries.is_empty()); + } + + #[test] + fn fails_on_missing_roots_files_and_aborts() { + let tree = Tree::new(&["file"]); + let missing = list( + &tree.0.join("missing").to_string_lossy(), + &options(), + &|| false, + ); + assert_eq!(missing.err().unwrap().code, "ENOENT"); + let file = list(&tree.0.join("file").to_string_lossy(), &options(), &|| { + false + }); + assert_eq!(file.err().unwrap().code, "ENOTDIR"); + let aborted = list(&tree.0.to_string_lossy(), &options(), &|| true); + assert_eq!(aborted.err().unwrap().code, "aborted"); + } +} diff --git a/packages/env/daemon/src/main.rs b/packages/env/daemon/src/main.rs index 4a9fb730a..4457fef4d 100644 --- a/packages/env/daemon/src/main.rs +++ b/packages/env/daemon/src/main.rs @@ -6,6 +6,7 @@ mod errors; mod exec; mod frame; mod fs; +mod list; mod output; mod pty; mod scan; @@ -29,6 +30,8 @@ use std::thread; use std::time::{Duration, SystemTime, UNIX_EPOCH}; const PROTOCOL: u64 = 1; +/// Operations added after protocol 1's first release; clients check `hello.features` before using them. +const FEATURES: &[&str] = &["listFiles"]; /// The npm package version this daemon ships with (build.rs). const VERSION: &str = env!("PI_ENV_VERSION"); const PING_INTERVAL: Duration = Duration::from_secs(5); @@ -153,6 +156,7 @@ impl Server { "cwd": sys::cwd(), "driveCwds": sys::drive_cwds(), "pid": std::process::id(), + "features": FEATURES, }))), "lstat" => plain(fs::lstat(field(json, "path")?)), "realpath" => plain(fs::realpath(field(json, "path")?)), @@ -239,6 +243,7 @@ impl Server { plain(Ok(json!({ "handle": handle }))) } "readdir" => plain(self.read_dir(json)), + "listFiles" => list::run(json, &|| control.aborted.load(Ordering::SeqCst)), "close" => { self.handles .lock() diff --git a/packages/env/docs/protocol.md b/packages/env/docs/protocol.md index e5d95c0bb..ae43fb767 100644 --- a/packages/env/docs/protocol.md +++ b/packages/env/docs/protocol.md @@ -47,7 +47,8 @@ contents and output travel as raw payload bytes. A cancel names one request. The daemon registers each request before running it, so a cancel sent right after its request is never lost. `exec` and `pty` kill the command's process group and finish with `aborted`; `{ mode: "kill" }` kills without aborting, so the command finishes with the killed process's status, as `cleanup()` does. `scanLines` stops -between chunks with `aborted`. `watch` stops and finishes with `{}`. Other operations are short and finish normally. +between chunks and `listFiles` between entries, with `aborted`. `watch` stops and finishes with `{}`. Other operations +are short and finish normally. ## Scheduling @@ -67,10 +68,13 @@ handles belong to the daemon that opened them, so requests on them fail instead ## Operations -`hello { protocol }` → `{ protocol, version, os, arch, home, tmpdir, cwd, driveCwds, pid }`. The first request. -`version` is the npm package version the daemon shipped with; `os` and `arch` are Rust's `std::env::consts` values; -`tmpdir` follows the remote Node's `os.tmpdir()` rules; `cwd` and `driveCwds` (Windows' `=C:` variables) are what -Node's `path.resolve` falls back to for drive-relative paths. +`hello { protocol }` → `{ protocol, version, os, arch, home, tmpdir, cwd, driveCwds, pid, features }`. The first +request. `version` is the npm package version the daemon shipped with; `os` and `arch` are Rust's `std::env::consts` +values (`linux`, `android`, `macos`, `windows`; `x86_64`, `aarch64`), which the client's `nodePlatform` maps to Node's +names; `tmpdir` follows the remote Node's `os.tmpdir()` rules; `cwd` and `driveCwds` (Windows' `=C:` variables) are +what Node's `path.resolve` falls back to for drive-relative paths. `features` lists operations added to protocol 1 +after its first release (`listFiles`); older daemons omit the field and refuse those operations as unknown, so clients +check it first. File operations take `{ path }` (and the listed fields) and return `{}` unless noted: @@ -94,6 +98,19 @@ File operations take `{ path }` (and the listed fields) and return `{}` unless n | `readdir` | `handle`, `max` | `{ entries, done }` | | `close` | `handle` | | +`listFiles { path, hidden?, gitignore?, exclude?, maxEntries?, maxBytes?, timeoutMs? }` lists the tree below the +directory `path` (feature `listFiles`): files, directories and symbolic links, which are not followed; other kinds of +files are left out, and entries that cannot be read are skipped. The payload holds one record per entry: a kind byte +(`f` file, `d` directory, `l` symbolic link), the path relative to `path` with `/` separators as UTF-8 (names that are +not UTF-8 get replacement characters), and a NUL byte. Records come depth first, siblings in byte order of their +names. The result is `{ count, truncated }`. Entries named `.git` and the names in `exclude` are skipped with +everything below them. `hidden` (default true) includes names starting with `.` (and Windows' hidden files). +`gitignore` (default true) skips, inside git work trees, what the `.gitignore` files of `path`, its subdirectories and +its parents, `.git/info/exclude` and the global excludes file ignore, using the `ignore` crate of `fd` and ripgrep. The +listing stops with `truncated: true` at the first entry beyond `maxEntries` (default 50 000), beyond `maxBytes` of +payload (default 4 MiB, at most the frame limit less 64 KiB), or after `timeoutMs` (default 10 000). A missing `path` +fails with `ENOENT`, a file with `ENOTDIR`. + `info` is `{ name, kind, size, mtimeSec, mtimeNsec, dev, ino }` with `kind` one of `file`, `directory`, `symlink`, `other`. `readdir` entries are `{ name, raw?, info }` or `{ name, raw?, error }` when the entry's `lstat` failed. Names are decoded as UTF-8 with replacement characters, and lstat'ed under that name, as Node does; `raw` holds the original