feat(env): listFiles for remote and local environments, nodePlatform

RemoteExecutionEnv.listFiles resolves the path and lists through the
daemon; a daemon without the listFiles feature yields not_supported.
LocalExecutionEnv is NodeExecutionEnv plus listFiles through a daemon on
the same machine, so local and SSH sessions list with the same code.
nodePlatform maps hello's Rust names to Node's (windows -> win32,
macos -> darwin, x86_64 -> x64, aarch64 -> arm64).

packages/env/README.md

@@ -33,6 +33,23 @@ checking against the application's own known-hosts file under a fixed alias, no
 connections or configured commands, and without forwarding the local locale. On Windows, detection and deployment go
 through PowerShell, and the daemon starts through the server's default shell (cmd.exe or PowerShell).
 
+`env.listFiles(path, options, context)` lists the tree below a directory in the daemon, as `fd --hidden --exclude .git`
+does without following symbolic links: files, directories and links as relative `/` paths, skipping what git ignores
+inside work trees, bounded by `maxEntries`, `maxBytes` and `timeoutMs` with a `truncated` flag, and stopped by aborting
+the context. It is Durable's optional `ExecutionEnv.listFiles`. A daemon released before it reports `not_supported`
+(`hello.features` lacks `listFiles`), and callers fall back to their own walk. For local sessions,
+`LocalExecutionEnv` is `NodeExecutionEnv` plus `listFiles` through a daemon on the same machine:
+
+```ts
+const local = new Connection({ command: [localDaemon()] }); // owned by the caller; close() it at shutdown
+const env = new LocalExecutionEnv({ cwd: "/home/me/project", connection: local });
+const listing = await env.listFiles(".", { exclude: ["node_modules"] }, context);
+// { ok: true, value: { entries: [{ path: "src", kind: "directory" }, { path: "src/a.ts", kind: "file" }], truncated } }
+```
+
+`nodePlatform(await connection.info())` gives the remote system in Node's names (`win32`, `darwin`, `x64`, `arm64`),
+as `process.platform` and `process.arch` spell them.
+
 `openTerminal(connection, options)` runs a program, by default the user's login shell, in a pseudo-terminal of the
 daemon: raw output, input, resizing and its exit code, for interactive terminals. `localDaemon()` is the daemon for this
 machine, for terminals without SSH (`new Connection({ command: [localDaemon()] })`). Terminals are not supported on

packages/env/docs/semantics.md

@@ -34,6 +34,9 @@ env conformance suite and `test/differential.test.ts` check each rule below.
 - `readBinaryFile`, `readTextFile` and `openTextLineReader` open any file like `open(path, "r")`: `/dev/null` reads
   empty, a FIFO blocks until it has a writer, and a directory fails when read (a line reader at `readLine`). A regular
   file is read up to its size at open; other files to their end.
+- `listFiles` has no `NodeExecutionEnv` counterpart: the daemon implements it for remote and local environments
+  alike (`LocalExecutionEnv` runs a daemon on the same machine), so both list with the same code. Unlike `fd`,
+  symbolic links are not followed; unlike `git ls-files`, a tracked file that matches an ignore rule is left out.
 - `readBinaryFile` returns a `Buffer`, reader reads a plain `Uint8Array`, as Node does.
 - Windows: error codes are libuv's translations; `rm` of a symbolic link or junction to a directory removes the link;
   creating a file where a directory exists fails with `EISDIR`; modification times before 1970 are negative.

packages/env/src/connection.ts

@@ -48,6 +48,19 @@ export interface RemoteInfo {
 	/** Windows: per-drive working directories (`C:` → `C:\\work`) of the `=C:` variables. */
 	driveCwds: Record<string, string>;
 	pid: number;
+	/** Operations added to protocol 1 that this daemon supports, such as `listFiles`; absent from older daemons. */
+	features?: string[];
+}
+
+/**
+ * Node's names for the daemon's system and architecture, which `hello` reports in Rust's spelling: `windows` is
+ * `win32`, `macos` is `darwin`, `x86_64` is `x64`, `aarch64` is `arm64`, so they compare with `process.platform` and
+ * `process.arch`. Other names are kept.
+ */
+export function nodePlatform(info: Pick<RemoteInfo, "os" | "arch">): { platform: string; arch: string } {
+	const platforms: Record<string, string> = { windows: "win32", macos: "darwin" };
+	const archs: Record<string, string> = { x86_64: "x64", aarch64: "arm64", x86: "ia32" };
+	return { platform: platforms[info.os] ?? info.os, arch: archs[info.arch] ?? info.arch };
 }
 
 export interface ConnectionOptions {

packages/env/src/index.ts

@@ -2,11 +2,13 @@ export {
 	Connection,
 	type ConnectionOptions,
 	isConnectionLost,
+	nodePlatform,
 	RemoteError,
 	type RemoteInfo,
 	type Reply,
 	type RequestOptions,
 } from "./connection.ts";
+export { LocalExecutionEnv, type LocalExecutionEnvOptions } from "./list-files.ts";
 export { RemoteExecutionEnv, type RemoteExecutionEnvOptions } from "./remote-env.ts";
 export {
 	acceptHostKey,

packages/env/src/list-files.ts

@@ -0,0 +1,102 @@
+import type { Context } from "@earendil-works/chord";
+import {
+	type ExecutionEnv,
+	err,
+	FileError,
+	type FileKind,
+	type FileListEntry,
+	type FileListing,
+	type ListFilesOptions,
+	ok,
+	type Result,
+} from "@earendil-works/pi-durable/env";
+import { NodeExecutionEnv } from "@earendil-works/pi-durable/env/node";
+import type { Connection } from "./connection.ts";
+import { abortResult, toFileError } from "./errors.ts";
+
+/** Kind bytes of `listFiles` records: `f`, `d`, `l`. */
+const KINDS: Record<number, FileKind> = { 102: "file", 100: "directory", 108: "symlink" };
+
+/**
+ * `listFiles` of an absolute path through the daemon: fails with `not_supported` if the daemon does not list files
+ * (a release before `listFiles`), so callers fall back to their own walk.
+ */
+export async function listFilesOver(
+	connection: Connection,
+	path: string,
+	options: ListFilesOptions | undefined,
+	context: Context,
+): Promise<Result<FileListing, FileError>> {
+	for (const key of ["maxEntries", "maxBytes", "timeoutMs"] as const) {
+		const value = options?.[key];
+		if (value !== undefined && (!Number.isSafeInteger(value) || value < 0)) {
+			return err(new FileError("invalid", `${key} must be a non-negative safe integer`, path));
+		}
+	}
+	const aborted = abortResult<FileListing>(context.abortSignal, path);
+	if (aborted) return aborted;
+	try {
+		if (!(await connection.info()).features?.includes("listFiles")) {
+			return err(new FileError("not_supported", "The pi-env daemon does not list files; update it", path));
+		}
+		const { json, payload } = await connection.request(
+			"listFiles",
+			// Fields left undefined are not sent; the daemon applies the defaults.
+			{
+				path,
+				hidden: options?.hidden,
+				gitignore: options?.gitignore,
+				exclude: options?.exclude,
+				maxEntries: options?.maxEntries,
+				maxBytes: options?.maxBytes,
+				timeoutMs: options?.timeoutMs,
+			},
+			context.abortSignal ? { signal: context.abortSignal } : {},
+		);
+		const afterAbort = abortResult<FileListing>(context.abortSignal, path);
+		if (afterAbort) return afterAbort;
+		// Records of a kind byte, the UTF-8 path, and a NUL.
+		const entries: FileListEntry[] = [];
+		const bytes = Buffer.from(payload.buffer, payload.byteOffset, payload.length);
+		for (let start = 0; start < bytes.length; ) {
+			const end = bytes.indexOf(0, start);
+			if (end === -1) break;
+			const kind = KINDS[bytes[start]!];
+			if (kind !== undefined) entries.push({ path: bytes.toString("utf8", start + 1, end), kind });
+			start = end + 1;
+		}
+		return ok({ entries, truncated: json.truncated === true });
+	} catch (error) {
+		return err(toFileError(error, path));
+	}
+}
+
+export type LocalExecutionEnvOptions = ConstructorParameters<typeof NodeExecutionEnv>[0] & {
+	/** A connection to this machine's daemon, `new Connection({ command: [localDaemon()] })`, for `listFiles`. */
+	connection: Connection;
+};
+
+/**
+ * `NodeExecutionEnv` with `listFiles` through a pi-env daemon on the same machine, so local and remote environments
+ * list files with the same code. Everything else is `NodeExecutionEnv`'s. The connection belongs to the caller, who
+ * closes it; several environments may share it.
+ */
+export class LocalExecutionEnv extends NodeExecutionEnv implements ExecutionEnv {
+	readonly #connection: Connection;
+
+	constructor(options: LocalExecutionEnvOptions) {
+		const { connection, ...node } = options;
+		super(node);
+		this.#connection = connection;
+	}
+
+	async listFiles(
+		path: string,
+		options: ListFilesOptions | undefined,
+		context: Context,
+	): Promise<Result<FileListing, FileError>> {
+		const resolved = await this.absolutePath(path, context);
+		if (!resolved.ok) return resolved;
+		return listFilesOver(this.#connection, resolved.value, options, context);
+	}
+}

packages/env/src/remote-env.ts

@@ -12,8 +12,10 @@ import {
 	FileError,
 	type FileInfo,
 	type FileKind,
+	type FileListing,
 	type FileWatcher,
 	type LineScan,
+	type ListFilesOptions,
 	ok,
 	type Result,
 	type ShellExecOptions,
@@ -26,8 +28,9 @@ import {
 	type WatchChange,
 	type WatchTarget,
 } from "@earendil-works/pi-durable/env";
-import { type Connection, type Json, RemoteError, type RemoteInfo } from "./connection.ts";
+import { type Connection, type Json, nodePlatform, RemoteError, type RemoteInfo } from "./connection.ts";
 import { abortResult, toFileError } from "./errors.ts";
+import { listFilesOver } from "./list-files.ts";
 import { RemoteWatcher, type RemoteWatchOptions } from "./watch.ts";
 
 const MAX_TIMEOUT_MS = 2_147_483_647;
@@ -63,7 +66,7 @@ type RemoteDirEntry = { name: string; raw?: number[]; info?: RemoteFileInfo; err
 
 /** Node's path rules of the remote system. */
 export async function remotePath(connection: Connection): Promise<PlatformPath> {
-	return (await connection.info()).os === "windows" ? win32 : posix;
+	return nodePlatform(await connection.info()).platform === "win32" ? win32 : posix;
 }
 
 function toInfo(path: string, remote: RemoteFileInfo, paths: PlatformPath): Result<FileInfo, FileError> {
@@ -370,7 +373,7 @@ export class RemoteExecutionEnv implements ExecutionEnv {
 	/** Node's `resolvePath` on the remote system, with its home directory and working directories. */
 	async #resolve(path: string): Promise<string> {
 		const info = await this.connection.info();
-		const windows = info.os === "windows";
+		const windows = nodePlatform(info).platform === "win32";
 		const paths = windows ? win32 : posix;
 		let normalized = path;
 		if (normalized === "~") {
@@ -717,6 +720,21 @@ export class RemoteExecutionEnv implements ExecutionEnv {
 		});
 	}
 
+	/** Through the daemon; `not_supported` with a daemon released before `listFiles`. */
+	async listFiles(
+		path: string,
+		options: ListFilesOptions | undefined,
+		context: Context,
+	): Promise<Result<FileListing, FileError>> {
+		let resolved: string;
+		try {
+			resolved = await this.#resolve(path);
+		} catch (error) {
+			return err(toFileError(error, path));
+		}
+		return listFilesOver(this.connection, resolved, options, context);
+	}
+
 	async #openDir(resolved: string): Promise<{ handle: Handle }> {
 		const reply = await this.connection.request("opendir", { path: resolved });
 		return { handle: new Handle(this, reply.json.handle as number, reply.session, resolved) };

packages/env/test/conformance.test.ts

@@ -4,6 +4,7 @@ import { join } from "node:path";
 import { registerEnvConformance } from "@earendil-works/pi-durable/testing";
 import { afterAll, describe, expect, it } from "vitest";
 import { Connection } from "../src/connection.ts";
+import { LocalExecutionEnv } from "../src/list-files.ts";
 import { RemoteExecutionEnv } from "../src/remote-env.ts";
 import { daemon } from "./daemon.ts";
 
@@ -26,5 +27,19 @@ registerEnvConformance(
 		}
 	},
 	// Git Bash's `ln -s` copies instead of linking unless native symlinks are enabled.
-	windows ? { shell: [gitBash, "-c"], symlinks: false } : {},
+	windows ? { shell: [gitBash, "-c"], symlinks: false, listFiles: true } : { listFiles: true },
+);
+
+registerEnvConformance(
+	{ describe, expect, it },
+	"LocalExecutionEnv",
+	async (use) => {
+		const cwd = mkdtempSync(join(tmpdir(), "pi-env-local-conformance-"));
+		try {
+			await use(new LocalExecutionEnv({ connection, cwd, watch: { pollIntervalMs: 100 } }));
+		} finally {
+			rmSync(cwd, { recursive: true, force: true, maxRetries: 50, retryDelay: 100 });
+		}
+	},
+	windows ? { shell: [gitBash, "-c"], symlinks: false, listFiles: true } : { listFiles: true },
 );

packages/env/test/old-daemon.ts

@@ -0,0 +1,48 @@
+/**
+ * A stand-in for a daemon released before `hello.features`: it answers `hello` without `features` and refuses every
+ * other operation as an unknown one, as such a daemon refuses `listFiles`. Started as `node old-daemon.ts serve --token
+ * <hex>`.
+ */
+const token = process.argv[process.argv.indexOf("--token") + 1];
+process.stdout.write(`PI-ENV ${token}\n`);
+
+function send(type: number, id: number, json: Record<string, unknown>): void {
+	const body = Buffer.from(JSON.stringify(json), "utf8");
+	const header = Buffer.alloc(13);
+	header.writeUInt32BE(9 + body.length, 0);
+	header.writeUInt8(type, 4);
+	header.writeUInt32BE(id, 5);
+	header.writeUInt32BE(body.length, 9);
+	process.stdout.write(Buffer.concat([header, body]));
+}
+
+let buffered = Buffer.alloc(0);
+process.stdin.on("data", (chunk: Buffer) => {
+	buffered = Buffer.concat([buffered, chunk]);
+	while (buffered.length >= 4 && buffered.length >= 4 + buffered.readUInt32BE(0)) {
+		const length = buffered.readUInt32BE(0);
+		const type = buffered.readUInt8(4);
+		const id = buffered.readUInt32BE(5);
+		const jsonLength = buffered.readUInt32BE(9);
+		const json = JSON.parse(buffered.subarray(13, 13 + jsonLength).toString("utf8")) as { op?: string };
+		buffered = buffered.subarray(4 + length);
+		if (type !== 1) continue;
+		if (json.op === "hello") {
+			send(2, id, {
+				protocol: 1,
+				version: "1.1.0",
+				os: "linux",
+				arch: "x86_64",
+				home: "/",
+				separator: "/",
+				tmpdir: "/tmp",
+				cwd: "/",
+				driveCwds: {},
+				pid: process.pid,
+			});
+		} else {
+			send(3, id, { code: "EINVAL", message: `unknown operation ${json.op}` });
+		}
+	}
+});
+process.stdin.on("end", () => process.exit(0));

packages/env/test/remote.test.ts

@@ -1,10 +1,10 @@
-import { mkdtempSync, rmSync } from "node:fs";
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { BACKGROUND_CONTEXT, withAbortSignal } from "@earendil-works/chord/context";
 import { getOrThrow, type ShellOutputInfo } from "@earendil-works/pi-durable/env";
 import { afterAll, describe, expect, it } from "vitest";
-import { Connection } from "../src/connection.ts";
+import { Connection, nodePlatform } from "../src/connection.ts";
 import { RemoteExecutionEnv } from "../src/remote-env.ts";
 import { daemon } from "./daemon.ts";
 
@@ -70,6 +70,40 @@ describe("RemoteExecutionEnv", () => {
 		expect(Date.now() - started).toBeLessThan(10_000);
 	}, 30_000);
 
+	it("reports the remote system in Node's names", async () => {
+		const { connection } = environment();
+		expect(nodePlatform(await connection.info())).toEqual({ platform: process.platform, arch: process.arch });
+		expect(nodePlatform({ os: "windows", arch: "aarch64" })).toEqual({ platform: "win32", arch: "arm64" });
+		expect(nodePlatform({ os: "macos", arch: "x86_64" })).toEqual({ platform: "darwin", arch: "x64" });
+	});
+
+	it("reports listFiles as not supported by daemons released before it", async () => {
+		const { env } = environment();
+		expect((await env.connection.info()).features).toContain("listFiles");
+		const connection = new Connection({ command: [process.execPath, join(import.meta.dirname, "old-daemon.ts")] });
+		connections.push(connection);
+		const old = new RemoteExecutionEnv({ connection, id: "pi-env:old", cwd: env.cwd });
+		const listed = await old.listFiles(".", undefined, context);
+		expect(listed.ok ? "ok" : listed.error.code).toBe("not_supported");
+	});
+
+	it("cancels a listing in the daemon", async () => {
+		const { env, connection } = environment();
+		for (let dir = 0; dir < 100; dir++) {
+			mkdirSync(join(env.cwd, `d${dir}`));
+			for (let file = 0; file < 100; file++) writeFileSync(join(env.cwd, `d${dir}`, `f${file}`), "");
+		}
+		// The cancel follows the request at once, bypassing the client's own abort check, so the daemon stops the walk.
+		const controller = new AbortController();
+		controller.abort();
+		const cancelled = connection.request("listFiles", { path: env.cwd }, { signal: controller.signal });
+		await expect(cancelled).rejects.toMatchObject({ code: "aborted" });
+		const listed = getOrThrow(await env.listFiles(".", undefined, context));
+		expect([listed.entries.length, listed.truncated]).toEqual([10_100, false]);
+		const bounded = getOrThrow(await env.listFiles(".", { maxEntries: 5000 }, context));
+		expect([bounded.entries.length, bounded.truncated]).toEqual([5000, true]);
+	});
+
 	it("times out a command that produces no output", async () => {
 		const { env } = environment();
 		const started = Date.now();

packages/env/test/ssh-external.test.ts

@@ -5,7 +5,7 @@ import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context";
 import { getOrThrow } from "@earendil-works/pi-durable/env";
 import { registerEnvConformance } from "@earendil-works/pi-durable/testing";
 import { afterAll, describe, expect, it } from "vitest";
-import type { Connection } from "../src/connection.ts";
+import { type Connection, nodePlatform } from "../src/connection.ts";
 import { RemoteExecutionEnv } from "../src/remote-env.ts";
 import {
 	acceptHostKey,
@@ -65,7 +65,7 @@ describe.skipIf(host === undefined)("SSH to this machine's server", () => {
 	it("reaches the remote system through its default shell", async () => {
 		const { connection, remote } = await connect();
 		const info = await connection.info();
-		expect(info.os).toBe(windows ? "windows" : process.platform === "darwin" ? "macos" : process.platform);
+		expect(nodePlatform(info).platform).toBe(process.platform);
 		if (windows) expect(remote.shell).toBe(process.env.PI_ENV_SSH_SHELL ?? "cmd");
 	}, 120_000);
 });
@@ -87,6 +87,6 @@ if (host !== undefined) {
 				await home.remove(cwd, { recursive: true, force: true }, BACKGROUND_CONTEXT);
 			}
 		},
-		windows ? { shell: [gitBash, "-c"], symlinks: false } : {},
+		windows ? { shell: [gitBash, "-c"], symlinks: false, listFiles: true } : { listFiles: true },
 	);
 }

packages/env/test/ssh.test.ts

@@ -16,6 +16,8 @@ import { join } from "node:path";
 import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context";
 import { getOrThrow } from "@earendil-works/pi-durable/env";
 import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import { Connection } from "../src/connection.ts";
+import { LocalExecutionEnv } from "../src/list-files.ts";
 import { RemoteExecutionEnv } from "../src/remote-env.ts";
 import {
 	acceptHostKey,
@@ -183,6 +185,39 @@ describe.skipIf(sshd === undefined)("SSH bootstrap", () => {
 		}
 	}, 60_000);
 
+	it("lists files over SSH as the local environment lists them", async () => {
+		const tree = join(home, "tree");
+		for (const file of ["a/b.txt", "a/c/d.txt", ".hidden", "out/x.js", "skip/y.txt", ".git/HEAD"]) {
+			mkdirSync(join(tree, file, ".."), { recursive: true });
+			writeFileSync(join(tree, file), "");
+		}
+		mkdirSync(join(tree, "empty"));
+		writeFileSync(join(tree, ".gitignore"), "out/\n");
+		const { connection } = await connectSsh({ ...target, binary: daemon });
+		const local = new Connection({ command: [daemon] });
+		try {
+			const remote = new RemoteExecutionEnv({ connection, id: "pi-env:test", cwd: home });
+			const listing = getOrThrow(await remote.listFiles("tree", { exclude: ["skip"] }, context));
+			expect(listing).toEqual({
+				entries: [
+					{ path: ".gitignore", kind: "file" },
+					{ path: ".hidden", kind: "file" },
+					{ path: "a", kind: "directory" },
+					{ path: "a/b.txt", kind: "file" },
+					{ path: "a/c", kind: "directory" },
+					{ path: "a/c/d.txt", kind: "file" },
+					{ path: "empty", kind: "directory" },
+				],
+				truncated: false,
+			});
+			const env = new LocalExecutionEnv({ connection: local, cwd: home });
+			expect(getOrThrow(await env.listFiles("tree", { exclude: ["skip"] }, context))).toEqual(listing);
+		} finally {
+			connection.close();
+			local.close();
+		}
+	}, 60_000);
+
 	it("starts the daemon through the login shell only when asked", async () => {
 		// The login shell is the account's shell: sh and bash read .profile, zsh (the macOS default) reads .zprofile.
 		writeFileSync(join(home, ".profile"), "export PI_ENV_LOGIN=yes\n");