feat(durable): optional ExecutionEnv.listFiles for tree listings

An environment may list the files, directories and symbolic links below a
directory: relative / paths, depth first in name order, without .git and
what git ignores in work trees, bounded with a truncated flag. Callers fall
back to their own walk when it is missing or fails with not_supported.
The env conformance suite checks it with the new listFiles option.

packages/durable/README.md

@@ -357,6 +357,12 @@ await env.exec(["git", "status", "--porcelain=v2", "-z"], {
 
 Abort the context to stop one call; `cleanup()` is for shutting the environment down. A custom environment can check itself with `registerEnvConformance()` from `@earendil-works/pi-durable/testing`, like storage below.
 
+Some environments also implement the optional `listFiles()`: the files and directories below a directory at any depth, as `fd` lists them, skipping what git ignores and stopping at bounds with `truncated`. `NodeExecutionEnv` does not; pi-env's `RemoteExecutionEnv` and `LocalExecutionEnv` do. Callers fall back to their own walk when the method is missing or fails with `not_supported`:
+
+```typescript
+const listing = await env.listFiles?.(".", { exclude: ["node_modules"], maxEntries: 50_000 }, context);
+```
+
 ## Reload
 
 Installing an extension with an installed name replaces it in place, in one step:

packages/durable/docs/spec.md

@@ -3152,7 +3152,13 @@ the newline count, the byte range of a span of lines, and the decoded sizes of
 that span and its first line, so a caller can count and locate lines without
 moving the file. `openDirReader`
 pages a directory in file-system order, reading metadata only for the entries
-it returns and skipping entries removed meanwhile. All operations stop when
+it returns and skipping entries removed meanwhile. The optional `listFiles`
+lists a whole tree for indexes such as file completion: files, directories and
+symbolic links (not followed), relative with `/` separators, depth first in
+byte order of names, without `.git` and, inside git work trees, without what
+git ignores; `maxEntries`, `maxBytes` and `timeoutMs` end it early with
+`truncated`. An environment that cannot list right now fails with
+`not_supported`, and callers fall back to their own walk. All operations stop when
 their context is aborted, and only that call's work stops: a timeout or abort
 kills only that command's processes. `cleanup()` kills every command the
 environment still runs and belongs to its owner's shutdown, never to a single

packages/durable/src/env/index.ts

@@ -167,6 +167,44 @@ export interface DirReader {
 	close(context: Context): Promise<void>;
 }
 
+/** What `FileSystem.listFiles` lists. */
+export interface ListFilesOptions {
+	/**
+	 * Include names starting with `.` (on Windows also files with the hidden attribute); default true. Entries named
+	 * `.git` are never listed.
+	 */
+	hidden?: boolean;
+	/**
+	 * Inside a git work tree, skip what git ignores: the `.gitignore` files of the directory, its subdirectories and its
+	 * parents, `.git/info/exclude`, and the global excludes file; default true. Unlike `git ls-files`, a tracked file that
+	 * matches an ignore rule is skipped too. Outside work trees, `.gitignore` files have no effect.
+	 */
+	gitignore?: boolean;
+	/** Names of entries to skip with everything below them, such as `node_modules`. */
+	exclude?: readonly string[];
+	/** Most entries to list; default 50 000. */
+	maxEntries?: number;
+	/** Most bytes of the listing: each entry's UTF-8 path plus two; default 4 MiB, at most 15.9 MiB. */
+	maxBytes?: number;
+	/** Time after which the listing stops; default 10 000 ms. */
+	timeoutMs?: number;
+}
+
+/** One entry of a `FileListing`. */
+export interface FileListEntry {
+	/** Relative to the listed directory, with `/` separators on every system. */
+	path: string;
+	/** A symbolic link is listed as such and not followed. */
+	kind: FileKind;
+}
+
+export interface FileListing {
+	/** Depth first, parents before their entries, siblings in byte order of their names. */
+	entries: FileListEntry[];
+	/** A bound in `ListFilesOptions` ended the listing early. */
+	truncated: boolean;
+}
+
 /** Portable filesystem capability. Operations return failures rather than throwing. */
 export interface FileSystem {
 	/**
@@ -204,6 +242,17 @@ export interface FileSystem {
 	renameFile(sourcePath: string, destinationPath: string, context: Context): Promise<Result<void, FileError>>;
 	fileInfo(path: string, context: Context): Promise<Result<FileInfo, FileError>>;
 	listDir(path: string, context: Context): Promise<Result<FileInfo[], FileError>>;
+	/**
+	 * Optional: the files, directories and symbolic links below a directory, at any depth, for indexes such as file
+	 * completion. Symbolic links are not followed; other kinds of files are not listed; entries that cannot be read are
+	 * skipped. See `ListFilesOptions` for hidden and ignored files and for the bounds. Fails with `not_supported` when
+	 * the environment cannot list right now (an older remote daemon), so callers fall back to their own walk.
+	 */
+	listFiles?(
+		path: string,
+		options: ListFilesOptions | undefined,
+		context: Context,
+	): Promise<Result<FileListing, FileError>>;
 	openDirReader(path: string, context: Context): Promise<Result<DirReader, FileError>>;
 	/**
 	 * Report changes to files and directories, for hosts that load resources from the environment. When the returned

packages/durable/src/testing/env-conformance.ts

@@ -4,6 +4,7 @@ import {
 	type ExecutionEnv,
 	type FileInfo,
 	getOrThrow,
+	type ListFilesOptions,
 	type Result,
 	type ShellOutputInfo,
 	type WatchChange,
@@ -46,6 +47,21 @@ async function readAll(
 	}
 }
 
+/** A listing as `<kind letter> <path>` lines (`f`, `d`, `l`), with its truncation. */
+async function listed(
+	env: ExecutionEnv,
+	path: string,
+	options?: ListFilesOptions,
+): Promise<{ entries: string[]; truncated: boolean }> {
+	if (env.listFiles === undefined) throw new Error("The environment does not implement listFiles");
+	const listing = getOrThrow(await env.listFiles(path, options, context));
+	const letter = { file: "f", directory: "d", symlink: "l" } as const;
+	return {
+		entries: listing.entries.map((entry) => `${letter[entry.kind]} ${entry.path}`),
+		truncated: listing.truncated,
+	};
+}
+
 /** Whether a change reports `path`: an overflow, or a reported path at or above it. */
 function covers(change: WatchChange, path: string): boolean {
 	if ("overflow" in change) return true;
@@ -465,6 +481,112 @@ export function createEnvConformance(options: EnvConformanceOptions): readonly E
 		}),
 	];
 
+	if (options.listFiles) {
+		const writeAll = async (env: ExecutionEnv, files: readonly string[]) => {
+			for (const file of files) getOrThrow(await env.writeFile(file, "", context));
+		};
+		cases.push(
+			createCase("listFiles lists files and directories depth first in name order", async (env) => {
+				await writeAll(env, ["b.txt", "a/z.txt", "a/b/c.txt", ".hidden/x", "A.txt"]);
+				getOrThrow(await env.createDir("empty", undefined, context));
+				assert.deepEqual(await listed(env, "."), {
+					entries: [
+						"d .hidden",
+						"f .hidden/x",
+						"f A.txt",
+						"d a",
+						"d a/b",
+						"f a/b/c.txt",
+						"f a/z.txt",
+						"f b.txt",
+						"d empty",
+					],
+					truncated: false,
+				});
+				assert.deepEqual((await listed(env, ".", { hidden: false })).entries, [
+					"f A.txt",
+					"d a",
+					"d a/b",
+					"f a/b/c.txt",
+					"f a/z.txt",
+					"f b.txt",
+					"d empty",
+				]);
+				const expected = { entries: ["d b", "f b/c.txt", "f z.txt"], truncated: false };
+				assert.deepEqual(await listed(env, "a"), expected);
+				assert.deepEqual(await listed(env, getOrThrow(await env.absolutePath("a", context))), expected);
+				assert.deepEqual(await listed(env, "empty"), { entries: [], truncated: false });
+			}),
+
+			createCase("listFiles skips what git ignores in work trees, .git, and excluded names", async (env) => {
+				await writeAll(env, ["keep.txt", "out/x.js", "sub/keep.txt", "sub/drop.log", "node_modules/m/i.js"]);
+				getOrThrow(await env.writeFile(".gitignore", "out/\n*.log\n", context));
+				getOrThrow(await env.writeFile("sub/.gitignore", "!keep.txt\n", context));
+				// Not a work tree yet: `.gitignore` files have no effect.
+				assert.ok((await listed(env, ".")).entries.includes("f out/x.js"));
+				getOrThrow(await env.writeFile(".git/HEAD", "ref: refs/heads/main\n", context));
+				getOrThrow(await env.writeFile(".git/info/exclude", "keep.txt\n", context));
+				assert.deepEqual((await listed(env, ".", { exclude: ["node_modules"] })).entries, [
+					"f .gitignore",
+					"d sub",
+					"f sub/.gitignore",
+					"f sub/keep.txt",
+				]);
+				// A directory inside the work tree applies its parents' rules.
+				assert.deepEqual((await listed(env, "sub")).entries, ["f .gitignore", "f keep.txt"]);
+				const all = (await listed(env, ".", { gitignore: false })).entries;
+				assert.ok(
+					all.includes("f out/x.js") && all.includes("f node_modules/m/i.js") && all.includes("f keep.txt"),
+				);
+				assert.ok(!all.some((entry) => entry.includes(".git/") || entry === "d .git"));
+			}),
+
+			createCase("listFiles stops at its bounds and reports truncated", async (env) => {
+				await writeAll(env, ["a", "b", "c"]);
+				assert.deepEqual(await listed(env, ".", { maxEntries: 2 }), { entries: ["f a", "f b"], truncated: true });
+				assert.deepEqual(await listed(env, ".", { maxEntries: 3 }), {
+					entries: ["f a", "f b", "f c"],
+					truncated: false,
+				});
+				// Each entry counts its path plus two bytes.
+				assert.deepEqual(await listed(env, ".", { maxBytes: 6 }), { entries: ["f a", "f b"], truncated: true });
+				assert.deepEqual(await listed(env, ".", { maxEntries: 0 }), { entries: [], truncated: true });
+				assert.strictEqual(errorCode(await env.listFiles!(".", { maxEntries: -1 }, context)), "invalid");
+				assert.strictEqual(errorCode(await env.listFiles!(".", { timeoutMs: 1.5 }, context)), "invalid");
+			}),
+
+			createCase("listFiles fails on missing directories, files, and aborts", async (env) => {
+				getOrThrow(await env.writeFile("file.txt", "", context));
+				assert.strictEqual(errorCode(await env.listFiles!("missing", undefined, context)), "not_found");
+				assert.strictEqual(errorCode(await env.listFiles!("file.txt", undefined, context)), "not_directory");
+				assert.strictEqual(errorCode(await env.listFiles!(".", undefined, abortedContext())), "aborted");
+			}),
+		);
+	}
+
+	if (symlinks && options.listFiles) {
+		cases.push(
+			createCase("listFiles lists symbolic links without following them", async (env) => {
+				getOrThrow(await env.writeFile("dir/inner.txt", "", context));
+				const linked = await env.exec(
+					[...shell, "ln -s dir link && ln -s missing dangling && ln -s .. dir/loop"],
+					undefined,
+					context,
+				);
+				assert.strictEqual(getOrThrow(linked).exitCode, 0);
+				assert.deepEqual((await listed(env, ".")).entries, [
+					"l dangling",
+					"d dir",
+					"f dir/inner.txt",
+					"l dir/loop",
+					"l link",
+				]);
+				// The listed directory itself may be a link.
+				assert.deepEqual((await listed(env, "link")).entries, ["f inner.txt", "l loop"]);
+			}),
+		);
+	}
+
 	if (symlinks) {
 		cases.push(
 			createCase("binary reader follows symlinks unless noFollow refuses the final one", async (env) => {

packages/durable/src/testing/runner.ts

@@ -29,7 +29,7 @@ export function registerEnvConformance(
 	runner: StorageConformanceRunner,
 	name: string,
 	withEnv: EnvConformanceProvider,
-	options: Pick<EnvConformanceOptions, "shell" | "symlinks"> = {},
+	options: Pick<EnvConformanceOptions, "shell" | "symlinks" | "listFiles"> = {},
 ): void {
 	const cases = createEnvConformance({ ...options, assertions: createExpectAssertions(runner.expect), withEnv });
 	runner.describe(name, () => {

packages/durable/src/testing/types.ts

@@ -34,6 +34,8 @@ export interface EnvConformanceOptions {
 	readonly shell?: readonly string[];
 	/** Whether the shell's `ln -s` creates symbolic links; default true. */
 	readonly symlinks?: boolean;
+	/** Whether the environment implements the optional `listFiles`, which the suite then checks; default false. */
+	readonly listFiles?: boolean;
 }
 
 export interface EnvConformanceCase {