From abfe62928b39414ea2be4d6feb65ae608c67674f Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Mon, 12 Oct 2026 15:13:10 +0000 Subject: [PATCH] feat(durable): optional ExecutionEnv.listFiles for tree listings An environment may list the files, directories and symbolic links below a directory: relative / paths, depth first in name order, without .git and what git ignores in work trees, bounded with a truncated flag. Callers fall back to their own walk when it is missing or fails with not_supported. The env conformance suite checks it with the new listFiles option. --- packages/durable/README.md | 6 + packages/durable/docs/spec.md | 8 +- packages/durable/src/env/index.ts | 49 +++++++ .../durable/src/testing/env-conformance.ts | 122 ++++++++++++++++++ packages/durable/src/testing/runner.ts | 2 +- packages/durable/src/testing/types.ts | 2 + 6 files changed, 187 insertions(+), 2 deletions(-) diff --git a/packages/durable/README.md b/packages/durable/README.md index 81b452c86..884ea5a32 100644 --- a/packages/durable/README.md +++ b/packages/durable/README.md @@ -357,6 +357,12 @@ await env.exec(["git", "status", "--porcelain=v2", "-z"], { Abort the context to stop one call; `cleanup()` is for shutting the environment down. A custom environment can check itself with `registerEnvConformance()` from `@earendil-works/pi-durable/testing`, like storage below. +Some environments also implement the optional `listFiles()`: the files and directories below a directory at any depth, as `fd` lists them, skipping what git ignores and stopping at bounds with `truncated`. `NodeExecutionEnv` does not; pi-env's `RemoteExecutionEnv` and `LocalExecutionEnv` do. Callers fall back to their own walk when the method is missing or fails with `not_supported`: + +```typescript +const listing = await env.listFiles?.(".", { exclude: ["node_modules"], maxEntries: 50_000 }, context); +``` + ## Reload Installing an extension with an installed name replaces it in place, in one step: diff --git a/packages/durable/docs/spec.md b/packages/durable/docs/spec.md index f8c7bc9e2..023f26243 100644 --- a/packages/durable/docs/spec.md +++ b/packages/durable/docs/spec.md @@ -3152,7 +3152,13 @@ the newline count, the byte range of a span of lines, and the decoded sizes of that span and its first line, so a caller can count and locate lines without moving the file. `openDirReader` pages a directory in file-system order, reading metadata only for the entries -it returns and skipping entries removed meanwhile. All operations stop when +it returns and skipping entries removed meanwhile. The optional `listFiles` +lists a whole tree for indexes such as file completion: files, directories and +symbolic links (not followed), relative with `/` separators, depth first in +byte order of names, without `.git` and, inside git work trees, without what +git ignores; `maxEntries`, `maxBytes` and `timeoutMs` end it early with +`truncated`. An environment that cannot list right now fails with +`not_supported`, and callers fall back to their own walk. All operations stop when their context is aborted, and only that call's work stops: a timeout or abort kills only that command's processes. `cleanup()` kills every command the environment still runs and belongs to its owner's shutdown, never to a single diff --git a/packages/durable/src/env/index.ts b/packages/durable/src/env/index.ts index 31b1a5349..89437d3c2 100644 --- a/packages/durable/src/env/index.ts +++ b/packages/durable/src/env/index.ts @@ -167,6 +167,44 @@ export interface DirReader { close(context: Context): Promise; } +/** What `FileSystem.listFiles` lists. */ +export interface ListFilesOptions { + /** + * Include names starting with `.` (on Windows also files with the hidden attribute); default true. Entries named + * `.git` are never listed. + */ + hidden?: boolean; + /** + * Inside a git work tree, skip what git ignores: the `.gitignore` files of the directory, its subdirectories and its + * parents, `.git/info/exclude`, and the global excludes file; default true. Unlike `git ls-files`, a tracked file that + * matches an ignore rule is skipped too. Outside work trees, `.gitignore` files have no effect. + */ + gitignore?: boolean; + /** Names of entries to skip with everything below them, such as `node_modules`. */ + exclude?: readonly string[]; + /** Most entries to list; default 50 000. */ + maxEntries?: number; + /** Most bytes of the listing: each entry's UTF-8 path plus two; default 4 MiB, at most 15.9 MiB. */ + maxBytes?: number; + /** Time after which the listing stops; default 10 000 ms. */ + timeoutMs?: number; +} + +/** One entry of a `FileListing`. */ +export interface FileListEntry { + /** Relative to the listed directory, with `/` separators on every system. */ + path: string; + /** A symbolic link is listed as such and not followed. */ + kind: FileKind; +} + +export interface FileListing { + /** Depth first, parents before their entries, siblings in byte order of their names. */ + entries: FileListEntry[]; + /** A bound in `ListFilesOptions` ended the listing early. */ + truncated: boolean; +} + /** Portable filesystem capability. Operations return failures rather than throwing. */ export interface FileSystem { /** @@ -204,6 +242,17 @@ export interface FileSystem { renameFile(sourcePath: string, destinationPath: string, context: Context): Promise>; fileInfo(path: string, context: Context): Promise>; listDir(path: string, context: Context): Promise>; + /** + * Optional: the files, directories and symbolic links below a directory, at any depth, for indexes such as file + * completion. Symbolic links are not followed; other kinds of files are not listed; entries that cannot be read are + * skipped. See `ListFilesOptions` for hidden and ignored files and for the bounds. Fails with `not_supported` when + * the environment cannot list right now (an older remote daemon), so callers fall back to their own walk. + */ + listFiles?( + path: string, + options: ListFilesOptions | undefined, + context: Context, + ): Promise>; openDirReader(path: string, context: Context): Promise>; /** * Report changes to files and directories, for hosts that load resources from the environment. When the returned diff --git a/packages/durable/src/testing/env-conformance.ts b/packages/durable/src/testing/env-conformance.ts index 442b9a444..a2c7d43ba 100644 --- a/packages/durable/src/testing/env-conformance.ts +++ b/packages/durable/src/testing/env-conformance.ts @@ -4,6 +4,7 @@ import { type ExecutionEnv, type FileInfo, getOrThrow, + type ListFilesOptions, type Result, type ShellOutputInfo, type WatchChange, @@ -46,6 +47,21 @@ async function readAll( } } +/** A listing as ` ` lines (`f`, `d`, `l`), with its truncation. */ +async function listed( + env: ExecutionEnv, + path: string, + options?: ListFilesOptions, +): Promise<{ entries: string[]; truncated: boolean }> { + if (env.listFiles === undefined) throw new Error("The environment does not implement listFiles"); + const listing = getOrThrow(await env.listFiles(path, options, context)); + const letter = { file: "f", directory: "d", symlink: "l" } as const; + return { + entries: listing.entries.map((entry) => `${letter[entry.kind]} ${entry.path}`), + truncated: listing.truncated, + }; +} + /** Whether a change reports `path`: an overflow, or a reported path at or above it. */ function covers(change: WatchChange, path: string): boolean { if ("overflow" in change) return true; @@ -465,6 +481,112 @@ export function createEnvConformance(options: EnvConformanceOptions): readonly E }), ]; + if (options.listFiles) { + const writeAll = async (env: ExecutionEnv, files: readonly string[]) => { + for (const file of files) getOrThrow(await env.writeFile(file, "", context)); + }; + cases.push( + createCase("listFiles lists files and directories depth first in name order", async (env) => { + await writeAll(env, ["b.txt", "a/z.txt", "a/b/c.txt", ".hidden/x", "A.txt"]); + getOrThrow(await env.createDir("empty", undefined, context)); + assert.deepEqual(await listed(env, "."), { + entries: [ + "d .hidden", + "f .hidden/x", + "f A.txt", + "d a", + "d a/b", + "f a/b/c.txt", + "f a/z.txt", + "f b.txt", + "d empty", + ], + truncated: false, + }); + assert.deepEqual((await listed(env, ".", { hidden: false })).entries, [ + "f A.txt", + "d a", + "d a/b", + "f a/b/c.txt", + "f a/z.txt", + "f b.txt", + "d empty", + ]); + const expected = { entries: ["d b", "f b/c.txt", "f z.txt"], truncated: false }; + assert.deepEqual(await listed(env, "a"), expected); + assert.deepEqual(await listed(env, getOrThrow(await env.absolutePath("a", context))), expected); + assert.deepEqual(await listed(env, "empty"), { entries: [], truncated: false }); + }), + + createCase("listFiles skips what git ignores in work trees, .git, and excluded names", async (env) => { + await writeAll(env, ["keep.txt", "out/x.js", "sub/keep.txt", "sub/drop.log", "node_modules/m/i.js"]); + getOrThrow(await env.writeFile(".gitignore", "out/\n*.log\n", context)); + getOrThrow(await env.writeFile("sub/.gitignore", "!keep.txt\n", context)); + // Not a work tree yet: `.gitignore` files have no effect. + assert.ok((await listed(env, ".")).entries.includes("f out/x.js")); + getOrThrow(await env.writeFile(".git/HEAD", "ref: refs/heads/main\n", context)); + getOrThrow(await env.writeFile(".git/info/exclude", "keep.txt\n", context)); + assert.deepEqual((await listed(env, ".", { exclude: ["node_modules"] })).entries, [ + "f .gitignore", + "d sub", + "f sub/.gitignore", + "f sub/keep.txt", + ]); + // A directory inside the work tree applies its parents' rules. + assert.deepEqual((await listed(env, "sub")).entries, ["f .gitignore", "f keep.txt"]); + const all = (await listed(env, ".", { gitignore: false })).entries; + assert.ok( + all.includes("f out/x.js") && all.includes("f node_modules/m/i.js") && all.includes("f keep.txt"), + ); + assert.ok(!all.some((entry) => entry.includes(".git/") || entry === "d .git")); + }), + + createCase("listFiles stops at its bounds and reports truncated", async (env) => { + await writeAll(env, ["a", "b", "c"]); + assert.deepEqual(await listed(env, ".", { maxEntries: 2 }), { entries: ["f a", "f b"], truncated: true }); + assert.deepEqual(await listed(env, ".", { maxEntries: 3 }), { + entries: ["f a", "f b", "f c"], + truncated: false, + }); + // Each entry counts its path plus two bytes. + assert.deepEqual(await listed(env, ".", { maxBytes: 6 }), { entries: ["f a", "f b"], truncated: true }); + assert.deepEqual(await listed(env, ".", { maxEntries: 0 }), { entries: [], truncated: true }); + assert.strictEqual(errorCode(await env.listFiles!(".", { maxEntries: -1 }, context)), "invalid"); + assert.strictEqual(errorCode(await env.listFiles!(".", { timeoutMs: 1.5 }, context)), "invalid"); + }), + + createCase("listFiles fails on missing directories, files, and aborts", async (env) => { + getOrThrow(await env.writeFile("file.txt", "", context)); + assert.strictEqual(errorCode(await env.listFiles!("missing", undefined, context)), "not_found"); + assert.strictEqual(errorCode(await env.listFiles!("file.txt", undefined, context)), "not_directory"); + assert.strictEqual(errorCode(await env.listFiles!(".", undefined, abortedContext())), "aborted"); + }), + ); + } + + if (symlinks && options.listFiles) { + cases.push( + createCase("listFiles lists symbolic links without following them", async (env) => { + getOrThrow(await env.writeFile("dir/inner.txt", "", context)); + const linked = await env.exec( + [...shell, "ln -s dir link && ln -s missing dangling && ln -s .. dir/loop"], + undefined, + context, + ); + assert.strictEqual(getOrThrow(linked).exitCode, 0); + assert.deepEqual((await listed(env, ".")).entries, [ + "l dangling", + "d dir", + "f dir/inner.txt", + "l dir/loop", + "l link", + ]); + // The listed directory itself may be a link. + assert.deepEqual((await listed(env, "link")).entries, ["f inner.txt", "l loop"]); + }), + ); + } + if (symlinks) { cases.push( createCase("binary reader follows symlinks unless noFollow refuses the final one", async (env) => { diff --git a/packages/durable/src/testing/runner.ts b/packages/durable/src/testing/runner.ts index c7d82d83f..ff9dc6ec7 100644 --- a/packages/durable/src/testing/runner.ts +++ b/packages/durable/src/testing/runner.ts @@ -29,7 +29,7 @@ export function registerEnvConformance( runner: StorageConformanceRunner, name: string, withEnv: EnvConformanceProvider, - options: Pick = {}, + options: Pick = {}, ): void { const cases = createEnvConformance({ ...options, assertions: createExpectAssertions(runner.expect), withEnv }); runner.describe(name, () => { diff --git a/packages/durable/src/testing/types.ts b/packages/durable/src/testing/types.ts index 1b7ec09a5..52c19739d 100644 --- a/packages/durable/src/testing/types.ts +++ b/packages/durable/src/testing/types.ts @@ -34,6 +34,8 @@ export interface EnvConformanceOptions { readonly shell?: readonly string[]; /** Whether the shell's `ln -s` creates symbolic links; default true. */ readonly symlinks?: boolean; + /** Whether the environment implements the optional `listFiles`, which the suite then checks; default false. */ + readonly listFiles?: boolean; } export interface EnvConformanceCase {