diff --git a/.github/workflows/durable-env.yml b/.github/workflows/durable-env.yml index 68cc05b86..36be86019 100644 --- a/.github/workflows/durable-env.yml +++ b/.github/workflows/durable-env.yml @@ -63,4 +63,4 @@ jobs: - name: Env tests working-directory: packages/durable - run: npx vitest --run test/env-node.test.ts test/env-node-spill.test.ts test/env-node-conformance.test.ts test/tools.test.ts + run: npx vitest --run test/env-node.test.ts test/env-node-spill.test.ts test/env-node-conformance.test.ts test/env-line-scan.test.ts test/tools.test.ts test/tools-read-differential.test.ts diff --git a/packages/durable/CHANGELOG.md b/packages/durable/CHANGELOG.md index b6ff3992f..dcd540cbb 100644 --- a/packages/durable/CHANGELOG.md +++ b/packages/durable/CHANGELOG.md @@ -5,6 +5,7 @@ ### Breaking Changes - `FileSystem` requires `openBinaryReader()` and `openDirReader()`; custom environments must implement them. +- `BinaryReader` requires `scanLines()`. - `Shell.exec()` accepts an argv array besides a shell string, and `ShellExecOptions.onOutput` receives a third `info` argument naming the stream (`stdout` or `stderr`); custom environments must accept both forms and pass the stream. ### Added @@ -15,11 +16,17 @@ - `createEnvConformance()` and `registerEnvConformance()` in `@earendil-works/pi-durable/testing` for checking custom `ExecutionEnv` implementations. - `settings.progress` with `partialIntervalMs` and `outputIntervalMs` configures how often generation partials and running tool output are committed; defaults stay 100 ms ([#10357](https://github.com/earendil-works/pi/issues/10357)) - `ShellExecOptions.window` and `ShellOutputInfo.skipped`: an environment may omit shell output outside the caller's retained tail and report the omission, so remote environments need not transfer output the caller drops. `ToolExecutionApi.outputWindow` provides the window and `output(chunk, skipped)` counts omissions; the `bash` tool passes them through. +- `BinaryReader.scanLines()` locates and measures a span of lines in one pass inside the environment; `LineScanner`, `StreamDecoder`, `rangeDecoder()` and `startsWithBom()` in `@earendil-works/pi-durable/env` let other environments decode and scan exactly like `NodeExecutionEnv`. + +### Changed + +- The `read` tool reads only the file's header, one scan, and the lines it shows, instead of loading the whole file; its results are unchanged. ### Fixed - `NodeExecutionEnv.flushFile()` on a directory fails with `is_directory` on Windows, as on POSIX. - Tail-retained tool output no longer depends on when progress commits happened: a progress snapshot compacted the stored output to the kept window, which could move where a later window's first line started. +- `NodeExecutionEnv` shell output and text line readers no longer drop a U+FEFF that follows a chunk boundary; Node's streaming `TextDecoder` with BOM handling dropped it, unlike decoding all of the bytes at once. ## [1.0.2] - 2026-10-04 diff --git a/packages/durable/docs/spec.md b/packages/durable/docs/spec.md index 362460b2c..45f94c03f 100644 --- a/packages/durable/docs/spec.md +++ b/packages/durable/docs/spec.md @@ -2846,7 +2846,11 @@ stdout and stderr apart collects them separately, bounds them itself, and aborts the call when it has enough. `openBinaryReader` opens one regular file for positional reads, so a host reads a bounded range instead of the whole file, and every read sees the file it opened even if the path is renamed; -`noFollow` refuses a symbolic link as the final path component. `openDirReader` +`noFollow` refuses a symbolic link as the final path component. Its +`scanLines` makes one pass over the file inside the environment and reports +the newline count, the byte range of a span of lines, and the decoded sizes of +that span and its first line, so a caller can count and locate lines without +moving the file. `openDirReader` pages a directory in file-system order, reading metadata only for the entries it returns and skipping entries removed meanwhile. All operations stop when their context is aborted, and only that call's work stops: a timeout or abort @@ -3053,7 +3057,14 @@ if any, and `details` is the tool's last reported value, if any. `@earendil-works/pi-durable/tools` provides `read`, `write`, `edit`, and `bash` factories, ported from the agent harness tools, and the `CodingTools` extension with all four. They use only `api.env`; nothing -installs them automatically. `read` does not return images yet. `edit` and +installs them automatically. `read` does not return images yet. It reads a +file through `openBinaryReader`: image detection reads the header (and a PNG's +chunk headers), `scanLines` counts and locates the selected lines, and only the +shown head is read and decoded, so its cost and transfer are bounded by the +output limits plus one pass over the file inside the environment. Its result is +exactly that of decoding the whole file, splitting it into lines, and +truncating the selection. A file that changes while it is read is read again +once, then fails. `edit` and `write` serialize their read-modify-write of one file within the process, keyed by the environment's `FileSystem.id` (equal ids see the same files at the same paths) and the canonical path, so two calls with fresh environment objects for diff --git a/packages/durable/src/env/decode.ts b/packages/durable/src/env/decode.ts new file mode 100644 index 000000000..b27190056 --- /dev/null +++ b/packages/durable/src/env/decode.ts @@ -0,0 +1,30 @@ +/** + * Decoding that matches `new TextDecoder().decode(bytes)` of the whole input when the input arrives in pieces. Node's + * streaming decoder with BOM handling can drop a U+FEFF that follows a chunk boundary, not only a leading byte-order + * mark, so these decoders turn BOM handling off and drop a leading mark themselves. + */ + +/** A streaming decoder for a byte range; callers that start at the beginning of a file skip a leading mark themselves. */ +export function rangeDecoder(): InstanceType { + return new TextDecoder("utf-8", { ignoreBOM: true }); +} + +/** Whether decoding the whole input drops its first three bytes as a byte-order mark. */ +export function startsWithBom(firstBytes: Uint8Array): boolean { + return firstBytes[0] === 0xef && firstBytes[1] === 0xbb && firstBytes[2] === 0xbf; +} + +/** Decodes one stream chunk by chunk exactly like decoding all of it at once. */ +export class StreamDecoder { + readonly #decoder = rangeDecoder(); + #started = false; + + /** Text for `bytes`, holding back an incomplete character; without `bytes`, the end of the stream. */ + decode(bytes?: Uint8Array): string { + const text = bytes === undefined ? this.#decoder.decode() : this.#decoder.decode(bytes, { stream: true }); + if (this.#started || text === "") return text; + this.#started = true; + // U+FEFF encodes only as EF BB BF, so a leading U+FEFF is exactly a byte-order mark at the stream's start. + return text.startsWith("\ufeff") ? text.slice(1) : text; + } +} diff --git a/packages/durable/src/env/index.ts b/packages/durable/src/env/index.ts index a410fa354..9f860ced7 100644 --- a/packages/durable/src/env/index.ts +++ b/packages/durable/src/env/index.ts @@ -98,9 +98,34 @@ export interface BinaryReader { info(context: Context): Promise>; /** Up to `length` bytes at `offset`; fewer only at end of file. */ read(offset: number, length: number, context: Context): Promise>; + /** + * One pass over the file that locates lines `[startLine, endLine)` (`endLine` absent: to the end), 0-based, where line + * `k` starts after the `k`-th newline byte. Decoded sizes are those of the text `new TextDecoder().decode(file)` would + * produce for that range, so a byte-order mark at the start of the file is not counted. + */ + scanLines(options: { startLine: number; endLine?: number }, context: Context): Promise>; close(context: Context): Promise; } +/** Where lines of a file are, as `BinaryReader.scanLines` found them. */ +export interface LineScan { + /** Newline bytes in the whole file; it has `newlines + 1` lines. */ + newlines: number; + /** + * Byte range of the selected lines: from the start of the first to the end of the last, without the newline that + * ends it. A selection past the last line is empty at the end of the file. + */ + start: number; + end: number; + /** Where the first selected line ends: its newline, or the end of the file. */ + firstLineEnd: number; + /** Where the last selected line starts. */ + lastLineStart: number; + /** UTF-8 byte length of the decoded selection and of its first line. */ + selectedBytes: number; + firstLineBytes: number; +} + /** Pages of one directory's entries. */ export interface DirReader { /** @@ -257,3 +282,6 @@ export interface Shell { } export interface ExecutionEnv extends FileSystem, Shell {} + +export { rangeDecoder, StreamDecoder, startsWithBom } from "./decode.ts"; +export { LineScanner } from "./line-scan.ts"; diff --git a/packages/durable/src/env/line-scan.ts b/packages/durable/src/env/line-scan.ts new file mode 100644 index 000000000..45880fd74 --- /dev/null +++ b/packages/durable/src/env/line-scan.ts @@ -0,0 +1,145 @@ +import { rangeDecoder, startsWithBom } from "./decode.ts"; +import type { LineScan } from "./index.ts"; + +const NEWLINE = 0x0a; +const encoder = new TextEncoder(); + +/** UTF-8 byte length of decoded text. */ +function decodedBytes(text: string): number { + return text === "" ? 0 : encoder.encode(text).length; +} + +/** + * Computes a `LineScan` from a file's bytes fed in order, so an environment can scan any size in bounded memory. Decoded + * sizes use the same streaming WHATWG decoder as decoding the whole file at once. + */ +export class LineScanner { + readonly #startLine: number; + readonly #endLine: number; + #position = 0; + #newlines = 0; + #lineStart = 0; + #start: number | undefined; + #end: number | undefined; + #firstLineEnd: number | undefined; + #lastLineStart: number | undefined; + #selectedBytes = 0; + #firstLineBytes = 0; + #selection: InstanceType | undefined; + #firstLine: InstanceType | undefined; + /** The first bytes, held until it is known whether they are a byte-order mark. */ + #head: number[] | undefined = []; + #bom = false; + + /** `startLine` and `endLine` must be non-negative integers with `endLine > startLine`; `endLine` absent: to the end. */ + constructor(startLine: number, endLine = Number.POSITIVE_INFINITY) { + if (!Number.isSafeInteger(startLine) || startLine < 0 || !(endLine > startLine)) { + throw new RangeError("Invalid line range"); + } + if (endLine !== Number.POSITIVE_INFINITY && !Number.isSafeInteger(endLine)) + throw new RangeError("Invalid line range"); + this.#startLine = startLine; + this.#endLine = endLine; + if (startLine === 0) this.#begin(0); + } + + push(chunk: Uint8Array): void { + if (this.#head !== undefined) { + const take = Math.min(3 - this.#head.length, chunk.length); + this.#head.push(...chunk.subarray(0, take)); + if (this.#head.length < 3) return; + this.#releaseHead(); + chunk = chunk.subarray(take); + } + this.#process(chunk); + } + + #releaseHead(): void { + const head = Uint8Array.from(this.#head ?? []); + this.#head = undefined; + this.#bom = startsWithBom(head); + this.#process(head); + } + + #process(chunk: Uint8Array): void { + const base = this.#position; + let from = 0; + for (let index = chunk.indexOf(NEWLINE); index !== -1; index = chunk.indexOf(NEWLINE, index + 1)) { + // The newline ends line `this.#newlines`. It belongs to the selection between selected lines only. + this.#feed(chunk, base, from, index); + const line = this.#newlines; + const position = base + index; + if (line === this.#startLine) this.#endFirstLine(position); + if (line === this.#endLine - 1) this.#endSelection(position); + this.#feed(chunk, base, index, index + 1); + from = index + 1; + this.#newlines++; + this.#lineStart = position + 1; + if (this.#newlines === this.#startLine) this.#begin(this.#lineStart); + if (this.#newlines === this.#endLine - 1) this.#lastLineStart = this.#lineStart; + } + this.#feed(chunk, base, from, chunk.length); + this.#position += chunk.length; + } + + finish(): LineScan { + if (this.#head !== undefined) this.#releaseHead(); + const size = this.#position; + if (this.#start === undefined) { + return { + newlines: this.#newlines, + start: size, + end: size, + firstLineEnd: size, + lastLineStart: size, + selectedBytes: 0, + firstLineBytes: 0, + }; + } + if (this.#firstLineEnd === undefined) this.#endFirstLine(size); + if (this.#end === undefined) this.#endSelection(size); + return { + newlines: this.#newlines, + start: this.#start, + end: this.#end ?? size, + firstLineEnd: this.#firstLineEnd ?? size, + // A selection that reaches past the last line ends with the last line. + lastLineStart: this.#lastLineStart ?? this.#lineStart, + selectedBytes: this.#selectedBytes, + firstLineBytes: this.#firstLineBytes, + }; + } + + #begin(start: number): void { + this.#start = start; + if (this.#startLine === this.#endLine - 1) this.#lastLineStart = start; + this.#selection = rangeDecoder(); + this.#firstLine = rangeDecoder(); + } + + #endFirstLine(position: number): void { + this.#firstLineEnd = position; + if (this.#firstLine !== undefined) this.#firstLineBytes += decodedBytes(this.#firstLine.decode()); + this.#firstLine = undefined; + } + + #endSelection(position: number): void { + this.#end = position; + if (this.#selection !== undefined) this.#selectedBytes += decodedBytes(this.#selection.decode()); + this.#selection = undefined; + } + + /** Feed `chunk[from, to)`, which starts at file offset `base`, to the decoders of the ranges still open. */ + #feed(chunk: Uint8Array, base: number, from: number, to: number): void { + // Decoding the whole file drops a leading byte-order mark. + if (this.#bom && base + from < 3) from = Math.min(to, 3 - base); + if (to <= from) return; + const bytes = chunk.subarray(from, to); + if (this.#selection !== undefined) { + this.#selectedBytes += decodedBytes(this.#selection.decode(bytes, { stream: true })); + } + if (this.#firstLine !== undefined) { + this.#firstLineBytes += decodedBytes(this.#firstLine.decode(bytes, { stream: true })); + } + } +} diff --git a/packages/durable/src/env/node.ts b/packages/durable/src/env/node.ts index 5ad7c7207..89c0be6c2 100644 --- a/packages/durable/src/env/node.ts +++ b/packages/durable/src/env/node.ts @@ -21,6 +21,7 @@ import { homedir, constants as osConstants, tmpdir } from "node:os"; import { basename, isAbsolute, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import type { Context } from "@earendil-works/chord"; +import { StreamDecoder } from "./decode.ts"; import { type BinaryReader, type DirReader, @@ -30,6 +31,7 @@ import { FileError, type FileInfo, type FileKind, + type LineScan, ok, type Result, type ShellExecOptions, @@ -39,6 +41,7 @@ import { type TextLineReader, toError, } from "./index.ts"; +import { LineScanner } from "./line-scan.ts"; const MAX_TIMEOUT_MS = 2_147_483_647; const MAX_TIMEOUT_SECONDS = MAX_TIMEOUT_MS / 1000; @@ -380,7 +383,7 @@ function waitForChildProcess( class NodeTextLineReader implements TextLineReader { private readonly file: Awaited>; private readonly path: string; - private readonly decoder = new TextDecoder(); + private readonly decoder = new StreamDecoder(); private readonly chunk = new Uint8Array(64 * 1024); private byteOffset = 0; private buffered = ""; @@ -421,7 +424,7 @@ class NodeTextLineReader implements TextLineReader { this.buffered += this.decoder.decode(); this.ended = true; } else { - this.buffered += this.decoder.decode(this.chunk.subarray(0, bytesRead), { stream: true }); + this.buffered += this.decoder.decode(this.chunk.subarray(0, bytesRead)); } } } catch (error) { @@ -498,6 +501,34 @@ class NodeBinaryReader implements BinaryReader { return ok(bytes); } + async scanLines( + options: { startLine: number; endLine?: number }, + context: Context, + ): Promise> { + const aborted = abortResult(context.abortSignal, this.path); + if (aborted) return aborted; + if (this.closed) return closedResult("Binary reader", this.path); + let scanner: LineScanner; + try { + scanner = new LineScanner(options.startLine, options.endLine); + } catch { + return err(new FileError("invalid", "Invalid line range", this.path)); + } + const chunk = new Uint8Array(64 * 1024); + try { + for (let position = 0; ; ) { + const { bytesRead } = await this.file.read(chunk, 0, chunk.length, position); + const afterReadAbort = abortResult(context.abortSignal, this.path); + if (afterReadAbort) return afterReadAbort; + if (bytesRead === 0) return ok(scanner.finish()); + scanner.push(chunk.subarray(0, bytesRead)); + position += bytesRead; + } + } catch (error) { + return err(toFileError(error, this.path)); + } + } + async close(_context: Context): Promise { if (this.closed) return; this.closed = true; @@ -653,8 +684,8 @@ export class NodeExecutionEnv implements ExecutionEnv { onAbort(); }; // One decoder per stream, so a character split across chunks of one stream survives interleaving. - const stdoutDecoder = new TextDecoder(); - const stderrDecoder = new TextDecoder(); + const stdoutDecoder = new StreamDecoder(); + const stderrDecoder = new StreamDecoder(); // No output reaches the caller after exec() settled, for example from a descendant holding stdio open. const emit = (text: string, stream: ShellOutputInfo["stream"]): void => { if (settled || text === "" || options?.onOutput === undefined || callbackError !== undefined) return; @@ -769,27 +800,25 @@ export class NodeExecutionEnv implements ExecutionEnv { else signal.addEventListener("abort", onAbort, { once: true }); } - const feed = - (source: ShellOutputInfo["stream"], decoder: InstanceType) => (chunk: Uint8Array) => { - emit(decoder.decode(chunk, { stream: true }), source); - const spill = options?.spill; - if (spill === undefined || chunk.length === 0) return; - if (spillStart !== undefined) { - startSpill(chunk); - return; - } - seenBytes += chunk.length; - for (let index = chunk.indexOf(0x0a); index !== -1; index = chunk.indexOf(0x0a, index + 1)) - seenNewlines++; - const lines = seenNewlines + (chunk[chunk.length - 1] === 0x0a ? 0 : 1); - if (seenBytes <= spill.afterBytes && lines <= spill.afterLines) { - spillPrefix.push(chunk); - return; - } - for (const prefix of spillPrefix) startSpill(prefix); - spillPrefix.length = 0; + const feed = (source: ShellOutputInfo["stream"], decoder: StreamDecoder) => (chunk: Uint8Array) => { + emit(decoder.decode(chunk), source); + const spill = options?.spill; + if (spill === undefined || chunk.length === 0) return; + if (spillStart !== undefined) { startSpill(chunk); - }; + return; + } + seenBytes += chunk.length; + for (let index = chunk.indexOf(0x0a); index !== -1; index = chunk.indexOf(0x0a, index + 1)) seenNewlines++; + const lines = seenNewlines + (chunk[chunk.length - 1] === 0x0a ? 0 : 1); + if (seenBytes <= spill.afterBytes && lines <= spill.afterLines) { + spillPrefix.push(chunk); + return; + } + for (const prefix of spillPrefix) startSpill(prefix); + spillPrefix.length = 0; + startSpill(chunk); + }; child.stdout?.on("data", feed("stdout", stdoutDecoder)); child.stderr?.on("data", feed("stderr", stderrDecoder)); @@ -922,7 +951,7 @@ export class NodeExecutionEnv implements ExecutionEnv { await file.close().catch(() => undefined); return err( stats.isDirectory() - ? new FileError("is_directory", "Is a directory", resolved) + ? new FileError("is_directory", "EISDIR: illegal operation on a directory, read", resolved) : new FileError("invalid", "Not a regular file", resolved), ); } diff --git a/packages/durable/src/testing/env-conformance.ts b/packages/durable/src/testing/env-conformance.ts index 29be58738..6a8a0ab97 100644 --- a/packages/durable/src/testing/env-conformance.ts +++ b/packages/durable/src/testing/env-conformance.ts @@ -85,6 +85,47 @@ export function createEnvConformance(options: EnvConformanceOptions): readonly E assert.strictEqual(errorCode(await reader.info(context)), "invalid"); }), + createCase("binary reader scans lines like decoding the whole file", async (env) => { + // A byte-order mark, an invalid sequence before a newline, an empty line, a later U+FEFF, and no final newline. + const bytes = Uint8Array.from([ + 0xef, 0xbb, 0xbf, 0x61, 0x0a, 0xe2, 0x82, 0x0a, 0x0a, 0xef, 0xbb, 0xbf, 0x62, 0x0a, 0xc3, 0xa9, + ]); + getOrThrow(await env.writeFile("lines.txt", bytes, context)); + const lines = new TextDecoder().decode(bytes).split("\n"); + const reader = getOrThrow(await env.openBinaryReader("lines.txt", undefined, context)); + try { + const ranges: [number, number | undefined][] = [ + [0, undefined], + [0, 1], + [1, 3], + [2, 3], + [3, undefined], + [4, 9], + ]; + for (const [startLine, endLine] of ranges) { + const scan = getOrThrow( + await reader.scanLines({ startLine, ...(endLine === undefined ? {} : { endLine }) }, context), + ); + const selected = lines.slice(startLine, endLine); + const range = (from: number, to: number) => + new TextDecoder("utf-8", { ignoreBOM: from > 0 }).decode(bytes.subarray(from, to)); + assert.strictEqual(scan.newlines, lines.length - 1); + assert.strictEqual(range(scan.start, scan.end), selected.join("\n")); + assert.strictEqual(scan.selectedBytes, encoder.encode(selected.join("\n")).length); + assert.strictEqual(range(scan.start, scan.firstLineEnd), lines[startLine]); + assert.strictEqual(scan.firstLineBytes, encoder.encode(lines[startLine]).length); + } + assert.partialDeepEqual(getOrThrow(await reader.scanLines({ startLine: 9 }, context)), { + start: bytes.length, + end: bytes.length, + selectedBytes: 0, + }); + assert.strictEqual(errorCode(await reader.scanLines({ startLine: 2, endLine: 2 }, context)), "invalid"); + } finally { + await reader.close(context); + } + }), + createCase("binary reader keeps reading the file it opened after a rename", async (env) => { getOrThrow(await env.writeFile("a.txt", "one", context)); const reader = getOrThrow(await env.openBinaryReader("a.txt", undefined, context)); diff --git a/packages/durable/src/tools/image.ts b/packages/durable/src/tools/image.ts index 554192b5f..dbb2f2a3b 100644 --- a/packages/durable/src/tools/image.ts +++ b/packages/durable/src/tools/image.ts @@ -1,4 +1,47 @@ const PNG_SIGNATURE = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]; +/** Bytes every check except the APNG chunk walk needs: BMP reads up to offset 29. */ +const HEADER_BYTES = 32; +const BLOCK_BYTES = 64 * 1024; + +/** Positional reads of a file of `size` bytes. */ +export type ByteSource = { + readonly size: number; + read(offset: number, length: number): Promise; +}; + +/** + * `detectSupportedImageMimeType` of a whole file, reading only its header and, for PNG, the chunk headers up to the + * first `acTL` or `IDAT`. + */ +export async function detectSupportedImageMimeTypeOf(source: ByteSource): Promise { + const header = await source.read(0, HEADER_BYTES); + if (!startsWith(header, PNG_SIGNATURE)) return detectSupportedImageMimeType(header); + return isPng(header) && !(await isAnimatedPngOf(source)) ? "image/png" : undefined; +} + +/** `isAnimatedPng` over a file read in blocks. */ +async function isAnimatedPngOf(source: ByteSource): Promise { + let block: Uint8Array = new Uint8Array(0); + let blockStart = 0; + const bytesAt = async (offset: number, length: number): Promise => { + if (offset < blockStart || offset + length > blockStart + block.length) { + blockStart = offset; + block = await source.read(offset, BLOCK_BYTES); + } + return block.subarray(offset - blockStart, offset - blockStart + length); + }; + let offset = PNG_SIGNATURE.length; + while (offset + 8 <= source.size) { + const chunkHeader = await bytesAt(offset, 8); + const chunkLength = readUint32BE(chunkHeader, 0); + if (startsWithAscii(chunkHeader, 4, "acTL")) return true; + if (startsWithAscii(chunkHeader, 4, "IDAT")) return false; + const nextOffset = offset + 8 + chunkLength + 4; + if (nextOffset <= offset || nextOffset > source.size) return false; + offset = nextOffset; + } + return false; +} export function detectSupportedImageMimeType(buffer: Uint8Array): string | undefined { if (startsWith(buffer, [0xff, 0xd8, 0xff])) return buffer[3] === 0xf7 ? undefined : "image/jpeg"; diff --git a/packages/durable/src/tools/read.ts b/packages/durable/src/tools/read.ts index 1fd914a47..f08b88fba 100644 --- a/packages/durable/src/tools/read.ts +++ b/packages/durable/src/tools/read.ts @@ -1,11 +1,26 @@ +import type { Context } from "@earendil-works/chord"; import { type Static, Type } from "typebox"; -import { getOrThrow } from "../env/index.ts"; +import { + type BinaryReader, + type FileInfo, + getOrThrow, + type LineScan, + rangeDecoder, + startsWithBom, +} from "../env/index.ts"; import { defineTool } from "../harness/define.ts"; import { characterEnd } from "../harness/output.ts"; import type { ToolDiagnostic, ToolRegistration } from "../harness/types.ts"; -import { DEFAULT_MAX_BYTES, DEFAULT_MAX_LINES, formatSize, type TruncationResult, truncateHead } from "../truncate.ts"; +import { + DEFAULT_MAX_BYTES, + DEFAULT_MAX_LINES, + formatSize, + type TruncationResult, + truncateHeadOf, + utf8ByteLength, +} from "../truncate.ts"; import { requireEnv } from "./env.ts"; -import { detectSupportedImageMimeType } from "./image.ts"; +import { detectSupportedImageMimeTypeOf } from "./image.ts"; import { resolveReadToolPath } from "./path-utils.ts"; const readSchema = Type.Object({ @@ -21,6 +36,39 @@ export type ReadToolDetails = { truncation?: Omit; }; +const READ_CHUNK = 64 * 1024; + +/** `Array.prototype.slice`'s conversion of an index: NaN is 0, other values truncate toward zero. */ +function sliceIndex(value: number): number { + return Number.isNaN(value) ? 0 : Math.trunc(value); +} + +/** + * The decoded start of bytes `[start, end)` of the file, decoded as part of the whole file: all of it, or enough for + * `truncateHeadOf` (more than `DEFAULT_MAX_BYTES + 1` bytes, or `DEFAULT_MAX_LINES` newlines). + */ +async function readHead( + reader: BinaryReader, + start: number, + end: number, + skipBom: boolean, + context: Context, +): Promise { + const decoder = rangeDecoder(); + let text = ""; + let newlines = 0; + for (let position = skipBom && start === 0 ? 3 : start; position < end; ) { + const bytes = getOrThrow(await reader.read(position, Math.min(READ_CHUNK, end - position), context)); + if (bytes.length === 0) break; + position += bytes.length; + const decoded = decoder.decode(bytes, { stream: true }); + text += decoded; + for (let index = decoded.indexOf("\n"); index !== -1; index = decoded.indexOf("\n", index + 1)) newlines++; + if (newlines >= DEFAULT_MAX_LINES || utf8ByteLength(text) > DEFAULT_MAX_BYTES + 1) return text; + } + return text + decoder.decode(); +} + /** Reads text files. Remarks about truncation and continuation are diagnostics; the content is only file text. */ export function createReadTool(): ToolRegistration { return defineTool({ @@ -31,81 +79,135 @@ export function createReadTool(): ToolRegistration= allLines.length) { - throw new Error(`Offset ${offset} is beyond end of file (${allLines.length} lines total)`); - } - - let selectedContent: string; - let userLimitedLines: number | undefined; - if (limit !== undefined) { - const endLine = Math.min(startLine + limit, allLines.length); - selectedContent = allLines.slice(startLine, endLine).join("\n"); - userLimitedLines = endLine - startLine; - } else { - selectedContent = allLines.slice(startLine).join("\n"); - } - - const { content: headText, ...truncation } = truncateHead(selectedContent); - const diagnostics: ToolDiagnostic[] = []; - let outputText = headText; - let details: ReadToolDetails | undefined; - if (truncation.firstLineExceedsLimit) { - // Show the start of the line, cut at the byte limit on a character boundary. - const lineBytes = new TextEncoder().encode(allLines[startLine]); - const end = characterEnd(lineBytes, DEFAULT_MAX_BYTES); - outputText = new TextDecoder().decode(lineBytes.subarray(0, end)); - diagnostics.push({ - severity: "warn", - code: "truncated", - message: `Line ${startLineDisplay} is ${formatSize(lineBytes.byteLength)}, exceeds the ${formatSize(DEFAULT_MAX_BYTES)} limit; showing its first ${formatSize(end)}. Use bash: sed -n '${startLineDisplay}p' ${path} | tail -c +${end + 1}`, - }); - details = { truncation: { ...truncation, outputBytes: end, outputLines: 1 } }; - } else if (truncation.truncated) { - const endLineDisplay = startLineDisplay + truncation.outputLines - 1; - const nextOffset = endLineDisplay + 1; - const limitText = truncation.truncatedBy === "lines" ? "" : ` (${formatSize(DEFAULT_MAX_BYTES)} limit)`; - diagnostics.push({ - severity: "info", - code: "truncated", - message: `Showing lines ${startLineDisplay}-${endLineDisplay} of ${totalFileLines}${limitText}. Use offset=${nextOffset} to continue.`, - }); - details = { truncation }; - } else if (userLimitedLines !== undefined && startLine + userLimitedLines < allLines.length) { - const remaining = allLines.length - (startLine + userLimitedLines); - const nextOffset = startLine + userLimitedLines + 1; - diagnostics.push({ - severity: "info", - message: `${remaining} more lines in file. Use offset=${nextOffset} to continue.`, - }); - } - - return { - content: outputText === "" ? [] : [{ type: "text", text: outputText }], - ...(details === undefined ? {} : { details }), - diagnostics, - }; }, }); } + +/** + * The read result for the opened file. It equals decoding the whole file with `TextDecoder`, splitting it on `\n`, + * and bounding the selected lines with `truncateHead`, while reading only one scan's worth of the file plus the head. + */ +async function readText( + reader: BinaryReader, + info: FileInfo, + path: string, + offset: number | undefined, + limit: number | undefined, + context: Context, +) { + const mimeType = await detectSupportedImageMimeTypeOf({ + size: info.size, + read: async (position, length) => getOrThrow(await reader.read(position, length, context)), + }); + if (mimeType) { + // Image content is not supported yet. + return { + content: [], + isError: true, + diagnostics: [ + { + severity: "error" as const, + code: "unsupported_image", + message: `${path} is an image (${mimeType}); reading images is not supported`, + }, + ], + }; + } + + const startLine = offset ? Math.max(0, offset - 1) : 0; + const startLineDisplay = startLine + 1; + // Lines are selected like `allLines.slice(startLine, endLine)`, which truncates fractional indices. + const sliceStart = sliceIndex(startLine); + // One pass finds the line count and the selection; a selection past the last line ends with it, as `slice` does, + // and an empty one (a zero or negative limit) is scanned as one line and then ignored. + // A start beyond any file is scanned from 0 only to count lines; the offset check below then fails as before. + const scanStart = Number.isSafeInteger(sliceStart) ? sliceStart : 0; + const requestedEnd = limit === undefined ? undefined : Math.max(scanStart + 1, sliceIndex(startLine + limit)); + const scanEnd = requestedEnd !== undefined && Number.isSafeInteger(requestedEnd) ? requestedEnd : undefined; + const scanOf = async (endLine: number | undefined): Promise => + getOrThrow( + await reader.scanLines({ startLine: scanStart, ...(endLine === undefined ? {} : { endLine }) }, context), + ); + let scan = await scanOf(scanEnd); + const totalFileLines = scan.newlines + 1; + if (startLine >= totalFileLines) { + throw new Error(`Offset ${offset} is beyond end of file (${totalFileLines} lines total)`); + } + + let userLimitedLines: number | undefined; + let selectedLineCount = totalFileLines - sliceStart; + if (limit !== undefined) { + const endLine = Math.min(startLine + limit, totalFileLines); + userLimitedLines = endLine - startLine; + // `slice` counts a negative end from the end of the lines, which only the line count tells; scan again for it. + const relativeEnd = sliceIndex(endLine); + const sliceEnd = relativeEnd < 0 ? Math.max(totalFileLines + relativeEnd, 0) : relativeEnd; + selectedLineCount = Math.max(0, sliceEnd - sliceStart); + if (selectedLineCount > 0 && relativeEnd < 0) scan = await scanOf(sliceEnd); + } + const empty = selectedLineCount === 0; + // Counted like `truncateHead`: a trailing newline adds no line, and empty text has none. + const endsWithNewline = !empty && scan.lastLineStart === scan.end && scan.lastLineStart > scan.start; + const totals = { + lines: empty || scan.selectedBytes === 0 ? 0 : selectedLineCount - (endsWithNewline ? 1 : 0), + bytes: empty ? 0 : scan.selectedBytes, + }; + const firstBytes = getOrThrow(await reader.read(0, 3, context)); + const head = empty ? "" : await readHead(reader, scan.start, scan.end, startsWithBom(firstBytes), context); + + const { content: headText, ...truncation } = truncateHeadOf(head, totals); + const diagnostics: ToolDiagnostic[] = []; + let outputText = headText; + let details: ReadToolDetails | undefined; + if (truncation.firstLineExceedsLimit) { + // Show the start of the line, cut at the byte limit on a character boundary. Like `allLines[startLine]`, a + // fractional start line names no line. + const integral = Number.isInteger(startLine); + const lineBytes = new TextEncoder().encode(integral ? (head.split("\n")[0] ?? "") : ""); + const lineSize = integral ? scan.firstLineBytes : 0; + const end = characterEnd(lineBytes, DEFAULT_MAX_BYTES); + outputText = new TextDecoder().decode(lineBytes.subarray(0, end)); + diagnostics.push({ + severity: "warn", + code: "truncated", + message: `Line ${startLineDisplay} is ${formatSize(lineSize)}, exceeds the ${formatSize(DEFAULT_MAX_BYTES)} limit; showing its first ${formatSize(end)}. Use bash: sed -n '${startLineDisplay}p' ${path} | tail -c +${end + 1}`, + }); + details = { truncation: { ...truncation, outputBytes: end, outputLines: 1 } }; + } else if (truncation.truncated) { + const endLineDisplay = startLineDisplay + truncation.outputLines - 1; + const nextOffset = endLineDisplay + 1; + const limitText = truncation.truncatedBy === "lines" ? "" : ` (${formatSize(DEFAULT_MAX_BYTES)} limit)`; + diagnostics.push({ + severity: "info", + code: "truncated", + message: `Showing lines ${startLineDisplay}-${endLineDisplay} of ${totalFileLines}${limitText}. Use offset=${nextOffset} to continue.`, + }); + details = { truncation }; + } else if (userLimitedLines !== undefined && startLine + userLimitedLines < totalFileLines) { + const remaining = totalFileLines - (startLine + userLimitedLines); + const nextOffset = startLine + userLimitedLines + 1; + diagnostics.push({ + severity: "info", + message: `${remaining} more lines in file. Use offset=${nextOffset} to continue.`, + }); + } + + return { + content: outputText === "" ? [] : [{ type: "text" as const, text: outputText }], + ...(details === undefined ? {} : { details }), + diagnostics, + }; +} diff --git a/packages/durable/src/truncate.ts b/packages/durable/src/truncate.ts index 43cdf412e..ed2041f78 100644 --- a/packages/durable/src/truncate.ts +++ b/packages/durable/src/truncate.ts @@ -106,17 +106,34 @@ export function formatSize(bytes: number): string { * returns empty content with firstLineExceedsLimit=true. */ export function truncateHead(content: string, options: TruncationOptions = {}): TruncationResult { + return truncateHeadOf( + content, + { lines: splitLinesForCounting(content).length, bytes: utf8ByteLength(content) }, + options, + ); +} + +/** + * `truncateHead` of a text known by a prefix and its totals (`lines` counted like `truncateHead`, ignoring a trailing + * newline). The prefix must be the whole text, or longer than `maxBytes + 1` UTF-8 bytes, or hold at least `maxLines` + * newlines; then the result equals `truncateHead` of the whole text. + */ +export function truncateHeadOf( + prefix: string, + totals: { readonly lines: number; readonly bytes: number }, + options: TruncationOptions = {}, +): TruncationResult { const maxLines = options.maxLines ?? DEFAULT_MAX_LINES; const maxBytes = options.maxBytes ?? DEFAULT_MAX_BYTES; - const totalBytes = utf8ByteLength(content); - const lines = splitLinesForCounting(content); - const totalLines = lines.length; + const totalBytes = totals.bytes; + const lines = splitLinesForCounting(prefix); + const totalLines = totals.lines; // Check if no truncation needed if (totalLines <= maxLines && totalBytes <= maxBytes) { return { - content, + content: prefix, truncated: false, truncatedBy: null, totalLines, diff --git a/packages/durable/test/env-line-scan.test.ts b/packages/durable/test/env-line-scan.test.ts new file mode 100644 index 000000000..538c59bc7 --- /dev/null +++ b/packages/durable/test/env-line-scan.test.ts @@ -0,0 +1,107 @@ +import { describe, expect, it } from "vitest"; +import { StreamDecoder } from "../src/env/decode.ts"; +import { LineScanner } from "../src/env/line-scan.ts"; + +function random(seed: number): () => number { + let state = seed >>> 0; + return () => { + state = (state + 0x6d2b79f5) >>> 0; + let t = state; + t = Math.imul(t ^ (t >>> 15), t | 1); + t ^= t + Math.imul(t ^ (t >>> 7), t | 61); + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} + +// Newlines, ASCII, a byte-order mark, valid multi-byte sequences, and bytes that form invalid or truncated sequences. +const PIECES = [ + [0x0a], + [0x0a], + [0x61], + [0x62, 0x63], + [0xef, 0xbb, 0xbf], + [0xc3, 0xa9], + [0xe2, 0x82, 0xac], + [0xf0, 0x9f, 0x98, 0x80], + [0xe2, 0x82], + [0xff], + [0x80], + [0xf0, 0x9f], + [0x0d, 0x0a], +]; + +function randomFile(next: () => number): Uint8Array { + const bytes: number[] = []; + const pieces = Math.floor(next() * 60); + for (let index = 0; index < pieces; index++) bytes.push(...PIECES[Math.floor(next() * PIECES.length)]!); + return Uint8Array.from(bytes); +} + +const utf8 = (text: string): number => new TextEncoder().encode(text).length; + +describe("StreamDecoder", () => { + // Node's streaming TextDecoder with BOM handling dropped this U+FEFF, which follows an invalid sequence. + it("keeps a U+FEFF that does not start the stream", () => { + const bytes = Uint8Array.from([0xe2, 0x82, 0xef, 0xbb, 0xbf, 0x61]); + const decoder = new StreamDecoder(); + const text = [...bytes].map((byte) => decoder.decode(Uint8Array.of(byte))).join("") + decoder.decode(); + expect(text).toBe(new TextDecoder().decode(bytes)); + expect(text).toBe("\ufffd\ufeffa"); + }); + + it("decodes any chunking like decoding the whole stream", () => { + for (let seed = 1; seed <= 5000; seed++) { + const next = random(seed); + const bytes = randomFile(next); + const decoder = new StreamDecoder(); + let text = ""; + for (let offset = 0; offset < bytes.length; ) { + const size = 1 + Math.floor(next() * 7); + text += decoder.decode(bytes.subarray(offset, offset + size)); + offset += size; + } + expect(text + decoder.decode(), `seed ${seed}`).toBe(new TextDecoder().decode(bytes)); + } + }); +}); + +describe("LineScanner", () => { + it("agrees with decoding and splitting the whole file", () => { + for (let seed = 1; seed <= 5000; seed++) { + const next = random(seed); + const file = randomFile(next); + const lines = new TextDecoder().decode(file).split("\n"); + const startLine = Math.floor(next() * (lines.length + 2)); + const endLine = next() < 0.3 ? undefined : startLine + 1 + Math.floor(next() * (lines.length + 1)); + const scanner = new LineScanner(startLine, endLine); + for (let offset = 0; offset < file.length; ) { + const size = 1 + Math.floor(next() * 7); + scanner.push(file.subarray(offset, offset + size)); + offset += size; + } + const scan = scanner.finish(); + const message = `seed ${seed}`; + expect(scan.newlines, message).toBe(lines.length - 1); + const selected = lines.slice(startLine, endLine); + const decode = (from: number, to: number) => + new TextDecoder("utf-8", { ignoreBOM: from > 0 }).decode(file.subarray(from, to)); + expect(decode(scan.start, scan.end), message).toBe(selected.join("\n")); + expect(scan.selectedBytes, message).toBe(utf8(selected.join("\n"))); + if (startLine < lines.length) { + expect(decode(scan.start, scan.firstLineEnd), message).toBe(lines[startLine]); + expect(scan.firstLineBytes, message).toBe(utf8(lines[startLine]!)); + const lastLine = Math.min(endLine ?? lines.length, lines.length) - 1; + const lastEnd = lastLine + 1 < lines.length ? file.indexOf(0x0a, scan.lastLineStart) : file.length; + expect(decode(scan.lastLineStart, lastEnd), message).toBe(lines[lastLine]); + } else { + expect(scan, message).toMatchObject({ start: file.length, end: file.length, selectedBytes: 0 }); + } + } + }); + + it("rejects empty or invalid ranges", () => { + expect(() => new LineScanner(2, 2)).toThrow(RangeError); + expect(() => new LineScanner(-1)).toThrow(RangeError); + expect(() => new LineScanner(1.5)).toThrow(RangeError); + }); +}); diff --git a/packages/durable/test/harness-tools.test.ts b/packages/durable/test/harness-tools.test.ts index a2a4c9226..82b6e3794 100644 --- a/packages/durable/test/harness-tools.test.ts +++ b/packages/durable/test/harness-tools.test.ts @@ -361,7 +361,7 @@ describe("tool results", () => { }); expect(windows).toContainEqual(undefined); // 8 bytes written, 8 skipped, then "x\n" dropped by the window: 3 lines, 18 bytes in all. - expect(resultText(results(entries)[0]!)).toBe( + expect(resultText(results(entries).find((result) => result.toolCallId === "c1")!)).toBe( "y\n|\n[warn] Output truncated to its end: 3 lines, 18 bytes dropped\n", ); await harness.close(context); diff --git a/packages/durable/test/tools-read-differential.test.ts b/packages/durable/test/tools-read-differential.test.ts new file mode 100644 index 000000000..c191d8a1f --- /dev/null +++ b/packages/durable/test/tools-read-differential.test.ts @@ -0,0 +1,210 @@ +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; +import { afterAll, describe, expect, it } from "vitest"; +import { NodeExecutionEnv } from "../src/env/node.ts"; +import { characterEnd } from "../src/harness/output.ts"; +import type { ToolDiagnostic, ToolExecutionApi } from "../src/harness/types.ts"; +import { detectSupportedImageMimeType } from "../src/tools/image.ts"; +import { createReadTool } from "../src/tools/read.ts"; +import { DEFAULT_MAX_BYTES, formatSize, truncateHead } from "../src/truncate.ts"; + +/** + * The `read` tool before bounded reads, kept as the reference: it decodes the whole file, splits it into lines, and + * truncates the selection. The tool must give the same result while reading only a bounded part of the file. + */ +function referenceRead(bytes: Uint8Array, path: string, offset: number | undefined, limit: number | undefined) { + const mimeType = detectSupportedImageMimeType(bytes); + if (mimeType) { + return { + content: [], + isError: true, + diagnostics: [ + { + severity: "error", + code: "unsupported_image", + message: `${path} is an image (${mimeType}); reading images is not supported`, + }, + ], + }; + } + const textContent = new TextDecoder().decode(bytes); + const allLines = textContent.split("\n"); + const totalFileLines = allLines.length; + const startLine = offset ? Math.max(0, offset - 1) : 0; + const startLineDisplay = startLine + 1; + if (startLine >= allLines.length) { + throw new Error(`Offset ${offset} is beyond end of file (${allLines.length} lines total)`); + } + let selectedContent: string; + let userLimitedLines: number | undefined; + if (limit !== undefined) { + const endLine = Math.min(startLine + limit, allLines.length); + selectedContent = allLines.slice(startLine, endLine).join("\n"); + userLimitedLines = endLine - startLine; + } else { + selectedContent = allLines.slice(startLine).join("\n"); + } + const { content: headText, ...truncation } = truncateHead(selectedContent); + const diagnostics: ToolDiagnostic[] = []; + let outputText = headText; + let details: object | undefined; + if (truncation.firstLineExceedsLimit) { + const lineBytes = new TextEncoder().encode(allLines[startLine]); + const end = characterEnd(lineBytes, DEFAULT_MAX_BYTES); + outputText = new TextDecoder().decode(lineBytes.subarray(0, end)); + diagnostics.push({ + severity: "warn", + code: "truncated", + message: `Line ${startLineDisplay} is ${formatSize(lineBytes.byteLength)}, exceeds the ${formatSize(DEFAULT_MAX_BYTES)} limit; showing its first ${formatSize(end)}. Use bash: sed -n '${startLineDisplay}p' ${path} | tail -c +${end + 1}`, + }); + details = { truncation: { ...truncation, outputBytes: end, outputLines: 1 } }; + } else if (truncation.truncated) { + const endLineDisplay = startLineDisplay + truncation.outputLines - 1; + const nextOffset = endLineDisplay + 1; + const limitText = truncation.truncatedBy === "lines" ? "" : ` (${formatSize(DEFAULT_MAX_BYTES)} limit)`; + diagnostics.push({ + severity: "info", + code: "truncated", + message: `Showing lines ${startLineDisplay}-${endLineDisplay} of ${totalFileLines}${limitText}. Use offset=${nextOffset} to continue.`, + }); + details = { truncation }; + } else if (userLimitedLines !== undefined && startLine + userLimitedLines < allLines.length) { + const remaining = allLines.length - (startLine + userLimitedLines); + const nextOffset = startLine + userLimitedLines + 1; + diagnostics.push({ + severity: "info", + message: `${remaining} more lines in file. Use offset=${nextOffset} to continue.`, + }); + } + return { + content: outputText === "" ? [] : [{ type: "text", text: outputText }], + ...(details === undefined ? {} : { details }), + diagnostics, + }; +} + +function random(seed: number): () => number { + let state = seed >>> 0; + return () => { + state = (state + 0x6d2b79f5) >>> 0; + let t = state; + t = Math.imul(t ^ (t >>> 15), t | 1); + t ^= t + Math.imul(t ^ (t >>> 7), t | 61); + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} + +const PIECES = [ + [0x0a], + [0x0a], + [0x61], + [0x62, 0x63, 0x64, 0x65], + [0xef, 0xbb, 0xbf], + [0xc3, 0xa9], + [0xe2, 0x82, 0xac], + [0xf0, 0x9f, 0x98, 0x80], + [0xe2, 0x82], + [0xff], + [0x0d, 0x0a], +]; + +/** Mostly small files, some over the line limit, some over the byte limit, some with one huge line. */ +function randomFile(next: () => number): Uint8Array { + const kind = next(); + if (kind < 0.1) return Uint8Array.from([0xef, 0xbb, 0xbf, ...new TextEncoder().encode("x\n".repeat(2100))]); + if (kind < 0.2) { + const line = new Uint8Array(DEFAULT_MAX_BYTES + 200 + Math.floor(next() * 400)).fill(0x61); + line[DEFAULT_MAX_BYTES - 1 + Math.floor(next() * 3)] = 0xc3; + return Uint8Array.from([...line, 0x0a, 0x62]); + } + if (kind < 0.3) return new TextEncoder().encode(`${"0123456789".repeat(30)}\n`.repeat(200)); + const bytes: number[] = []; + const pieces = Math.floor(next() * 80); + for (let index = 0; index < pieces; index++) bytes.push(...PIECES[Math.floor(next() * PIECES.length)]!); + return Uint8Array.from(bytes); +} + +const OFFSETS = [undefined, 0, 1, 2, 3, 2.5, -4, 50, 2001, 3000, 1e20, Number.NaN]; +const LIMITS = [undefined, 0, 1, 2, -3, -1e20, 1.5, 7, 2000, 2500, 1e20, Number.NaN]; + +function apiFor(env: NodeExecutionEnv): ToolExecutionApi { + return { + env, + output: () => {}, + outputWindow: undefined, + diagnostic: () => {}, + details: async () => {}, + } as unknown as ToolExecutionApi; +} + +const dirs: string[] = []; +afterAll(() => { + for (const dir of dirs) rmSync(dir, { recursive: true, force: true }); +}); + +async function outcome(run: () => T | Promise): Promise { + try { + return await run(); + } catch (error) { + return { error: (error as Error).message }; + } +} + +describe("read tool", () => { + it("returns exactly what reading the whole file returned", async () => { + const cwd = mkdtempSync(join(tmpdir(), "pi-durable-read-")); + dirs.push(cwd); + const env = new NodeExecutionEnv({ cwd }); + const tool = createReadTool(); + for (let seed = 1; seed <= 400; seed++) { + const next = random(seed); + const file = randomFile(next); + writeFileSync(join(cwd, "f.txt"), file); + for (let trial = 0; trial < 4; trial++) { + const offset = OFFSETS[Math.floor(next() * OFFSETS.length)]; + const limit = LIMITS[Math.floor(next() * LIMITS.length)]; + const args = { + path: "f.txt", + ...(offset === undefined ? {} : { offset }), + ...(limit === undefined ? {} : { limit }), + }; + const actual = await outcome(() => tool.execute(args, apiFor(env), BACKGROUND_CONTEXT)); + const expected = await outcome(() => referenceRead(file, "f.txt", offset, limit)); + expect(actual, `seed ${seed} offset ${offset} limit ${limit}`).toEqual(expected); + } + } + }, 120_000); + + it("detects animated PNGs whose acTL chunk lies far beyond the header", async () => { + const cwd = mkdtempSync(join(tmpdir(), "pi-durable-read-")); + dirs.push(cwd); + const chunk = (type: string, length: number): number[] => [ + (length >>> 24) & 0xff, + (length >>> 16) & 0xff, + (length >>> 8) & 0xff, + length & 0xff, + ...new TextEncoder().encode(type), + ...new Array(length + 4).fill(0), + ]; + const png = Uint8Array.from([ + 0x89, + 0x50, + 0x4e, + 0x47, + 0x0d, + 0x0a, + 0x1a, + 0x0a, + ...chunk("IHDR", 13), + ...chunk("iCCP", 200_000), + ...chunk("acTL", 8), + ...chunk("IDAT", 10), + ]); + writeFileSync(join(cwd, "a.png"), png); + const env = new NodeExecutionEnv({ cwd }); + const actual = await outcome(() => createReadTool().execute({ path: "a.png" }, apiFor(env), BACKGROUND_CONTEXT)); + expect(actual).toEqual(await outcome(() => referenceRead(png, "a.png", undefined, undefined))); + }); +});