diff --git a/packages/durable/CHANGELOG.md b/packages/durable/CHANGELOG.md index 4032be47c..1b3515e89 100644 --- a/packages/durable/CHANGELOG.md +++ b/packages/durable/CHANGELOG.md @@ -178,6 +178,7 @@ - Fixed a commit that Storage rejects after adopting it (a failed JSONL reclaim) publishing nothing: it is published and its waits resolve before the Session fails, so a replica built from `subscribeCommits()` stays equal to Storage and `pi.durable.tasks.active` stays balanced. - Fixed a throwing telemetry adapter (a commit's `pi.durable.entry` or `pi.durable.submission` event, `pi.durable.tasks.active`) rejecting a durable commit and leaving the views and the scheduler without it: it is reported as `host_error` with callback `"telemetry"`, and the commit resolves. A throwing report metric or report event is dropped. - Fixed a host that crashes during a model request resending it forever, adding an aborted entry each time: the interrupted request is a failed attempt under the retry policy, and the run ends with `runFailure` reason `interrupted` once no retry is left; other work cut off `restartLimit` times ends instead of keeping its conversation busy ([#11139](https://github.com/earendil-works/pi/issues/11139)) +- Fixed the built-in `read` resolving to `""` for an image in Code Mode scripts, so `image(await tools.read(...))` threw: `read` declares a `structuredOutputSchema`, a text file's text or an image's `ImageContent`, as the coding agent's `read` does ([#11015](https://github.com/earendil-works/pi/issues/11015)) - Fixed the Code Mode extension loading pi-codemode's WASM QuickJS runtime (`quickjs-wasi`) although it only runs scripts in the sandbox the host passes: it imports `toCodemodeIdentifier` from `@earendil-works/pi-codemode/declarations` instead of `/portable` ([#11254](https://github.com/earendil-works/pi/issues/11254)) ## [1.4.0] - 2026-10-21 diff --git a/packages/durable/README.md b/packages/durable/README.md index 71c05bb48..5c2b28b68 100644 --- a/packages/durable/README.md +++ b/packages/durable/README.md @@ -292,7 +292,7 @@ const lines = defineTool({ }); ``` -A missing, undeclared, or invalid `structuredOutput` turns a nested call's result into an error result with an `invalid_structured_output` diagnostic; for a call the model made, which never sees it, it is dropped and reported through `onReport`. An error result may omit it, and programs then find its text in a `tool_error` diagnostic; error results the Harness writes itself (blocked, invalid, interrupted, aborted) carry none, even when an `afterTool` hook returns one, and their `diagnostics` say what went wrong; one a hook turns into a success gives a schema-less tool's program its output. An `afterTool` hook that redacts a schema tool's `output` must redact its `structuredOutput` too. The built-in `bash` returns `{ output, truncated, fullOutputPath?, exitCode }`, where `output` is the retained tail the model sees, read with `api.retainedOutput()`, and answers a nonzero exit with an error result that still carries it. +A missing, undeclared, or invalid `structuredOutput` turns a nested call's result into an error result with an `invalid_structured_output` diagnostic; for a call the model made, which never sees it, it is dropped and reported through `onReport`. An error result may omit it, and programs then find its text in a `tool_error` diagnostic; error results the Harness writes itself (blocked, invalid, interrupted, aborted) carry none, even when an `afterTool` hook returns one, and their `diagnostics` say what went wrong; one a hook turns into a success gives a schema-less tool's program its output. An `afterTool` hook that redacts a schema tool's `output` must redact its `structuredOutput` too. The built-in `bash` returns `{ output, truncated, fullOutputPath?, exitCode }`, where `output` is the retained tail the model sees, read with `api.retainedOutput()`, and answers a nonzero exit with an error result that still carries it. The built-in `read` returns a text file's text as a string and an image as its `ImageContent`, which code mode's `image()` shows. A later extension's tool with the same name replaces an earlier one where both are selected, and `wrapTool()` decorates whichever tool won: diff --git a/packages/durable/src/tools/read.ts b/packages/durable/src/tools/read.ts index cede96ef3..b0312f3b3 100644 --- a/packages/durable/src/tools/read.ts +++ b/packages/durable/src/tools/read.ts @@ -39,6 +39,15 @@ const readSchema = Type.Object({ export type ReadToolInput = Static; +/** + * What programs such as code mode scripts get: the text of a text file, or the image block of an image, which code + * mode's `image()` shows. + */ +const readStructuredOutputSchema = Type.Union([ + Type.String(), + Type.Object({ type: Type.Literal("image"), data: Type.String(), mimeType: Type.String() }), +]); + export type ReadToolDetails = { /** How the shown text was cut; the text itself is the result content. */ truncation?: Omit; @@ -84,7 +93,7 @@ export type ReadToolOptions = { /** * Reads text files and images. Remarks about truncation, continuation, and image processing are diagnostics; the content - * is only file text, or one image, which programs get as its `ImageContent`. + * is only file text, or one image. Programs get the text, or the image's `ImageContent`. */ export function createReadTool(options: ReadToolOptions = {}): ToolRegistration { const formats = options.images === undefined ? "jpg, png, gif, webp" : "jpg, png, gif, webp, bmp"; @@ -92,6 +101,7 @@ export function createReadTool(options: ReadToolOptions = {}): ToolRegistration< name: "read", description: `Read the contents of a file. Supports text files and images (${formats}). Images are sent as attachments. For text files, output is truncated to ${DEFAULT_MAX_LINES} lines or ${DEFAULT_MAX_BYTES / 1024}KB (whichever is hit first). Use offset/limit for large files. When you need the full file, continue with offset until complete.`, parameters: readSchema, + structuredOutputSchema: readStructuredOutputSchema, async execute(args, api, context) { const { path, offset, limit } = args; const env = requireEnv(api); @@ -187,7 +197,8 @@ async function readImage( const diagnostics: ToolDiagnostic[] = [ { severity: "info", code: "image", message: [`Read image file [${image.mimeType}].`, ...notes].join(" ") }, ]; - return { output: [{ type: "image", data: image.data, mimeType: image.mimeType }], diagnostics }; + const block = { type: "image" as const, data: image.data, mimeType: image.mimeType }; + return { output: [block], structuredOutput: block, diagnostics }; } /** What the conversation's model takes: whether images, and within which limits. */ @@ -306,6 +317,7 @@ async function readText( return { output: outputText === "" ? [] : [{ type: "text" as const, text: outputText }], + structuredOutput: outputText, ...(details === undefined ? {} : { details }), diagnostics, }; diff --git a/packages/durable/test/codemode.test.ts b/packages/durable/test/codemode.test.ts index 460836d72..e96b9b773 100644 --- a/packages/durable/test/codemode.test.ts +++ b/packages/durable/test/codemode.test.ts @@ -1,4 +1,4 @@ -import { mkdtemp, readdir, readFile, rm } from "node:fs/promises"; +import { mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import type { Context } from "@earendil-works/chord"; @@ -24,6 +24,7 @@ import { import type { CodemodeNamespace } from "../src/codemode/describe.ts"; import { NodeExecutionEnv } from "../src/env/node.ts"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; +import { createReadTool } from "../src/tools/read.ts"; import { allEntries, type ChatSetup, chatSetup, openChat } from "./chat-support.ts"; import { addHooks, addTool } from "./harness-support.ts"; import { ControlledStorage, context } from "./session-support.ts"; @@ -1047,6 +1048,29 @@ describe("durable code mode output and tool search", () => { expect(message.content.some((item) => item.type === "image")).toBe(true); }); + // #11015 + it("resolves the built-in read with an image's block, which image() shows, and with a text file's text", async () => { + const png = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg=="; + const files = await mkdtemp(join(tmpdir(), "pi-codemode-read-")); + directories.add(files); + await writeFile(join(files, "pixel.png"), Buffer.from(png, "base64")); + await writeFile(join(files, "note.txt"), "hello"); + const chat = codemodeChat(); + addTool(chat.setup.registry, createReadTool()); + const { text, message } = await runScript( + `const picture = await tools.read({ path: ${JSON.stringify(join(files, "pixel.png"))} }); + text([picture.type, picture.mimeType, picture.data.length].join(" ")); + image(picture); + text(await tools.read({ path: ${JSON.stringify(join(files, "note.txt"))} }));`, + chat, + ); + expect(message.isError).toBe(false); + expect(text).toContain(`image image/png ${png.length}`); + expect(text).toMatch(/\[Image saved to \S+ \(image\/png, \d+ bytes\)\]/); + expect(text).toContain("hello"); + expect(message.content.some((item) => item.type === "image")).toBe(true); + }); + it("calls a deferred tool the model is not offered, and finds it in ALL_TOOLS", async () => { const chat = codemodeChat(); addTool( diff --git a/packages/durable/test/tools-read-differential.test.ts b/packages/durable/test/tools-read-differential.test.ts index e8486be10..b1e30fe4c 100644 --- a/packages/durable/test/tools-read-differential.test.ts +++ b/packages/durable/test/tools-read-differential.test.ts @@ -17,8 +17,10 @@ import { DEFAULT_MAX_BYTES, formatSize, truncateHead } from "../src/truncate.ts" function referenceRead(bytes: Uint8Array, path: string, offset: number | undefined, limit: number | undefined) { const mimeType = detectSupportedImageMimeType(bytes); if (mimeType) { + const block = { type: "image", data: Buffer.from(bytes).toString("base64"), mimeType }; return { - output: [{ type: "image", data: Buffer.from(bytes).toString("base64"), mimeType }], + output: [block], + structuredOutput: block, diagnostics: [{ severity: "info", code: "image", message: `Read image file [${mimeType}].` }], }; } @@ -73,6 +75,7 @@ function referenceRead(bytes: Uint8Array, path: string, offset: number | undefin } return { output: outputText === "" ? [] : [{ type: "text", text: outputText }], + structuredOutput: outputText, ...(details === undefined ? {} : { details }), diagnostics, };