diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 217ff31b2..b8abc0270 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -5,6 +5,7 @@ ### Fixed - Fixed syntax highlighting losing colors after the first line of multiline strings and comments in fenced code blocks ([#10143](https://github.com/earendil-works/pi/issues/10143)) +- Fixed codemode scripts not receiving images from `read`: `tools.read()` now resolves to an image block for image files, which `image()` shows ([#10251](https://github.com/earendil-works/pi/issues/10251)) ## [1.0.3] - 2026-10-05 diff --git a/packages/coding-agent/docs/codemode.md b/packages/coding-agent/docs/codemode.md index 7b189afb0..764413ce2 100644 --- a/packages/coding-agent/docs/codemode.md +++ b/packages/coding-agent/docs/codemode.md @@ -42,7 +42,8 @@ What a call resolves to depends on the tool: - Tools with an output schema resolve to a structured value. `bash` resolves to `{ output, truncated, full_output_path?, exit_code, wall_time_seconds }`, also for non-zero exit codes. Its `output` is not limited to the 2000 lines or 50KB the model sees: it holds up to 1 MiB, and longer output keeps its first and last 512 KiB around an omission marker, with `truncated` set and the full output in `full_output_path`. - MCP tools resolve to their `CallToolResult`, including `isError` and `structuredContent`. -- Other tools, such as `read`, `edit`, and `write`, resolve to their text output. +- `read` resolves to the file's text, or for an image to an image block `{ type: "image", data, mimeType, note }` that `image()` shows. `data` is the base64 image the model would see and `note` the text that goes with it, such as resize hints. +- Other tools, such as `edit` and `write`, resolve to their text output. A call that fails, is blocked, or gets invalid arguments rejects with an `Error` that carries the tool's error text. Use `Promise.allSettled()` to keep the results of the calls that succeed. diff --git a/packages/coding-agent/src/core/tools/read.ts b/packages/coding-agent/src/core/tools/read.ts index 91034861b..ebe2c2517 100644 --- a/packages/coding-agent/src/core/tools/read.ts +++ b/packages/coding-agent/src/core/tools/read.ts @@ -24,6 +24,19 @@ export const readToolSystemPromptContribution = { export type ReadToolInput = Static; +/** + * Result for programmatic callers such as codemode scripts: the text for text files, and an image + * block for images that codemode's `image()` accepts. `note` is the text that goes with the image, + * such as resize hints. Property descriptions are left out so the type stays on one line in tool + * descriptions. + */ +const readOutputSchema = Type.Union([ + Type.String(), + Type.Object({ type: Type.Literal("image"), data: Type.String(), mimeType: Type.String(), note: Type.String() }), +]); + +export type ReadToolOutput = Static; + export interface ReadToolDetails { truncation?: TruncationResult; } @@ -56,6 +69,13 @@ export interface ReadToolOptions { operations?: ReadOperations; } +/** The image block and its note, or the text for text files and images that could not be processed. */ +function toReadOutput(content: (TextContent | ImageContent)[]): ReadToolOutput { + const text = content.find((block) => block.type === "text")?.text ?? ""; + const image = content.find((block) => block.type === "image"); + return image ? { type: "image", data: image.data, mimeType: image.mimeType, note: text } : text; +} + function getNonVisionImageNote(model: Model | undefined): string | undefined { if (!model || model.input.includes("image")) { return undefined; @@ -77,6 +97,7 @@ export function createReadToolDefinition( promptSnippet: readToolSystemPromptContribution.snippet, promptGuidelines: [...readToolSystemPromptContribution.guidelines], parameters: readSchema, + outputSchema: readOutputSchema, constrainedSampling: { type: "json_schema", strict: "prefer" }, async execute( _toolCallId, @@ -192,7 +213,7 @@ export function createReadToolDefinition( } })(); }, - ); + ).then((result) => ({ ...result, structuredContent: toReadOutput(result.content) })); }, ...readRenderers, }; diff --git a/packages/coding-agent/test/suite/agent-session-codemode.test.ts b/packages/coding-agent/test/suite/agent-session-codemode.test.ts index a4621e378..58054a493 100644 --- a/packages/coding-agent/test/suite/agent-session-codemode.test.ts +++ b/packages/coding-agent/test/suite/agent-session-codemode.test.ts @@ -1,4 +1,5 @@ -import { readFileSync, rmSync } from "node:fs"; +import { readFileSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; import type { AgentTool } from "@earendil-works/pi-agent-core"; import { type AssistantImages, @@ -497,6 +498,24 @@ describe("codemode options and store", () => { expect(resultText(result)).toBe('["out\\n",3,"number"]'); }); + // https://github.com/earendil-works/pi/issues/10251 + it("resolves read calls to text for text files and to image blocks that image() shows", async () => { + const harness = await createHarness({ + initialActiveToolNames: ["codemode", "read"], + extensionFactories: [createCodemodeExtension()], + }); + harnesses.push(harness); + writeFileSync(join(harness.tempDir, "notes.txt"), "hello"); + writeFileSync(join(harness.tempDir, "pixel.png"), Buffer.from(TINY_PNG_BASE64, "base64")); + const result = await run( + harness, + 'text(await tools.read({ path: "notes.txt" }));\nconst shot = await tools.read({ path: "pixel.png" });\ntext(shot.note);\nimage(shot);', + ); + expect(result.isError).toBe(false); + expect(checkSavedImages(resultText(result))).toBe("hello\nRead image file [image/png]\n\n"); + expect(result.content.at(-1)).toEqual({ type: "image", data: TINY_PNG_BASE64, mimeType: "image/png" }); + }); + it("persists store() writes as custom entries for later calls", async () => { const harness = await setup(); expect(resultText(await run(harness, increment))).toBe("1"); diff --git a/packages/coding-agent/test/tools.test.ts b/packages/coding-agent/test/tools.test.ts index eb9e0a8ad..ae1644396 100644 --- a/packages/coding-agent/test/tools.test.ts +++ b/packages/coding-agent/test/tools.test.ts @@ -89,6 +89,7 @@ describe("Coding Agent Tools", () => { // No truncation message since file fits within limits expect(getTextOutput(result)).not.toContain("Use offset="); expect(result.details).toBeUndefined(); + expect(result.structuredContent).toBe(content); }); it("should handle non-existent files", async () => { @@ -217,6 +218,13 @@ describe("Coding Agent Tools", () => { expect(imageBlock?.mimeType).toBe("image/png"); expect(typeof imageBlock?.data).toBe("string"); expect((imageBlock?.data ?? "").length).toBeGreaterThan(0); + // Programmatic callers get the same image as a block. https://github.com/earendil-works/pi/issues/10251 + expect(result.structuredContent).toEqual({ + type: "image", + data: imageBlock?.data, + mimeType: "image/png", + note: getTextOutput(result), + }); }); it("should read BMP files from disk as PNG image attachments", async () => {