diff --git a/packages/codemode/CHANGELOG.md b/packages/codemode/CHANGELOG.md index 3303fff6c..c70784b48 100644 --- a/packages/codemode/CHANGELOG.md +++ b/packages/codemode/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Added + +- Added `console: true` to text output items produced by `console.*`, so hosts can tell them apart from `text()` output + ## [1.0.4] - 2026-10-05 ### Changed diff --git a/packages/codemode/README.md b/packages/codemode/README.md index d2878650e..059a4b90c 100644 --- a/packages/codemode/README.md +++ b/packages/codemode/README.md @@ -39,7 +39,7 @@ await sandbox.close(); - `tools.(args)` returns a promise. Arguments and results make a JSON round trip. A tool that throws rejects with an `Error` carrying the same message. Tool names are also exposed as identifiers: characters that are not valid in identifiers become `_` (`toCodemodeIdentifier`), so `my-tool` is `tools.my_tool` as well as `tools["my-tool"]`. - `ALL_TOOLS` lists `{ name, description }` for every tool, with `name` as the identifier. -- `text(value)` appends a text item to `result.output`; values other than strings are JSON-stringified. `console.log/info/warn/error/debug` append text items too. +- `text(value)` appends a text item to `result.output`; values other than strings are JSON-stringified. `console.log/info/warn/error/debug` append text items with `console: true`. - `image(urlOrItem)` appends an image item. It accepts a base64 `data:` URL, `{ image_url }`, or an MCP `ImageContent` block (`{ type: "image", data, mimeType }`). Remote URLs are rejected. - `exit()` ends the script successfully right away, keeping its output and store writes. - `globals` passed to the sandbox are called as top-level functions, for example a host helper `image(ref)`. They behave like tools but are not recorded in `result.calls`. A name like `models.classify` puts the function on a frozen `models` object. With `spread: true`, `execute` receives all call arguments as an array instead of the first one, and `signature` replaces the declaration generated from the schemas. diff --git a/packages/codemode/src/runtime/prelude-source.ts b/packages/codemode/src/runtime/prelude-source.ts index fa2bd0c46..491608d88 100644 --- a/packages/codemode/src/runtime/prelude-source.ts +++ b/packages/codemode/src/runtime/prelude-source.ts @@ -10,7 +10,7 @@ * Output helpers: `text(value)` appends a text item (non-strings are * JSON-stringified), `image(urlOrItem)` appends an image from a base64 `data:` URL, an * `{ image_url }` object, or an MCP `ImageContent` block, and `exit()` ends the script - * successfully. `console.*` appends text items like `text()`. + * successfully. `console.*` appends text items marked as console output. * * `store(key, value)` and `load(key)` are synchronous: they work on a snapshot of * JSON text passed in as `storeJson`, and the keys the script wrote are reported @@ -23,7 +23,7 @@ * or I/O in the VM, nothing can ever resume it. * `globalsJson` lists `{ name, spread }`; `a.b` names are grouped into a frozen `a` object. * `bridge(kind, a, b, c)` with kind "call" or "global" (id, name, argsJson), - * "output" ("text", text) or ("image", data, mimeType), or "done" (ok, valueJsonOrErrorJson, writesJson). + * "output" ("text" or "console", text) or ("image", data, mimeType), or "done" (ok, valueJsonOrErrorJson, writesJson). * * Before anything else the prelude freezes the built-ins and makes the built-in globals read-only. * The prelude shares them with the script, so a script that patched one (for example @@ -441,7 +441,7 @@ export const PRELUDE_SOURCE: string = `(function (bridge, toolsJson, globalsJson const console = {}; for (const level of ["log", "info", "warn", "error", "debug"]) { console[level] = (...args) => { - output("text", args.map(format).join(" ")); + output("console", args.map(format).join(" ")); }; } Object.freeze(console); diff --git a/packages/codemode/src/runtime/worker.ts b/packages/codemode/src/runtime/worker.ts index 5450e0f59..85c7798fe 100644 --- a/packages/codemode/src/runtime/worker.ts +++ b/packages/codemode/src/runtime/worker.ts @@ -80,7 +80,9 @@ async function main(data: WorkerData): Promise { item: a.toString() === "image" ? { type: "image", data: b.toString(), mimeType: c.toString() } - : { type: "text", text: b.toString() }, + : a.toString() === "console" + ? { type: "text", text: b.toString(), console: true } + : { type: "text", text: b.toString() }, }); break; case "done": diff --git a/packages/codemode/src/types.ts b/packages/codemode/src/types.ts index 4bc4f3222..b59f819fb 100644 --- a/packages/codemode/src/types.ts +++ b/packages/codemode/src/types.ts @@ -42,9 +42,11 @@ export interface CodemodeTool { /** * One item of the script's output, in the order the script produced it: `text()` and `console.*` - * produce text items, `image()` image items. `data` is base64. + * produce text items, with `console: true` for `console.*`, and `image()` image items. `data` is base64. */ -export type CodemodeOutputItem = { type: "text"; text: string } | { type: "image"; data: string; mimeType: string }; +export type CodemodeOutputItem = + | { type: "text"; text: string; console?: true } + | { type: "image"; data: string; mimeType: string }; export type CodemodeCallStatus = "ok" | "error" | "cancelled"; diff --git a/packages/codemode/test/sandbox.test.ts b/packages/codemode/test/sandbox.test.ts index 7645dcfc1..951a57467 100644 --- a/packages/codemode/test/sandbox.test.ts +++ b/packages/codemode/test/sandbox.test.ts @@ -66,7 +66,7 @@ describe("script execution", () => { `); expect(result.ok).toBe(true); expect(result.output.slice(0, -1)).toEqual([ - { type: "text", text: 'hello 1 {"a":1}' }, + { type: "text", text: 'hello 1 {"a":1}', console: true }, { type: "text", text: '{"json":true}' }, { type: "text", text: "undefined" }, { type: "text", text: "7" }, @@ -77,7 +77,11 @@ describe("script execution", () => { { type: "image", data: WEBP, mimeType: "image/webp" }, { type: "image", data: PNG, mimeType: "image/png" }, ]); - expect(result.output.at(-1)).toMatchObject({ type: "text", text: expect.stringMatching(/^Error: bad/) }); + expect(result.output.at(-1)).toMatchObject({ + type: "text", + text: expect.stringMatching(/^Error: bad/), + console: true, + }); }); it("rejects invalid text() and image() arguments", async () => { diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 791e9d0a5..f233300ff 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -19,6 +19,7 @@ - Fixed standalone binaries loading `.env`, `.env.local`, and `.env.development` from the launch directory into Pi's environment ([#10473](https://github.com/earendil-works/pi/issues/10473)) - Fixed `!!` command headers losing their dim color once output arrives ([#10557](https://github.com/earendil-works/pi/pull/10557) by [@rwachtler](https://github.com/rwachtler)) - Fixed the codemode description not marking `searchTools()`, `describeTool()`, and `describeNamespace()` as async, which led models to serialize the unawaited promise as `{}` ([#10555](https://github.com/earendil-works/pi/issues/10555)) +- Fixed codemode output items running together, so models could not tell where one `text()` or `console.log()` output ended and the next began. With several text items, each now starts with a `==> text N/M <==` line, and `console` calls follow the other output in one `` block with one line per call ## [1.0.4] - 2026-10-05 diff --git a/packages/coding-agent/docs/codemode.md b/packages/coding-agent/docs/codemode.md index d79f987c9..1a873e36f 100644 --- a/packages/coding-agent/docs/codemode.md +++ b/packages/coding-agent/docs/codemode.md @@ -15,7 +15,7 @@ A script may start with an options line: - `max_output_tokens` (default 10000) limits the output. Longer output keeps its start and end, and the full text is written to a temp file whose path is included in the result. A script fails when its output passes 16777216 characters of text and base64 image data or 100000 `text()`, `image()`, and `console` calls; write large data to a file with a tool instead. - `timeout_ms` is a hard deadline for the whole script. It is unset by default. Image generation can take minutes, so do not set a short deadline for scripts that generate images. -The result starts with `Script completed` or `Script failed`, the wall time, and the output. A failed script keeps its partial output, followed by `Script error:` and the error. Tool calls are real: calls made before a failure are not undone. Calls still running when the script ends are cancelled, and unawaited promises are discarded. +The result starts with `Script completed` or `Script failed`, the wall time, and the output. Text and image items appear in order, each on its own line. When the output has more than one text item (from `text()` or `return`), each starts with a `==> text N/M <==` line. `console` calls follow in one `` block with one line per call. A failed script keeps its partial output, followed by `Script error:` and the error. Tool calls are real: calls made before a failure are not undone. Calls still running when the script ends are cancelled, and unawaited promises are discarded. ## Globals @@ -24,7 +24,7 @@ The result starts with `Script completed` or `Script failed`, the wall time, and | `tools.(args)` | Call a tool. See [Call tools](#call-tools). | | `text(value)` | Add a text item to the output. Strings are added as is, other values as JSON. | | `image(value)` | Add an image to the output: a base64 `data:` URL, an `{ image_url }` object, or an image block `{ type: "image", data, mimeType }` such as those returned by MCP tools and `models.generateImages()`. Remote URLs are not supported. PNG, JPEG, GIF, and WebP are accepted. Each image is also saved to a temp file, and the result names the path before the image. | -| `console.log(...)` | Like `text()`; `info`, `warn`, `error`, and `debug` do the same. | +| `console.log(...)` | Add a line to the `` block after the other output. Arguments are joined with spaces; `info`, `warn`, `error`, and `debug` do the same. | | `return value` | A top-level `return` adds the value like `text()`. | | `exit()` | End the script successfully. | | `store(key, value)` / `load(key)` | Keep small JSON values across `codemode` calls. See [Store values](#store-values). | diff --git a/packages/coding-agent/src/extensions/codemode/execute.ts b/packages/coding-agent/src/extensions/codemode/execute.ts index 630f831eb..306af4552 100644 --- a/packages/coding-agent/src/extensions/codemode/execute.ts +++ b/packages/coding-agent/src/extensions/codemode/execute.ts @@ -15,6 +15,7 @@ import type { Usage, } from "@earendil-works/pi-ai"; import { + type CodemodeOutputItem, type CodemodeResult, CodemodeSandbox, type CodemodeTool, @@ -238,6 +239,48 @@ function valueText(value: unknown): string { return JSON.stringify(value) ?? String(value); } +/** + * Lay out the script's output so the model can tell items apart: providers join adjacent text + * blocks with a newline or with nothing. With more than one text item (`text()` or the returned + * value), each starts with a `==> text N/M <==` line. `console.*` lines follow all other output in + * one `` block. + */ +function formatOutput(output: readonly CodemodeOutputItem[]): (TextContent | ImageContent)[] { + const total = output.filter((item) => item.type === "text" && !item.console).length; + const items: (TextContent | ImageContent)[] = []; + const consoleLines: string[] = []; + let index = 0; + for (const item of output) { + if (item.type === "image") { + items.push(item); + } else if (item.console) { + consoleLines.push(item.text); + } else { + index++; + items.push({ type: "text", text: total > 1 ? `==> text ${index}/${total} <==\n${item.text}` : item.text }); + } + } + if (consoleLines.length > 0) { + items.push({ type: "text", text: `\n${consoleLines.join("\n")}\n` }); + } + return items; +} + +/** Join adjacent text items into one, each part starting on its own line. */ +function joinAdjacentText(items: (TextContent | ImageContent)[]): (TextContent | ImageContent)[] { + const joined: (TextContent | ImageContent)[] = []; + for (const item of items) { + const last = joined.at(-1); + if (item.type === "text" && last?.type === "text") { + const separator = last.text === "" || last.text.endsWith("\n") ? "" : "\n"; + joined[joined.length - 1] = { type: "text", text: `${last.text}${separator}${item.text}` }; + } else { + joined.push(item); + } + } + return joined; +} + function formatCallSummary(calls: readonly CodemodeNestedCall[]): string { if (calls.length === 0) return "No tool calls were made."; return `Tool calls made before the failure (they are not undone): ${calls.map((call) => `${call.name} (${call.status})`).join(", ")}`; @@ -442,19 +485,17 @@ export async function executeCodemode( if (call.status === "running") call.status = "cancelled"; } - const items: (TextContent | ImageContent)[] = result.output.map((item) => - item.type === "text" ? { type: "text", text: item.text } : item, - ); + const scriptOutput = [...result.output]; if (result.ok) { const { set, delete: deleted } = result.storeWrites; if (Object.keys(set).length > 0 || deleted.length > 0) { options.appendEntry?.(CODEMODE_STORE_ENTRY_TYPE, { set, delete: deleted }); } // pi extension: a returned value is appended like text(). - if (result.value !== undefined) items.push({ type: "text", text: valueText(result.value) }); - } else { - items.push({ type: "text", text: `Script error:\n${formatError(result, calls)}` }); + if (result.value !== undefined) scriptOutput.push({ type: "text", text: valueText(result.value) }); } + const items = formatOutput(scriptOutput); + if (!result.ok) items.push({ type: "text", text: `Script error:\n${formatError(result, calls)}` }); if (generatedImages > 0 && !items.some((item) => item.type === "image")) { items.push({ type: "text", @@ -462,10 +503,13 @@ export async function executeCodemode( }); } - const truncated = await truncateOutput(items, sourceOptions.maxOutputTokens ?? DEFAULT_MAX_OUTPUT_TOKENS); + const truncated = await truncateOutput( + joinAdjacentText(items), + sourceOptions.maxOutputTokens ?? DEFAULT_MAX_OUTPUT_TOKENS, + ); // After truncation, which joins the text items and moves images after them, so each path stays // next to its image and is never cut. - const output = await saveImages(truncated.items); + const output = joinAdjacentText(await saveImages(truncated.items)); const wallTime = ((performance.now() - startedAt) / 1000).toFixed(1); const header = `${result.ok ? "Script completed" : "Script failed"}\nWall time ${wallTime} seconds\nOutput:\n`; const details = snapshot(); diff --git a/packages/coding-agent/src/extensions/codemode/tool.ts b/packages/coding-agent/src/extensions/codemode/tool.ts index 2aaf16262..ae8285a3e 100644 --- a/packages/coding-agent/src/extensions/codemode/tool.ts +++ b/packages/coding-agent/src/extensions/codemode/tool.ts @@ -142,7 +142,7 @@ const DESCRIPTION_INTRO = `Run JavaScript that calls other tools. The input is r function describeGlobals(models: boolean): string { const lines = [ "Globals:", - "- `text(value)`, `image(dataUrlOrImageBlock)`, `console.log(...)`, and top-level `return` add output; `exit()` ends the script. `image()` also saves the image to a temp file and the result names its path.", + "- `text(value)`, `image(dataUrlOrImageBlock)`, `console.log(...)`, and top-level `return` add output; `exit()` ends the script. With several text items, each starts with a `==> text N/M <==` line, and `console` lines follow the other output in one `` block. `image()` also saves the image to a temp file and the result names its path.", "- `store(key, value)` and `load(key)` keep JSON values across codemode calls.", "- `ALL_TOOLS`, `await searchTools(query, { limit?, namespace? })`, `await describeTool(name)`, `await describeNamespace(name)`: find unlisted tools, such as MCP tools.", ]; diff --git a/packages/coding-agent/test/suite/agent-session-codemode.test.ts b/packages/coding-agent/test/suite/agent-session-codemode.test.ts index 5da2d884e..c4cf0b94b 100644 --- a/packages/coding-agent/test/suite/agent-session-codemode.test.ts +++ b/packages/coding-agent/test/suite/agent-session-codemode.test.ts @@ -231,7 +231,7 @@ describe("AgentSession codemode tool", () => { const result = codemodeResult(harness); expect(result.isError).toBe(false); expect(resultText(result)).toBe( - 'files 2\necho,stats,screenshot\n{"a":"echo: one","b":"echo: two","names":["a","b"]}', + '==> text 1/2 <==\necho,stats,screenshot\n==> text 2/2 <==\n{"a":"echo: one","b":"echo: two","names":["a","b"]}\n\nfiles 2\n', ); const details = result.details as unknown as CodemodeToolDetails; expect(details.calls.map((call) => [call.name, call.status])).toEqual([ @@ -379,9 +379,42 @@ describe("AgentSession codemode tool", () => { const result = codemodeResult(harness); // The same image shown twice is saved once, so both labels name one file. const lines = resultText(result).split("\n"); - expect(lines).toEqual(["captured", lines[1], "", lines[1], "", "after"]); - expect(checkSavedImages(lines[1])).toBe(""); - expect(result.content[3]).toEqual({ type: "image", data: TINY_PNG_BASE64, mimeType: "image/png" }); + expect(lines).toEqual([ + "==> text 1/2 <==", + "captured", + lines[2], + "", + lines[2], + "", + "==> text 2/2 <==", + "after", + ]); + expect(checkSavedImages(lines[2])).toBe(""); + expect(result.content[2]).toEqual({ type: "image", data: TINY_PNG_BASE64, mimeType: "image/png" }); + }); + + it("marks where each text item starts and puts console lines last in one text block", async () => { + const harness = await setup(); + harness.setResponses([ + fauxAssistantMessage( + [ + fauxToolCall("codemode", { + code: 'text("one\\ntwo");\nconsole.log("a");\nconsole.log("b");\ntext("three\\n");\nreturn 4;', + }), + ], + { stopReason: "toolUse" }, + ), + fauxAssistantMessage("done"), + ]); + + await harness.session.prompt("go"); + + // Providers join adjacent text blocks with nothing or a newline, so the output is one block. + const result = codemodeResult(harness); + expect(result.content).toHaveLength(2); + expect(resultText(result)).toBe( + "==> text 1/3 <==\none\ntwo\n==> text 2/3 <==\nthree\n==> text 3/3 <==\n4\n\na\nb\n", + ); }); it("reports script failures as results that keep partial output and the calls that ran", async () => { @@ -390,7 +423,7 @@ describe("AgentSession codemode tool", () => { fauxAssistantMessage( [ fauxToolCall("codemode", { - code: `text("partial");\nawait tools.echo({ text: "x" });\nthrow new Error("boom");`, + code: `text("partial");\nconsole.log("log");\nawait tools.echo({ text: "x" });\nthrow new Error("boom");`, }), ], { stopReason: "toolUse" }, @@ -404,8 +437,8 @@ describe("AgentSession codemode tool", () => { expect(result.isError).toBe(true); expect((result.content[0] as { text: string }).text).toMatch(/^Script failed\n/); const text = resultText(result); - expect(text).toMatch(/^partial\nScript error:\nError: boom\n/); - expect(text).toContain("codemode.js:3"); + expect(text).toMatch(/^partial\n\nlog\n<\/console_output>\nScript error:\nError: boom\n/); + expect(text).toContain("codemode.js:4"); expect(text).toContain("Tool calls made before the failure (they are not undone): echo (ok)"); expect((result.details as unknown as CodemodeToolDetails).calls.map((call) => call.name)).toEqual(["echo"]); }); @@ -481,7 +514,7 @@ describe("codemode options and store", () => { const harness = await setup(); const result = await run( harness, - `// @options: {"max_output_tokens": 10}\nfor (let i = 0; i < 100; i++) text("row " + i);\nimage("data:image/png;base64,${TINY_PNG_BASE64}");`, + `// @options: {"max_output_tokens": 30}\nfor (let i = 0; i < 100; i++) text("row " + i);\nimage("data:image/png;base64,${TINY_PNG_BASE64}");`, ); const details = result.details as unknown as CodemodeToolDetails; const path = details.fullOutputPath; @@ -497,7 +530,9 @@ describe("codemode options and store", () => { // Images follow the truncated text, each after the path it was saved to. expect(result.content.at(-1)).toEqual({ type: "image", data: TINY_PNG_BASE64, mimeType: "image/png" }); expect(checkSavedImages(text.split("\n").at(-2) ?? "")).toBe(""); - expect(readFileSync(path, "utf8")).toBe(Array.from({ length: 100 }, (_, i) => `row ${i}`).join("\n")); + expect(readFileSync(path, "utf8")).toBe( + Array.from({ length: 100 }, (_, i) => `==> text ${i + 1}/100 <==\nrow ${i}`).join("\n"), + ); } finally { rmSync(path, { force: true }); } @@ -534,7 +569,9 @@ describe("codemode options and store", () => { 'text(await tools.read({ path: "notes.txt" }));\nconst shot = await tools.read({ path: "pixel.png" });\ntext(shot.note);\nimage(shot);', ); expect(result.isError).toBe(false); - expect(checkSavedImages(resultText(result))).toBe("hello\nRead image file [image/png]\n\n"); + expect(checkSavedImages(resultText(result))).toBe( + "==> text 1/2 <==\nhello\n==> text 2/2 <==\nRead image file [image/png]\n\n", + ); expect(result.content.at(-1)).toEqual({ type: "image", data: TINY_PNG_BASE64, mimeType: "image/png" }); }); @@ -816,18 +853,20 @@ describe("codemode models", () => { `, ); expect(result.isError).toBe(false); - const [text, ...rest] = checkSavedImages(resultText(result)).split("\n"); + const [header, text, ...rest] = checkSavedImages(resultText(result)).split("\n"); + expect(header).toBe("==> text 1/2 <=="); expect(text).toBe("painted a fox"); expect(rest[0]).toBe(""); expect(rest[1]).toBe(""); - expect(JSON.parse(rest.slice(2).join("\n"))).toEqual({ + expect(rest[2]).toBe("==> text 2/2 <=="); + expect(JSON.parse(rest.slice(3).join("\n"))).toEqual({ id: "painter", stopReason: "stop", failed: ["error", "painter exploded"], wrongType: '"scorer/judge" is a classifier model, not an image model. List the image models you can use with models.getAvailableOfType("image").', }); - expect(result.content[3]).toEqual({ type: "image", data: TINY_PNG_BASE64, mimeType: "image/png" }); + expect(result.content[2]).toEqual({ type: "image", data: TINY_PNG_BASE64, mimeType: "image/png" }); expect(imageRequests.map((request) => [request.baseUrl, request.apiKey])).toEqual([ ["https://images.test/v1", "secret-key"], ["https://images.test/v1", "secret-key"],