From 4cf69332c3355c0023982d90650830a66617be60 Mon Sep 17 00:00:00 2001 From: "robopipi[bot]" <336283572+robopipi[bot]@users.noreply.github.com> Date: Wed, 4 Nov 2026 09:36:54 +0100 Subject: [PATCH] fix(coding-agent): resize images in queued steering and follow-up messages (#11782) Images in idle prompts go through `processImage`: they are resized to the model's limits, and a coordinate hint is added to the text. Images in messages queued while the agent was busy skipped this step. That covers `steer()`, `followUp()`, `prompt()` with `streamingBehavior`, and RPC `steer`/`follow_up`. Those images were sent and saved at full size. For example, a 1200x9000 PNG stayed 1200x9000, which is over Anthropic's 8000 px limit, and because it is saved in the session it was replayed on every later request. What the change does: - `_queueSteer`/`_queueFollowUp` now mark queued user messages that carry images. - When such a message is delivered (`message_start`), `AgentSession` normalizes its images in place with `_normalizePromptImages`, the same code the idle prompt path uses, and appends the resize hints to its text. This runs before extensions and listeners see the message, before it is saved, and before it is added to the request context. - Doing it at delivery keeps enqueueing synchronous. A slow resize therefore cannot leave a message queued after the run has ended. I hit exactly that when I first tried normalizing at enqueue time. Delivery is also when the resize uses the model current at that moment. - The queue display still matches and removes the message, because matching runs before the text is changed. Testing: - New `test/agent-session-queued-images.test.ts`: a 1200x9000 PNG is queued while the agent is streaming, through `steer()`, `followUp()` and `prompt(..., { streamingBehavior: "followUp" })`. The test checks three things: each saved message has a resized image and the "original 1200x9000, displayed at 267x2000" hint; no request sent to the model contains the original image; and the queue is empty afterwards. It fails without the change and passes with it. - `npm run check` passes. The `agent-session*` tests, the queue- and image-related tests and `test/suite` pass. - Before the change, the saved steer and follow-up messages had `image/png 1200x9000` while the idle prompt had a resized JPEG. After it, all three are resized. Fixes #11738 Requested by @badlogic Co-authored-by: triagent --- packages/coding-agent/CHANGELOG.md | 1 + .../coding-agent/src/core/agent-session.ts | 36 +++++-- .../test/agent-session-queued-images.test.ts | 94 +++++++++++++++++++ 3 files changed, 125 insertions(+), 6 deletions(-) create mode 100644 packages/coding-agent/test/agent-session-queued-images.test.ts diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index f9ca79449..39aaadcf0 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -14,6 +14,7 @@ ### Fixed +- Images in steering and follow-up messages, including `prompt()` while streaming, are now resized like images in idle prompts instead of being sent and saved at full size ([#11738](https://github.com/earendil-works/pi/issues/11738)) - A failed token refresh of an MCP connection no longer registers an OAuth client or replaces the PKCE verifier of a sign-in in progress, which made that sign-in fail. MCP sign-in now refreshes the stored tokens with the stored client even when the loopback callback port changed, instead of registering a new client and opening the browser. Ctrl+C in `/mcp` and the `--timeout` of `pi mcp login` now also stop a sign-in while it waits up to 25 seconds for another pi process's token refresh, and a refresh lock that fails for another reason than that wait now fails the sign-in instead of opening the browser. - Fixed RPC mode ignoring an extension's `ctx.shutdown()` while idle, for example from a timer: the process kept running until the client sent another command. pi now shuts down right away when idle, and a shutdown requested by an extension command waits until the command's `prompt` response is written ([#11599](https://github.com/earendil-works/pi/issues/11599)) - Fixed changelog notes repeating on every fresh launch when the installed package version has a prerelease suffix, such as `0.1.5-20261020.7`: the saved version was read as `0.1.0`, so the `0.1.1` to `0.1.5` notes counted as new each time. A saved version without `major.minor.patch` now shows no notes and is replaced by the installed version, instead of showing the whole changelog ([#11703](https://github.com/earendil-works/pi/issues/11703)) diff --git a/packages/coding-agent/src/core/agent-session.ts b/packages/coding-agent/src/core/agent-session.ts index dbe36c365..5d341c4d7 100644 --- a/packages/coding-agent/src/core/agent-session.ts +++ b/packages/coding-agent/src/core/agent-session.ts @@ -43,6 +43,7 @@ import type { TextContent, ToolResultMessage, Usage, + UserMessage, } from "@earendil-works/pi-ai/compat"; import { clampThinkingLevel, @@ -408,6 +409,8 @@ export class AgentSession { private _steeringMessages: string[] = []; /** Tracks pending follow-up messages for UI display. Removed when delivered. */ private _followUpMessages: string[] = []; + /** Queued user messages whose images are normalized when they are delivered. */ + private _queuedMessagesWithImages = new WeakSet(); /** Messages queued to be included with the next user prompt as context ("asides"). */ private _pendingNextTurnMessages: CustomMessage[] = []; /** Context-only custom messages queued during a run, flushed once the current turn's tool results are in. */ @@ -1153,6 +1156,12 @@ export class AgentSession { } } } + // Normalize queued images at delivery, before events and persistence see the message, + // so they get the same resize as idle prompts with the current model's limits (#11738). + if (this._queuedMessagesWithImages.has(event.message)) { + this._queuedMessagesWithImages.delete(event.message); + await this._normalizeQueuedMessageImages(event.message); + } } // Emit to extensions first, then notify public listeners. @@ -2327,11 +2336,9 @@ export class AgentSession { if (images) { content.push(...images); } - this.agent.steer({ - role: "user", - content, - timestamp: Date.now(), - }); + const message: AgentMessage = { role: "user", content, timestamp: Date.now() }; + if (images?.length) this._queuedMessagesWithImages.add(message); + this.agent.steer(message); } /** @@ -2344,7 +2351,24 @@ export class AgentSession { if (images) { content.push(...images); } - this.agent.followUp({ role: "user", content, timestamp: Date.now() }); + const message: AgentMessage = { role: "user", content, timestamp: Date.now() }; + if (images?.length) this._queuedMessagesWithImages.add(message); + this.agent.followUp(message); + } + + /** Replace a queued user message's images in place with normalized ones, appending hints to its text. */ + private async _normalizeQueuedMessageImages(message: UserMessage): Promise { + if (typeof message.content === "string") return; + const [first, ...rest] = message.content; + if (first?.type !== "text") return; + const images = rest.filter((block): block is ImageContent => block.type === "image"); + const normalized = await this._normalizePromptImages(images); + const text = normalized.hints.length > 0 ? `${first.text}\n\n${normalized.hints.join("\n")}` : first.text; + message.content = [ + { type: "text", text }, + ...rest.filter((block) => block.type !== "image"), + ...normalized.images, + ]; } /** diff --git a/packages/coding-agent/test/agent-session-queued-images.test.ts b/packages/coding-agent/test/agent-session-queued-images.test.ts new file mode 100644 index 000000000..800445bf3 --- /dev/null +++ b/packages/coding-agent/test/agent-session-queued-images.test.ts @@ -0,0 +1,94 @@ +import { deflateSync } from "node:zlib"; +import type { ImageContent, TextContent } from "@earendil-works/pi-ai"; +import { afterEach, describe, expect, it } from "vitest"; +import { createHarness, type Harness } from "./test-harness.ts"; + +function crc32(data: Buffer): number { + let crc = 0xffffffff; + for (const byte of data) { + crc ^= byte; + for (let i = 0; i < 8; i++) crc = (crc >>> 1) ^ (0xedb88320 & -(crc & 1)); + } + return (crc ^ 0xffffffff) >>> 0; +} + +function pngChunk(type: string, data: Buffer): Buffer { + const length = Buffer.alloc(4); + length.writeUInt32BE(data.length); + const body = Buffer.concat([Buffer.from(type, "ascii"), data]); + const crc = Buffer.alloc(4); + crc.writeUInt32BE(crc32(body)); + return Buffer.concat([length, body, crc]); +} + +/** A solid gray RGB PNG of the given size, base64 encoded. */ +function createPng(width: number, height: number): string { + const header = Buffer.alloc(13); + header.writeUInt32BE(width, 0); + header.writeUInt32BE(height, 4); + header[8] = 8; + header[9] = 2; + const row = Buffer.alloc(1 + width * 3, 0x80); + row[0] = 0; + const raw = Buffer.concat(Array.from({ length: height }, () => row)); + return Buffer.concat([ + Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), + pngChunk("IHDR", header), + pngChunk("IDAT", deflateSync(raw)), + pngChunk("IEND", Buffer.alloc(0)), + ]).toString("base64"); +} + +function persistedUserContent(harness: Harness, text: string): (TextContent | ImageContent)[] { + for (const entry of harness.sessionManager.getEntries()) { + if (entry.type !== "message" || entry.message.role !== "user") continue; + const content = entry.message.content; + if (typeof content === "string") continue; + if (content[0]?.type === "text" && content[0].text.split("\n")[0] === text) return content; + } + throw new Error(`No user message "${text}"`); +} + +async function waitFor(condition: () => boolean): Promise { + for (let i = 0; i < 200 && !condition(); i++) await new Promise((resolve) => setTimeout(resolve, 25)); +} + +describe("AgentSession queued message images", () => { + let harness: Harness | undefined; + + afterEach(() => { + harness?.cleanup(); + harness = undefined; + }); + + // Regression test for #11738: queued messages skipped the image resize idle prompts get. + it("resizes images in steering, follow-up, and streaming prompt messages", async () => { + harness = await createHarness({ responses: [{ text: "first", delayMs: 300 }, "second", "third", "fourth"] }); + const image: ImageContent = { type: "image", data: createPng(1200, 9000), mimeType: "image/png" }; + + const run = harness.session.prompt("idle"); + await waitFor(() => harness!.session.isStreaming); + await harness.session.steer("steer", [image]); + await harness.session.followUp("follow up", [image]); + await harness.session.prompt("streaming prompt", { images: [image], streamingBehavior: "followUp" }); + await run; + await waitFor(() => !harness!.session.isStreaming && harness!.faux.callCount >= 4); + + for (const text of ["steer", "follow up", "streaming prompt"]) { + const content = persistedUserContent(harness, text); + const images = content.filter((block): block is ImageContent => block.type === "image"); + expect(images).toHaveLength(1); + expect(images[0].data).not.toBe(image.data); + expect((content[0] as TextContent).text).toContain("original 1200x9000, displayed at 267x2000"); + } + const sentImages = harness.faux.contexts + .flatMap((context) => context.messages) + .flatMap((message): readonly { type: string }[] => + typeof message.content === "string" ? [] : message.content, + ) + .filter((block): block is ImageContent => block.type === "image"); + expect(sentImages.length).toBeGreaterThan(0); + expect(sentImages.some((block) => block.data === image.data)).toBe(false); + expect(harness.session.pendingMessageCount).toBe(0); + }); +});