diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index f9ca79449..39aaadcf0 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -14,6 +14,7 @@ ### Fixed +- Images in steering and follow-up messages, including `prompt()` while streaming, are now resized like images in idle prompts instead of being sent and saved at full size ([#11738](https://github.com/earendil-works/pi/issues/11738)) - A failed token refresh of an MCP connection no longer registers an OAuth client or replaces the PKCE verifier of a sign-in in progress, which made that sign-in fail. MCP sign-in now refreshes the stored tokens with the stored client even when the loopback callback port changed, instead of registering a new client and opening the browser. Ctrl+C in `/mcp` and the `--timeout` of `pi mcp login` now also stop a sign-in while it waits up to 25 seconds for another pi process's token refresh, and a refresh lock that fails for another reason than that wait now fails the sign-in instead of opening the browser. - Fixed RPC mode ignoring an extension's `ctx.shutdown()` while idle, for example from a timer: the process kept running until the client sent another command. pi now shuts down right away when idle, and a shutdown requested by an extension command waits until the command's `prompt` response is written ([#11599](https://github.com/earendil-works/pi/issues/11599)) - Fixed changelog notes repeating on every fresh launch when the installed package version has a prerelease suffix, such as `0.1.5-20261020.7`: the saved version was read as `0.1.0`, so the `0.1.1` to `0.1.5` notes counted as new each time. A saved version without `major.minor.patch` now shows no notes and is replaced by the installed version, instead of showing the whole changelog ([#11703](https://github.com/earendil-works/pi/issues/11703)) diff --git a/packages/coding-agent/src/core/agent-session.ts b/packages/coding-agent/src/core/agent-session.ts index dbe36c365..5d341c4d7 100644 --- a/packages/coding-agent/src/core/agent-session.ts +++ b/packages/coding-agent/src/core/agent-session.ts @@ -43,6 +43,7 @@ import type { TextContent, ToolResultMessage, Usage, + UserMessage, } from "@earendil-works/pi-ai/compat"; import { clampThinkingLevel, @@ -408,6 +409,8 @@ export class AgentSession { private _steeringMessages: string[] = []; /** Tracks pending follow-up messages for UI display. Removed when delivered. */ private _followUpMessages: string[] = []; + /** Queued user messages whose images are normalized when they are delivered. */ + private _queuedMessagesWithImages = new WeakSet(); /** Messages queued to be included with the next user prompt as context ("asides"). */ private _pendingNextTurnMessages: CustomMessage[] = []; /** Context-only custom messages queued during a run, flushed once the current turn's tool results are in. */ @@ -1153,6 +1156,12 @@ export class AgentSession { } } } + // Normalize queued images at delivery, before events and persistence see the message, + // so they get the same resize as idle prompts with the current model's limits (#11738). + if (this._queuedMessagesWithImages.has(event.message)) { + this._queuedMessagesWithImages.delete(event.message); + await this._normalizeQueuedMessageImages(event.message); + } } // Emit to extensions first, then notify public listeners. @@ -2327,11 +2336,9 @@ export class AgentSession { if (images) { content.push(...images); } - this.agent.steer({ - role: "user", - content, - timestamp: Date.now(), - }); + const message: AgentMessage = { role: "user", content, timestamp: Date.now() }; + if (images?.length) this._queuedMessagesWithImages.add(message); + this.agent.steer(message); } /** @@ -2344,7 +2351,24 @@ export class AgentSession { if (images) { content.push(...images); } - this.agent.followUp({ role: "user", content, timestamp: Date.now() }); + const message: AgentMessage = { role: "user", content, timestamp: Date.now() }; + if (images?.length) this._queuedMessagesWithImages.add(message); + this.agent.followUp(message); + } + + /** Replace a queued user message's images in place with normalized ones, appending hints to its text. */ + private async _normalizeQueuedMessageImages(message: UserMessage): Promise { + if (typeof message.content === "string") return; + const [first, ...rest] = message.content; + if (first?.type !== "text") return; + const images = rest.filter((block): block is ImageContent => block.type === "image"); + const normalized = await this._normalizePromptImages(images); + const text = normalized.hints.length > 0 ? `${first.text}\n\n${normalized.hints.join("\n")}` : first.text; + message.content = [ + { type: "text", text }, + ...rest.filter((block) => block.type !== "image"), + ...normalized.images, + ]; } /** diff --git a/packages/coding-agent/test/agent-session-queued-images.test.ts b/packages/coding-agent/test/agent-session-queued-images.test.ts new file mode 100644 index 000000000..800445bf3 --- /dev/null +++ b/packages/coding-agent/test/agent-session-queued-images.test.ts @@ -0,0 +1,94 @@ +import { deflateSync } from "node:zlib"; +import type { ImageContent, TextContent } from "@earendil-works/pi-ai"; +import { afterEach, describe, expect, it } from "vitest"; +import { createHarness, type Harness } from "./test-harness.ts"; + +function crc32(data: Buffer): number { + let crc = 0xffffffff; + for (const byte of data) { + crc ^= byte; + for (let i = 0; i < 8; i++) crc = (crc >>> 1) ^ (0xedb88320 & -(crc & 1)); + } + return (crc ^ 0xffffffff) >>> 0; +} + +function pngChunk(type: string, data: Buffer): Buffer { + const length = Buffer.alloc(4); + length.writeUInt32BE(data.length); + const body = Buffer.concat([Buffer.from(type, "ascii"), data]); + const crc = Buffer.alloc(4); + crc.writeUInt32BE(crc32(body)); + return Buffer.concat([length, body, crc]); +} + +/** A solid gray RGB PNG of the given size, base64 encoded. */ +function createPng(width: number, height: number): string { + const header = Buffer.alloc(13); + header.writeUInt32BE(width, 0); + header.writeUInt32BE(height, 4); + header[8] = 8; + header[9] = 2; + const row = Buffer.alloc(1 + width * 3, 0x80); + row[0] = 0; + const raw = Buffer.concat(Array.from({ length: height }, () => row)); + return Buffer.concat([ + Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), + pngChunk("IHDR", header), + pngChunk("IDAT", deflateSync(raw)), + pngChunk("IEND", Buffer.alloc(0)), + ]).toString("base64"); +} + +function persistedUserContent(harness: Harness, text: string): (TextContent | ImageContent)[] { + for (const entry of harness.sessionManager.getEntries()) { + if (entry.type !== "message" || entry.message.role !== "user") continue; + const content = entry.message.content; + if (typeof content === "string") continue; + if (content[0]?.type === "text" && content[0].text.split("\n")[0] === text) return content; + } + throw new Error(`No user message "${text}"`); +} + +async function waitFor(condition: () => boolean): Promise { + for (let i = 0; i < 200 && !condition(); i++) await new Promise((resolve) => setTimeout(resolve, 25)); +} + +describe("AgentSession queued message images", () => { + let harness: Harness | undefined; + + afterEach(() => { + harness?.cleanup(); + harness = undefined; + }); + + // Regression test for #11738: queued messages skipped the image resize idle prompts get. + it("resizes images in steering, follow-up, and streaming prompt messages", async () => { + harness = await createHarness({ responses: [{ text: "first", delayMs: 300 }, "second", "third", "fourth"] }); + const image: ImageContent = { type: "image", data: createPng(1200, 9000), mimeType: "image/png" }; + + const run = harness.session.prompt("idle"); + await waitFor(() => harness!.session.isStreaming); + await harness.session.steer("steer", [image]); + await harness.session.followUp("follow up", [image]); + await harness.session.prompt("streaming prompt", { images: [image], streamingBehavior: "followUp" }); + await run; + await waitFor(() => !harness!.session.isStreaming && harness!.faux.callCount >= 4); + + for (const text of ["steer", "follow up", "streaming prompt"]) { + const content = persistedUserContent(harness, text); + const images = content.filter((block): block is ImageContent => block.type === "image"); + expect(images).toHaveLength(1); + expect(images[0].data).not.toBe(image.data); + expect((content[0] as TextContent).text).toContain("original 1200x9000, displayed at 267x2000"); + } + const sentImages = harness.faux.contexts + .flatMap((context) => context.messages) + .flatMap((message): readonly { type: string }[] => + typeof message.content === "string" ? [] : message.content, + ) + .filter((block): block is ImageContent => block.type === "image"); + expect(sentImages.length).toBeGreaterThan(0); + expect(sentImages.some((block) => block.data === image.data)).toBe(false); + expect(harness.session.pendingMessageCount).toBe(0); + }); +});