From 15e7bdc14cae38786c4cde24870ffce888e062cd Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Sun, 11 Oct 2026 13:53:05 +0200 Subject: [PATCH] fix(coding-agent,durable): omit thinking from compaction summaries Replayed thinking is often not part of the live context (for example, radius drops it, and Anthropic drops blocks that do not match their request), but serializeConversation included all of it. A Pi Mobile session at ~996K of 1M context tokens produced 1.38M-token summarization requests that failed on every attempt. Providers only return summaries of the reasoning anyway; visible text, tool calls, and tool results record what happened. --- packages/coding-agent/CHANGELOG.md | 1 + packages/coding-agent/docs/compaction.md | 4 +-- .../coding-agent/src/core/compaction/utils.ts | 14 ++++----- .../test/compaction-serialization.test.ts | 30 +++++++++++++++++++ packages/durable/CHANGELOG.md | 1 + packages/durable/docs/spec.md | 8 +++-- packages/durable/src/harness/compaction.ts | 9 ++++-- .../durable/test/harness-compaction.test.ts | 5 ++-- 8 files changed, 55 insertions(+), 17 deletions(-) diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 096d245a0..6d3cbda15 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -9,6 +9,7 @@ ### Changed - Changed the `showHardwareCursor` setting to use only the terminal cursor instead of also drawing Pi's reverse-video cursor +- Changed `serializeConversation()` (compaction, branch summaries, bug reports) to omit thinking blocks; replayed thinking is often not in the live context, so including it could make a summarization request larger than the context it summarizes and fail ### Fixed diff --git a/packages/coding-agent/docs/compaction.md b/packages/coding-agent/docs/compaction.md index a702f7b6e..72a7c9409 100644 --- a/packages/coding-agent/docs/compaction.md +++ b/packages/coding-agent/docs/compaction.md @@ -279,7 +279,6 @@ Before summarization, messages are serialized to text via [`serializeConversatio ``` [User]: What they said -[Assistant thinking]: Internal reasoning [Assistant]: Response text [Assistant tool calls]: read(path="foo.ts"); edit(path="bar.ts", ...) [Tool result]: Output from tool @@ -287,6 +286,8 @@ Before summarization, messages are serialized to text via [`serializeConversatio This prevents the model from treating it as a conversation to continue. +Thinking blocks are omitted. Providers return summaries of the real reasoning, and replayed thinking is often not part of the model's live context, so including it could make the summarization request larger than the context it summarizes. + Tool results are truncated to 2000 characters during serialization. Content beyond that limit is replaced with a marker indicating how many characters were truncated. This keeps summarization requests within reasonable token budgets, since tool results (especially from `read` and `bash`) are typically the largest contributors to context size. ## Custom Summarization via Extensions @@ -346,7 +347,6 @@ pi.on("session_before_compact", async (event, ctx) => { ); // Returns: // [User]: message text - // [Assistant thinking]: thinking content // [Assistant]: response text // [Assistant tool calls]: read(path="..."); bash(command="...") // [Tool result]: output text diff --git a/packages/coding-agent/src/core/compaction/utils.ts b/packages/coding-agent/src/core/compaction/utils.ts index a40a5d14b..1f05a14b4 100644 --- a/packages/coding-agent/src/core/compaction/utils.ts +++ b/packages/coding-agent/src/core/compaction/utils.ts @@ -110,6 +110,12 @@ function truncateForSummary(text: string, maxChars: number): string { * * Tool results are truncated to keep the summarization request within * reasonable token budgets. Full content is not needed for summarization. + * + * Thinking is omitted. Providers return summaries of the real reasoning, and + * replayed thinking is often not part of the model's live context: for example, + * a block that does not match its request is dropped. Including it can make the + * summarization request larger than the context it summarizes. Visible text, + * tool calls, and tool results record what happened. */ export function serializeConversation(messages: Message[]): string { const parts: string[] = []; @@ -119,13 +125,10 @@ export function serializeConversation(messages: Message[]): string { const content = contentText(msg.content, ""); if (content) parts.push(`[User]: ${content}`); } else if (msg.role === "assistant") { - const thinkingParts: string[] = []; const toolCalls: string[] = []; for (const block of msg.content) { - if (block.type === "thinking") { - thinkingParts.push(block.thinking); - } else if (block.type === "toolCall") { + if (block.type === "toolCall") { const args = block.arguments as Record; const argsStr = Object.entries(args) .map(([k, v]) => `${k}=${JSON.stringify(v)}`) @@ -134,9 +137,6 @@ export function serializeConversation(messages: Message[]): string { } } - if (thinkingParts.length > 0) { - parts.push(`[Assistant thinking]: ${thinkingParts.join("\n")}`); - } if (msg.content.some((block) => block.type === "text")) { parts.push(`[Assistant]: ${contentText(msg.content)}`); } diff --git a/packages/coding-agent/test/compaction-serialization.test.ts b/packages/coding-agent/test/compaction-serialization.test.ts index 91ca9251a..f7f479266 100644 --- a/packages/coding-agent/test/compaction-serialization.test.ts +++ b/packages/coding-agent/test/compaction-serialization.test.ts @@ -76,4 +76,34 @@ describe("serializeConversation", () => { expect(result).not.toContain("truncated"); expect(result).toContain(longText); }); + + it("should omit thinking", () => { + const messages: Message[] = [ + { + role: "assistant", + content: [ + { type: "thinking", thinking: "private reasoning", thinkingSignature: "sig" }, + { type: "text", text: "visible reply" }, + { type: "toolCall", id: "tc1", name: "read", arguments: { path: "a.ts" } }, + ], + api: "anthropic", + provider: "anthropic", + model: "test", + usage: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + stopReason: "toolUse", + timestamp: Date.now(), + }, + ]; + + const result = serializeConversation(messages); + + expect(result).toBe('[Assistant]: visible reply\n\n[Assistant tool calls]: read(path="a.ts")'); + }); }); diff --git a/packages/durable/CHANGELOG.md b/packages/durable/CHANGELOG.md index f9bb6f61b..019703771 100644 --- a/packages/durable/CHANGELOG.md +++ b/packages/durable/CHANGELOG.md @@ -25,6 +25,7 @@ ### Fixed +- Compaction no longer includes thinking in the summarization request. Replayed thinking is often not in the live context, so a request that included it could be larger than the context it summarized and fail with "prompt is too long" on every attempt. - A throwing `onReport` no longer replaces the error being reported or becomes an unhandled rejection, a throwing `now()` is reported once and `Date.now` used, and a watch listener's error is reported as well as ending the watch with `listener_error`. - A task whose fault write failed no longer runs again in the same process. - The scheduler no longer visits every live task on each pass: abort cascades start only from tasks with cancellation intent, owned live work and idle scopes are walks down an in-memory ownership index, and reservation, finalization, and failFast checks keep their own candidate sets. A chain of 500 owned tasks that took 12 s to settle takes 50 ms, 20,000 take 1.5 s; a tool making 8000 parallel nested calls finishes in 2.7 s instead of 18.8 s, with one document per nested call instead of one growing index (`npm run bench:scheduler-fanout`). diff --git a/packages/durable/docs/spec.md b/packages/durable/docs/spec.md index c2a1d3b64..e2a1bd94e 100644 --- a/packages/durable/docs/spec.md +++ b/packages/durable/docs/spec.md @@ -4104,9 +4104,11 @@ Phases: captured tail, and the cut as `firstKept`. - `summarize` derives the summarized messages again from the context at `tail`, which is immutable, and serializes them to text: `[User]: ...`, - `[Assistant thinking]: ...`, `[Assistant]: ...`, - `[Assistant tool calls]: name(key=json, ...)`, and `[Tool result]: ...` - truncated to 2000 characters; system messages are omitted. The request has no + `[Assistant]: ...`, `[Assistant tool calls]: name(key=json, ...)`, and + `[Tool result]: ...` truncated to 2000 characters; system messages and + thinking are omitted (replayed thinking is often not part of the live + context, so it could make the request larger than the context it + summarizes). The request has no tools and two messages: a system message with the built-in summarization system prompt, and a user message with the serialized text in `` tags followed by the built-in summarization prompt, and diff --git a/packages/durable/src/harness/compaction.ts b/packages/durable/src/harness/compaction.ts index db700163b..dbc164b15 100644 --- a/packages/durable/src/harness/compaction.ts +++ b/packages/durable/src/harness/compaction.ts @@ -347,7 +347,12 @@ function summaryPrompt(messages: readonly Message[], instructions: string | unde return `\n${serializeConversation(messages)}\n\n\n${SUMMARIZATION_PROMPT}${focus}`; } -/** Messages as plain text, so the summarizer reads a transcript instead of continuing it. System messages are omitted. */ +/** + * Messages as plain text, so the summarizer reads a transcript instead of continuing it. System messages are omitted. + * Thinking is omitted too: providers return summaries of the real reasoning, and replayed thinking is often not part of + * the live context (a block that does not match its request is dropped), so including it can make the request larger + * than the context it summarizes. Visible text, tool calls, and tool results record what happened. + */ export function serializeConversation(messages: readonly Message[]): string { const parts: string[] = []; for (const message of messages) { @@ -355,7 +360,6 @@ export function serializeConversation(messages: readonly Message[]): string { const text = contentText(message.content); if (text.length > 0) parts.push(`[User]: ${text}`); } else if (message.role === "assistant") { - const thinking = message.content.flatMap((content) => (content.type === "thinking" ? [content.thinking] : [])); const text = message.content.flatMap((content) => (content.type === "text" ? [content.text] : [])); const calls = message.content.flatMap((content) => content.type === "toolCall" @@ -366,7 +370,6 @@ export function serializeConversation(messages: readonly Message[]): string { ] : [], ); - if (thinking.length > 0) parts.push(`[Assistant thinking]: ${thinking.join("\n")}`); if (text.length > 0) parts.push(`[Assistant]: ${text.join("\n")}`); if (calls.length > 0) parts.push(`[Assistant tool calls]: ${calls.join("; ")}`); } else if (message.role === "toolResult") { diff --git a/packages/durable/test/harness-compaction.test.ts b/packages/durable/test/harness-compaction.test.ts index 182ac0ad8..7698c06eb 100644 --- a/packages/durable/test/harness-compaction.test.ts +++ b/packages/durable/test/harness-compaction.test.ts @@ -344,7 +344,7 @@ describe("range selection", () => { }); describe("serialization", () => { - it("writes a transcript, truncates tool results, and omits system messages", () => { + it("writes a transcript, truncates tool results, and omits system messages and thinking", () => { const call = fauxToolCall("read", { path: "a.ts" }, { id: "c" }); const messages: Message[] = [ { role: "system", content: "", sections: { s: "hidden" }, timestamp: 0 }, @@ -357,7 +357,8 @@ describe("serialization", () => { const serialized = serializeConversation(messages); expect(serialized).not.toContain("hidden"); expect(serialized).toContain("[User]: hello"); - expect(serialized).toContain("[Assistant thinking]: hmm"); + expect(serialized).not.toContain("hmm"); + expect(serialized).not.toContain("[Assistant thinking]"); expect(serialized).toContain("[Assistant]: sure"); expect(serialized).toContain('[Assistant tool calls]: read(path="a.ts")'); expect(serialized).toContain(`[Tool result]: ${"y".repeat(2000)}\n\n[... 500 more characters truncated]`);