From 962b130e0cb58033045f85e2180e5ba1483e7c40 Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Tue, 6 Oct 2026 23:55:17 +0200 Subject: [PATCH] feat(coding-agent): add in-context compaction With compaction.inContext enabled, compaction can generate the summary inside the cached conversation: it repeats the request the next turn would send, appends the instructions as a system message plus a user message, and sets toolChoice none. The recent tail is kept as before. A gate requires a verified model, a warm cache from the same model and thinking level, room in the context window, and a lower estimated cost than the standalone summary. Failed or rejected attempts fall back to the standalone summary, never execute tools, and are recorded in details.inContext with their usage. pi-ai adds the supportsInContextCompaction compat setting and test/in-context-compaction-probe.ts, which measures whether a model keeps its prompt cache for this request shape. The generated catalog flags the models that passed. --- packages/ai/README.md | 3 + packages/ai/scripts/generate-models.ts | 54 +++ packages/ai/src/api/openai-completions.ts | 2 + packages/ai/src/api/openai-responses.ts | 11 +- packages/ai/src/types.ts | 8 + .../ai/test/in-context-compaction-probe.ts | 449 ++++++++++++++++++ ...penai-completions-thinking-as-text.test.ts | 1 + ...nai-completions-tool-result-images.test.ts | 1 + packages/ai/test/providers.test.ts | 31 ++ packages/coding-agent/docs/compaction.md | 26 +- packages/coding-agent/docs/settings.md | 1 + .../coding-agent/src/core/agent-session.ts | 125 ++++- .../coding-agent/src/core/cache-warmer.ts | 9 +- .../src/core/compaction/compaction.ts | 290 +++++++++++ .../coding-agent/src/core/settings-manager.ts | 6 + .../src/modes/interactive/interactive-mode.ts | 29 +- .../test/compaction-in-context.test.ts | 159 +++++++ .../test/interactive-mode-compaction.test.ts | 76 +++ ...gent-session-in-context-compaction.test.ts | 259 ++++++++++ 19 files changed, 1517 insertions(+), 23 deletions(-) create mode 100644 packages/ai/test/in-context-compaction-probe.ts create mode 100644 packages/coding-agent/test/compaction-in-context.test.ts create mode 100644 packages/coding-agent/test/suite/agent-session-in-context-compaction.test.ts diff --git a/packages/ai/README.md b/packages/ai/README.md index 0ad23aa31..a3ba68762 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -1395,6 +1395,7 @@ interface OpenAICompletionsCompat { supportsOpenAIGrammarTools?: boolean; // Whether to emit OpenAI custom Lark/regex grammar tools; false falls back to normal function tools (default: false; the generated catalog enables it for capable models) supportsMidConvoSystemMessages?: boolean; // Whether the model accepts system messages after the conversation started; false folds them into the leading prompt (default: false; the generated catalog enables it for verified models) supportsMidConvoToolAdditions?: boolean; // Whether system messages can add tools mid-conversation via Kimi-style `tools` system messages; requires supportsMidConvoSystemMessages (default: false) + supportsInContextCompaction?: boolean; // Whether repeating the conversation with an appended system message, user message, and `toolChoice: "none"` still reads the cached prefix; informational for callers such as coding-agent compaction (default: false; the generated catalog enables it for models validated in real pi sessions) sendSessionAffinityHeaders?: boolean; // Send session-affinity data from `sessionId` (default: true for OpenRouter, false otherwise) sessionAffinityFormat?: 'openai' | 'openai-nosession' | 'openrouter'; // Format for session affinity: 'openai' uses `prompt_cache_key`, `session_id`, `x-client-request-id`, and `x-session-affinity`; 'openai-nosession' uses `prompt_cache_key`, `x-client-request-id`, and `x-session-affinity`; 'openrouter' uses `x-session-id` (default: auto-detected) maxTokensField?: 'max_completion_tokens' | 'max_tokens'; // Which field name to use (default: max_completion_tokens) @@ -1591,6 +1592,8 @@ A custom `Provider` or `ProviderStreams` implementation reads the prompt and too Models that accept system messages mid-conversation (`supportsMidConvoSystemMessages` in the model's compat settings, set by the generated catalog for verified models) receive each later system message in place, so the cached prefix stays intact; section changes are framed by name for the model. Every other model receives `collapseSystemMessages(transcript)`: the replayed prompt and current tools as the leading system message, with later system messages dropped. Anthropic models that also set `supportsMidConvoToolChanges` send tool changes as native blocks (`inline-tools-2026-09-15` beta): the top-level tool list holds the initial tools plus a stable deferred placeholder and never changes, later tools are defined by value in `tool_addition` blocks (a new definition under an existing name replaces the old one), and removals are `tool_removal` references, so tool changes do not invalidate the prompt cache. That needs at least one initial tool; otherwise the current tool list is sent at the top level with the system text only. OpenAI Responses models with `supportsAdditionalTools` or `supportsToolSearch` anchor additive tool changes at their message; everything else sends the current tool list at the top level. +`supportsInContextCompaction` marks models whose prompt cache survives a request that repeats the conversation, appends a system message and a user message, and sets `toolChoice: "none"`. pi-ai does not read it; callers use it to decide whether a summary can be generated inside the cached conversation. The generated catalog sets it only for models validated in real pi sessions: after a tool-heavy turn with native reasoning, the compaction request must still read the cached prompt. Some reasoning models drop the last turn's reasoning when a user message follows it, which breaks the cache at the start of that turn. + ## Context Serialization The `Context` object can be easily serialized and deserialized using standard JSON methods, making it simple to persist conversations, implement chat history, or transfer contexts between services: diff --git a/packages/ai/scripts/generate-models.ts b/packages/ai/scripts/generate-models.ts index 8deeb1014..145b6a6ac 100644 --- a/packages/ai/scripts/generate-models.ts +++ b/packages/ai/scripts/generate-models.ts @@ -976,6 +976,59 @@ function applyOpenAIExplicitPromptCacheMetadata(model: Model): void { }; } +// Models whose prompt cache survives in-context compaction in real pi sessions: after a tool-heavy +// turn with native reasoning, the compaction request (same conversation plus a system message and +// a user message, `toolChoice: "none"`) reads at least 90% of the last turn's prompt from cache. +// Validated 2026-10-06 by driving pi in RPC mode and compacting directly after the tool turn. +// test/in-context-compaction-probe.ts alone is not enough: its synthetic history has no native +// reasoning, so it passes models that fail in real sessions. +// Failures stay unflagged. openai-codex/gpt-5.5, moonshotai/kimi-k2.6, and GPT-5.4 and GPT-5.4 mini +// on opencode and openrouter read 0-9%: the appended user message makes the provider drop the +// last turn's reasoning from the rendered prompt, so the cache breaks at the start of that turn. +// deepseek/deepseek-v4-pro read 0% and opencode-go/kimi-k3 read 89% in the probe. +// opencode/claude-opus-4-8 and opencode-go/gpt-5.6-luna were unreachable (model access disabled); +// they stay flagged because the same models pass through other providers. +const IN_CONTEXT_COMPACTION_VERIFIED_MODELS = new Set([ + "anthropic/claude-fable-5", + "anthropic/claude-fable-5-1", + "anthropic/claude-opus-4-8", + "anthropic/claude-opus-5-5", + "anthropic/claude-sonnet-5-5", + "moonshotai/kimi-k2.7-code", + "moonshotai/kimi-k2.7-code-highspeed", + "moonshotai/kimi-k3", + "openai-codex/gpt-5.6-luna", + "openai-codex/gpt-5.6-sol", + "openai-codex/gpt-5.6-terra", + "openai-codex/gpt-6-astra", + "openai-codex/gpt-6-luna", + "openai-codex/gpt-6-sol", + "openai-codex/gpt-6.1-sol", + "opencode-go/gpt-5.6-luna", + "opencode-go/gpt-6-luna", + "opencode/claude-fable-5-1", + "opencode/claude-opus-4-8", + "opencode/claude-opus-5-5", + "opencode/claude-sonnet-5-5", + "opencode/kimi-k3", + "openrouter/openai/gpt-5.5", + "openrouter/openai/gpt-5.6-luna", + "openrouter/openai/gpt-5.6-sol", + "openrouter/openai/gpt-5.6-terra", + "openrouter/openai/gpt-6-astra", + "openrouter/openai/gpt-6-luna", + "openrouter/openai/gpt-6-sol", + "openrouter/openai/gpt-6.1-sol", +]); + +function applyInContextCompactionMetadata(model: Model): void { + if (!IN_CONTEXT_COMPACTION_VERIFIED_MODELS.has(`${model.provider}/${model.id}`)) return; + const compat = model.compat as (AnthropicMessagesCompat & OpenAIResponsesCompat & OpenAICompletionsCompat) | undefined; + // In-context compaction sends the instructions as a mid-conversation system message. + if (compat?.supportsMidConvoSystemMessages !== true) return; + model.compat = { ...compat, supportsInContextCompaction: true }; +} + // Anthropic ephemeral entries have a hard five-minute lifetime; `ttl: "1h"` // extends it to one hour. Only direct Anthropic is annotated so cache warming // does not assume equivalent behavior through proxies. @@ -3483,6 +3536,7 @@ async function generateModels() { applyOpenAICompletionsTranscriptMetadata(model); applyOpenAIResponsesTranscriptMetadata(model); applyOpenAIExplicitPromptCacheMetadata(model); + applyInContextCompactionMetadata(model); applyPromptCacheMetadata(model); applyImageInputMetadata(model); } diff --git a/packages/ai/src/api/openai-completions.ts b/packages/ai/src/api/openai-completions.ts index 96cb23ab3..590517b1f 100644 --- a/packages/ai/src/api/openai-completions.ts +++ b/packages/ai/src/api/openai-completions.ts @@ -174,6 +174,7 @@ type ResolvedOpenAICompletionsCompat = Omit< | "thinkingTokenBudgetField" | "supportsMidConvoSystemMessages" | "supportsMidConvoToolAdditions" + | "supportsInContextCompaction" | "vllmPriority" > & { cacheControlFormat?: OpenAICompletionsCompat["cacheControlFormat"]; @@ -181,6 +182,7 @@ type ResolvedOpenAICompletionsCompat = Omit< thinkingTokenBudgetField?: OpenAICompletionsCompat["thinkingTokenBudgetField"]; supportsMidConvoSystemMessages?: OpenAICompletionsCompat["supportsMidConvoSystemMessages"]; supportsMidConvoToolAdditions?: OpenAICompletionsCompat["supportsMidConvoToolAdditions"]; + supportsInContextCompaction?: OpenAICompletionsCompat["supportsInContextCompaction"]; vllmPriority?: OpenAICompletionsCompat["vllmPriority"]; }; diff --git a/packages/ai/src/api/openai-responses.ts b/packages/ai/src/api/openai-responses.ts index 07e5624e9..ae22c978a 100644 --- a/packages/ai/src/api/openai-responses.ts +++ b/packages/ai/src/api/openai-responses.ts @@ -79,7 +79,10 @@ function resolveCacheRetention(cacheRetention?: CacheRetention, env?: ProviderEn return "short"; } -function getCompat(model: Model<"openai-responses">): Required { +/** Compat settings this API reads. `supportsInContextCompaction` is for callers and has no request effect. */ +type ResolvedOpenAIResponsesCompat = Omit, "supportsInContextCompaction">; + +function getCompat(model: Model<"openai-responses">): ResolvedOpenAIResponsesCompat { return { supportsDeveloperRole: model.compat?.supportsDeveloperRole ?? true, supportsMidConvoSystemMessages: model.compat?.supportsMidConvoSystemMessages ?? false, @@ -95,7 +98,7 @@ function getCompat(model: Model<"openai-responses">): Required, + compat: ResolvedOpenAIResponsesCompat, cacheRetention: CacheRetention, ): "24h" | undefined { return cacheRetention === "long" && compat.supportsLongCacheRetention && !compat.supportsExplicitPromptCacheMode @@ -104,7 +107,7 @@ function getPromptCacheRetention( } function getPromptCacheOptions( - compat: Required, + compat: ResolvedOpenAIResponsesCompat, cacheRetention: CacheRetention, ): ResponseCreateParamsStreaming["prompt_cache_options"] { if (!compat.supportsExplicitPromptCacheMode) return undefined; @@ -303,7 +306,7 @@ function buildParams( model: Model<"openai-responses">, context: TranscriptContext, options: OpenAIResponsesOptions | undefined, - compat: Required = getCompat(model), + compat: ResolvedOpenAIResponsesCompat = getCompat(model), grammarToolInputProperties: ReadonlyMap = createGrammarToolInputProperties( getDeclaredTools(context.messages), compat.supportsOpenAIGrammarTools, diff --git a/packages/ai/src/types.ts b/packages/ai/src/types.ts index 7469f3038..005d5b533 100644 --- a/packages/ai/src/types.ts +++ b/packages/ai/src/types.ts @@ -846,6 +846,8 @@ export interface OpenAICompletionsCompat { supportsOpenAIGrammarTools?: boolean; /** Whether the exact model accepts system or developer messages after the conversation has started. When false, later system messages are folded into the leading system message. Default: false; the generated model catalog enables it for verified models. */ supportsMidConvoSystemMessages?: boolean; + /** Whether the exact model keeps its prompt cache for in-context compaction: a request that repeats the conversation, appends a system message and a user message, and sets `toolChoice: "none"` still reads the cached prefix. Requires `supportsMidConvoSystemMessages`. Default: false; the generated model catalog enables it for models validated in real pi sessions. */ + supportsInContextCompaction?: boolean; /** Whether system messages can introduce additional tools mid-conversation. Requires `supportsMidConvoSystemMessages`. Default: false; the generated model catalog enables it for capable models. */ supportsMidConvoToolAdditions?: boolean; /** Whether the provider supports the `strict` field in tool definitions. Default: false; generated capable models enable it explicitly. */ @@ -873,6 +875,8 @@ export interface OpenAIResponsesCompat { supportsDeveloperRole?: boolean; /** Whether the exact model accepts developer or system messages after the conversation has started. When false, later system messages are folded into the leading system message. Default: false; the generated model catalog enables it for verified models. */ supportsMidConvoSystemMessages?: boolean; + /** Whether the exact model keeps its prompt cache for in-context compaction: a request that repeats the conversation, appends a system message and a user message, and sets `toolChoice: "none"` still reads the cached prefix. Requires `supportsMidConvoSystemMessages`. Default: false; the generated model catalog enables it for models validated in real pi sessions. */ + supportsInContextCompaction?: boolean; /** Session-affinity header format: `openai` sends `session_id` and `x-client-request-id`; `openai-nosession` sends `x-client-request-id`; `openrouter` sends `x-session-id`. Does not affect the `prompt_cache_key` body param, which is governed by cache retention. Default: auto-detected. */ sessionAffinityFormat?: SessionAffinityFormat; /** Whether the provider supports long prompt cache retention. This uses `prompt_cache_options.ttl: "30m"` on GPT-5.6+ and `prompt_cache_retention: "24h"` on earlier models. Default: true. */ @@ -945,6 +949,8 @@ export interface AnthropicMessagesCompat { supportsMidConvoEffort?: boolean; /** Whether the exact model accepts system-role messages inside the conversation. When false, later system messages are folded into the top-level system prompt. Default: false. */ supportsMidConvoSystemMessages?: boolean; + /** Whether the exact model keeps its prompt cache for in-context compaction: a request that repeats the conversation, appends a system message and a user message, and sets `toolChoice: "none"` still reads the cached prefix. Requires `supportsMidConvoSystemMessages`. Default: false; the generated model catalog enables it for models validated in real pi sessions. */ + supportsInContextCompaction?: boolean; /** Whether the exact model accepts mid-conversation `tool_addition` blocks with inline tool definitions (`inline-tools-2026-09-15`) and `tool_removal` blocks. Requires `supportsMidConvoSystemMessages`. Default: false. */ supportsMidConvoToolChanges?: boolean; /** @@ -966,6 +972,8 @@ export interface BedrockCompat { export interface MistralConversationsCompat { /** Whether the exact model accepts system messages after the conversation has started. When false, later system messages are folded into the leading system message. Default: false. */ supportsMidConvoSystemMessages?: boolean; + /** Whether the exact model keeps its prompt cache for in-context compaction: a request that repeats the conversation, appends a system message and a user message, and sets `toolChoice: "none"` still reads the cached prefix. Requires `supportsMidConvoSystemMessages`. Default: false; the generated model catalog enables it for models validated in real pi sessions. */ + supportsInContextCompaction?: boolean; } /** diff --git a/packages/ai/test/in-context-compaction-probe.ts b/packages/ai/test/in-context-compaction-probe.ts new file mode 100644 index 000000000..75b4df0b7 --- /dev/null +++ b/packages/ai/test/in-context-compaction-probe.ts @@ -0,0 +1,449 @@ +#!/usr/bin/env node +/** + * Live probe for in-context compaction cache stability. + * + * In-context compaction (coding-agent `compaction.inContext`) repeats the request the next + * turn would send, appends a system message and a user message, and sets + * `toolChoice: "none"`. It only saves money when the provider still reads the cached prefix + * of that request. For each model this probe: + * + * 1. sends a tool-using conversation of about 12k tokens to write the prompt cache, + * 2. sends the in-context compaction request on top of it with identical options, + * 3. passes when that request reads at least 90% of the first prompt from cache, stops + * normally, calls no tools, and returns text. + * + * When a model fails, a control request (a plain user message with default tool choice) + * separates "the compaction request breaks the cache" from "the cache did not hold". + * Passing is necessary but not sufficient for `supportsInContextCompaction`: the synthetic history + * has no native reasoning, so it misses models that drop the last turn's reasoning when a user + * message follows it (openai-codex/gpt-5.5, moonshotai/kimi-k2.6, GPT-5.4 on opencode and + * openrouter). Validate candidates in real pi sessions that compact right after a tool turn. + * + * Run from packages/ai with the source resolver so workspace packages load from source: + * + * node --import ../coding-agent/src/experimental/source-resolver.ts \ + * test/in-context-compaction-probe.ts --all + */ + +import { randomUUID } from "node:crypto"; +import { writeFileSync } from "node:fs"; +import { + appendInContextCompactionRequest, + IN_CONTEXT_COMPACTION_INSTRUCTIONS, + IN_CONTEXT_COMPACTION_PROMPT, +} from "../../coding-agent/src/core/compaction/compaction.ts"; +import { ModelRuntime } from "../../coding-agent/src/core/model-runtime.ts"; +import { normalizeContext } from "../src/compat.ts"; +import type { + Api, + AssistantMessage, + Message, + Model, + SimpleStreamOptions, + ThinkingLevel, + Tool, + ToolCall, + Usage, +} from "../src/types.ts"; +import { contentText } from "../src/utils/text.ts"; + +const PASS_RATIO = 0.9; +const FILE_COUNT = 8; +const FILE_LINES = 90; + +interface Args { + all: boolean; + providers: Set; + models: string[]; + reasoning: ThinkingLevel | "off"; + attempts: number; + shapes: boolean; + onlyShapes: Set; + json?: string; +} + +type Verdict = "pass" | "fail" | "inconclusive" | "error"; + +interface ProbeResult { + model: string; + verdict: Verdict; + detail: string; + ratio?: number; + controlRatio?: number; + firstPromptTokens?: number; + compactionUsage?: Usage; +} + +function parseArgs(argv: string[]): Args { + const args: Args = { + all: false, + providers: new Set(), + models: [], + reasoning: "low", + attempts: 2, + shapes: false, + onlyShapes: new Set(), + }; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + switch (arg) { + case "--all": + args.all = true; + break; + case "--provider": + args.providers.add(required(argv[++i], arg)); + break; + case "--reasoning": + args.reasoning = required(argv[++i], arg) as Args["reasoning"]; + break; + case "--attempts": + args.attempts = Math.max(1, Number.parseInt(required(argv[++i], arg), 10)); + break; + case "--shapes": + args.shapes = true; + break; + case "--shape": + args.shapes = true; + args.onlyShapes.add(required(argv[++i], arg)); + break; + case "--json": + args.json = required(argv[++i], arg); + break; + case "--help": + printHelp(); + process.exit(0); + break; + default: + if (arg.startsWith("--")) throw new Error(`Unknown argument: ${arg}`); + args.models.push(arg); + } + } + if (!args.all && args.models.length === 0 && args.providers.size === 0) { + printHelp(); + process.exit(1); + } + return args; +} + +function required(value: string | undefined, flag: string): string { + if (!value) throw new Error(`Missing value for ${flag}`); + return value; +} + +function printHelp(): void { + console.log(`Usage: node --import ../coding-agent/src/experimental/source-resolver.ts test/in-context-compaction-probe.ts [options] [provider/model ...] + +Options: + --all Probe every model with supportsMidConvoSystemMessages that has configured auth + --provider Probe those models of one provider (repeatable) + --reasoning off | minimal | low | medium | high | xhigh | max. Default: low + --attempts Attempts per model before reporting a cache miss. Default: 2 + --shapes Compare request shapes (system or user instructions, with or without toolChoice "none") + instead of probing the production shape; includes models without mid-conversation system messages + --shape Compare only this shape, in the given order (repeatable): system+none | system | user+none | user + --json Write results as JSON +`); +} + +function supportsMidConvoSystemMessages(model: Model): boolean { + return ( + (model.compat as { supportsMidConvoSystemMessages?: boolean } | undefined)?.supportsMidConvoSystemMessages === + true + ); +} + +const TOOLS: Tool[] = [ + { + name: "read_file", + description: "Read a file from the project. Returns the file contents.", + parameters: { + type: "object", + properties: { path: { type: "string", description: "Project-relative file path" } }, + required: ["path"], + } as Tool["parameters"], + }, + { + name: "search", + description: "Search the project for a regular expression. Returns matching lines.", + parameters: { + type: "object", + properties: { pattern: { type: "string" }, path: { type: "string" } }, + required: ["pattern"], + } as Tool["parameters"], + }, +]; + +const MODULES = ["config", "retry", "transport", "cache", "parser", "scheduler", "metrics", "format"]; + +function moduleSource(index: number): string { + const name = MODULES[index % MODULES.length]; + const lines = [ + `// src/${name}.ts: ${name} module of the probe project`, + `export const ${name}Version = ${index + 1};`, + ]; + for (let line = 0; line < FILE_LINES; line++) { + lines.push( + `export function ${name}Step${line}(input: number): number { return (input * ${line + 3} + ${index * 31 + line}) % 1009; }`, + ); + } + return lines.join("\n"); +} + +function foreignAssistant( + content: AssistantMessage["content"], + stopReason: AssistantMessage["stopReason"], +): AssistantMessage { + return { + role: "assistant", + content, + api: "probe", + provider: "probe", + model: "probe-history", + usage: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + stopReason, + timestamp: 1, + }; +} + +/** A deterministic tool-using conversation that ends with a user question, like a real last turn. */ +function buildConversation(nonce: string): Message[] { + const systemPrompt = [ + `Probe session ${nonce}.`, + "You are a coding assistant working in a small TypeScript project. Use the tools to inspect files.", + "Answer concisely. Preserve exact file paths and function names.", + ].join("\n"); + const messages: Message[] = [ + { role: "system", content: systemPrompt, toolsAdded: TOOLS, timestamp: 1 }, + { role: "user", content: "Read every module in src/ and tell me what the project does.", timestamp: 1 }, + ]; + for (let index = 0; index < FILE_COUNT; index++) { + const path = `src/${MODULES[index]}.ts`; + const call: ToolCall = { type: "toolCall", id: `probe_call_${index}`, name: "read_file", arguments: { path } }; + messages.push(foreignAssistant([{ type: "text", text: `Reading ${path}.` }, call], "toolUse")); + messages.push({ + role: "toolResult", + toolCallId: call.id, + toolName: call.name, + content: [{ type: "text", text: moduleSource(index) }], + isError: false, + timestamp: 1, + }); + } + messages.push( + foreignAssistant( + [ + { + type: "text", + text: `The project has ${FILE_COUNT} modules: ${MODULES.join(", ")}. Each exports numeric step functions.`, + }, + ], + "stop", + ), + ); + messages.push({ role: "user", content: "In one sentence, which module would handle retries?", timestamp: 1 }); + return messages; +} + +function promptTokens(usage: Usage): number { + return usage.input + usage.cacheRead + usage.cacheWrite; +} + +function toolCalls(message: AssistantMessage): string[] { + return message.content.filter((block) => block.type === "toolCall").map((block) => (block as ToolCall).name); +} + +async function probeOnce(runtime: ModelRuntime, model: Model, reasoning: Args["reasoning"]): Promise { + const id = `${model.provider}/${model.id}`; + const options: SimpleStreamOptions = { + sessionId: `pi-in-context-compaction-probe-${randomUUID()}`, + ...(model.reasoning && reasoning !== "off" ? { reasoning } : {}), + }; + const request = (messages: Message[], extra: SimpleStreamOptions = {}) => + runtime.completeSimple(model, normalizeContext({ messages }), { ...options, ...extra }); + + const history = buildConversation(randomUUID()); + const first = await request(history); + if (first.stopReason === "error" || first.stopReason === "aborted") { + return { model: id, verdict: "error", detail: `first request: ${first.errorMessage ?? first.stopReason}` }; + } + const firstPrompt = promptTokens(first.usage); + const reply = + first.stopReason === "stop" && toolCalls(first).length === 0 + ? first + : foreignAssistant([{ type: "text", text: "The retry module handles retries." }], "stop"); + const conversation = [...history, reply]; + + const compaction = await request(appendInContextCompactionRequest(conversation), { toolChoice: "none" }); + if (compaction.stopReason === "error" || compaction.stopReason === "aborted") { + return { + model: id, + verdict: "fail", + detail: `compaction request: ${compaction.errorMessage ?? compaction.stopReason}`, + }; + } + const ratio = firstPrompt > 0 ? compaction.usage.cacheRead / firstPrompt : 0; + const base = { model: id, ratio, firstPromptTokens: firstPrompt, compactionUsage: compaction.usage }; + if (compaction.stopReason !== "stop") + return { ...base, verdict: "fail", detail: `stop reason ${compaction.stopReason}` }; + if (toolCalls(compaction).length > 0) + return { ...base, verdict: "fail", detail: `called ${toolCalls(compaction).join(", ")}` }; + if (contentText(compaction.content).trim().length === 0) + return { ...base, verdict: "fail", detail: "no summary text" }; + if (ratio >= PASS_RATIO) return { ...base, verdict: "pass", detail: "cache held" }; + + const control = await request([...conversation, { role: "user", content: "Reply with OK.", timestamp: 1 }]); + const controlRatio = firstPrompt > 0 ? control.usage.cacheRead / firstPrompt : 0; + return controlRatio >= PASS_RATIO + ? { + ...base, + controlRatio, + verdict: "fail", + detail: "compaction request missed the cache; a plain follow-up hit it", + } + : { + ...base, + controlRatio, + verdict: "inconclusive", + detail: "the cache did not hold for a plain follow-up either", + }; +} + +type Shape = "system+none" | "system" | "user+none" | "user"; +const SHAPES: Shape[] = ["system+none", "system", "user+none", "user"]; + +/** Compare how each compaction request shape reads the cache, after a plain follow-up confirms it works. */ +async function probeShapes( + runtime: ModelRuntime, + model: Model, + reasoning: Args["reasoning"], + only: Set, +): Promise { + const id = `${model.provider}/${model.id}`; + const options: SimpleStreamOptions = { + sessionId: `pi-in-context-compaction-probe-${randomUUID()}`, + ...(model.reasoning && reasoning !== "off" ? { reasoning } : {}), + }; + const request = (messages: Message[], extra: SimpleStreamOptions = {}) => + runtime.completeSimple(model, normalizeContext({ messages }), { ...options, ...extra }); + const history = buildConversation(randomUUID()); + const first = await request(history); + if (first.stopReason === "error" || first.stopReason === "aborted") + return `${id} first request: ${first.errorMessage}`; + const firstPrompt = promptTokens(first.usage); + const reply = + first.stopReason === "stop" && toolCalls(first).length === 0 + ? first + : foreignAssistant([{ type: "text", text: "The retry module handles retries." }], "stop"); + const conversation = [...history, reply]; + const describe = (label: string, message: AssistantMessage) => { + if (message.stopReason === "error") return `${label}=error(${message.errorMessage?.slice(0, 80)})`; + const ratio = firstPrompt > 0 ? ((message.usage.cacheRead / firstPrompt) * 100).toFixed(0) : "?"; + const calls = toolCalls(message); + const summary = label === "control" || contentText(message.content).includes("## Goal") ? "" : "no-summary"; + const flags = [ + message.stopReason !== "stop" ? message.stopReason : "", + calls.length > 0 ? `tools:${calls.join("+")}` : "", + summary, + ]; + return `${label}=${ratio}%${flags + .filter(Boolean) + .map((flag) => `(${flag})`) + .join("")}`; + }; + const results = [ + describe("control", await request([...conversation, { role: "user", content: "Reply with OK.", timestamp: 1 }])), + ]; + for (const shape of only.size > 0 ? ([...only] as Shape[]) : SHAPES) { + const messages = shape.startsWith("system") + ? appendInContextCompactionRequest(conversation) + : [ + ...conversation, + { + role: "user" as const, + content: `${IN_CONTEXT_COMPACTION_INSTRUCTIONS}\n\n${IN_CONTEXT_COMPACTION_PROMPT}`, + timestamp: 1, + }, + ]; + results.push(describe(shape, await request(messages, shape.endsWith("+none") ? { toolChoice: "none" } : {}))); + } + const midConvo = supportsMidConvoSystemMessages(model) ? "mid-convo-system" : "collapsed-system"; + return `${id.padEnd(44)} prompt=${firstPrompt} ${midConvo} ${results.join(" ")}`; +} + +async function probe(runtime: ModelRuntime, model: Model, args: Args): Promise { + let result: ProbeResult | undefined; + for (let attempt = 0; attempt < args.attempts; attempt++) { + try { + result = await probeOnce(runtime, model, args.reasoning); + } catch (error) { + result = { + model: `${model.provider}/${model.id}`, + verdict: "error", + detail: error instanceof Error ? error.message : String(error), + }; + } + // Retry only cache misses: provider caches are best-effort and can miss on routing. + if (result.verdict === "pass" || result.verdict === "error" || result.ratio === undefined) break; + } + return result!; +} + +function format(result: ProbeResult): string { + const ratio = result.ratio === undefined ? "" : ` read=${(result.ratio * 100).toFixed(1)}%`; + const control = result.controlRatio === undefined ? "" : ` control=${(result.controlRatio * 100).toFixed(1)}%`; + const prompt = result.firstPromptTokens === undefined ? "" : ` prompt=${result.firstPromptTokens}`; + return `${result.verdict.toUpperCase().padEnd(12)} ${result.model}${prompt}${ratio}${control} ${result.detail}`; +} + +const args = parseArgs(process.argv.slice(2)); +const runtime = await ModelRuntime.create(); +const selected = new Map>(); +for (const spec of args.models) { + const slash = spec.indexOf("/"); + const model = slash > 0 ? runtime.getModel(spec.slice(0, slash), spec.slice(slash + 1)) : undefined; + if (!model) throw new Error(`Unknown model: ${spec}`); + selected.set(spec, model); +} +if (args.all || args.providers.size > 0) { + for (const model of runtime.getModels()) { + if (!args.shapes && !supportsMidConvoSystemMessages(model)) continue; + if (args.providers.size > 0 && !args.providers.has(model.provider)) continue; + if (!runtime.hasConfiguredAuth(model.provider)) continue; + selected.set(`${model.provider}/${model.id}`, model); + } +} + +// Providers run in parallel; models of one provider run in sequence to respect rate limits. +const byProvider = new Map[]>(); +for (const model of selected.values()) { + byProvider.set(model.provider, [...(byProvider.get(model.provider) ?? []), model]); +} +const results: ProbeResult[] = []; +await Promise.all( + [...byProvider.values()].map(async (models) => { + for (const model of models) { + if (args.shapes) { + console.log( + await probeShapes(runtime, model, args.reasoning, args.onlyShapes).catch( + (error) => `${model.provider}/${model.id} ${error}`, + ), + ); + continue; + } + const result = await probe(runtime, model, args); + results.push(result); + console.log(format(result)); + } + }), +); +results.sort((left, right) => left.model.localeCompare(right.model)); +console.log(`\n${results.filter((result) => result.verdict === "pass").length}/${results.length} passed`); +if (args.json) writeFileSync(args.json, `${JSON.stringify(results, null, 2)}\n`); +process.exit(0); diff --git a/packages/ai/test/openai-completions-thinking-as-text.test.ts b/packages/ai/test/openai-completions-thinking-as-text.test.ts index e609b1453..9ae831f37 100644 --- a/packages/ai/test/openai-completions-thinking-as-text.test.ts +++ b/packages/ai/test/openai-completions-thinking-as-text.test.ts @@ -45,6 +45,7 @@ const compat = { supportsOpenAIGrammarTools: false, supportsMidConvoSystemMessages: false, supportsMidConvoToolAdditions: false, + supportsInContextCompaction: false, cacheControlFormat: undefined, sendSessionAffinityHeaders: false, sessionAffinityFormat: "openai", diff --git a/packages/ai/test/openai-completions-tool-result-images.test.ts b/packages/ai/test/openai-completions-tool-result-images.test.ts index 4d303865e..cf812e87e 100644 --- a/packages/ai/test/openai-completions-tool-result-images.test.ts +++ b/packages/ai/test/openai-completions-tool-result-images.test.ts @@ -37,6 +37,7 @@ const compat: Omit, "thinkingTokenBudgetField" supportsOpenAIGrammarTools: false, supportsMidConvoSystemMessages: false, supportsMidConvoToolAdditions: false, + supportsInContextCompaction: false, cacheControlFormat: "anthropic", sendSessionAffinityHeaders: false, sessionAffinityFormat: "openai", diff --git a/packages/ai/test/providers.test.ts b/packages/ai/test/providers.test.ts index 41309b28c..5dbdc55ab 100644 --- a/packages/ai/test/providers.test.ts +++ b/packages/ai/test/providers.test.ts @@ -218,6 +218,37 @@ describe("builtin providers", () => { } }); + it("enables in-context compaction only for validated models", () => { + const models = builtinModels(); + for (const [provider, modelId] of [ + ["anthropic", "claude-opus-5-5"], + ["openai-codex", "gpt-6-astra"], + ["moonshotai", "kimi-k3"], + ["opencode", "claude-sonnet-5-5"], + ["openrouter", "openai/gpt-5.6-terra"], + ] as const) { + expect(models.getModel(provider, modelId), `${provider}/${modelId}`).toHaveProperty( + "compat.supportsInContextCompaction", + true, + ); + } + // These missed the cache in validation, or were never validated. + for (const [provider, modelId] of [ + ["openai-codex", "gpt-5.5"], + ["moonshotai", "kimi-k2.6"], + ["openrouter", "openai/gpt-5.4-mini"], + ["deepseek", "deepseek-v4-pro"], + ["opencode-go", "kimi-k3"], + ["anthropic", "claude-opus-5"], + ["github-copilot", "claude-opus-5"], + ["anthropic", "claude-sonnet-4-5"], + ] as const) { + expect(models.getModel(provider, modelId), `${provider}/${modelId}`).not.toHaveProperty( + "compat.supportsInContextCompaction", + ); + } + }); + it("routes proxied tool changes through verified transports only", () => { const models = builtinModels(); for (const [provider, modelId] of [ diff --git a/packages/coding-agent/docs/compaction.md b/packages/coding-agent/docs/compaction.md index a702f7b6e..954604dbf 100644 --- a/packages/coding-agent/docs/compaction.md +++ b/packages/coding-agent/docs/compaction.md @@ -82,6 +82,28 @@ What the LLM sees: On repeated compactions, the summarized span starts at the previous compaction's kept boundary (`firstKeptEntryId`), not at the compaction entry itself, falling back to the entry after the previous compaction if that kept entry cannot be found in the path. A retain-none compaction records its own ID as `firstKeptEntryId`; repeated compaction starts after that entry. This preserves messages that survived the earlier compaction by including them in the next summarization pass as well. Pi also recalculates `tokensBefore` from the rebuilt, context-edited session projection before writing the new `CompactionEntry`, so the token count reflects the actual pre-compaction context being replaced. Omitted raw entries remain stored but do not affect cut selection, summaries, checkpoints, or token estimates. +### In-context compaction + +With `compaction.inContext` enabled, Pi can generate the summary inside the live conversation instead of in a separate request. It repeats the request the next turn would send, appends a system message with the summary instructions and a user message that asks for the summary, and sets `toolChoice: "none"`. The provider reads the conversation from its prompt cache, and the model sees the full conversation, including the recent messages that stay verbatim, so the summary describes the current state. The kept tail, `firstKeptEntryId`, and the resulting `CompactionEntry` are the same as for a standalone summary. + +Pi uses in-context compaction only when all of these hold; otherwise it uses the standalone summary: + +- The model has `supportsInContextCompaction` in its catalog compat settings. The catalog sets it only for models where real pi sessions showed that this request still reads the cached prefix right after a tool-heavy turn. +- The model is not a virtual model, and the compaction is not overflow recovery. +- The last response came from the same model and thinking level, reported prompt cache activity, and is younger than 90% of the model's cache lifetime (five minutes when the catalog has none). Cache warming refreshes the entry and counts as a newer response. +- The repeated conversation, instructions, and summary budget fit the context window. +- The estimated prompt cost is lower than the standalone request: cached tokens at the cache-read price plus new tokens at the cache-write or input price, against the standalone prompt at the input price. Models without prices weight cached tokens at 10%. + +If the in-context request fails, is truncated, or calls a tool, Pi discards the response without running any tool and falls back to the standalone summary. Payload hooks such as `before_provider_request` run for the in-context request, since they can change the cached prefix. When a `before_agent_start` handler replaced the system prompt for the last run, a compaction after that run (for example `/compact`) repeats that prompt, because the provider cached it. + +With the setting on, every compaction records the attempt in `CompactionEntry.details.inContext`: + +- `{ "status": "used" }`: the summary was generated in context. +- `{ "status": "skipped", "reason": "..." }`: the decision chose the standalone summary, for example `"prompt cache has likely expired"` or `"standalone summary is cheaper"`. +- `{ "status": "failed", "stage": "rebuild" | "request", "error": "..." }`: an attempt failed and Pi fell back. `rebuild` failures happen while reproducing the next-turn request, before anything is sent; `request` failures come from the provider or the response. + +A failed attempt is billed together with the standalone summary in the entry's `usage`. Interactive mode warns when an attempt failed, and the compaction cost notice (`showCacheMissNotices`) notes whether the summary was generated in context and why not. + ### Overflow and Length Recovery Ordering Recovery preserves the existing lifecycle and queue order. The completed attempt remains visible to `turn_end` and `agent_end`; post-run recovery then repairs persisted model context before a fresh retry: @@ -423,7 +445,8 @@ Configure compaction in `~/.pi/agent/settings.json` or `/.pi/settin "compaction": { "enabled": true, "reserveTokens": 16384, - "keepRecentTokens": 20000 + "keepRecentTokens": 20000, + "inContext": false } } ``` @@ -433,6 +456,7 @@ Configure compaction in `~/.pi/agent/settings.json` or `/.pi/settin | `enabled` | `true` | Enable auto-compaction | | `reserveTokens` | `16384` | Tokens to reserve for LLM response | | `keepRecentTokens` | `20000` | Recent tokens to keep (not summarized) | +| `inContext` | `false` | Generate the summary inside the cached conversation when that is cheaper. See [In-context compaction](#in-context-compaction) | Disable auto-compaction with `"enabled": false`. You can still compact manually with `/compact`. diff --git a/packages/coding-agent/docs/settings.md b/packages/coding-agent/docs/settings.md index e7063179e..393c423d2 100644 --- a/packages/coding-agent/docs/settings.md +++ b/packages/coding-agent/docs/settings.md @@ -70,6 +70,7 @@ CLI tool options override this setting for one invocation. `--tools` with only ` | `compaction.enabled` | boolean | `true` | Enable automatic compaction. | | `compaction.reserveTokens` | number | `16384` | Tokens reserved for the model response. | | `compaction.keepRecentTokens` | number | `20000` | Recent tokens retained without summarization. | +| `compaction.inContext` | boolean | `false` | Generate summaries inside the cached conversation on verified models when that is cheaper. See [In-context compaction](compaction.md#in-context-compaction). | | `compaction.modelOverrides` | object | None | Per-model token settings keyed by exact `provider/modelId`. | diff --git a/packages/coding-agent/src/core/agent-session.ts b/packages/coding-agent/src/core/agent-session.ts index 7388024e0..bc3f2c93a 100644 --- a/packages/coding-agent/src/core/agent-session.ts +++ b/packages/coding-agent/src/core/agent-session.ts @@ -36,6 +36,7 @@ import type { AssistantMessage, AuthResult, ImageContent, + Message, Model, ProviderHeaders, SystemMessage, @@ -65,15 +66,20 @@ import { type BashResult, executeBashWithOperations } from "./bash-executor.ts"; import { generateBugReportSummary } from "./bug-report.ts"; import type { CacheWarmer, CacheWarmingStatus } from "./cache-warmer.ts"; import { + type CompactionDetails, type CompactionPreparation, type CompactionResult, calculateContextTokens, collectEntriesForBranchSummary, compact, + compactInContext, estimateContextTokens, estimateProjectedContextTokens, estimateTokens, + evaluateInContextCompaction, generateBranchSummary, + InContextCompactionError, + type InContextCompactionOutcome, prepareCompaction, shouldCompact, } from "./compaction/index.ts"; @@ -371,6 +377,18 @@ function estimateMessagesTokens(messages: AgentMessage[]): number { return tokens; } +/** Replace every system message with one leading forced prompt that keeps the current tool declarations. */ +function withForcedSystemPrompt(messages: AgentMessage[], forced: string): AgentMessage[] { + const current = getCurrentSystemMessage(messages); + const head: SystemMessage = { + role: "system", + content: forced, + ...(current?.toolsAdded ? { toolsAdded: current.toolsAdded } : {}), + timestamp: current?.timestamp ?? Date.now(), + }; + return [head, ...messages.filter((message) => message.role !== "system")]; +} + // ============================================================================ // AgentSession Class // ============================================================================ @@ -481,6 +499,8 @@ export class AgentSession { private _baseSystemPromptOptions!: NormalizedBuildSystemPromptOptions; /** Prompt options after before_agent_start mutations for the active run. */ private _runSystemPromptOptions?: NormalizedBuildSystemPromptOptions; + /** Forced system prompt of the last finished run, which its cached provider requests used. */ + private _lastRunForcedSystemPrompt?: string; constructor(config: AgentSessionConfig) { this.agent = config.agent; @@ -1787,15 +1807,7 @@ export class AgentSession { this.agent.transformContext = async (messages, signal) => { const transformed = previousTransformContext ? await previousTransformContext(messages, signal) : messages; const forced = this._runSystemPromptOptions?.forceSystemPrompt; - if (forced === undefined) return transformed; - const current = getCurrentSystemMessage(transformed); - const head: SystemMessage = { - role: "system", - content: forced, - ...(current?.toolsAdded ? { toolsAdded: current.toolsAdded } : {}), - timestamp: current?.timestamp ?? Date.now(), - }; - return [head, ...transformed.filter((message) => message.role !== "system")]; + return forced === undefined ? transformed : withForcedSystemPrompt(transformed, forced); }; } @@ -1841,6 +1853,7 @@ export class AgentSession { } finally { if (this._agentRunAbortRequested) this._finishCancelledRetry(); this._failedResponse = undefined; + this._lastRunForcedSystemPrompt = this._runSystemPromptOptions?.forceSystemPrompt; this._runSystemPromptOptions = undefined; this._flushPendingBashMessages(); this._flushPendingCustomMessages(); @@ -2721,9 +2734,11 @@ export class AgentSession { signal: AbortSignal, reason: "manual" | "threshold" | "overflow", ): Promise { + const attempt = await this._runInContextCompaction(preparation, model, customInstructions, signal, reason); + if (attempt?.result) return attempt.result; // Resolve the request only when Pi summarizes itself: routing may call models or fail. const request = await this._getSummarizationRequestAuth(model, signal); - return compact( + const result = await compact( preparation, request.model, request.apiKey, @@ -2737,6 +2752,96 @@ export class AgentSession { this._summarizationRetryCallbacks({ source: "compaction", reason }), undefined, // sessionId ); + if (!attempt) return result; + // Keep the attempt visible in the entry, and bill a rejected in-context response with the summary. + const usage = attempt.usage && result.usage ? combineUsage(attempt.usage, result.usage) : result.usage; + const details: CompactionDetails = { ...(result.details as CompactionDetails), inContext: attempt.outcome }; + return { ...result, usage, details }; + } + + /** + * Try to summarize inside the live conversation when `compaction.inContext` is on. Returns + * undefined when the setting is off. Otherwise `result` is set when the attempt succeeded, and + * `outcome` records why it was skipped or how it failed, so the caller can fall back to + * `compact()` without hiding the reason. Aborts propagate. + */ + private async _runInContextCompaction( + preparation: CompactionPreparation, + model: Model, + customInstructions: string | undefined, + signal: AbortSignal, + reason: "manual" | "threshold" | "overflow", + ): Promise<{ result?: CompactionResult; outcome: InContextCompactionOutcome; usage?: Usage } | undefined> { + if (!this.settingsManager.getCompactionInContext()) return undefined; + const skip = (skipReason: string) => ({ outcome: { status: "skipped" as const, reason: skipReason } }); + // Overflow recovery cannot repeat a context that already exceeded the window. + if (reason === "overflow") return skip("overflow recovery"); + // A virtual model routes each request, so the next turn may not use the cached model. + if (isVirtualModel(model)) return skip("virtual model"); + const projection = this.sessionManager.buildSessionProjection(); + const thinkingLevel = this.agent.state.thinkingLevel; + const decision = evaluateInContextCompaction({ + model, + projection, + branchEntries: this.sessionManager.getBranch(), + preparation, + thinkingLevel, + }); + if (!decision.use) return skip(decision.reason); + + const fail = (stage: "rebuild" | "request", error: unknown) => { + if (signal.aborted) throw error; + const message = error instanceof Error ? error.message : String(error); + return { + outcome: { status: "failed" as const, stage, error: message }, + usage: error instanceof InContextCompactionError ? error.usage : undefined, + }; + }; + + // Rebuild the request the next turn would send (see _installAgentRequestProjection and the + // agent loop), including payload hooks: they can rewrite the cached prefix. + let messages: Message[]; + let apiKey: string | undefined; + try { + const transformed = this.agent.transformContext + ? await this.agent.transformContext(projection.messages, signal) + : projection.messages; + // Between runs the forced prompt projection is off, but the cached requests used it. + const forced = this._runSystemPromptOptions ? undefined : this._lastRunForcedSystemPrompt; + messages = await this.agent.convertToLlm( + forced === undefined ? transformed : withForcedSystemPrompt(transformed, forced), + ); + apiKey = await this.agent.getApiKey?.(model.provider); + } catch (error) { + return fail("rebuild", error); + } + + try { + const result = await compactInContext( + preparation, + model, + messages, + { + apiKey, + sessionId: this.agent.sessionId, + reasoning: thinkingLevel === "off" ? undefined : thinkingLevel, + thinkingBudgets: this.agent.thinkingBudgets, + transport: this.agent.transport, + maxRetryDelayMs: this.agent.maxRetryDelayMs, + onPayload: this.agent.onPayload, + onResponse: this.agent.onResponse, + onProviderStreamEvent: this.agent.onProviderStreamEvent, + signal, + }, + this.agent.streamFunction, + customInstructions, + this.settingsManager.getRetrySettings(), + this._summarizationRetryCallbacks({ source: "compaction", reason }), + ); + return { result, outcome: { status: "used" } }; + } catch (error) { + return fail("request", error); + } } private _clearManualCompactionState(): void { diff --git a/packages/coding-agent/src/core/cache-warmer.ts b/packages/coding-agent/src/core/cache-warmer.ts index f4f885f23..5a1d529a2 100644 --- a/packages/coding-agent/src/core/cache-warmer.ts +++ b/packages/coding-agent/src/core/cache-warmer.ts @@ -69,7 +69,8 @@ function lastPromptTokens(entries: SessionEntry[]): number { return 0; } -function price( +/** Price token counts with the model's rates, including tiered pricing. */ +export function priceTokens( model: Model, tokens: Partial>, ): number { @@ -378,12 +379,12 @@ export class CacheWarmer { private evaluate(run: ActiveRun): CacheWarmingDecision { const model = run.model; const promptTokens = lastPromptTokens(this.sessionManager.getBranch()); - const cacheHitCost = price(model, { cacheRead: promptTokens }); - const cacheMissCost = price( + const cacheHitCost = priceTokens(model, { cacheRead: promptTokens }); + const cacheMissCost = priceTokens( model, model.cost.cacheWrite > 0 ? { cacheWrite: promptTokens } : { input: promptTokens }, ); - const warmCost = price(model, { cacheRead: promptTokens, output: 1 }); + const warmCost = priceTokens(model, { cacheRead: promptTokens, output: 1 }); const missCost = Math.max(0, cacheMissCost - cacheHitCost); const continuationProbability = run.phase === "idle" ? IDLE_CONTINUATION_PROBABILITY : 1; const economicsAvailable = promptTokens > 0 && (cacheHitCost > 0 || cacheMissCost > 0); diff --git a/packages/coding-agent/src/core/compaction/compaction.ts b/packages/coding-agent/src/core/compaction/compaction.ts index 6bcfbb2b5..02e890629 100644 --- a/packages/coding-agent/src/core/compaction/compaction.ts +++ b/packages/coding-agent/src/core/compaction/compaction.ts @@ -16,7 +16,9 @@ import { uuidv7, } from "@earendil-works/pi-ai"; import type { + Api, AssistantMessage, + Message, Model, SimpleStreamOptions, SystemMessage, @@ -24,6 +26,7 @@ import type { Usage, } from "@earendil-works/pi-ai/compat"; import { completeSimple } from "@earendil-works/pi-ai/compat"; +import { getPromptCacheTtlMs, priceTokens } from "../cache-warmer.ts"; import { convertToLlm } from "../messages.ts"; import { buildSessionProjection, @@ -52,6 +55,29 @@ import { export interface CompactionDetails { readFiles: string[]; modifiedFiles: string[]; + /** Outcome of the in-context attempt. Absent when `compaction.inContext` is off. */ + inContext?: InContextCompactionOutcome; +} + +/** + * What happened to the in-context attempt of one compaction. `skipped` carries the reason the + * decision chose a standalone summary; `failed` means an attempt was made and Pi fell back. + * `rebuild` failures happen while reproducing the next-turn request, before anything is sent. + */ +export type InContextCompactionOutcome = + | { status: "used" } + | { status: "skipped"; reason: string } + | { status: "failed"; stage: "rebuild" | "request"; error: string }; + +/** Read the in-context outcome from compaction details, which extensions may shape freely. */ +export function getInContextCompactionOutcome(details: unknown): InContextCompactionOutcome | undefined { + if (typeof details !== "object" || details === null || !("inContext" in details)) return undefined; + const outcome = details.inContext; + if (typeof outcome !== "object" || outcome === null || !("status" in outcome)) return undefined; + const status = outcome.status; + return status === "used" || status === "skipped" || status === "failed" + ? (outcome as InContextCompactionOutcome) + : undefined; } /** @@ -1117,3 +1143,267 @@ async function generateTurnPrefixSummary( usage: response.usage, }; } + +// ============================================================================ +// In-context compaction +// ============================================================================ + +/** + * Instructions appended as a system message to the live conversation. The model sees the + * whole conversation, including the recent messages that compaction keeps verbatim, so the + * summary describes the current state instead of the state at the cut point. + */ +export const IN_CONTEXT_COMPACTION_INSTRUCTIONS = `# Context compaction + +The conversation is about to be compacted. Everything above will be replaced by a checkpoint summary that you write now. The most recent messages will be kept verbatim after the summary, but the summary must stand on its own for everything earlier. + +Do NOT continue the task. Do NOT call tools. Do NOT answer questions from the conversation. Respond only with the summary. + +The system prompt and project instructions stay in place after compaction. Do not restate them; record only constraints and preferences that come from the conversation. + +If the conversation contains an earlier compaction summary, merge it: preserve its information, move finished items to Done, and update Next Steps. + +Use this EXACT format: + +## Goal +[What is the user trying to accomplish? Can be multiple items if the session covers different tasks.] + +## Constraints & Preferences +- [Any constraints, preferences, or requirements mentioned by user] +- [Or "(none)" if none were mentioned] + +## Progress +### Done +- [x] [Completed tasks/changes] + +### In Progress +- [ ] [Current work] + +### Blocked +- [Issues preventing progress, if any] + +## Key Decisions +- **[Decision]**: [Brief rationale] + +## Next Steps +1. [Ordered list of what should happen next] + +## Critical Context +- [Any data, examples, or references needed to continue] +- [Or "(none)" if not applicable] + +Keep each section concise. Preserve exact file paths, function names, and error messages.`; + +/** User message that asks for the in-context summary after the instructions. */ +export const IN_CONTEXT_COMPACTION_PROMPT = "Write the context checkpoint summary now."; + +/** Fallback lifetime for models without catalog cache lifetimes: the shortest documented provider lifetime. */ +const DEFAULT_PROMPT_CACHE_TTL_MS = 5 * 60_000; +/** Share of the cache lifetime after which the entry counts as expired. Requests are timed from their start. */ +const PROMPT_CACHE_TTL_SAFETY = 0.9; +/** Cost of a cached prompt token relative to an uncached one when the model has no prices. */ +const UNPRICED_CACHE_READ_WEIGHT = 0.1; + +/** + * Append the compaction request to the provider transcript of the live conversation. + * The system message carries the instructions; models that accept system messages + * mid-conversation receive it in place, so the conversation prefix stays cached. + */ +export function appendInContextCompactionRequest(messages: Message[], customInstructions?: string): Message[] { + const timestamp = Date.now(); + const prompt = customInstructions + ? `${IN_CONTEXT_COMPACTION_PROMPT}\n\nAdditional focus: ${customInstructions}` + : IN_CONTEXT_COMPACTION_PROMPT; + return [ + ...messages, + { role: "system", content: IN_CONTEXT_COMPACTION_INSTRUCTIONS, timestamp }, + { role: "user", content: [{ type: "text", text: prompt }], timestamp }, + ]; +} + +/** Whether the model catalog marks the model as keeping its prompt cache for in-context compaction. */ +export function supportsInContextCompaction(model: Model): boolean { + const compat = model.compat as + | { supportsMidConvoSystemMessages?: boolean; supportsInContextCompaction?: boolean } + | undefined; + return compat?.supportsMidConvoSystemMessages === true && compat.supportsInContextCompaction === true; +} + +/** Estimated cost of both summary strategies, excluding output, which is similar for both. */ +export interface InContextCompactionCosts { + /** Prompt tokens of the last response, expected to be read from the cache. */ + cachedTokens: number; + /** Prompt tokens added since the last response, including the compaction request. */ + uncachedTokens: number; + /** Estimated prompt tokens of the standalone summary request(s). */ + standaloneTokens: number; + /** In dollars when `priced`, otherwise in uncached-token equivalents. */ + inContextCost: number; + standaloneCost: number; + priced: boolean; +} + +export type InContextCompactionDecision = ({ use: true } & InContextCompactionCosts) | { use: false; reason: string }; + +export interface InContextCompactionInput { + /** The model that would answer the next turn. */ + model: Model; + projection: SessionProjection; + branchEntries: SessionEntry[]; + preparation: CompactionPreparation; + /** The session's current thinking level, which the next turn would send. */ + thinkingLevel: ThinkingLevel; + env?: Record; + now?: number; +} + +function serializedTokens(messages: AgentMessage[]): number { + return Math.ceil(serializeConversation(convertToLlm(messages)).length / 4); +} + +/** Estimate the prompt tokens of the standalone requests `compact()` would send. */ +function estimateStandaloneSummaryTokens(preparation: CompactionPreparation): number { + const { messagesToSummarize, turnPrefixMessages, isSplitTurn, previousSummary } = preparation; + const splitTurn = isSplitTurn && turnPrefixMessages.length > 0; + let tokens = 0; + if (!splitTurn || messagesToSummarize.length > 0) { + const prompt = previousSummary ? UPDATE_SUMMARIZATION_PROMPT : SUMMARIZATION_PROMPT; + const chars = SUMMARIZATION_SYSTEM_PROMPT.length + prompt.length + (previousSummary?.length ?? 0); + tokens += Math.ceil(chars / 4) + serializedTokens(messagesToSummarize); + } + if (splitTurn) { + const chars = SUMMARIZATION_SYSTEM_PROMPT.length + TURN_PREFIX_SUMMARIZATION_PROMPT.length; + tokens += Math.ceil(chars / 4) + serializedTokens(turnPrefixMessages); + } + return tokens; +} + +/** + * Decide whether to summarize inside the live conversation instead of in a standalone request. + * + * In-context compaction repeats the request the next turn would send and appends the + * compaction instructions. It is only cheaper while the provider still caches that + * request's prefix, so the decision requires recent cache activity from the same model + * and thinking level, and compares the estimated prompt cost of both strategies. + */ +export function evaluateInContextCompaction(input: InContextCompactionInput): InContextCompactionDecision { + const { model, projection, branchEntries, preparation, thinkingLevel } = input; + const now = input.now ?? Date.now(); + if (!supportsInContextCompaction(model)) { + return { use: false, reason: "model is not verified for in-context compaction" }; + } + + const estimate = estimateProjectedContextTokens(projection, branchEntries); + const usageMessage = estimate.lastUsageIndex === null ? undefined : projection.messages[estimate.lastUsageIndex]; + if (!usageMessage || usageMessage.role !== "assistant") { + return { use: false, reason: "no provider usage for the current context" }; + } + if (usageMessage.provider !== model.provider || usageMessage.model !== model.id) { + return { use: false, reason: "last response came from a different model" }; + } + if (usageMessage.thinkingLevel !== undefined && usageMessage.thinkingLevel !== thinkingLevel) { + return { use: false, reason: "thinking level changed since the last response" }; + } + const usage = usageMessage.usage; + if (usage.cacheRead + usage.cacheWrite === 0) { + return { use: false, reason: "last response reported no prompt caching" }; + } + + // Cache warming re-sends the request and refreshes the entry. + let refreshedAt = usageMessage.timestamp; + for (const entry of branchEntries) { + if (entry.type !== "usage" || entry.kind !== "cache_warm" || entry.provider !== model.provider) continue; + refreshedAt = Math.max(refreshedAt, new Date(entry.timestamp).getTime()); + } + const ttlMs = getPromptCacheTtlMs(model, { env: input.env }) ?? DEFAULT_PROMPT_CACHE_TTL_MS; + if (now - refreshedAt > ttlMs * PROMPT_CACHE_TTL_SAFETY) { + return { use: false, reason: "prompt cache has likely expired" }; + } + + const requestTokens = Math.ceil( + (IN_CONTEXT_COMPACTION_INSTRUCTIONS.length + IN_CONTEXT_COMPACTION_PROMPT.length) / 4, + ); + const promptTokens = estimate.tokens + requestTokens; + const outputBudget = Math.floor(0.8 * preparation.settings.reserveTokens); + if (promptTokens + outputBudget > model.contextWindow) { + return { use: false, reason: "context is too full for an in-context summary" }; + } + + const cachedTokens = Math.min(usage.input + usage.cacheRead + usage.cacheWrite, promptTokens); + const uncachedTokens = promptTokens - cachedTokens; + const standaloneTokens = estimateStandaloneSummaryTokens(preparation); + const priced = model.cost.input > 0 && model.cost.cacheRead > 0; + const inContextCost = priced + ? priceTokens(model, { + cacheRead: cachedTokens, + ...(model.cost.cacheWrite > 0 ? { cacheWrite: uncachedTokens } : { input: uncachedTokens }), + }) + : cachedTokens * UNPRICED_CACHE_READ_WEIGHT + uncachedTokens; + const standaloneCost = priced ? priceTokens(model, { input: standaloneTokens }) : standaloneTokens; + const costs = { cachedTokens, uncachedTokens, standaloneTokens, inContextCost, standaloneCost, priced }; + if (inContextCost >= standaloneCost) { + return { use: false, reason: "standalone summary is cheaper" }; + } + return { use: true, ...costs }; +} + +/** A rejected in-context response. Carries the billed usage so callers can account for it. */ +export class InContextCompactionError extends Error { + readonly usage: Usage; + + constructor(message: string, usage: Usage) { + super(message); + this.name = "InContextCompactionError"; + this.usage = usage; + } +} + +/** + * Generate the compaction summary inside the live conversation. + * + * `messages` and `options` must reproduce the request the next turn would send, so the + * provider reads the cached prefix. Tools stay declared to keep that prefix intact; + * `toolChoice: "none"` stops the model from calling them. The kept tail is unchanged, + * so the result has the same shape as `compact()`. + * + * @throws {InContextCompactionError} When the response fails, is truncated, calls a tool, or has no + * text. The response is discarded; callers fall back to `compact()`. + */ +export async function compactInContext( + preparation: CompactionPreparation, + model: Model, + messages: Message[], + options: SimpleStreamOptions, + streamFn: StreamFn, + customInstructions?: string, + retry?: RetryPolicy, + callbacks?: RetryCallbacks, +): Promise { + const context = normalizeContext({ messages: appendInContextCompactionRequest(messages, customInstructions) }); + const requestOptions: SimpleStreamOptions = { ...options, toolChoice: "none" }; + const response = await retryAssistantCall( + async () => (await streamFn(model, context, requestOptions)).result(), + retry, + requestOptions.signal, + callbacks, + ); + + const failure = getSummarizationFailure(response, "In-context summarization"); + if (failure) throw new InContextCompactionError(failure, response.usage); + if (response.content.some((block) => block.type === "toolCall")) { + throw new InContextCompactionError("In-context summarization attempted to call a tool", response.usage); + } + const text = contentText(response.content).trim(); + if (text.length === 0) { + throw new InContextCompactionError("In-context summarization returned no text", response.usage); + } + + const { readFiles, modifiedFiles } = computeFileLists(preparation.fileOps); + return { + summary: text + formatFileOperations(readFiles, modifiedFiles), + firstKeptEntryId: preparation.firstKeptEntryId, + tokensBefore: preparation.tokensBefore, + usage: response.usage, + details: { readFiles, modifiedFiles, inContext: { status: "used" } } satisfies CompactionDetails, + }; +} diff --git a/packages/coding-agent/src/core/settings-manager.ts b/packages/coding-agent/src/core/settings-manager.ts index 2dd248eb4..e7ff1a1f5 100644 --- a/packages/coding-agent/src/core/settings-manager.ts +++ b/packages/coding-agent/src/core/settings-manager.ts @@ -29,6 +29,7 @@ export interface CompactionSettings { enabled?: boolean; // default: true reserveTokens?: number; // default: 16384 keepRecentTokens?: number; // default: 20000 + inContext?: boolean; // default: false - summarize inside the cached conversation when cheaper modelOverrides?: Record; // exact "provider/modelId" keys } @@ -947,6 +948,11 @@ export class SettingsManager { this.save(); } + /** Whether compaction may summarize inside the live conversation on models verified to keep their prompt cache. */ + getCompactionInContext(): boolean { + return this.settings.compaction?.inContext ?? false; + } + private getCompactionTokenSetting( field: keyof CompactionModelOverride, model?: Pick, "provider" | "id">, diff --git a/packages/coding-agent/src/modes/interactive/interactive-mode.ts b/packages/coding-agent/src/modes/interactive/interactive-mode.ts index 0928f918f..4a71eb50a 100644 --- a/packages/coding-agent/src/modes/interactive/interactive-mode.ts +++ b/packages/coding-agent/src/modes/interactive/interactive-mode.ts @@ -76,6 +76,7 @@ import { detectCacheMiss, } from "../../core/cache-stats.ts"; import { formatCacheWarmingStatus, formatCacheWarmingUsage } from "../../core/cache-warmer.ts"; +import { getInContextCompactionOutcome } from "../../core/compaction/index.ts"; import { findExtensionStackMatches, recordCrash, takeUnnotifiedCrash } from "../../core/crash-log.ts"; import { DEFAULT_THINKING_LEVEL, THINKING_LEVEL_OPTIONS } from "../../core/defaults.ts"; import type { @@ -272,8 +273,19 @@ type CompactionCostNotice = { type: "compaction_cost"; kind: "compaction" | "branch_summary"; usage: Usage; + /** Compaction details, which record whether the summary was generated in context. */ + details?: unknown; }; +/** Short note for the compaction cost notice that says how the summary was generated. */ +function describeInContextOutcome(details: unknown): string | undefined { + const outcome = getInContextCompactionOutcome(details); + if (!outcome) return undefined; + if (outcome.status === "used") return "in-context"; + if (outcome.status === "skipped") return `standalone: ${outcome.reason}`; + return `standalone after in-context ${outcome.stage} failure`; +} + type RenderSessionItem = AgentMessage | Extract | CompactionCostNotice; function isCustomSessionEntry(item: RenderSessionItem): item is Extract { @@ -3456,6 +3468,7 @@ export class InteractiveMode { type: "compaction_cost", kind: "compaction", usage: event.entry.usage, + details: event.entry.details, }); } this.renderSessionEntries(retainedEntries.filter((entry) => entriesAfterCompaction.has(entry.id))); @@ -3697,8 +3710,15 @@ export class InteractiveMode { type: "compaction_cost", kind: "compaction", usage: event.result.usage, + details: event.result.details, }); } + const inContext = getInContextCompactionOutcome(event.result.details); + if (inContext?.status === "failed") { + this.showWarning( + `In-context compaction failed (${inContext.stage}): ${inContext.error}. Used a standalone summary instead.`, + ); + } this.footer.invalidate(); } else if (event.errorMessage) { if (event.reason === "manual") { @@ -4079,7 +4099,8 @@ export class InteractiveMode { } const messages = sessionEntryToContextMessages(entry); if ((entry.type === "compaction" || entry.type === "branch_summary") && entry.usage && messages.length > 0) { - return [...messages, { type: "compaction_cost", kind: entry.type, usage: entry.usage }]; + const details = entry.type === "compaction" ? entry.details : undefined; + return [...messages, { type: "compaction_cost", kind: entry.type, usage: entry.usage, details }]; } return messages; }); @@ -4104,10 +4125,10 @@ export class InteractiveMode { const tokens = usage.input + usage.output + usage.cacheRead + usage.cacheWrite; const cost = usage.cost.total >= 0.01 ? ` (~$${usage.cost.total.toFixed(2)})` : ""; const label = notice.kind === "compaction" ? "Compaction" : "Branch summary"; + const note = describeInContextOutcome(notice.details); + const text = `${label}: ${formatTokens(tokens)} tokens billed${cost}${note ? ` (${note})` : ""}`; this.chatContainer.addChild(new Spacer(1)); - this.chatContainer.addChild( - new ThemedText(() => theme.fg("warning", `${label}: ${formatTokens(tokens)} tokens billed${cost}`), 1, 0), - ); + this.chatContainer.addChild(new ThemedText(() => theme.fg("warning", text), 1, 0)); } private static countDroppedThinkingBlocks(message: AssistantMessage): number { diff --git a/packages/coding-agent/test/compaction-in-context.test.ts b/packages/coding-agent/test/compaction-in-context.test.ts new file mode 100644 index 000000000..351c195b7 --- /dev/null +++ b/packages/coding-agent/test/compaction-in-context.test.ts @@ -0,0 +1,159 @@ +import type { AssistantMessage, Model, Usage } from "@earendil-works/pi-ai"; +import { describe, expect, it } from "vitest"; +import { + appendInContextCompactionRequest, + evaluateInContextCompaction, + IN_CONTEXT_COMPACTION_INSTRUCTIONS, + IN_CONTEXT_COMPACTION_PROMPT, + type InContextCompactionInput, + prepareCompaction, +} from "../src/core/compaction/index.ts"; +import { SessionManager } from "../src/core/session-manager.ts"; + +function createModel(overrides: Partial> = {}): Model<"anthropic-messages"> { + return { + id: "cached-model", + name: "Cached Model", + api: "anthropic-messages", + provider: "anthropic", + baseUrl: "https://api.anthropic.com", + reasoning: true, + input: ["text"], + cost: { input: 5, output: 25, cacheRead: 0.5, cacheWrite: 6.25 }, + promptCache: { short: 300, long: 3600 }, + contextWindow: 200000, + maxTokens: 64000, + compat: { supportsMidConvoSystemMessages: true, supportsInContextCompaction: true }, + ...overrides, + }; +} + +function usage(input: number, cacheRead: number, cacheWrite: number): Usage { + return { + input, + output: 100, + cacheRead, + cacheWrite, + totalTokens: input + cacheRead + cacheWrite + 100, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }; +} + +interface SessionOptions { + historyChars?: number; + lastUsage?: Usage; + ageMs?: number; + responseProvider?: string; + cacheWarmed?: boolean; +} + +/** Three turns of history; the last response reports usage that `evaluateInContextCompaction` trusts. */ +function createInput( + model = createModel(), + options: SessionOptions = {}, + thinkingLevel: InContextCompactionInput["thinkingLevel"] = "high", +): InContextCompactionInput { + const now = Date.now(); + const sessionManager = SessionManager.inMemory(); + const historyChars = options.historyChars ?? 40000; + for (let turn = 0; turn < 3; turn++) { + sessionManager.appendMessage({ + role: "user", + content: [{ type: "text", text: `Question ${turn}. ${"x".repeat(historyChars)}` }], + timestamp: now - 60_000, + }); + const response: AssistantMessage = { + role: "assistant", + content: [{ type: "text", text: `Answer ${turn}.` }], + api: model.api, + provider: turn === 2 ? (options.responseProvider ?? model.provider) : model.provider, + model: model.id, + thinkingLevel: "high", + usage: turn === 2 ? (options.lastUsage ?? usage(100, 40000, 500)) : usage(100, 0, 0), + stopReason: "stop", + timestamp: now - (turn === 2 ? (options.ageMs ?? 30_000) : 60_000), + }; + sessionManager.appendMessage(response); + } + if (options.cacheWarmed) sessionManager.appendUsage("cache_warm", model.provider, model.id, usage(0, 40000, 0)); + const branchEntries = sessionManager.getBranch(); + const preparation = prepareCompaction(branchEntries, { enabled: true, reserveTokens: 16384, keepRecentTokens: 100 }); + if (!preparation) throw new Error("expected a compactable session"); + return { + model, + projection: sessionManager.buildSessionProjection(), + branchEntries, + preparation, + thinkingLevel, + now, + }; +} + +describe("evaluateInContextCompaction", () => { + it("uses in-context compaction while the cache is warm and cheaper", () => { + const decision = evaluateInContextCompaction(createInput()); + expect(decision.use).toBe(true); + if (!decision.use) return; + expect(decision.priced).toBe(true); + expect(decision.cachedTokens).toBe(40600); + expect(decision.inContextCost).toBeLessThan(decision.standaloneCost); + }); + + it.each<[string, () => InContextCompactionInput, string]>([ + [ + "models without the catalog flag", + () => createInput(createModel({ compat: { supportsMidConvoSystemMessages: true } })), + "not verified", + ], + [ + "responses from another model", + () => createInput(createModel(), { responseProvider: "other" }), + "different model", + ], + ["changed thinking levels", () => createInput(createModel(), {}, "low"), "thinking level changed"], + [ + "responses without cache activity", + () => createInput(createModel(), { lastUsage: usage(40600, 0, 0) }), + "no prompt caching", + ], + ["expired caches", () => createInput(createModel(), { ageMs: 10 * 60_000 }), "expired"], + ["full contexts", () => createInput(createModel({ contextWindow: 50000 })), "too full"], + [ + "small histories where the standalone prompt is cheaper", + () => createInput(createModel(), { historyChars: 1000 }), + "standalone summary is cheaper", + ], + ])("falls back for %s", (_label, input, reason) => { + const decision = evaluateInContextCompaction(input()); + expect(decision.use).toBe(false); + if (!decision.use) expect(decision.reason).toContain(reason); + }); + + it("counts cache warming as a cache refresh", () => { + const decision = evaluateInContextCompaction( + createInput(createModel(), { ageMs: 10 * 60_000, cacheWarmed: true }), + ); + expect(decision.use).toBe(true); + }); + + it("weights cached tokens when the model has no prices", () => { + const free = createModel({ cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 } }); + const decision = evaluateInContextCompaction(createInput(free)); + expect(decision.use).toBe(true); + if (decision.use) expect(decision.priced).toBe(false); + }); +}); + +describe("appendInContextCompactionRequest", () => { + it("appends the instructions as a system message and the request as a user message", () => { + const messages = appendInContextCompactionRequest( + [{ role: "user", content: "hello", timestamp: 1 }], + "focus on tests", + ); + expect(messages.map((message) => message.role)).toEqual(["user", "system", "user"]); + expect(messages[1].content).toBe(IN_CONTEXT_COMPACTION_INSTRUCTIONS); + expect(messages[2].content).toEqual([ + { type: "text", text: `${IN_CONTEXT_COMPACTION_PROMPT}\n\nAdditional focus: focus on tests` }, + ]); + }); +}); diff --git a/packages/coding-agent/test/interactive-mode-compaction.test.ts b/packages/coding-agent/test/interactive-mode-compaction.test.ts index 5b0980ba7..4d9b747e9 100644 --- a/packages/coding-agent/test/interactive-mode-compaction.test.ts +++ b/packages/coding-agent/test/interactive-mode-compaction.test.ts @@ -22,6 +22,7 @@ describe("InteractiveMode compaction events", () => { type: "compaction_cost"; kind: "compaction" | "branch_summary"; usage: Usage; + details?: unknown; }, ) => void; @@ -36,9 +37,24 @@ describe("InteractiveMode compaction events", () => { kind: "branch_summary", usage, }); + for (const inContext of [ + { status: "used" }, + { status: "skipped", reason: "prompt cache has likely expired" }, + { status: "failed", stage: "request", error: "boom" }, + ]) { + addCompactionCostNotice.call(enabled, { + type: "compaction_cost", + kind: "compaction", + usage, + details: { readFiles: [], modifiedFiles: [], inContext }, + }); + } const output = stripAnsi(enabled.chatContainer.render(120).join("\n")); expect(output).toContain("Compaction: 100 tokens billed (~$0.13)"); expect(output).toContain("Branch summary: 100 tokens billed (~$0.13)"); + expect(output).toContain("Compaction: 100 tokens billed (~$0.13) (in-context)"); + expect(output).toContain("(standalone: prompt cache has likely expired)"); + expect(output).toContain("(standalone after in-context request failure)"); const disabled = { chatContainer: new Container(), @@ -197,6 +213,66 @@ describe("InteractiveMode compaction events", () => { expect(fakeThis.flushCompactionQueue).toHaveBeenCalledWith({ willRetry: false }); }); + test("warns when an in-context attempt failed before the standalone summary", async () => { + const compaction: SessionEntry = { + type: "compaction", + id: "latest", + parentId: null, + timestamp: "2025-01-02T00:00:00Z", + summary: "summary", + firstKeptEntryId: "kept", + tokensBefore: 123, + }; + const fakeThis = { + isInitialized: true, + footer: { invalidate: vi.fn() }, + autoCompactionEscapeHandler: undefined as (() => void) | undefined, + defaultEditor: {}, + chatContainer: { clear: vi.fn() }, + sessionManager: { buildContextEntries: vi.fn().mockReturnValue([compaction]) }, + renderSessionEntries: vi.fn(), + addMessageToChat: vi.fn(), + addCompactionCostNotice: vi.fn(), + showWarning: vi.fn(), + showError: vi.fn(), + showStatus: vi.fn(), + clearStatusIndicator: vi.fn(), + flushCompactionQueue: vi.fn().mockResolvedValue(undefined), + settingsManager: { getShowTerminalProgress: () => false }, + ui: { requestRender: vi.fn(), terminal: { setProgress: vi.fn() } }, + }; + const handleEvent = Reflect.get(InteractiveMode.prototype, "handleEvent") as ( + this: typeof fakeThis, + event: { + type: "compaction_end"; + reason: "threshold"; + result: { tokensBefore: number; summary: string; details: unknown }; + aborted: boolean; + willRetry: boolean; + }, + ) => Promise; + + await handleEvent.call(fakeThis, { + type: "compaction_end", + reason: "threshold", + result: { + tokensBefore: 123, + summary: "summary", + details: { + readFiles: [], + modifiedFiles: [], + inContext: { status: "failed", stage: "request", error: "400 tool_choice not supported" }, + }, + }, + aborted: false, + willRetry: false, + }); + + expect(fakeThis.showWarning).toHaveBeenCalledWith( + "In-context compaction failed (request): 400 tool_choice not supported. Used a standalone summary instead.", + ); + }); + test("updates the working state when the same agent run resumes after compaction", async () => { const fakeThis = { isInitialized: true, diff --git a/packages/coding-agent/test/suite/agent-session-in-context-compaction.test.ts b/packages/coding-agent/test/suite/agent-session-in-context-compaction.test.ts new file mode 100644 index 000000000..a9ea5f736 --- /dev/null +++ b/packages/coding-agent/test/suite/agent-session-in-context-compaction.test.ts @@ -0,0 +1,259 @@ +import type { AgentTool } from "@earendil-works/pi-agent-core"; +import { + fauxAssistantMessage, + fauxToolCall, + type Message, + type SimpleStreamOptions, + type TranscriptContext, + type Usage, +} from "@earendil-works/pi-ai"; +import { Type } from "typebox"; +import { afterEach, describe, expect, it } from "vitest"; +import { + getInContextCompactionOutcome, + IN_CONTEXT_COMPACTION_INSTRUCTIONS, + IN_CONTEXT_COMPACTION_PROMPT, +} from "../../src/core/compaction/index.ts"; +import { SUMMARIZATION_SYSTEM_PROMPT } from "../../src/core/compaction/utils.ts"; +import { createHarness, type Harness, type HarnessOptions } from "./harness.ts"; + +const SESSION_ID = "in-context-compaction-session"; + +interface CapturedRequest { + messages: Message[]; + options: SimpleStreamOptions | undefined; +} + +/** The faux provider reports the full prompt as input plus cache reads; cache writes overlap input. */ +function fauxPromptTokens(usage: Usage): number { + return usage.input + usage.cacheRead; +} + +function text(message: Message): string { + if (typeof message.content === "string") return message.content; + return message.content.map((block) => (block.type === "text" ? block.text : "")).join(""); +} + +async function createInContextHarness( + inContext = true, + extensionFactories?: HarnessOptions["extensionFactories"], + tools?: AgentTool[], +): Promise { + const harness = await createHarness({ + settings: { compaction: { keepRecentTokens: 200, inContext } }, + extensionFactories, + tools, + }); + // The faux catalog has no compat settings; mark the session model as probe-verified. + Object.assign(harness.session.model!, { + compat: { supportsMidConvoSystemMessages: true, supportsInContextCompaction: true }, + }); + // The faux provider simulates a prefix cache per session id, like real providers. + harness.session.agent.sessionId = SESSION_ID; + return harness; +} + +/** Run three turns with large messages so most of the context is summarizable and cached. */ +async function runConversation(harness: Harness): Promise { + const requests: CapturedRequest[] = []; + const reply = (answer: string) => (context: TranscriptContext, options: SimpleStreamOptions | undefined) => { + requests.push({ messages: context.messages, options }); + return fauxAssistantMessage(answer); + }; + harness.setResponses([reply(`First answer. ${"a".repeat(3000)}`), reply("Second answer."), reply("Third answer.")]); + await harness.session.prompt(`First question. ${"q".repeat(6000)}`); + await harness.session.prompt(`Second question. ${"r".repeat(6000)}`); + await harness.session.prompt("Third question."); + return requests; +} + +describe("AgentSession in-context compaction", () => { + const harnesses: Harness[] = []; + + afterEach(() => { + while (harnesses.length > 0) harnesses.pop()?.cleanup(); + }); + + it("summarizes inside the cached conversation and keeps the recent tail", async () => { + const harness = await createInContextHarness(); + harnesses.push(harness); + const turns = await runConversation(harness); + const lastTurn = turns.at(-1)!; + const lastResponse = harness.session.messages.findLast((message) => message.role === "assistant"); + if (lastResponse?.role !== "assistant") throw new Error("expected an assistant response"); + + let compactionRequest: CapturedRequest | undefined; + harness.appendResponses([ + (context, options) => { + compactionRequest = { messages: context.messages, options }; + return fauxAssistantMessage("## Goal\nAnswer three questions."); + }, + ]); + const result = await harness.session.compact(); + + expect(harness.getPendingResponseCount()).toBe(0); + expect(compactionRequest).toBeDefined(); + const { messages, options } = compactionRequest!; + expect(options?.toolChoice).toBe("none"); + expect(options?.sessionId).toBe(SESSION_ID); + // The request repeats the last turn's request, then the reply, then the compaction request. + expect(JSON.stringify(messages.slice(0, lastTurn.messages.length))).toBe(JSON.stringify(lastTurn.messages)); + const tail = messages.slice(lastTurn.messages.length); + expect(tail.map((message) => message.role)).toEqual(["assistant", "system", "user"]); + expect(text(tail[1])).toBe(IN_CONTEXT_COMPACTION_INSTRUCTIONS); + expect(text(tail[2])).toBe(IN_CONTEXT_COMPACTION_PROMPT); + // The simulated provider cache reads the whole previous prompt. + expect(result.usage?.cacheRead).toBeGreaterThanOrEqual(fauxPromptTokens(lastResponse.usage)); + + expect(result.summary).toContain("## Goal\nAnswer three questions."); + expect(getInContextCompactionOutcome(result.details)).toEqual({ status: "used" }); + const userTexts = harness.session.messages.filter((message) => message.role === "user").map(text); + expect(userTexts.at(-1)).toBe("Third question."); + expect(userTexts.some((value) => value.startsWith("First question."))).toBe(false); + }); + + it("repeats the forced system prompt of the last run after the run ends", async () => { + const forced = "Forced prompt from before_agent_start."; + const harness = await createInContextHarness(true, [ + (pi) => { + pi.on("before_agent_start", () => ({ systemPrompt: forced })); + }, + ]); + harnesses.push(harness); + const turns = await runConversation(harness); + expect(text(turns.at(-1)!.messages[0])).toBe(forced); + const lastResponse = harness.session.messages.findLast((message) => message.role === "assistant"); + if (lastResponse?.role !== "assistant") throw new Error("expected an assistant response"); + + let compactionRequest: CapturedRequest | undefined; + harness.appendResponses([ + (context, options) => { + compactionRequest = { messages: context.messages, options }; + return fauxAssistantMessage("Summary."); + }, + ]); + // Manual compaction runs after the run ended, when the forced prompt projection is off. + const result = await harness.session.compact(); + + expect(text(compactionRequest!.messages[0])).toBe(forced); + expect(result.usage?.cacheRead).toBeGreaterThanOrEqual(fauxPromptTokens(lastResponse.usage)); + }); + + it("never executes a tool call from the in-context response and falls back", async () => { + let executions = 0; + const readTool: AgentTool = { + name: "read", + label: "Read", + description: "Read a file", + parameters: Type.Object({ path: Type.String() }), + execute: async () => { + executions++; + return { content: [{ type: "text", text: "file contents" }], details: {} }; + }, + }; + const harness = await createInContextHarness(true, undefined, [readTool]); + harnesses.push(harness); + await runConversation(harness); + const messageCount = harness.session.messages.length; + + const requests: CapturedRequest[] = []; + harness.appendResponses([ + (context, options) => { + requests.push({ messages: context.messages, options }); + return fauxAssistantMessage([fauxToolCall("read", { path: "x" })], { stopReason: "toolUse" }); + }, + (context, options) => { + requests.push({ messages: context.messages, options }); + return fauxAssistantMessage("Standalone summary."); + }, + ]); + const result = await harness.session.compact(); + + expect(requests).toHaveLength(2); + expect(requests[0].options?.toolChoice).toBe("none"); + expect(text(requests[1].messages[0])).toBe(SUMMARIZATION_SYSTEM_PROMPT); + expect(result.summary).toContain("Standalone summary."); + expect(getInContextCompactionOutcome(result.details)).toEqual({ + status: "failed", + stage: "request", + error: "In-context summarization attempted to call a tool", + }); + // The standalone request disables caching; cache reads come from the rejected attempt it also bills. + expect(result.usage?.cacheRead).toBeGreaterThan(0); + // The rejected response never reaches the agent loop or the session. + expect(executions).toBe(0); + expect(harness.session.messages.some((message) => message.role === "toolResult")).toBe(false); + const persisted = harness.sessionManager.getEntries().filter((entry) => entry.type === "message"); + expect(persisted.some((entry) => entry.type === "message" && entry.message.role === "toolResult")).toBe(false); + expect(harness.session.messages.length).toBeLessThan(messageCount); + }); + + it("uses a standalone summary when the setting is off", async () => { + const harness = await createInContextHarness(false); + harnesses.push(harness); + await runConversation(harness); + + const requests: CapturedRequest[] = []; + harness.appendResponses([ + (context, options) => { + requests.push({ messages: context.messages, options }); + return fauxAssistantMessage("Standalone summary."); + }, + ]); + const result = await harness.session.compact(); + + expect(requests).toHaveLength(1); + expect(text(requests[0].messages[0])).toBe(SUMMARIZATION_SYSTEM_PROMPT); + expect(requests[0].options?.toolChoice).toBeUndefined(); + expect(getInContextCompactionOutcome(result.details)).toBeUndefined(); + }); + + it("records why the decision skipped in-context compaction", async () => { + const harness = await createInContextHarness(); + harnesses.push(harness); + // Without a session id the faux provider caches nothing, so no response reports cache activity. + harness.session.agent.sessionId = undefined; + await runConversation(harness); + + harness.appendResponses([fauxAssistantMessage("Standalone summary.")]); + const result = await harness.session.compact(); + + expect(result.summary).toContain("Standalone summary."); + expect(getInContextCompactionOutcome(result.details)).toEqual({ + status: "skipped", + reason: "last response reported no prompt caching", + }); + const entry = harness.sessionManager.getEntries().findLast((candidate) => candidate.type === "compaction"); + expect(getInContextCompactionOutcome(entry?.type === "compaction" ? entry.details : undefined)).toEqual({ + status: "skipped", + reason: "last response reported no prompt caching", + }); + }); + + it("separates request rebuild failures from provider failures", async () => { + const harness = await createInContextHarness(); + harnesses.push(harness); + await runConversation(harness); + harness.session.agent.convertToLlm = () => { + throw new Error("broken conversion"); + }; + + const requests: CapturedRequest[] = []; + harness.appendResponses([ + (context, options) => { + requests.push({ messages: context.messages, options }); + return fauxAssistantMessage("Standalone summary."); + }, + ]); + const result = await harness.session.compact(); + + // Nothing was sent for the in-context attempt. + expect(requests).toHaveLength(1); + expect(text(requests[0].messages[0])).toBe(SUMMARIZATION_SYSTEM_PROMPT); + expect(getInContextCompactionOutcome(result.details)).toEqual({ + status: "failed", + stage: "rebuild", + error: "broken conversion", + }); + }); +});