From 836aee6d38f60428ab6bd2679f93dce43a55dab3 Mon Sep 17 00:00:00 2001 From: Vegard Stikbakke Date: Tue, 18 Aug 2026 12:56:34 +0200 Subject: [PATCH 01/10] feat(coding-agent): show compaction usage notices --- packages/coding-agent/CHANGELOG.md | 4 + packages/coding-agent/docs/settings.md | 2 +- .../coding-agent/src/core/settings-manager.ts | 2 +- .../components/settings-selector.ts | 2 +- .../src/modes/interactive/interactive-mode.ts | 55 ++++++- .../test/interactive-mode-compaction.test.ts | 148 +++++++++++++++++- 6 files changed, 202 insertions(+), 11 deletions(-) diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index d64c60d22..e857000f5 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Added + +- Added transcript usage notices for compaction and branch summaries when cache miss notices are enabled. + ### Fixed - Fixed the subagent example repeatedly prompting before running project-local agents in trusted repositories ([#8261](https://github.com/earendil-works/pi/issues/8261)). diff --git a/packages/coding-agent/docs/settings.md b/packages/coding-agent/docs/settings.md index 585160806..c56340b1d 100644 --- a/packages/coding-agent/docs/settings.md +++ b/packages/coding-agent/docs/settings.md @@ -31,7 +31,7 @@ Use `/trust` in interactive mode to save a project trust decision for future ses | `defaultModel` | string | - | Default model ID | | `defaultThinkingLevel` | string | - | `"off"`, `"minimal"`, `"low"`, `"medium"`, `"high"`, `"xhigh"`, `"max"` | | `hideThinkingBlock` | boolean | `false` | Hide thinking blocks in output | -| `showCacheMissNotices` | boolean | `false` | Show transcript notices for significant prompt-cache misses | +| `showCacheMissNotices` | boolean | `false` | Show transcript notices for significant prompt-cache misses and compaction or branch-summary usage | | `thinkingBudgets` | object | - | Custom token budgets per thinking level | #### thinkingBudgets diff --git a/packages/coding-agent/src/core/settings-manager.ts b/packages/coding-agent/src/core/settings-manager.ts index 9a744f027..22e72b8df 100644 --- a/packages/coding-agent/src/core/settings-manager.ts +++ b/packages/coding-agent/src/core/settings-manager.ts @@ -100,7 +100,7 @@ export interface Settings { branchSummary?: BranchSummarySettings; retry?: RetrySettings; hideThinkingBlock?: boolean; - showCacheMissNotices?: boolean; // default: false - show transcript notices for significant prompt-cache misses + showCacheMissNotices?: boolean; // default: false - show prompt-cache miss and compaction cost notices externalEditor?: string; // Command for Ctrl+G external editor; takes precedence over VISUAL/EDITOR shellPath?: string; // Custom shell path (e.g., for Cygwin users on Windows); supports leading ~ expansion quietStartup?: boolean; diff --git a/packages/coding-agent/src/modes/interactive/components/settings-selector.ts b/packages/coding-agent/src/modes/interactive/components/settings-selector.ts index 20c6b7586..1d626270e 100644 --- a/packages/coding-agent/src/modes/interactive/components/settings-selector.ts +++ b/packages/coding-agent/src/modes/interactive/components/settings-selector.ts @@ -549,7 +549,7 @@ export class SettingsSelectorComponent extends Container { { id: "cache-miss-notices", label: "Cache miss notices", - description: "Show transcript notices for significant prompt-cache misses", + description: "Show transcript notices for significant prompt-cache misses and compaction costs", currentValue: config.showCacheMissNotices ? "true" : "false", values: ["true", "false"], }, diff --git a/packages/coding-agent/src/modes/interactive/interactive-mode.ts b/packages/coding-agent/src/modes/interactive/interactive-mode.ts index 999220398..76c69b544 100644 --- a/packages/coding-agent/src/modes/interactive/interactive-mode.ts +++ b/packages/coding-agent/src/modes/interactive/interactive-mode.ts @@ -9,7 +9,7 @@ import * as os from "node:os"; import * as path from "node:path"; import type { AgentMessage } from "@earendil-works/pi-agent-core"; import type { AuthEvent, AuthPrompt } from "@earendil-works/pi-ai"; -import type { AssistantMessage, ImageContent, Message, Model } from "@earendil-works/pi-ai/compat"; +import type { AssistantMessage, ImageContent, Message, Model, Usage } from "@earendil-works/pi-ai/compat"; import type { AutocompleteItem, AutocompleteProvider, @@ -204,12 +204,22 @@ type CompactionQueuedMessage = { mode: "steer" | "followUp"; }; -type RenderSessionItem = AgentMessage | Extract; +type CompactionCostNotice = { + type: "compaction_cost"; + kind: "compaction" | "branch_summary"; + usage: Usage; +}; + +type RenderSessionItem = AgentMessage | Extract | CompactionCostNotice; function isCustomSessionEntry(item: RenderSessionItem): item is Extract { return "type" in item && item.type === "custom"; } +function isCompactionCostNotice(item: RenderSessionItem): item is CompactionCostNotice { + return "type" in item && item.type === "compaction_cost"; +} + const DEAD_TERMINAL_ERROR_CODES = new Set(["EIO", "EPIPE", "ENOTCONN"]); function isDeadTerminalError(error: unknown): boolean { @@ -3349,8 +3359,13 @@ export class InteractiveMode { this.showStatus("Auto-compaction cancelled"); } } else if (event.result) { + const entries = this.sessionManager.buildContextEntries(); + if (entries[0]?.type !== "compaction") { + throw new Error("Completed compaction is missing from the session context"); + } this.chatContainer.clear(); - this.rebuildChatFromMessages(); + // The latest compaction is prepended for model context; append it below at its chronological position. + this.renderSessionEntries(entries.slice(1)); this.addMessageToChat( createCompactionSummaryMessage( event.result.summary, @@ -3358,6 +3373,13 @@ export class InteractiveMode { new Date().toISOString(), ), ); + if (event.result.usage) { + this.addCompactionCostNotice({ + type: "compaction_cost", + kind: "compaction", + usage: event.result.usage, + }); + } this.footer.invalidate(); } else if (event.errorMessage) { if (event.reason === "manual") { @@ -3629,6 +3651,10 @@ export class InteractiveMode { this.addCustomEntryToChat(item); continue; } + if (isCompactionCostNotice(item)) { + this.addCompactionCostNotice(item); + continue; + } const message = item; // Assistant messages need special handling for tool calls @@ -3706,11 +3732,32 @@ export class InteractiveMode { if (entry.type === "custom") { return [entry]; } - return sessionEntryToContextMessages(entry); + const messages = sessionEntryToContextMessages(entry); + if ((entry.type === "compaction" || entry.type === "branch_summary") && entry.usage && messages.length > 0) { + return [...messages, { type: "compaction_cost", kind: entry.type, usage: entry.usage }]; + } + return messages; }); this.renderSessionItems(items, options); } + /** + * Render billing usage for a compaction or branch summary. The notice is derived + * from persisted summary usage and is not stored as a separate session entry. + */ + private addCompactionCostNotice(notice: CompactionCostNotice): void { + if (!this.settingsManager.getShowCacheMissNotices()) return; + + const { usage } = notice; + const tokens = usage.input + usage.output + usage.cacheRead + usage.cacheWrite; + const cost = usage.cost.total >= 0.01 ? ` (~$${usage.cost.total.toFixed(2)})` : ""; + const label = notice.kind === "compaction" ? "Compaction" : "Branch summary"; + this.chatContainer.addChild(new Spacer(1)); + this.chatContainer.addChild( + new Text(theme.fg("warning", `${label}: ${formatTokens(tokens)} tokens billed${cost}`), 1, 0), + ); + } + /** * Show a transcript notice when a completed assistant message paid for a * significant cache miss. Only states observable facts: the miss itself, diff --git a/packages/coding-agent/test/interactive-mode-compaction.test.ts b/packages/coding-agent/test/interactive-mode-compaction.test.ts index dd019f8f4..ece2d43ca 100644 --- a/packages/coding-agent/test/interactive-mode-compaction.test.ts +++ b/packages/coding-agent/test/interactive-mode-compaction.test.ts @@ -1,8 +1,140 @@ +import type { Usage } from "@earendil-works/pi-ai"; +import { Container } from "@earendil-works/pi-tui"; import { describe, expect, test, vi } from "vitest"; +import type { SessionEntry } from "../src/core/session-manager.ts"; import { InteractiveMode } from "../src/modes/interactive/interactive-mode.ts"; +import { initTheme } from "../src/modes/interactive/theme/theme.ts"; +import { stripAnsi } from "../src/utils/ansi.ts"; describe("InteractiveMode compaction events", () => { - test("rebuilds chat and appends a synthetic compaction summary at the bottom", async () => { + test("uses the cache miss notice setting for compaction and branch summary costs", () => { + const usage: Usage = { + input: 10, + output: 20, + cacheRead: 30, + cacheWrite: 40, + totalTokens: 100, + cost: { input: 0.01, output: 0.02, cacheRead: 0.03, cacheWrite: 0.065, total: 0.125 }, + }; + const addCompactionCostNotice = Reflect.get(InteractiveMode.prototype, "addCompactionCostNotice") as ( + this: { chatContainer: Container; settingsManager: { getShowCacheMissNotices(): boolean } }, + notice: { + type: "compaction_cost"; + kind: "compaction" | "branch_summary"; + usage: Usage; + }, + ) => void; + + initTheme("dark"); + const enabled = { + chatContainer: new Container(), + settingsManager: { getShowCacheMissNotices: () => true }, + }; + addCompactionCostNotice.call(enabled, { type: "compaction_cost", kind: "compaction", usage }); + addCompactionCostNotice.call(enabled, { + type: "compaction_cost", + kind: "branch_summary", + usage, + }); + const output = stripAnsi(enabled.chatContainer.render(120).join("\n")); + expect(output).toContain("Compaction: 100 tokens billed (~$0.13)"); + expect(output).toContain("Branch summary: 100 tokens billed (~$0.13)"); + + const disabled = { + chatContainer: new Container(), + settingsManager: { getShowCacheMissNotices: () => false }, + }; + addCompactionCostNotice.call(disabled, { type: "compaction_cost", kind: "compaction", usage }); + expect(disabled.chatContainer.children).toHaveLength(0); + }); + + test("renders each compaction cost after its summary", () => { + const currentUsage: Usage = { + input: 10, + output: 20, + cacheRead: 30, + cacheWrite: 40, + totalTokens: 100, + cost: { input: 0.01, output: 0.02, cacheRead: 0.03, cacheWrite: 0.04, total: 0.1 }, + }; + const previousUsage: Usage = { + input: 1, + output: 2, + cacheRead: 3, + cacheWrite: 4, + totalTokens: 10, + cost: { input: 0.001, output: 0.002, cacheRead: 0.003, cacheWrite: 0.004, total: 0.01 }, + }; + const entries: SessionEntry[] = [ + { + type: "compaction", + id: "current", + parentId: "previous", + timestamp: "2025-01-02T00:00:00Z", + summary: "current summary", + firstKeptEntryId: "kept", + tokensBefore: 200, + usage: currentUsage, + }, + { + type: "compaction", + id: "previous", + parentId: null, + timestamp: "2025-01-01T00:00:00Z", + summary: "previous summary", + firstKeptEntryId: "kept", + tokensBefore: 100, + usage: previousUsage, + }, + ]; + const fakeThis = { renderSessionItems: vi.fn() }; + const renderSessionEntries = Reflect.get(InteractiveMode.prototype, "renderSessionEntries") as ( + this: typeof fakeThis, + entries: SessionEntry[], + ) => void; + + renderSessionEntries.call(fakeThis, entries); + + expect(fakeThis.renderSessionItems).toHaveBeenCalledWith( + [ + expect.objectContaining({ role: "compactionSummary", summary: "current summary" }), + { type: "compaction_cost", kind: "compaction", usage: currentUsage }, + expect.objectContaining({ role: "compactionSummary", summary: "previous summary" }), + { type: "compaction_cost", kind: "compaction", usage: previousUsage }, + ], + {}, + ); + }); + + test("renders retained entries and appends the latest summary cost at the bottom", async () => { + const usage: Usage = { + input: 10, + output: 20, + cacheRead: 30, + cacheWrite: 40, + totalTokens: 100, + cost: { input: 0.01, output: 0.02, cacheRead: 0.03, cacheWrite: 0.065, total: 0.125 }, + }; + const latestCompaction: SessionEntry = { + type: "compaction", + id: "latest", + parentId: "previous", + timestamp: "2025-01-02T00:00:00Z", + summary: "summary", + firstKeptEntryId: "kept", + tokensBefore: 123, + usage, + }; + const previousCompaction: SessionEntry = { + type: "compaction", + id: "previous", + parentId: null, + timestamp: "2025-01-01T00:00:00Z", + summary: "previous summary", + firstKeptEntryId: "kept", + tokensBefore: 100, + usage, + }; const fakeThis = { isInitialized: true, footer: { invalidate: vi.fn() }, @@ -11,8 +143,10 @@ describe("InteractiveMode compaction events", () => { defaultEditor: {}, statusContainer: { clear: vi.fn() }, chatContainer: { clear: vi.fn() }, - rebuildChatFromMessages: vi.fn(), + sessionManager: { buildContextEntries: vi.fn().mockReturnValue([latestCompaction, previousCompaction]) }, + renderSessionEntries: vi.fn(), addMessageToChat: vi.fn(), + addCompactionCostNotice: vi.fn(), showError: vi.fn(), showStatus: vi.fn(), clearStatusIndicator: vi.fn(), @@ -26,7 +160,7 @@ describe("InteractiveMode compaction events", () => { event: { type: "compaction_end"; reason: "manual" | "threshold" | "overflow"; - result: { tokensBefore: number; summary: string } | undefined; + result: { tokensBefore: number; summary: string; usage?: Usage } | undefined; aborted: boolean; willRetry: boolean; errorMessage?: string; @@ -39,13 +173,14 @@ describe("InteractiveMode compaction events", () => { result: { tokensBefore: 123, summary: "summary", + usage, }, aborted: false, willRetry: false, }); expect(fakeThis.chatContainer.clear).toHaveBeenCalledTimes(1); - expect(fakeThis.rebuildChatFromMessages).toHaveBeenCalledTimes(1); + expect(fakeThis.renderSessionEntries).toHaveBeenCalledWith([previousCompaction]); expect(fakeThis.addMessageToChat).toHaveBeenCalledTimes(1); expect(fakeThis.addMessageToChat).toHaveBeenCalledWith( expect.objectContaining({ @@ -54,6 +189,11 @@ describe("InteractiveMode compaction events", () => { summary: "summary", }), ); + expect(fakeThis.addCompactionCostNotice).toHaveBeenCalledWith({ + type: "compaction_cost", + kind: "compaction", + usage, + }); expect(fakeThis.flushCompactionQueue).toHaveBeenCalledWith({ willRetry: false }); }); From ef8dc7385b1ed41c40cd0a4bbf612326ec18bd23 Mon Sep 17 00:00:00 2001 From: Vegard Stikbakke Date: Tue, 18 Aug 2026 14:17:38 +0200 Subject: [PATCH 02/10] feat(coding-agent): centralize compaction summary requests --- .../coding-agent/src/core/agent-session.ts | 44 ++++++++++++++----- .../src/core/compaction/compaction.ts | 41 ++++++++--------- .../test/compaction-summary-reasoning.test.ts | 21 ++++++++- .../suite/agent-session-compaction.test.ts | 40 ++++++++++++++++- 4 files changed, 111 insertions(+), 35 deletions(-) diff --git a/packages/coding-agent/src/core/agent-session.ts b/packages/coding-agent/src/core/agent-session.ts index 5914a3712..74280d564 100644 --- a/packages/coding-agent/src/core/agent-session.ts +++ b/packages/coding-agent/src/core/agent-session.ts @@ -54,6 +54,7 @@ import { normalizeToolResultImages } from "../utils/tool-result-images.ts"; import { formatNoApiKeyFoundMessage, formatNoModelSelectedMessage } from "./auth-guidance.ts"; import { type BashResult, executeBashWithOperations } from "./bash-executor.ts"; import { + type CompactionPreparation, type CompactionResult, calculateContextTokens, collectEntriesForBranchSummary, @@ -1789,6 +1790,33 @@ export class AgentSession { // Compaction // ========================================================================= + /** Generate Pi's built-in compaction summary for manual and automatic compaction. */ + private async _runDefaultCompaction( + preparation: CompactionPreparation, + requestModel: Model, + apiKey: string | undefined, + headers: Record | undefined, + customInstructions: string | undefined, + signal: AbortSignal, + env: Record | undefined, + reason: "manual" | "threshold" | "overflow", + ): Promise { + return compact( + preparation, + requestModel, + apiKey, + headers, + customInstructions, + signal, + this.thinkingLevel, + this.agent.streamFunction, + env, + this.settingsManager.getRetrySettings(), + this._summarizationRetryCallbacks({ source: "compaction", reason }), + undefined, // sessionId + ); + } + /** * Manually compact the session context. * @@ -1868,19 +1896,15 @@ export class AgentSession { details = extensionCompaction.details; } else { // Shared default summary generator, also used by automatic compaction. - const result = await compact( + const result = await this._runDefaultCompaction( preparation, requestModel, apiKey, headers, customInstructions, this._compactionAbortController.signal, - this.thinkingLevel, - this.agent.streamFunction, env, - this.settingsManager.getRetrySettings(), - this._summarizationRetryCallbacks({ source: "compaction", reason: "manual" }), - undefined, // sessionId + "manual", ); summary = result.summary; firstKeptEntryId = result.firstKeptEntryId; @@ -2181,19 +2205,15 @@ export class AgentSession { details = extensionCompaction.details; } else { // Shared default summary generator, also used by manual compaction. - const compactResult = await compact( + const compactResult = await this._runDefaultCompaction( preparation, requestModel, apiKey, headers, undefined, this._autoCompactionAbortController.signal, - this.thinkingLevel, - this.agent.streamFunction, env, - this.settingsManager.getRetrySettings(), - this._summarizationRetryCallbacks({ source: "compaction", reason }), - undefined, // sessionId + reason, ); summary = compactResult.summary; firstKeptEntryId = compactResult.firstKeptEntryId; diff --git a/packages/coding-agent/src/core/compaction/compaction.ts b/packages/coding-agent/src/core/compaction/compaction.ts index 4a3734961..5fbeefc2a 100644 --- a/packages/coding-agent/src/core/compaction/compaction.ts +++ b/packages/coding-agent/src/core/compaction/compaction.ts @@ -508,9 +508,7 @@ Use this EXACT format: Keep each section concise. Preserve exact file paths, function names, and error messages.`; -const UPDATE_SUMMARIZATION_PROMPT = `The messages above are NEW conversation messages to incorporate into the existing summary provided in tags. - -Update the existing structured summary with new information. RULES: +const UPDATE_SUMMARIZATION_INSTRUCTIONS = `Update the existing structured summary with new information. RULES: - PRESERVE all existing information from the previous summary - ADD new progress, decisions, and context from the new messages - UPDATE the Progress section: move items from "In Progress" to "Done" when completed @@ -547,6 +545,10 @@ Use this EXACT format: Keep each section concise. Preserve exact file paths, function names, and error messages.`; +const UPDATE_SUMMARIZATION_PROMPT = `The messages above are NEW conversation messages to incorporate into the existing summary provided in tags. + +${UPDATE_SUMMARIZATION_INSTRUCTIONS}`; + function createSummarizationOptions( model: Model, maxTokens: number, @@ -638,6 +640,20 @@ export async function generateSummary( ).text; } +/** Build the provider context for a standalone summary request. */ +function buildSummarizationContext(promptText: string): Context { + return { + systemPrompt: SUMMARIZATION_SYSTEM_PROMPT, + messages: [ + { + role: "user", + content: [{ type: "text", text: promptText }], + timestamp: Date.now(), + }, + ], + }; +} + /** Generate or update a conversation summary and return its provider usage. */ export async function generateSummaryWithUsage( currentMessages: AgentMessage[], @@ -678,14 +694,6 @@ export async function generateSummaryWithUsage( } promptText += basePrompt; - const summarizationMessages = [ - { - role: "user" as const, - content: [{ type: "text" as const, text: promptText }], - timestamp: Date.now(), - }, - ]; - const completionOptions = createSummarizationOptions( model, maxTokens, @@ -699,7 +707,7 @@ export async function generateSummaryWithUsage( const response = await completeSummarization( model, - { systemPrompt: SUMMARIZATION_SYSTEM_PROMPT, messages: summarizationMessages }, + buildSummarizationContext(promptText), completionOptions, streamFn, retry, @@ -980,17 +988,10 @@ async function generateTurnPrefixSummary( const llmMessages = convertToLlm(messages); const conversationText = serializeConversation(llmMessages); const promptText = `\n${conversationText}\n\n\n${TURN_PREFIX_SUMMARIZATION_PROMPT}`; - const summarizationMessages = [ - { - role: "user" as const, - content: [{ type: "text" as const, text: promptText }], - timestamp: Date.now(), - }, - ]; const response = await completeSummarization( model, - { systemPrompt: SUMMARIZATION_SYSTEM_PROMPT, messages: summarizationMessages }, + buildSummarizationContext(promptText), createSummarizationOptions(model, maxTokens, apiKey, headers, env, signal, thinkingLevel, sessionId), streamFn, retry, diff --git a/packages/coding-agent/test/compaction-summary-reasoning.test.ts b/packages/coding-agent/test/compaction-summary-reasoning.test.ts index 2d5f155f7..340d5c1f6 100644 --- a/packages/coding-agent/test/compaction-summary-reasoning.test.ts +++ b/packages/coding-agent/test/compaction-summary-reasoning.test.ts @@ -1,5 +1,5 @@ import type { AgentMessage } from "@earendil-works/pi-agent-core"; -import type { AssistantMessage, Model } from "@earendil-works/pi-ai"; +import type { AssistantMessage, Context, Model } from "@earendil-works/pi-ai"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { type CompactionPreparation, @@ -129,6 +129,25 @@ describe("generateSummary reasoning options", () => { }); }); + it("preserves the standalone split-turn summary prompt", async () => { + const preparation: CompactionPreparation = { + firstKeptEntryId: "entry-keep", + messagesToSummarize: [], + turnPrefixMessages: messages, + isSplitTurn: true, + tokensBefore: 100, + fileOps: { read: new Set(), written: new Set(), edited: new Set() }, + settings: { enabled: true, reserveTokens: 2000, keepRecentTokens: 20 }, + }; + + await compact(preparation, createModel(false), "test-key"); + + const requestContext = completeSimpleMock.mock.calls[0][1] as Context; + const prompt = JSON.stringify(requestContext.messages); + expect(prompt).toContain("This is the PREFIX of a turn that was too large to keep"); + expect(prompt).toContain(""); + }); + it("rejects tool calls from conversation summaries", async () => { completeSimpleMock.mockResolvedValueOnce(mockToolCallResponse); diff --git a/packages/coding-agent/test/suite/agent-session-compaction.test.ts b/packages/coding-agent/test/suite/agent-session-compaction.test.ts index 5e9e7bcc8..7d81358c5 100644 --- a/packages/coding-agent/test/suite/agent-session-compaction.test.ts +++ b/packages/coding-agent/test/suite/agent-session-compaction.test.ts @@ -1,8 +1,11 @@ +import type { AgentMessage } from "@earendil-works/pi-agent-core"; import { type AssistantMessage, + type Context, createAssistantMessageEventStream, fauxAssistantMessage, type Model, + type SimpleStreamOptions, } from "@earendil-works/pi-ai"; import { afterEach, describe, expect, it, vi } from "vitest"; import { estimateTokens } from "../../src/core/compaction/index.ts"; @@ -47,10 +50,15 @@ function createAssistant( }; } -function useSummaryStreamFn(harness: Harness, summary: string): () => number { +function useSummaryStreamFn( + harness: Harness, + summary: string, + onRequest?: (context: Context, options: SimpleStreamOptions | undefined) => void, +): () => number { let callCount = 0; - harness.session.agent.streamFunction = (model) => { + harness.session.agent.streamFunction = (model, context, options) => { callCount++; + onRequest?.(context, options); const stream = createAssistantMessageEventStream(); queueMicrotask(() => { const message: AssistantMessage = { @@ -246,6 +254,34 @@ describe("AgentSession compaction characterization", () => { expect(harness.faux.state.callCount).toBe(1); }); + it("uses the standalone compaction request context", async () => { + const harness = await createHarness({ settings: { compaction: { keepRecentTokens: 1 } } }); + harnesses.push(harness); + seedCompactableSession(harness); + + const transformContext = vi.fn(async (messages: AgentMessage[]) => messages); + harness.session.agent.transformContext = transformContext; + harness.session.agent.sessionId = "active-routing-session"; + harness.session.agent.transport = "websocket"; + + let requestContext: Context | undefined; + let requestOptions: SimpleStreamOptions | undefined; + useSummaryStreamFn(harness, "standalone summary", (context, options) => { + requestContext = context; + requestOptions = options; + }); + + await harness.session.compact(); + + expect(transformContext).not.toHaveBeenCalled(); + expect(requestContext?.systemPrompt).not.toBe(harness.session.agent.state.systemPrompt); + expect(requestContext?.tools).toBeUndefined(); + expect(JSON.stringify(requestContext?.messages)).toContain(""); + expect(requestOptions).toMatchObject({ cacheRetention: "none" }); + expect(requestOptions?.sessionId).not.toBe("active-routing-session"); + expect(requestOptions?.transport).toBeUndefined(); + }); + it("persists usage from pi-generated manual compaction", async () => { const harness = await createHarness({ withConfiguredAuth: false }); harnesses.push(harness); From cff1cf52c6c73ef873dcb6148238ed68f69e1ead Mon Sep 17 00:00:00 2001 From: Vegard Stikbakke Date: Tue, 18 Aug 2026 14:25:28 +0200 Subject: [PATCH 03/10] feat(coding-agent): add cache-friendly compaction primitives Add provider-context prefix preparation and optional summary request settings needed by cache-friendly compaction. Existing AgentSession callers continue to pass undefined, so standalone summarization remains the default until the feature is wired in. --- packages/ai/src/index.ts | 1 + packages/coding-agent/docs/compaction.md | 2 + .../coding-agent/src/core/agent-session.ts | 2 +- .../src/core/compaction/compaction.ts | 255 +++++++++++++++--- packages/coding-agent/src/index.ts | 1 + .../test/compaction-summary-reasoning.test.ts | 227 +++++++++++++++- packages/coding-agent/test/compaction.test.ts | 85 +++++- 7 files changed, 528 insertions(+), 45 deletions(-) diff --git a/packages/ai/src/index.ts b/packages/ai/src/index.ts index 82bac9738..64142c999 100644 --- a/packages/ai/src/index.ts +++ b/packages/ai/src/index.ts @@ -37,6 +37,7 @@ export * from "./providers/faux.ts"; export * from "./session-resources.ts"; export * from "./types.ts"; export * from "./utils/diagnostics.ts"; +export { estimateContextTokens } from "./utils/estimate.ts"; export * from "./utils/event-stream.ts"; export * from "./utils/json-parse.ts"; export * from "./utils/overflow.ts"; diff --git a/packages/coding-agent/docs/compaction.md b/packages/coding-agent/docs/compaction.md index 3a9dff1c6..1ed15fcd6 100644 --- a/packages/coding-agent/docs/compaction.md +++ b/packages/coding-agent/docs/compaction.md @@ -281,7 +281,9 @@ pi.on("session_before_compact", async (event, ctx) => { const { preparation, branchEntries, customInstructions, reason, willRetry, signal } = event; // preparation.messagesToSummarize - messages to summarize + // preparation.sourceMessages - optional active-context history prefix, including any previous summary // preparation.turnPrefixMessages - split turn prefix (if isSplitTurn) + // preparation.turnPrefixSourceMessages - optional active-context prefix through the split turn // preparation.previousSummary - previous compaction summary // preparation.fileOps - extracted file operations // preparation.tokensBefore - context tokens before compaction diff --git a/packages/coding-agent/src/core/agent-session.ts b/packages/coding-agent/src/core/agent-session.ts index 74280d564..4d02d1943 100644 --- a/packages/coding-agent/src/core/agent-session.ts +++ b/packages/coding-agent/src/core/agent-session.ts @@ -1813,7 +1813,7 @@ export class AgentSession { env, this.settingsManager.getRetrySettings(), this._summarizationRetryCallbacks({ source: "compaction", reason }), - undefined, // sessionId + undefined, // cacheFriendly ); } diff --git a/packages/coding-agent/src/core/compaction/compaction.ts b/packages/coding-agent/src/core/compaction/compaction.ts index 5fbeefc2a..a2a1c063f 100644 --- a/packages/coding-agent/src/core/compaction/compaction.ts +++ b/packages/coding-agent/src/core/compaction/compaction.ts @@ -6,7 +6,14 @@ */ import type { AgentMessage, StreamFn, ThinkingLevel } from "@earendil-works/pi-agent-core"; -import { contentText, type RetryCallbacks, type RetryPolicy, retryAssistantCall, uuidv7 } from "@earendil-works/pi-ai"; +import { + contentText, + estimateContextTokens as estimateProviderContextTokens, + type RetryCallbacks, + type RetryPolicy, + retryAssistantCall, + uuidv7, +} from "@earendil-works/pi-ai"; import type { AssistantMessage, Context, Model, SimpleStreamOptions, Usage } from "@earendil-works/pi-ai/compat"; import { completeSimple } from "@earendil-works/pi-ai/compat"; import { convertToLlm } from "../messages.ts"; @@ -95,6 +102,27 @@ function getMessageFromEntryForCompaction(entry: SessionEntry): AgentMessage | u return sessionEntryToContextMessages(entry)[0]; } +/** Build an active-context prefix, placing the previous compaction summary before its retained messages. */ +function collectSourceMessages( + entries: SessionEntry[], + startIndex: number, + endIndex: number, + previousCompactionIndex: number, +): AgentMessage[] { + const messages: AgentMessage[] = []; + if (previousCompactionIndex >= 0) { + messages.push(...sessionEntryToContextMessages(entries[previousCompactionIndex])); + } + for (let i = startIndex; i < endIndex; i++) { + // The latest compaction was moved to the front above. Keep older compaction + // entries because buildSessionContext retains them in the active provider prefix. + if (i !== previousCompactionIndex) { + messages.push(...sessionEntryToContextMessages(entries[i])); + } + } + return messages; +} + /** Result from compact() - SessionManager adds uuid/parentUuid when saving */ export interface CompactionResult { summary: string; @@ -140,6 +168,19 @@ export interface CompactionSettings { keepRecentTokens: number; } +/** Active provider contexts and request settings used to preserve cacheable compaction prefixes. */ +export interface CacheFriendlySummaryOptions { + /** Exact provider context prefix containing the history to summarize. */ + sourceContext?: Context; + /** Exact provider context prefix containing a split turn's prefix. */ + turnPrefixSourceContext?: Context; + /** Provider request settings copied from the active agent request path. */ + requestOptions?: Pick< + SimpleStreamOptions, + "sessionId" | "onPayload" | "onResponse" | "transport" | "thinkingBudgets" | "maxRetryDelayMs" + >; +} + export const DEFAULT_COMPACTION_SETTINGS: CompactionSettings = { enabled: true, reserveTokens: 16384, @@ -549,6 +590,10 @@ const UPDATE_SUMMARIZATION_PROMPT = `The messages above are NEW conversation mes ${UPDATE_SUMMARIZATION_INSTRUCTIONS}`; +const SOURCE_CONTEXT_UPDATE_SUMMARIZATION_PROMPT = `The messages above contain an existing structured summary of earlier conversation history followed by NEW conversation messages. + +${UPDATE_SUMMARIZATION_INSTRUCTIONS}`; + function createSummarizationOptions( model: Model, maxTokens: number, @@ -557,9 +602,18 @@ function createSummarizationOptions( env: Record | undefined, signal: AbortSignal | undefined, thinkingLevel: ThinkingLevel | undefined, - sessionId: string | undefined, + requestOptions: CacheFriendlySummaryOptions["requestOptions"] | undefined, + cacheRetention: SimpleStreamOptions["cacheRetention"] | undefined, ): SimpleStreamOptions { - const options: SimpleStreamOptions = { maxTokens, signal, apiKey, headers, env, sessionId }; + const options: SimpleStreamOptions = { + ...requestOptions, + maxTokens, + signal, + apiKey, + headers, + env, + cacheRetention, + }; const refusalFallbacks = getAnthropicSummarizationFallback(model); if (refusalFallbacks) { options.refusalFallbacks = refusalFallbacks; @@ -585,12 +639,15 @@ export async function completeSummarization( retry?: RetryPolicy, callbacks?: RetryCallbacks, ): Promise { - // Avoid cache writes for one-off summaries. Reuse caller-supplied routing when available; - // callers without a session ID, including branch summaries, receive a fresh routing ID. + // Standalone one-off summaries default to no prompt caching. Cache-friendly callers + // explicitly request short retention so providers can reuse the active prefix. + // Callers without a session ID, including standalone branch summaries, receive a fresh routing ID. const requestOptions: SimpleStreamOptions = { ...options, - cacheRetention: "none", + cacheRetention: options.cacheRetention ?? "none", sessionId: options.sessionId ?? uuidv7(), + // Anthropic invalidates the messages cache when tool_choice changes. Its 20-content-block lookup + // can still reuse an earlier user-message checkpoint, but long tool-heavy turns may need reprocessing. toolChoice: "none", }; const produce = async (): Promise => @@ -618,7 +675,7 @@ export async function generateSummary( env?: Record, retry?: RetryPolicy, callbacks?: RetryCallbacks, - sessionId?: string, + cacheFriendly?: Pick, ): Promise { return ( await generateSummaryWithUsage( @@ -635,25 +692,47 @@ export async function generateSummary( env, retry, callbacks, - sessionId, + cacheFriendly, ) ).text; } -/** Build the provider context for a standalone summary request. */ -function buildSummarizationContext(promptText: string): Context { +/** Build a standalone summary request or append its instruction to an existing provider context. */ +function buildSummarizationContext(promptText: string, sourceContext?: Context): Context { + const instructionMessage = { + role: "user" as const, + content: [{ type: "text" as const, text: promptText }], + timestamp: Date.now(), + }; + + if (sourceContext) { + return { + ...sourceContext, + messages: [...sourceContext.messages, instructionMessage], + }; + } + return { systemPrompt: SUMMARIZATION_SYSTEM_PROMPT, - messages: [ - { - role: "user", - content: [{ type: "text", text: promptText }], - timestamp: Date.now(), - }, - ], + messages: [instructionMessage], }; } +/** + * Extra room for provider framing and tokenizer variance omitted by the heuristic context estimate. + * This matches the 4096-token margin used when normal simple requests clamp maxTokens to their context window. + */ +const CACHE_FRIENDLY_CONTEXT_SAFETY_TOKENS = 4096; + +/** Whether the source context leaves room for the requested summary output and provider safety margin. */ +function cacheFriendlyContextFits(model: Model, context: Context, maxTokens: number): boolean { + return ( + model.contextWindow <= 0 || + estimateProviderContextTokens(context).tokens + maxTokens + CACHE_FRIENDLY_CONTEXT_SAFETY_TOKENS <= + model.contextWindow + ); +} + /** Generate or update a conversation summary and return its provider usage. */ export async function generateSummaryWithUsage( currentMessages: AgentMessage[], @@ -669,31 +748,55 @@ export async function generateSummaryWithUsage( env?: Record, retry?: RetryPolicy, callbacks?: RetryCallbacks, - sessionId?: string, + cacheFriendly?: Pick, ): Promise<{ text: string; usage: Usage }> { const maxTokens = Math.min( Math.floor(0.8 * reserveTokens), model.maxTokens > 0 ? model.maxTokens : Number.POSITIVE_INFINITY, ); + // Provider-visible history prefix to reuse instead of serializing messages into a standalone prompt. + let sourceContext = cacheFriendly?.sourceContext; - // Use update prompt if we have a previous summary, otherwise initial prompt - let basePrompt = previousSummary ? UPDATE_SUMMARIZATION_PROMPT : SUMMARIZATION_PROMPT; + // Cache-friendly source contexts already contain the previous compaction summary, + // but still need iterative-update instructions so prior information is preserved. + let basePrompt = previousSummary + ? sourceContext + ? SOURCE_CONTEXT_UPDATE_SUMMARIZATION_PROMPT + : UPDATE_SUMMARIZATION_PROMPT + : SUMMARIZATION_PROMPT; if (customInstructions) { basePrompt = `${basePrompt}\n\nAdditional focus: ${customInstructions}`; } - // Serialize conversation to text so model doesn't try to continue it - // Convert to LLM messages first (handles custom types like bashExecution, custom, etc.) - const llmMessages = convertToLlm(currentMessages); - const conversationText = serializeConversation(llmMessages); + // A valid prefix can still be too large to leave the intended summary budget, + // especially during overflow recovery or after switching to a smaller-context model. + // In that case, use standalone serialization, which truncates large tool results. + if ( + sourceContext && + !cacheFriendlyContextFits(model, buildSummarizationContext(basePrompt, sourceContext), maxTokens) + ) { + sourceContext = undefined; + basePrompt = previousSummary ? UPDATE_SUMMARIZATION_PROMPT : SUMMARIZATION_PROMPT; + if (customInstructions) { + basePrompt = `${basePrompt}\n\nAdditional focus: ${customInstructions}`; + } + } - // Build the prompt with conversation wrapped in tags - let promptText = `\n${conversationText}\n\n\n`; - if (previousSummary) { + // Source contexts already contain the conversation. Standalone requests serialize it + // so the model treats those messages as data rather than continuing the conversation. + let promptText = ""; + if (!sourceContext) { + const llmMessages = convertToLlm(currentMessages); + const conversationText = serializeConversation(llmMessages); + promptText = `\n${conversationText}\n\n\n`; + } + if (previousSummary && !sourceContext) { promptText += `\n${previousSummary}\n\n\n`; } promptText += basePrompt; + const sourceRequestOptions = sourceContext ? cacheFriendly?.requestOptions : undefined; + const sourceCacheRetention = sourceContext ? "short" : undefined; const completionOptions = createSummarizationOptions( model, maxTokens, @@ -702,12 +805,13 @@ export async function generateSummaryWithUsage( env, signal, thinkingLevel, - sessionId, + sourceRequestOptions, + sourceCacheRetention, ); const response = await completeSummarization( model, - buildSummarizationContext(promptText), + buildSummarizationContext(promptText, sourceContext), completionOptions, streamFn, retry, @@ -735,8 +839,15 @@ export interface CompactionPreparation { firstKeptEntryId: string; /** Messages that will be summarized and discarded */ messagesToSummarize: AgentMessage[]; + /** + * Active-context prefix for the history summary. + * Includes the previous compaction summary before messages retained by that compaction. + */ + sourceMessages?: AgentMessage[]; /** Messages that will be turned into turn prefix summary (if splitting) */ turnPrefixMessages: AgentMessage[]; + /** Active-context prefix through the split-turn prefix, or empty when not splitting. */ + turnPrefixSourceMessages?: AgentMessage[]; /** Whether this is a split turn (cut point in middle of turn) */ isSplitTurn: boolean; tokensBefore: number; @@ -794,6 +905,8 @@ export function prepareCompaction( if (msg) messagesToSummarize.push(msg); } + const sourceMessages = collectSourceMessages(pathEntries, boundaryStart, historyEnd, prevCompactionIndex); + // Messages for turn prefix summary (if splitting a turn) const turnPrefixMessages: AgentMessage[] = []; if (cutPoint.isSplitTurn) { @@ -802,6 +915,9 @@ export function prepareCompaction( if (msg) turnPrefixMessages.push(msg); } } + const turnPrefixSourceMessages = cutPoint.isSplitTurn + ? collectSourceMessages(pathEntries, boundaryStart, cutPoint.firstKeptEntryIndex, prevCompactionIndex) + : []; if (messagesToSummarize.length === 0 && turnPrefixMessages.length === 0) { return undefined; @@ -820,7 +936,9 @@ export function prepareCompaction( return { firstKeptEntryId, messagesToSummarize, + sourceMessages, turnPrefixMessages, + turnPrefixSourceMessages, isSplitTurn: cutPoint.isSplitTurn, tokensBefore, previousSummary, @@ -848,13 +966,30 @@ Summarize the prefix to provide context for the retained suffix: Be concise. Focus on what's needed to understand the kept suffix.`; +const SOURCE_CONTEXT_TURN_PREFIX_SUMMARIZATION_PROMPT = `The final turn in the source conversation was too large to keep in full. Its SUFFIX (recent work) is retained. + +The source conversation may also contain complete earlier turns for background. Summarize only the final, incomplete turn. It begins with the last user-role request before this instruction. Do not summarize earlier turns except for details needed to understand this final turn's prefix. + +Summarize the prefix to provide context for the retained suffix: + +## Original Request +[What did the user ask for in this turn?] + +## Early Progress +- [Key decisions and work done in the prefix] + +## Context for Suffix +- [Information needed to understand the retained recent work] + +Be concise. Focus on what's needed to understand the kept suffix.`; + /** * Generate summaries for compaction using prepared data. * Returns CompactionResult - SessionManager adds uuid/parentUuid when saving. * * @param preparation - Pre-calculated preparation from prepareCompaction() * @param customInstructions - Optional custom focus for the summary - * @param sessionId - Optional routing session ID forwarded without enabling prompt caching + * @param cacheFriendly - Active provider contexts and request settings for cache-friendly summarization */ export async function compact( preparation: CompactionPreparation, @@ -868,7 +1003,7 @@ export async function compact( env?: Record, retry?: RetryPolicy, callbacks?: RetryCallbacks, - sessionId?: string, + cacheFriendly?: CacheFriendlySummaryOptions, ): Promise { const { firstKeptEntryId, @@ -903,7 +1038,10 @@ export async function compact( env, retry, callbacks, - sessionId, + { + sourceContext: cacheFriendly?.sourceContext, + requestOptions: cacheFriendly?.requestOptions, + }, ); historyText = historyResult.text; historyUsage = historyResult.usage; @@ -920,7 +1058,10 @@ export async function compact( streamFn, retry, callbacks, - sessionId, + { + sourceContext: cacheFriendly?.turnPrefixSourceContext, + requestOptions: cacheFriendly?.requestOptions, + }, ); // Merge into single summary summary = `${historyText}\n\n---\n\n**Turn Context (split turn):**\n\n${turnPrefixResult.text}`; @@ -941,7 +1082,10 @@ export async function compact( env, retry, callbacks, - sessionId, + { + sourceContext: cacheFriendly?.sourceContext, + requestOptions: cacheFriendly?.requestOptions, + }, ); summary = result.text; summaryUsage = result.usage; @@ -979,20 +1123,51 @@ async function generateTurnPrefixSummary( streamFn?: StreamFn, retry?: RetryPolicy, callbacks?: RetryCallbacks, - sessionId?: string, + cacheFriendly?: Pick, ): Promise<{ text: string; usage: Usage }> { const maxTokens = Math.min( Math.floor(0.5 * reserveTokens), model.maxTokens > 0 ? model.maxTokens : Number.POSITIVE_INFINITY, ); // Smaller budget for turn prefix - const llmMessages = convertToLlm(messages); - const conversationText = serializeConversation(llmMessages); - const promptText = `\n${conversationText}\n\n\n${TURN_PREFIX_SUMMARIZATION_PROMPT}`; + // Reuse the provider-visible split-turn prefix only when it leaves room for the summary; + // otherwise serialize and truncate the messages in a standalone prompt. + let sourceContext = cacheFriendly?.sourceContext; + if ( + sourceContext && + !cacheFriendlyContextFits( + model, + buildSummarizationContext(SOURCE_CONTEXT_TURN_PREFIX_SUMMARIZATION_PROMPT, sourceContext), + maxTokens, + ) + ) { + sourceContext = undefined; + } + let promptText: string; + if (sourceContext) { + promptText = SOURCE_CONTEXT_TURN_PREFIX_SUMMARIZATION_PROMPT; + } else { + const llmMessages = convertToLlm(messages); + const conversationText = serializeConversation(llmMessages); + promptText = `\n${conversationText}\n\n\n${TURN_PREFIX_SUMMARIZATION_PROMPT}`; + } + + const sourceRequestOptions = sourceContext ? cacheFriendly?.requestOptions : undefined; + const sourceCacheRetention = sourceContext ? "short" : undefined; const response = await completeSummarization( model, - buildSummarizationContext(promptText), - createSummarizationOptions(model, maxTokens, apiKey, headers, env, signal, thinkingLevel, sessionId), + buildSummarizationContext(promptText, sourceContext), + createSummarizationOptions( + model, + maxTokens, + apiKey, + headers, + env, + signal, + thinkingLevel, + sourceRequestOptions, + sourceCacheRetention, + ), streamFn, retry, callbacks, diff --git a/packages/coding-agent/src/index.ts b/packages/coding-agent/src/index.ts index 6d64d70c8..7c9c596a4 100644 --- a/packages/coding-agent/src/index.ts +++ b/packages/coding-agent/src/index.ts @@ -28,6 +28,7 @@ export { readStoredCredential } from "./core/auth-storage.ts"; export { type BranchPreparation, type BranchSummaryResult, + type CacheFriendlySummaryOptions, type CollectEntriesResult, type CompactionResult, type CutPointResult, diff --git a/packages/coding-agent/test/compaction-summary-reasoning.test.ts b/packages/coding-agent/test/compaction-summary-reasoning.test.ts index 340d5c1f6..9bafc9d5d 100644 --- a/packages/coding-agent/test/compaction-summary-reasoning.test.ts +++ b/packages/coding-agent/test/compaction-summary-reasoning.test.ts @@ -115,7 +115,7 @@ describe("generateSummary reasoning options", () => { expect(sessionIds[0]).not.toBe(sessionIds[1]); }); - it("honors a caller-supplied routing session without prompt caching", async () => { + it("honors caller-supplied cache retention and routing", async () => { await completeSummarization( createModel(false), { systemPrompt: "Summarize", messages: [] }, @@ -124,7 +124,7 @@ describe("generateSummary reasoning options", () => { expect(completeSimpleMock.mock.calls[0][2]).toMatchObject({ sessionId: "current-routing-session", - cacheRetention: "none", + cacheRetention: "long", toolChoice: "none", }); }); @@ -148,6 +148,229 @@ describe("generateSummary reasoning options", () => { expect(prompt).toContain(""); }); + it("appends instructions to a cache-friendly source context", async () => { + const sourceContext: Context = { + systemPrompt: "You are a coding agent.", + messages: [{ role: "user", content: "Previous summary and original request", timestamp: 1 }], + tools: [], + }; + const onPayload = async (payload: unknown) => payload; + const onResponse = async () => {}; + const preparation: CompactionPreparation = { + firstKeptEntryId: "entry-keep", + messagesToSummarize: messages, + turnPrefixMessages: [], + isSplitTurn: false, + tokensBefore: 100, + previousSummary: "Previous summary", + fileOps: { read: new Set(), written: new Set(), edited: new Set() }, + settings: { enabled: true, reserveTokens: 2000, keepRecentTokens: 20 }, + }; + + await compact( + preparation, + createModel(false), + "test-key", + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + { + sourceContext, + requestOptions: { + sessionId: "routing-session", + onPayload, + onResponse, + transport: "websocket", + thinkingBudgets: { low: 1234 }, + maxRetryDelayMs: 4321, + }, + }, + ); + + const requestContext = completeSimpleMock.mock.calls[0][1] as Context; + expect(requestContext.systemPrompt).toBe(sourceContext.systemPrompt); + expect(requestContext.tools).toBe(sourceContext.tools); + expect(requestContext.messages.slice(0, -1)).toEqual(sourceContext.messages); + const instruction = JSON.stringify(requestContext.messages.at(-1)); + expect(instruction).toContain("existing structured summary of earlier conversation history"); + expect(instruction).toContain("PRESERVE all existing information from the previous summary"); + expect(instruction).not.toContain(""); + expect(instruction).not.toContain(""); + expect(completeSimpleMock.mock.calls[0][2]).toMatchObject({ + cacheRetention: "short", + sessionId: "routing-session", + toolChoice: "none", + transport: "websocket", + thinkingBudgets: { low: 1234 }, + maxRetryDelayMs: 4321, + }); + expect(completeSimpleMock.mock.calls[0][2]?.onPayload).toBe(onPayload); + expect(completeSimpleMock.mock.calls[0][2]?.onResponse).toBe(onResponse); + expect(sourceContext.messages).toHaveLength(1); + }); + + it("falls back to standalone summarization when a cache-friendly source cannot leave the summary budget", async () => { + const oversizedToolResult: AgentMessage = { + role: "toolResult", + toolCallId: "tool-call-1", + toolName: "read", + content: [{ type: "text", text: "x".repeat(800_000) }], + isError: false, + timestamp: 1, + }; + const sourceContext: Context = { + systemPrompt: "You are a coding agent.", + messages: [oversizedToolResult], + tools: [], + }; + const preparation: CompactionPreparation = { + firstKeptEntryId: "entry-keep", + messagesToSummarize: [oversizedToolResult], + turnPrefixMessages: [], + isSplitTurn: false, + tokensBefore: 250_000, + fileOps: { read: new Set(), written: new Set(), edited: new Set() }, + settings: { enabled: true, reserveTokens: 2000, keepRecentTokens: 20 }, + }; + + await compact( + preparation, + createModel(false), + "test-key", + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + { sourceContext, requestOptions: { sessionId: "routing-session" } }, + ); + + const requestContext = completeSimpleMock.mock.calls[0][1] as Context; + const prompt = JSON.stringify(requestContext.messages); + expect(requestContext.systemPrompt).not.toBe(sourceContext.systemPrompt); + expect(requestContext.tools).toBeUndefined(); + expect(prompt).toContain(""); + expect(prompt).toContain("more characters truncated"); + expect(prompt).not.toContain("x".repeat(3000)); + expect(completeSimpleMock.mock.calls[0][2]).toMatchObject({ cacheRetention: "none", toolChoice: "none" }); + expect(completeSimpleMock.mock.calls[0][2]?.sessionId).not.toBe("routing-session"); + }); + + it("falls back to standalone summarization for an oversized split-turn source", async () => { + const splitUser = { role: "user" as const, content: "Large final request", timestamp: 1 }; + const oversizedToolResult: AgentMessage = { + role: "toolResult", + toolCallId: "tool-call-1", + toolName: "read", + content: [{ type: "text", text: "x".repeat(800_000) }], + isError: false, + timestamp: 2, + }; + const turnPrefixSourceContext: Context = { + systemPrompt: "You are a coding agent.", + messages: [splitUser, oversizedToolResult], + tools: [], + }; + const preparation: CompactionPreparation = { + firstKeptEntryId: "entry-keep", + messagesToSummarize: [], + turnPrefixMessages: [splitUser, oversizedToolResult], + isSplitTurn: true, + tokensBefore: 250_000, + fileOps: { read: new Set(), written: new Set(), edited: new Set() }, + settings: { enabled: true, reserveTokens: 2000, keepRecentTokens: 20 }, + }; + + await compact( + preparation, + createModel(false), + "test-key", + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + { turnPrefixSourceContext, requestOptions: { sessionId: "routing-session" } }, + ); + + const requestContext = completeSimpleMock.mock.calls[0][1] as Context; + const prompt = JSON.stringify(requestContext.messages); + expect(requestContext.systemPrompt).not.toBe(turnPrefixSourceContext.systemPrompt); + expect(requestContext.tools).toBeUndefined(); + expect(prompt).toContain("This is the PREFIX of a turn that was too large to keep"); + expect(prompt).toContain("more characters truncated"); + expect(prompt).not.toContain("source conversation may also contain complete earlier turns"); + expect(completeSimpleMock.mock.calls[0][2]).toMatchObject({ cacheRetention: "none", toolChoice: "none" }); + expect(completeSimpleMock.mock.calls[0][2]?.sessionId).not.toBe("routing-session"); + }); + + it("limits a cache-friendly split-turn summary to the final incomplete turn", async () => { + const earlierUser = { role: "user" as const, content: "Earlier request", timestamp: 1 }; + const earlierAssistant: AssistantMessage = { + ...mockSummaryResponse, + content: [{ type: "text", text: "Earlier work completed" }], + timestamp: 2, + }; + const splitUser = { role: "user" as const, content: "Large final request", timestamp: 3 }; + const earlyAssistant: AssistantMessage = { + ...mockSummaryResponse, + content: [{ type: "text", text: "Early work in final turn" }], + timestamp: 4, + }; + const turnPrefixSourceContext: Context = { + systemPrompt: "You are a coding agent.", + messages: [earlierUser, earlierAssistant, splitUser, earlyAssistant], + tools: [], + }; + const preparation: CompactionPreparation = { + firstKeptEntryId: "entry-keep", + messagesToSummarize: [], + turnPrefixMessages: [splitUser, earlyAssistant], + isSplitTurn: true, + tokensBefore: 100, + fileOps: { read: new Set(), written: new Set(), edited: new Set() }, + settings: { enabled: true, reserveTokens: 2000, keepRecentTokens: 20 }, + }; + + await compact( + preparation, + createModel(false), + "test-key", + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + { turnPrefixSourceContext }, + ); + + const requestContext = completeSimpleMock.mock.calls[0][1] as Context; + expect(requestContext.messages.slice(0, -1)).toEqual(turnPrefixSourceContext.messages); + const instruction = JSON.stringify(requestContext.messages.at(-1)); + expect(instruction).toContain("Summarize only the final, incomplete turn"); + expect(instruction).toContain("last user-role request before this instruction"); + expect(instruction).toContain("Do not summarize earlier turns"); + expect(instruction).not.toContain("Earlier request"); + expect(completeSimpleMock.mock.calls[0][2]).toMatchObject({ + cacheRetention: "short", + toolChoice: "none", + }); + }); + it("rejects tool calls from conversation summaries", async () => { completeSimpleMock.mockResolvedValueOnce(mockToolCallResponse); diff --git a/packages/coding-agent/test/compaction.test.ts b/packages/coding-agent/test/compaction.test.ts index 12c9d0be2..8fe754ce6 100644 --- a/packages/coding-agent/test/compaction.test.ts +++ b/packages/coding-agent/test/compaction.test.ts @@ -5,6 +5,7 @@ import { readFileSync } from "fs"; import { join } from "path"; import { beforeEach, describe, expect, it } from "vitest"; import { + type CompactionPreparation, type CompactionSettings, calculateContextTokens, compact, @@ -150,6 +151,17 @@ function createCustomMessageEntry(content: string): CustomMessageEntry { return entry; } +function requireSourceMessages( + preparation: CompactionPreparation | undefined, +): asserts preparation is CompactionPreparation & { + sourceMessages: AgentMessage[]; + turnPrefixSourceMessages: AgentMessage[]; +} { + if (!preparation?.sourceMessages || !preparation.turnPrefixSourceMessages) { + throw new Error("Expected compaction source messages"); + } +} + function extractText(messages: AgentMessage[]): string { return messages .map((message) => { @@ -461,6 +473,44 @@ describe("buildSessionContext", () => { }); }); +describe("prepareCompaction source messages", () => { + it("provides the active history prefix for a normal compaction", () => { + const u1 = createMessageEntry(createUserMessage("old user ".repeat(20))); + const a1 = createMessageEntry(createAssistantMessage("old assistant ".repeat(20))); + const u2 = createMessageEntry(createUserMessage("recent user")); + const a2 = createMessageEntry(createAssistantMessage("ok")); + const settings: CompactionSettings = { ...DEFAULT_COMPACTION_SETTINGS, keepRecentTokens: 3 }; + + const preparation = prepareCompaction([u1, a1, u2, a2], settings); + + expect(preparation).toBeDefined(); + requireSourceMessages(preparation); + expect(preparation.isSplitTurn).toBe(false); + expect(preparation.sourceMessages).toEqual(preparation.messagesToSummarize); + expect(extractText(preparation.sourceMessages)).toContain("old user"); + expect(extractText(preparation.sourceMessages)).not.toContain("recent user"); + expect(preparation.turnPrefixSourceMessages).toEqual([]); + }); + + it("provides the active prefix through a split turn", () => { + const user = createMessageEntry(createUserMessage("large request ".repeat(20))); + const earlyAssistant = createMessageEntry(createAssistantMessage("early work ".repeat(20))); + const keptAssistant = createMessageEntry(createAssistantMessage("kept")); + const settings: CompactionSettings = { ...DEFAULT_COMPACTION_SETTINGS, keepRecentTokens: 1 }; + + const preparation = prepareCompaction([user, earlyAssistant, keptAssistant], settings); + + expect(preparation).toBeDefined(); + requireSourceMessages(preparation); + expect(preparation.isSplitTurn).toBe(true); + expect(preparation.messagesToSummarize).toEqual([]); + expect(preparation.sourceMessages).toEqual([]); + expect(preparation.turnPrefixSourceMessages).toEqual(preparation.turnPrefixMessages); + expect(extractText(preparation.turnPrefixSourceMessages)).toContain("large request"); + expect(extractText(preparation.turnPrefixSourceMessages)).not.toContain("kept"); + }); +}); + describe("prepareCompaction with previous compaction", () => { it("should skip repeated compactions when kept messages still fit", () => { const u1 = createMessageEntry(createUserMessage("user msg 1 (summarized by compaction1)")); @@ -479,6 +529,33 @@ describe("prepareCompaction with previous compaction", () => { expect(preparation).toBeUndefined(); }); + it("preserves retained older summaries in the active source prefix", () => { + const u1 = createMessageEntry(createUserMessage("user 1")); + const a1 = createMessageEntry(createAssistantMessage("assistant 1")); + const u2 = createMessageEntry(createUserMessage("user 2 ".repeat(20))); + const a2 = createMessageEntry(createAssistantMessage("assistant 2 ".repeat(20))); + const compaction1 = createCompactionEntry("First summary", u1.id); + const u3 = createMessageEntry(createUserMessage("user 3 ".repeat(20))); + const a3 = createMessageEntry(createAssistantMessage("assistant 3 ".repeat(20))); + const compaction2 = createCompactionEntry("Second summary", u2.id); + const u4 = createMessageEntry(createUserMessage("user 4 ".repeat(20))); + const a4 = createMessageEntry(createAssistantMessage("kept")); + const settings: CompactionSettings = { ...DEFAULT_COMPACTION_SETTINGS, keepRecentTokens: 1 }; + const pathEntries = [u1, a1, u2, a2, compaction1, u3, a3, compaction2, u4, a4]; + + const preparation = prepareCompaction(pathEntries, settings); + + expect(preparation).toBeDefined(); + requireSourceMessages(preparation); + const activeMessages = buildSessionContext(pathEntries).messages; + expect(preparation.sourceMessages).toEqual(activeMessages.slice(0, preparation.sourceMessages.length)); + expect(preparation.turnPrefixSourceMessages).toEqual( + activeMessages.slice(0, preparation.turnPrefixSourceMessages.length), + ); + const sourceSummaries = preparation.sourceMessages.filter((message) => message.role === "compactionSummary"); + expect(sourceSummaries.map((message) => message.summary)).toEqual(["Second summary", "First summary"]); + }); + it("should re-summarize previously kept messages when the recent window moves past them", () => { const u1 = createMessageEntry(createUserMessage("user msg 1 (summarized by compaction1)".repeat(4))); const a1 = createMessageEntry(createAssistantMessage("assistant msg 1".repeat(4))); @@ -497,11 +574,15 @@ describe("prepareCompaction with previous compaction", () => { const preparation = prepareCompaction([u1, a1, u2, a2, u3, a3, compaction1, u4, a4], settings); expect(preparation).toBeDefined(); - const summarizedText = extractText(preparation!.messagesToSummarize); + requireSourceMessages(preparation); + const summarizedText = extractText(preparation.messagesToSummarize); expect(summarizedText).toContain("user msg 2 - kept by compaction1"); expect(summarizedText).toContain("user msg 3 - kept by compaction1"); expect(summarizedText).not.toContain("First summary"); - expect(preparation!.previousSummary).toBe("First summary"); + expect(preparation.previousSummary).toBe("First summary"); + expect(preparation.sourceMessages[0]?.role).toBe("compactionSummary"); + expect(extractText(preparation.sourceMessages)).toContain("First summary"); + expect(preparation.sourceMessages.slice(1)).toEqual(preparation.messagesToSummarize); }); }); From a6c6f801802a7083609541dbd49eec203804d8fc Mon Sep 17 00:00:00 2001 From: Cristina Poncela Cubeiro <140309543+cristinaponcela@users.noreply.github.com> Date: Tue, 18 Aug 2026 15:28:12 +0200 Subject: [PATCH 04/10] fix(ai): anthropic fallback usage (#8308) * fix(ai): anthropic fallback usage * changeloggg --- packages/ai/CHANGELOG.md | 1 + packages/ai/src/api/anthropic-messages.ts | 11 +++++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index b4de62a00..e15dab0cf 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -13,6 +13,7 @@ ### Fixed +- Fixed Anthropic server-side fallback responses being priced with the requested model instead of the returned fallback model ([#8285](https://github.com/earendil-works/pi/issues/8285)). - Fixed Azure OpenAI Responses ignoring `toolChoice` in provider-specific stream requests. - Added `deepseek-v4-pro-0813` to the Qwen Token Plan Individual catalog ([#8194](https://github.com/earendil-works/pi/issues/8194)). - Fixed Amazon Bedrock `after_provider_response`/`onResponse` to forward the raw response headers instead of only the synthesized request id header ([#8234](https://github.com/earendil-works/pi/issues/8234)). diff --git a/packages/ai/src/api/anthropic-messages.ts b/packages/ai/src/api/anthropic-messages.ts index f0e89a558..f96d94052 100644 --- a/packages/ai/src/api/anthropic-messages.ts +++ b/packages/ai/src/api/anthropic-messages.ts @@ -8,6 +8,7 @@ import type { RefusalStopDetails, } from "@anthropic-ai/sdk/resources/messages.js"; import { calculateCost } from "../models.ts"; +import { ANTHROPIC_MODELS } from "../providers/anthropic.models.ts"; import type { AnthropicMessagesCompat, AnthropicRefusalFallback, @@ -540,6 +541,7 @@ export const stream: StreamFunction<"anthropic-messages", AnthropicOptions> = ( try { let client: Anthropic; let isOAuth: boolean; + let usageModel = model; if (options?.client) { client = options.client; @@ -602,6 +604,11 @@ export const stream: StreamFunction<"anthropic-messages", AnthropicOptions> = ( if (event.type === "message_start") { output.responseId = event.message.id; output.model = event.message.model; + usageModel = + model.provider === "anthropic" + ? ((ANTHROPIC_MODELS as Record | undefined>)[output.model] ?? + model) + : model; // Capture initial token usage from message_start event // This ensures we have input token counts even if the stream is aborted early output.usage.input = event.message.usage.input_tokens || 0; @@ -612,7 +619,7 @@ export const stream: StreamFunction<"anthropic-messages", AnthropicOptions> = ( // Anthropic doesn't provide total_tokens, compute from components output.usage.totalTokens = output.usage.input + output.usage.output + output.usage.cacheRead + output.usage.cacheWrite; - calculateCost(model, output.usage); + calculateCost(usageModel, output.usage); } else if (event.type === "content_block_start") { if (event.content_block.type === "text") { const block: Block = { @@ -769,7 +776,7 @@ export const stream: StreamFunction<"anthropic-messages", AnthropicOptions> = ( // Anthropic doesn't provide total_tokens, compute from components output.usage.totalTokens = output.usage.input + output.usage.output + output.usage.cacheRead + output.usage.cacheWrite; - calculateCost(model, output.usage); + calculateCost(usageModel, output.usage); } } From b237412699fa33773921d295d2af14bf3031315e Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 18 Aug 2026 17:35:35 +0300 Subject: [PATCH 05/10] feat(ai): generalize openai-completions thinking token budget fields (#8275) #7638 hardcoded vLLM's thinking_token_budget. Map thinkingBudgets through thinkingTokenBudgetField, keep the boolean as a vLLM alias, clamp so the answer keeps 1024 tokens, and document it. fixes #8274 Co-authored-by: Cursor Co-authored-by: Mario Zechner --- packages/ai/README.md | 6 +- packages/ai/scripts/generate-models.ts | 7 +- packages/ai/src/api/openai-completions.ts | 71 ++++++---- packages/ai/src/api/simple-options.ts | 31 +++-- packages/ai/src/types.ts | 19 ++- ...penai-completions-thinking-as-text.test.ts | 7 +- ...-completions-thinking-token-budget.test.ts | 130 ++++++++++++++---- ...nai-completions-tool-result-images.test.ts | 4 +- packages/coding-agent/docs/custom-provider.md | 7 +- packages/coding-agent/docs/models.md | 10 +- packages/coding-agent/docs/settings.md | 2 +- 11 files changed, 218 insertions(+), 76 deletions(-) diff --git a/packages/ai/README.md b/packages/ai/README.md index e010fdf25..dde9c83b4 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -1183,8 +1183,10 @@ interface OpenAICompletionsCompat { requiresThinkingAsText?: boolean; // Whether thinking blocks must be converted to text (default: false) requiresReasoningContentOnAssistantMessages?: boolean; // Whether all replayed assistant messages must include empty reasoning_content when reasoning is enabled (default: auto-detected for DeepSeek) thinkingFormat?: 'openai' | 'openrouter' | 'deepseek' | 'together' | 'baseten' | 'zai' | 'qwen' | 'chat-template' | 'qwen-chat-template' | 'string-thinking' | 'ant-ling'; // Format for reasoning param: 'openai' uses reasoning_effort, 'openrouter' uses reasoning: { effort }, 'deepseek' uses thinking: { type } plus reasoning_effort when supported, 'together' uses reasoning: { enabled } plus reasoning_effort when supported, 'baseten' uses configurable chat_template_args plus reasoning_effort when supported, 'zai' uses thinking: { type }, 'qwen' uses enable_thinking, 'chat-template' uses configurable chat_template_kwargs, 'qwen-chat-template' uses chat_template_kwargs.enable_thinking and preserve_thinking, 'string-thinking' uses top-level thinking, 'ant-ling' uses reasoning: { effort } only for mapped efforts (default: openai) - chatTemplateKwargs?: Record; // chat_template_kwargs values; use $var for pi-controlled thinking values - chatTemplateArgs?: Record; // chat_template_args values for thinkingFormat: 'baseten'; use $var for pi-controlled thinking values + chatTemplateKwargs?: Record; // chat_template_kwargs values; use $var for pi-controlled thinking values + chatTemplateArgs?: Record; // chat_template_args values for thinkingFormat: 'baseten'; use $var for pi-controlled thinking values + thinkingTokenBudgetField?: 'thinking_token_budget' | 'thinking_budget' | 'thinking_budget_tokens'; // Top-level field that caps reasoning tokens from thinkingBudgets (vLLM / Qwen / llama.cpp). Off by default. + supportsThinkingTokenBudget?: boolean; // Alias for thinkingTokenBudgetField: 'thinking_token_budget' (vLLM). Prefer thinkingTokenBudgetField. Default: false. cacheControlFormat?: 'anthropic'; // Anthropic-style cache_control on system prompt, last tool, and last user/assistant text content openRouterRouting?: OpenRouterRouting; // OpenRouter routing preferences (default: {}) vercelGatewayRouting?: VercelGatewayRouting; // Vercel AI Gateway routing preferences (default: {}) diff --git a/packages/ai/scripts/generate-models.ts b/packages/ai/scripts/generate-models.ts index 2b09d9bd0..418b0a6d9 100644 --- a/packages/ai/scripts/generate-models.ts +++ b/packages/ai/scripts/generate-models.ts @@ -594,7 +594,12 @@ const OPENAI_COMPLETIONS_DEFAULT_COMPAT = { supportsOpenAIGrammarTools: false, sendSessionAffinityHeaders: false, supportsLongCacheRetention: true, -} satisfies Required> & { +} satisfies Required< + Omit< + OpenAICompletionsCompat, + "cacheControlFormat" | "deferredToolsMode" | "supportsThinkingTokenBudget" | "thinkingTokenBudgetField" + > +> & { cacheControlFormat?: OpenAICompletionsCompat["cacheControlFormat"]; deferredToolsMode?: OpenAICompletionsCompat["deferredToolsMode"]; }; diff --git a/packages/ai/src/api/openai-completions.ts b/packages/ai/src/api/openai-completions.ts index d261e82a4..8d968521a 100644 --- a/packages/ai/src/api/openai-completions.ts +++ b/packages/ai/src/api/openai-completions.ts @@ -30,6 +30,7 @@ import type { TextContent, ThinkingBudgets, ThinkingContent, + ThinkingTokenBudgetField, Tool, ToolCall, ToolResultMessage, @@ -54,7 +55,7 @@ import { } from "./constrained-sampling.ts"; import { buildCopilotDynamicHeaders, hasCopilotVisionInput } from "./github-copilot-headers.ts"; import { clampOpenAIPromptCacheKey } from "./openai-prompt-cache.ts"; -import { buildBaseOptions, clampReasoning, MIN_ANSWER_TOKENS } from "./simple-options.ts"; +import { buildBaseOptions, clampThinkingBudgetToAnswerRoom, thinkingBudgetForLevel } from "./simple-options.ts"; import { transformMessages } from "./transform-messages.ts"; /** @@ -144,7 +145,7 @@ function isEncryptedReasoningDetail(detail: unknown): detail is OpenAIEncryptedR export interface OpenAICompletionsOptions extends StreamOptions { toolChoice?: OpenAI.Chat.Completions.ChatCompletionToolChoiceOption; reasoningEffort?: "minimal" | "low" | "medium" | "high" | "xhigh" | "max"; - /** Token budgets per thinking level. Only used when `compat.supportsThinkingTokenBudget` is set. */ + /** Token budgets per thinking level. Used when `compat.thinkingTokenBudgetField` or `compat.supportsThinkingTokenBudget` is set, or by `{ "$var": "thinking.budget" }`. */ thinkingBudgets?: ThinkingBudgets; } @@ -159,11 +160,12 @@ interface OpenAICompatCacheControl { type ResolvedOpenAICompletionsCompat = Omit< Required, - "cacheControlFormat" | "deferredToolsMode" | "supportsThinkingTokenBudget" + "cacheControlFormat" | "deferredToolsMode" | "supportsThinkingTokenBudget" | "thinkingTokenBudgetField" > & { cacheControlFormat?: OpenAICompletionsCompat["cacheControlFormat"]; deferredToolsMode?: OpenAICompletionsCompat["deferredToolsMode"]; supportsThinkingTokenBudget?: OpenAICompletionsCompat["supportsThinkingTokenBudget"]; + thinkingTokenBudgetField?: OpenAICompletionsCompat["thinkingTokenBudgetField"]; }; type ResolvedChatTemplateKwargValue = string | number | boolean | null; @@ -752,6 +754,9 @@ function buildParams( params.tool_choice = options.toolChoice; } + const thinkingTokenBudgetField = resolveThinkingTokenBudgetField(compat); + const thinkingBudget = resolveClampedThinkingBudget(model, options, params); + if (compat.thinkingFormat === "zai" && model.reasoning) { const zaiParams = params as Omit & { thinking?: { type: "enabled" | "disabled"; clear_thinking?: boolean }; @@ -779,7 +784,7 @@ function buildParams( preserve_thinking: true, }; } else if (compat.thinkingFormat === "chat-template" && model.reasoning) { - const chatTemplateKwargs = buildChatTemplateValues(model, options, compat.chatTemplateKwargs); + const chatTemplateKwargs = buildChatTemplateValues(model, options, compat.chatTemplateKwargs, thinkingBudget); if (chatTemplateKwargs) { (params as any).chat_template_kwargs = chatTemplateKwargs; } @@ -788,7 +793,7 @@ function buildParams( chat_template_args?: Record; reasoning_effort?: string; }; - const chatTemplateArgs = buildChatTemplateValues(model, options, compat.chatTemplateArgs); + const chatTemplateArgs = buildChatTemplateValues(model, options, compat.chatTemplateArgs, thinkingBudget); if (chatTemplateArgs) { basetenParams.chat_template_args = chatTemplateArgs; } @@ -851,25 +856,12 @@ function buildParams( } } - // vLLM caps reasoning with a top-level thinking_token_budget. Independent of - // thinkingFormat: the same server can serve zai, qwen or chat-template models. - // Reasoning and the answer share max_tokens here, so an uncapped reasoning - // phase can consume the whole response and leave no answer and no tool call. - if (compat.supportsThinkingTokenBudget && options?.reasoningEffort && model.reasoning) { - const level = clampReasoning(options.reasoningEffort)!; - const budgets: ThinkingBudgets = { - minimal: 1024, - low: 2048, - medium: 8192, - high: 16384, - ...options.thinkingBudgets, - }; - const ceiling = (params as { max_tokens?: number }).max_tokens ?? params.max_completion_tokens ?? model.maxTokens; - // Always leave room for the answer, otherwise the budget recreates the bug it prevents. - const budget = Math.min(budgets[level]!, Math.max(0, ceiling - MIN_ANSWER_TOKENS)); - if (budget > 0) { - (params as { thinking_token_budget?: number }).thinking_token_budget = budget; - } + // Cap reasoning with a top-level budget field. Independent of thinkingFormat: the + // same server can serve zai, qwen or chat-template models. Reasoning and the answer + // share max_tokens here, so an uncapped reasoning phase can consume the whole + // response and leave no answer and no tool call. + if (thinkingTokenBudgetField && thinkingBudget !== undefined) { + Object.assign(params, { [thinkingTokenBudgetField]: thinkingBudget }); } // OpenRouter provider routing preferences @@ -896,15 +888,38 @@ function buildParams( return params; } +function resolveThinkingTokenBudgetField( + compat: Pick, +): ThinkingTokenBudgetField | undefined { + if (compat.thinkingTokenBudgetField) return compat.thinkingTokenBudgetField; + if (compat.supportsThinkingTokenBudget) return "thinking_token_budget"; + return undefined; +} + +function resolveClampedThinkingBudget( + model: Model<"openai-completions">, + options: OpenAICompletionsOptions | undefined, + params: { max_tokens?: number | null; max_completion_tokens?: number | null }, +): number | undefined { + if (!options?.reasoningEffort || !model.reasoning) return undefined; + const ceiling = params.max_tokens ?? params.max_completion_tokens ?? model.maxTokens; + const budget = clampThinkingBudgetToAnswerRoom( + thinkingBudgetForLevel(options.reasoningEffort, options.thinkingBudgets), + ceiling, + ); + return budget > 0 ? budget : undefined; +} + function buildChatTemplateValues( model: Model<"openai-completions">, options: OpenAICompletionsOptions | undefined, values: Record, + thinkingBudget?: number, ): Record | undefined { const resolvedValues: Record = {}; for (const [key, value] of Object.entries(values)) { - const resolved = resolveChatTemplateKwargValue(model, options, value); + const resolved = resolveChatTemplateKwargValue(model, options, value, thinkingBudget); if (resolved !== undefined) { resolvedValues[key] = resolved; } @@ -917,6 +932,7 @@ function resolveChatTemplateKwargValue( model: Model<"openai-completions">, options: OpenAICompletionsOptions | undefined, value: ChatTemplateKwargValue, + thinkingBudget?: number, ): ResolvedChatTemplateKwargValue | undefined { if (typeof value !== "object" || value === null) { return value; @@ -929,6 +945,9 @@ function resolveChatTemplateKwargValue( if (value.$var === "thinking.enabled") { return !!reasoningEffort; } + if (value.$var === "thinking.budget") { + return thinkingBudget; + } const mappedValue = reasoningEffort ? model.thinkingLevelMap?.[reasoningEffort] : model.thinkingLevelMap?.off; return mappedValue === undefined ? reasoningEffort : typeof mappedValue === "string" ? mappedValue : undefined; @@ -1532,6 +1551,7 @@ function detectCompat(model: Model<"openai-completions">): ResolvedOpenAIComplet chatTemplateArgs: {}, zaiToolStream: false, supportsThinkingTokenBudget: false, + thinkingTokenBudgetField: undefined, supportsStrictMode: !isMoonshot && !isTogether && !isCloudflareAiGateway && !isNvidia, supportsOpenAIGrammarTools: false, cacheControlFormat, @@ -1577,6 +1597,7 @@ function getCompat(model: Model<"openai-completions">): ResolvedOpenAICompletion chatTemplateArgs: model.compat.chatTemplateArgs ?? detected.chatTemplateArgs, zaiToolStream: model.compat.zaiToolStream ?? detected.zaiToolStream, supportsThinkingTokenBudget: model.compat.supportsThinkingTokenBudget ?? detected.supportsThinkingTokenBudget, + thinkingTokenBudgetField: model.compat.thinkingTokenBudgetField ?? detected.thinkingTokenBudgetField, supportsStrictMode: model.compat.supportsStrictMode ?? detected.supportsStrictMode, supportsOpenAIGrammarTools: model.compat.supportsOpenAIGrammarTools ?? detected.supportsOpenAIGrammarTools, cacheControlFormat: model.compat.cacheControlFormat ?? detected.cacheControlFormat, diff --git a/packages/ai/src/api/simple-options.ts b/packages/ai/src/api/simple-options.ts index 067d01e59..2a30265ad 100644 --- a/packages/ai/src/api/simple-options.ts +++ b/packages/ai/src/api/simple-options.ts @@ -54,10 +54,28 @@ export function buildBaseOptions( /** Tokens always left for the answer when a thinking budget shares the response ceiling. */ export const MIN_ANSWER_TOKENS = 1024; +export const DEFAULT_THINKING_BUDGETS: ThinkingBudgets = { + minimal: 1024, + low: 2048, + medium: 8192, + high: 16384, +}; + export function clampReasoning(effort: ThinkingLevel | undefined): Exclude | undefined { return effort === "xhigh" || effort === "max" ? "high" : effort; } +export function thinkingBudgetForLevel(reasoningLevel: ThinkingLevel, customBudgets?: ThinkingBudgets): number { + const budgets = { ...DEFAULT_THINKING_BUDGETS, ...customBudgets }; + const level = clampReasoning(reasoningLevel)!; + return budgets[level]!; +} + +/** Cap a thinking budget so at least MIN_ANSWER_TOKENS remain under a shared response ceiling. */ +export function clampThinkingBudgetToAnswerRoom(thinkingBudget: number, ceiling: number): number { + return Math.min(thinkingBudget, Math.max(0, ceiling - MIN_ANSWER_TOKENS)); +} + export function adjustMaxTokensForThinking( // Undefined means no explicit caller cap. Use the model cap and fit thinking inside it. baseMaxTokens: number | undefined, @@ -65,21 +83,12 @@ export function adjustMaxTokensForThinking( reasoningLevel: ThinkingLevel, customBudgets?: ThinkingBudgets, ): { maxTokens: number; thinkingBudget: number } { - const defaultBudgets: ThinkingBudgets = { - minimal: 1024, - low: 2048, - medium: 8192, - high: 16384, - }; - const budgets = { ...defaultBudgets, ...customBudgets }; - - const level = clampReasoning(reasoningLevel)!; - let thinkingBudget = budgets[level]!; + let thinkingBudget = thinkingBudgetForLevel(reasoningLevel, customBudgets); const maxTokens = baseMaxTokens === undefined ? modelMaxTokens : Math.min(baseMaxTokens + thinkingBudget, modelMaxTokens); if (maxTokens <= thinkingBudget) { - thinkingBudget = Math.max(0, maxTokens - MIN_ANSWER_TOKENS); + thinkingBudget = clampThinkingBudgetToAnswerRoom(thinkingBudget, maxTokens); } return { maxTokens, thinkingBudget }; diff --git a/packages/ai/src/types.ts b/packages/ai/src/types.ts index 832222133..e7df833d5 100644 --- a/packages/ai/src/types.ts +++ b/packages/ai/src/types.ts @@ -89,10 +89,13 @@ export type ChatTemplateKwargValue = | boolean | null | { - $var: "thinking.enabled" | "thinking.effort"; + $var: "thinking.enabled" | "thinking.effort" | "thinking.budget"; omitWhenOff?: boolean; }; +/** Top-level request field used to cap reasoning tokens on OpenAI-compatible servers. */ +export type ThinkingTokenBudgetField = "thinking_token_budget" | "thinking_budget" | "thinking_budget_tokens"; + /** Token budgets for each thinking level (token-based providers only) */ export interface ThinkingBudgets { minimal?: number; @@ -583,9 +586,9 @@ export interface OpenAICompletionsCompat { | "qwen-chat-template" | "string-thinking" | "ant-ling"; - /** Kwargs to send as `chat_template_kwargs` when `thinkingFormat` is `chat-template`. Use `{ "$var": "thinking.enabled" }` or `{ "$var": "thinking.effort" }` for pi-controlled thinking values. */ + /** Kwargs to send as `chat_template_kwargs` when `thinkingFormat` is `chat-template`. Use `{ "$var": "thinking.enabled" }`, `{ "$var": "thinking.effort" }`, or `{ "$var": "thinking.budget" }` for pi-controlled thinking values. */ chatTemplateKwargs?: Record; - /** Arguments to send as `chat_template_args` when `thinkingFormat` is `baseten`. Use `{ "$var": "thinking.enabled" }` or `{ "$var": "thinking.effort" }` for pi-controlled thinking values. */ + /** Arguments to send as `chat_template_args` when `thinkingFormat` is `baseten`. Use `{ "$var": "thinking.enabled" }`, `{ "$var": "thinking.effort" }`, or `{ "$var": "thinking.budget" }` for pi-controlled thinking values. */ chatTemplateArgs?: Record; /** OpenRouter-compatible routing preferences sent as the `provider` request field. */ openRouterRouting?: OpenRouterRouting; @@ -593,7 +596,15 @@ export interface OpenAICompletionsCompat { vercelGatewayRouting?: VercelGatewayRouting; /** Whether z.ai supports top-level `tool_stream: true` for streaming tool call deltas. Default: false. */ zaiToolStream?: boolean; - /** Whether the provider supports top-level `thinking_token_budget` to cap reasoning tokens (vLLM). Reasoning and the answer share `max_tokens` on these endpoints, so without a budget a reasoning-heavy turn can consume the whole response and emit no answer. Default: false. */ + /** + * Top-level request field used to cap reasoning tokens from `thinkingBudgets`. + * Reasoning and the answer share `max_tokens` on these endpoints, so without a budget a + * reasoning-heavy turn can consume the whole response and emit no answer. + * `"thinking_token_budget"` is vLLM, `"thinking_budget"` is Qwen/DashScope/SGLang, + * `"thinking_budget_tokens"` is llama.cpp. Off by default; not set on the generated catalog. + */ + thinkingTokenBudgetField?: ThinkingTokenBudgetField; + /** Alias for `thinkingTokenBudgetField: "thinking_token_budget"` (vLLM). Prefer `thinkingTokenBudgetField`. Default: false. */ supportsThinkingTokenBudget?: boolean; /** Whether the provider supports OpenAI custom tools with Lark/regex grammar formats. When false, grammar-constrained tools fall back to normal function tools. Default: false; the generated model catalog enables it for capable models. */ supportsOpenAIGrammarTools?: boolean; diff --git a/packages/ai/test/openai-completions-thinking-as-text.test.ts b/packages/ai/test/openai-completions-thinking-as-text.test.ts index 2e451f6b7..44e14416b 100644 --- a/packages/ai/test/openai-completions-thinking-as-text.test.ts +++ b/packages/ai/test/openai-completions-thinking-as-text.test.ts @@ -39,15 +39,20 @@ const compat = { chatTemplateArgs: {}, zaiToolStream: false, supportsThinkingTokenBudget: false, + thinkingTokenBudgetField: undefined, supportsStrictMode: true, supportsOpenAIGrammarTools: false, cacheControlFormat: undefined, sendSessionAffinityHeaders: false, sessionAffinityFormat: "openai", supportsLongCacheRetention: true, -} satisfies Omit, "cacheControlFormat" | "deferredToolsMode"> & { +} satisfies Omit< + Required, + "cacheControlFormat" | "deferredToolsMode" | "thinkingTokenBudgetField" +> & { cacheControlFormat?: OpenAICompletionsCompat["cacheControlFormat"]; deferredToolsMode?: OpenAICompletionsCompat["deferredToolsMode"]; + thinkingTokenBudgetField?: OpenAICompletionsCompat["thinkingTokenBudgetField"]; }; function buildModel(baseUrl = "http://127.0.0.1:1"): Model<"openai-completions"> { diff --git a/packages/ai/test/openai-completions-thinking-token-budget.test.ts b/packages/ai/test/openai-completions-thinking-token-budget.test.ts index 2ca0c3ce0..3f5cfa4ea 100644 --- a/packages/ai/test/openai-completions-thinking-token-budget.test.ts +++ b/packages/ai/test/openai-completions-thinking-token-budget.test.ts @@ -41,21 +41,35 @@ vi.mock("openai", () => { return { default: FakeOpenAI }; }); -// vLLM-served reasoning model: reasoning and the answer share max_tokens. -const vllmModel: Model<"openai-completions"> = { - id: "zai-org/glm-5.2", - name: "GLM 5.2 (local vLLM)", - api: "openai-completions", - provider: "local-vllm", - baseUrl: "http://localhost:8000/v1", - reasoning: true, - input: ["text"], - cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, - contextWindow: 262144, - maxTokens: 16384, - compat: { thinkingFormat: "zai", supportsThinkingTokenBudget: true }, +type CapturedParams = { + thinking_token_budget?: number; + thinking_budget?: number; + thinking_budget_tokens?: number; + thinking?: unknown; + chat_template_kwargs?: Record; }; +function vllmModel( + compat: Model<"openai-completions">["compat"] = { + thinkingFormat: "zai", + supportsThinkingTokenBudget: true, + }, +): Model<"openai-completions"> { + return { + id: "zai-org/glm-5.2", + name: "GLM 5.2 (local vLLM)", + api: "openai-completions", + provider: "local-vllm", + baseUrl: "http://localhost:8000/v1", + reasoning: true, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 262144, + maxTokens: 16384, + compat, + }; +} + async function capture( model: Model<"openai-completions">, options?: { @@ -63,7 +77,7 @@ async function capture( thinkingBudgets?: ThinkingBudgets; maxTokens?: number; }, -): Promise<{ thinking_token_budget?: number; thinking?: unknown }> { +): Promise { let payload: unknown; await streamSimple( @@ -80,45 +94,109 @@ async function capture( }, ).result(); - return (payload ?? mockState.lastParams) as { thinking_token_budget?: number; thinking?: unknown }; + return (payload ?? mockState.lastParams) as CapturedParams; } -describe("openai-completions thinking_token_budget", () => { +describe("openai-completions thinking token budget", () => { beforeEach(() => { mockState.lastParams = undefined; }); it("sends the configured budget for the requested level", async () => { - const params = await capture(vllmModel, { reasoning: "medium", thinkingBudgets: { medium: 4096 } }); + const params = await capture(vllmModel(), { reasoning: "medium", thinkingBudgets: { medium: 4096 } }); expect(params.thinking_token_budget).toBe(4096); }); - it("omits the budget when the compat flag is not set", async () => { - const model = { ...vllmModel, compat: { thinkingFormat: "zai" } } as Model<"openai-completions">; - const params = await capture(model, { reasoning: "medium", thinkingBudgets: { medium: 4096 } }); + it("omits the budget when neither the field nor the alias is set", async () => { + const params = await capture(vllmModel({ thinkingFormat: "zai" }), { + reasoning: "medium", + thinkingBudgets: { medium: 4096 }, + }); expect(params.thinking_token_budget).toBeUndefined(); + expect(params.thinking_budget).toBeUndefined(); + expect(params.thinking_budget_tokens).toBeUndefined(); }); it("omits the budget when thinking is off", async () => { - const params = await capture(vllmModel, { reasoning: undefined, thinkingBudgets: { high: 8192 } }); + const params = await capture(vllmModel(), { reasoning: undefined, thinkingBudgets: { high: 8192 } }); expect(params.thinking_token_budget).toBeUndefined(); }); it("clamps xhigh and max to the high budget", async () => { - const xhigh = await capture(vllmModel, { reasoning: "xhigh", thinkingBudgets: { high: 8192 } }); - const max = await capture(vllmModel, { reasoning: "max", thinkingBudgets: { high: 8192 } }); + const xhigh = await capture(vllmModel(), { reasoning: "xhigh", thinkingBudgets: { high: 8192 } }); + const max = await capture(vllmModel(), { reasoning: "max", thinkingBudgets: { high: 8192 } }); expect(xhigh.thinking_token_budget).toBe(8192); expect(max.thinking_token_budget).toBe(8192); }); it("leaves room for the answer when the budget meets the response ceiling", async () => { - // Default high budget (16384) equals the model ceiling, which would leave no answer. - const params = await capture(vllmModel, { reasoning: "high" }); + const params = await capture(vllmModel(), { reasoning: "high" }); expect(params.thinking_token_budget).toBe(16384 - 1024); }); it("uses the caller max_tokens as the ceiling when it is lower than the model cap", async () => { - const params = await capture(vllmModel, { reasoning: "high", thinkingBudgets: { high: 8192 }, maxTokens: 4096 }); + const params = await capture(vllmModel(), { + reasoning: "high", + thinkingBudgets: { high: 8192 }, + maxTokens: 4096, + }); expect(params.thinking_token_budget).toBe(4096 - 1024); }); + + it.each(["thinking_budget", "thinking_budget_tokens"] as const)( + "sends %s when thinkingTokenBudgetField is set", + async (field) => { + const params = await capture(vllmModel({ thinkingFormat: "qwen", thinkingTokenBudgetField: field }), { + reasoning: "medium", + thinkingBudgets: { medium: 4096 }, + }); + expect(params[field]).toBe(4096); + expect(params.thinking_token_budget).toBeUndefined(); + }, + ); + + it("lets thinkingTokenBudgetField win over the boolean alias", async () => { + const params = await capture( + vllmModel({ + thinkingFormat: "zai", + supportsThinkingTokenBudget: true, + thinkingTokenBudgetField: "thinking_budget", + }), + { reasoning: "medium", thinkingBudgets: { medium: 4096 } }, + ); + expect(params.thinking_budget).toBe(4096); + expect(params.thinking_token_budget).toBeUndefined(); + }); + + it("puts the clamped budget in chat_template_kwargs when $var is thinking.budget", async () => { + const params = await capture( + vllmModel({ + thinkingFormat: "chat-template", + chatTemplateKwargs: { + enable_thinking: { $var: "thinking.enabled" }, + thinking_budget: { $var: "thinking.budget" }, + }, + }), + { reasoning: "high" }, + ); + expect(params.chat_template_kwargs).toEqual({ + enable_thinking: true, + thinking_budget: 16384 - 1024, + }); + expect(params.thinking_token_budget).toBeUndefined(); + }); + + it("omits thinking.budget from chat_template_kwargs when thinking is off", async () => { + const params = await capture( + vllmModel({ + thinkingFormat: "chat-template", + chatTemplateKwargs: { + enable_thinking: { $var: "thinking.enabled" }, + thinking_budget: { $var: "thinking.budget" }, + }, + }), + { reasoning: undefined }, + ); + expect(params.chat_template_kwargs).toEqual({ enable_thinking: false }); + }); }); diff --git a/packages/ai/test/openai-completions-tool-result-images.test.ts b/packages/ai/test/openai-completions-tool-result-images.test.ts index 6beb7f284..de692a665 100644 --- a/packages/ai/test/openai-completions-tool-result-images.test.ts +++ b/packages/ai/test/openai-completions-tool-result-images.test.ts @@ -19,8 +19,9 @@ const emptyUsage: Usage = { cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, }; -const compat: Omit, "deferredToolsMode"> & { +const compat: Omit, "deferredToolsMode" | "thinkingTokenBudgetField"> & { deferredToolsMode?: OpenAICompletionsCompat["deferredToolsMode"]; + thinkingTokenBudgetField?: OpenAICompletionsCompat["thinkingTokenBudgetField"]; } = { supportsStore: true, supportsDeveloperRole: true, @@ -39,6 +40,7 @@ const compat: Omit, "deferredToolsMode"> & { chatTemplateArgs: {}, zaiToolStream: false, supportsThinkingTokenBudget: false, + thinkingTokenBudgetField: undefined, supportsStrictMode: true, supportsOpenAIGrammarTools: false, cacheControlFormat: "anthropic", diff --git a/packages/coding-agent/docs/custom-provider.md b/packages/coding-agent/docs/custom-provider.md index 858b992ef..cdf823eb0 100644 --- a/packages/coding-agent/docs/custom-provider.md +++ b/packages/coding-agent/docs/custom-provider.md @@ -752,8 +752,10 @@ interface ProviderModelConfig { requiresThinkingAsText?: boolean; requiresReasoningContentOnAssistantMessages?: boolean; thinkingFormat?: "openai" | "openrouter" | "deepseek" | "together" | "baseten" | "zai" | "qwen" | "chat-template" | "qwen-chat-template" | "string-thinking" | "ant-ling"; - chatTemplateKwargs?: Record; - chatTemplateArgs?: Record; + chatTemplateKwargs?: Record; + chatTemplateArgs?: Record; + thinkingTokenBudgetField?: "thinking_token_budget" | "thinking_budget" | "thinking_budget_tokens"; + supportsThinkingTokenBudget?: boolean; cacheControlFormat?: "anthropic"; sessionAffinityFormat?: "openai" | "openai-nosession" | "openrouter"; sendSessionAffinityHeaders?: boolean; @@ -771,4 +773,5 @@ interface ProviderModelConfig { ``` `openrouter` sends `reasoning: { effort }`. `deepseek` sends `thinking: { type: "enabled" | "disabled" }` and `reasoning_effort` when enabled. `together` sends `reasoning: { enabled }` and also `reasoning_effort` when `supportsReasoningEffort` is enabled. `qwen` is for DashScope-style top-level `enable_thinking`. Use `qwen-chat-template` for local Qwen-compatible servers that read `chat_template_kwargs.enable_thinking` and need `preserve_thinking`. Use `chat-template` for configurable `chat_template_kwargs`, for example DeepSeek V3.x behind vLLM with `chatTemplateKwargs: { "thinking": { "$var": "thinking.enabled" } }`. Use `thinkingFormat: "baseten"` with `chatTemplateArgs` when the provider expects toggle values under `chat_template_args` and optionally supports top-level `reasoning_effort`. +`thinkingTokenBudgetField` sends a clamped per-level thinking budget as a top-level request field (`thinking_token_budget` on vLLM, `thinking_budget` on Qwen/SGLang, `thinking_budget_tokens` on llama.cpp). `supportsThinkingTokenBudget: true` is an alias for the vLLM field name. Do not combine it with `reasoning_effort` on DashScope Qwen models. `cacheControlFormat: "anthropic"` applies Anthropic-style `cache_control` markers to the system prompt, last tool definition, and last user, assistant, or tool-result text content. diff --git a/packages/coding-agent/docs/models.md b/packages/coding-agent/docs/models.md index 53c702e13..76a5bf291 100644 --- a/packages/coding-agent/docs/models.md +++ b/packages/coding-agent/docs/models.md @@ -254,6 +254,8 @@ Current behavior: Only OpenAI-compatible APIs apply it (`openai-completions`, `openai-responses`, `azure-openai-responses`); other APIs ignore it. Keys override pi's named request fields (for example a `temperature` key here beats the request-level temperature), so prefer it as the single source of sampling truth for a model. In `modelOverrides`, `samplingParams` merges per key with the base model's value. +A constant thinking-token cap can go here too, but it will not follow `thinkingBudgets` or leave room for the answer. Prefer `compat.thinkingTokenBudgetField` (or the `supportsThinkingTokenBudget` alias) for that. + ### Thinking Level Map Use `thinkingLevelMap` on a model to describe model-specific thinking controls. Keys are pi thinking levels: `off`, `minimal`, `low`, `medium`, `high`, `xhigh`, `max`. Maps may contain holes; for example, a model can expose `high` and `max` without exposing `xhigh`. @@ -467,8 +469,10 @@ For providers with partial OpenAI compatibility, use the `compat` field. | `requiresThinkingAsText` | Convert thinking blocks to plain text | | `requiresReasoningContentOnAssistantMessages` | Include empty `reasoning_content` on all replayed assistant messages when reasoning is enabled | | `thinkingFormat` | Use `reasoning_effort`, `openrouter`, `deepseek`, `together`, `baseten`, `zai`, `qwen`, `chat-template`, or `qwen-chat-template` thinking parameters | -| `chatTemplateKwargs` | `chat_template_kwargs` values for `thinkingFormat: "chat-template"`; use `{ "$var": "thinking.enabled" }` or `{ "$var": "thinking.effort" }` for pi-controlled thinking values | -| `chatTemplateArgs` | `chat_template_args` values for `thinkingFormat: "baseten"`; use `{ "$var": "thinking.enabled" }` or `{ "$var": "thinking.effort" }` for pi-controlled thinking values | +| `chatTemplateKwargs` | `chat_template_kwargs` values for `thinkingFormat: "chat-template"`; use `{ "$var": "thinking.enabled" }`, `{ "$var": "thinking.effort" }`, or `{ "$var": "thinking.budget" }` for pi-controlled thinking values | +| `chatTemplateArgs` | `chat_template_args` values for `thinkingFormat: "baseten"`; use `{ "$var": "thinking.enabled" }`, `{ "$var": "thinking.effort" }`, or `{ "$var": "thinking.budget" }` for pi-controlled thinking values | +| `thinkingTokenBudgetField` | Top-level request field used to cap reasoning tokens from `thinkingBudgets`, clamped so at least 1024 tokens remain for the answer. `"thinking_token_budget"` (vLLM), `"thinking_budget"` (Qwen/DashScope/SGLang), `"thinking_budget_tokens"` (llama.cpp). Off by default; not set on the generated catalog. | +| `supportsThinkingTokenBudget` | Alias for `thinkingTokenBudgetField: "thinking_token_budget"` (vLLM). Prefer `thinkingTokenBudgetField`. Default: `false`. | | `cacheControlFormat` | Use Anthropic-style `cache_control` markers on the system prompt, last tool definition, and last user, assistant, or tool-result text content. Currently only `anthropic` is supported. | | `sendSessionAffinityHeaders` | For `openai-completions`, send session-affinity headers from the session id when caching is enabled. Default: `false`. | | `sessionAffinityFormat` | For `openai-completions` and `openai-responses`, the session-affinity header format: `openai` sends `session_id`/`x-client-request-id` (completions also `x-session-affinity`), `openai-nosession` omits the underscore-containing `session_id` header, `openrouter` sends `x-session-id`. Does not affect the `prompt_cache_key` body param. Default: auto-detected. | @@ -481,6 +485,8 @@ For providers with partial OpenAI compatibility, use the `compat` field. `openrouter` uses `reasoning: { effort }`. `together` uses `reasoning: { enabled }` and also `reasoning_effort` when `supportsReasoningEffort` is enabled. `qwen` uses top-level `enable_thinking`. Use `qwen-chat-template` for local Qwen-compatible servers that require `chat_template_kwargs.enable_thinking` and `preserve_thinking`. Use `chat-template` for vLLM/Hugging Face chat templates that need configurable `chat_template_kwargs`, such as `chatTemplateKwargs: { "thinking": { "$var": "thinking.enabled" } }` for DeepSeek V3.x templates. Use `thinkingFormat: "baseten"` with `chatTemplateArgs` for providers that expose toggle controls through `chat_template_args` and optionally support top-level `reasoning_effort`. +`thinkingTokenBudgetField` is independent of `thinkingFormat`. Do not enable it on the generated Qwen catalog: those models already send `reasoning_effort`, and DashScope rejects `thinking_budget` together with `reasoning_effort`. + `cacheControlFormat: "anthropic"` is for OpenAI-compatible providers that expose Anthropic-style prompt caching through `cache_control` markers on text content and tool definitions. Example: diff --git a/packages/coding-agent/docs/settings.md b/packages/coding-agent/docs/settings.md index c56340b1d..b23392843 100644 --- a/packages/coding-agent/docs/settings.md +++ b/packages/coding-agent/docs/settings.md @@ -32,7 +32,7 @@ Use `/trust` in interactive mode to save a project trust decision for future ses | `defaultThinkingLevel` | string | - | `"off"`, `"minimal"`, `"low"`, `"medium"`, `"high"`, `"xhigh"`, `"max"` | | `hideThinkingBlock` | boolean | `false` | Hide thinking blocks in output | | `showCacheMissNotices` | boolean | `false` | Show transcript notices for significant prompt-cache misses and compaction or branch-summary usage | -| `thinkingBudgets` | object | - | Custom token budgets per thinking level | +| `thinkingBudgets` | object | - | Custom token budgets per thinking level. Anthropic, Google, and Bedrock use these natively. OpenAI-compatible models use them when `compat.thinkingTokenBudgetField` (or `supportsThinkingTokenBudget`) is set. | #### thinkingBudgets From c0613289814c54a28c649add556f509d9c79714a Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Tue, 18 Aug 2026 16:29:26 +0200 Subject: [PATCH 06/10] fix(coding-agent): load extensions in Node SEA hosts, closes #8237 --- .../src/core/extensions/loader.ts | 13 +++-- .../8237-node-sea-extension-loading.test.ts | 50 +++++++++++++++++++ 2 files changed, 58 insertions(+), 5 deletions(-) create mode 100644 packages/coding-agent/test/suite/regressions/8237-node-sea-extension-loading.test.ts diff --git a/packages/coding-agent/src/core/extensions/loader.ts b/packages/coding-agent/src/core/extensions/loader.ts index 74d38f12c..2f00807f2 100644 --- a/packages/coding-agent/src/core/extensions/loader.ts +++ b/packages/coding-agent/src/core/extensions/loader.ts @@ -46,7 +46,7 @@ import type { ToolDefinition, } from "./types.ts"; -/** Modules available to extensions via virtualModules (for compiled Bun binary) */ +/** Modules available to extensions via virtualModules (for compiled binaries) */ const VIRTUAL_MODULES: Record = { typebox: _bundledTypebox, "typebox/compile": _bundledTypeboxCompile, @@ -75,11 +75,14 @@ const VIRTUAL_MODULES: Record = { const require = createRequire(import.meta.url); +const isNodeSeaBinary = + ("sea" in process.features && process.features.sea === true) || + process.getBuiltinModule("node:sea")?.isSea() === true; const isTypeScriptSourceRuntime = !isBunBinary && path.extname(fileURLToPath(import.meta.url)) === ".ts"; /** * Get aliases for jiti (used in built Node.js mode). - * In Bun binary mode, virtualModules is used instead. + * In compiled binary mode, virtualModules is used instead. */ let _aliases: Record | null = null; @@ -448,9 +451,9 @@ async function loadExtensionModule(extensionPath: string, cacheToken?: Extension const jiti = createJiti(import.meta.url, { moduleCache: false, - // Bun uses modules embedded in the executable. Source TypeScript reuses the - // host-resolved modules and root tsconfig paths. Built Node uses dist aliases. - ...(isBunBinary + // Compiled binaries use modules embedded in the executable. Source TypeScript + // reuses host modules and root tsconfig paths. Built Node uses dist aliases. + ...(isBunBinary || isNodeSeaBinary ? { virtualModules: VIRTUAL_MODULES, tryNative: false } : isTypeScriptSourceRuntime ? { virtualModules: VIRTUAL_MODULES, tsconfigPaths: true } diff --git a/packages/coding-agent/test/suite/regressions/8237-node-sea-extension-loading.test.ts b/packages/coding-agent/test/suite/regressions/8237-node-sea-extension-loading.test.ts new file mode 100644 index 000000000..160bf7dc3 --- /dev/null +++ b/packages/coding-agent/test/suite/regressions/8237-node-sea-extension-loading.test.ts @@ -0,0 +1,50 @@ +import { afterAll, describe, expect, it, vi } from "vitest"; + +const state = vi.hoisted(() => { + const originalGetBuiltinModule = Object.getOwnPropertyDescriptor(process, "getBuiltinModule"); + const getBuiltinModule = process.getBuiltinModule.bind(process); + Object.defineProperty(process, "getBuiltinModule", { + configurable: true, + value: (id: string) => (id === "node:sea" ? { isSea: () => true } : getBuiltinModule(id)), + }); + return { + originalGetBuiltinModule, + createJiti: vi.fn((_id: unknown, _options: unknown) => ({ + import: vi.fn(async () => () => {}), + })), + }; +}); + +vi.mock("jiti/static", () => ({ createJiti: state.createJiti })); + +import { loadExtensions } from "../../../src/core/extensions/loader.ts"; + +interface JitiOptionsProbe { + alias?: unknown; + tryNative?: boolean; + virtualModules?: Record; +} + +afterAll(() => { + if (state.originalGetBuiltinModule) { + Object.defineProperty(process, "getBuiltinModule", state.originalGetBuiltinModule); + } +}); + +describe("Node SEA extension loading", () => { + it("uses bundled virtual modules instead of filesystem aliases", async () => { + const result = await loadExtensions(["/extension.ts"], "/"); + + expect(result.errors).toEqual([]); + expect(result.extensions).toHaveLength(1); + expect(state.createJiti).toHaveBeenCalledOnce(); + + const options = state.createJiti.mock.calls[0][1] as JitiOptionsProbe; + // Source TypeScript also uses virtual modules, so tryNative: false is what + // proves the compiled-binary branch took precedence over the source branch. + expect(options.tryNative).toBe(false); + expect(options.alias).toBeUndefined(); + expect(options.virtualModules?.typebox).toBeDefined(); + expect(options.virtualModules?.["@earendil-works/pi-coding-agent"]).toBeDefined(); + }); +}); From f0c5d86d20f85bbeee957b0e9b06ee085ea40c80 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Tue, 18 Aug 2026 16:49:14 +0200 Subject: [PATCH 07/10] fix(tui): fit text padding to narrow widths, closes #8252 --- packages/tui/CHANGELOG.md | 1 + packages/tui/src/components/text.ts | 9 +++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/tui/CHANGELOG.md b/packages/tui/CHANGELOG.md index 6c0416351..33dfa5812 100644 --- a/packages/tui/CHANGELOG.md +++ b/packages/tui/CHANGELOG.md @@ -5,6 +5,7 @@ ### Fixed - Fixed duplicate fullscreen right-click paste in VS Code-based terminals on Windows ([#8186](https://github.com/earendil-works/pi/issues/8186)). +- Fixed padded text exceeding narrow terminal widths ([#8252](https://github.com/earendil-works/pi/issues/8252)). ## [0.84.2] - 2026-08-14 diff --git a/packages/tui/src/components/text.ts b/packages/tui/src/components/text.ts index 3809a48a8..7a50e2721 100644 --- a/packages/tui/src/components/text.ts +++ b/packages/tui/src/components/text.ts @@ -60,15 +60,16 @@ export class Text implements Component { // Replace tabs with 3 spaces const normalizedText = this.text.replace(/\t/g, " "); - // Calculate content width (subtract left/right margins) - const contentWidth = Math.max(1, width - this.paddingX * 2); + // Reduce margins when necessary so content and padding fit within the available width. + const paddingX = Math.min(this.paddingX, Math.max(0, Math.floor((width - 1) / 2))); + const contentWidth = Math.max(1, width - paddingX * 2); // Wrap text (this preserves ANSI codes but does NOT pad) const wrappedLines = wrapTextWithAnsi(normalizedText, contentWidth); // Add margins and background to each line - const leftMargin = " ".repeat(this.paddingX); - const rightMargin = " ".repeat(this.paddingX); + const leftMargin = " ".repeat(paddingX); + const rightMargin = " ".repeat(paddingX); const contentLines: string[] = []; for (const line of wrappedLines) { From 59a71b235dadb4ad0d67557a8abb0aaa093e68b4 Mon Sep 17 00:00:00 2001 From: Cristina Poncela Cubeiro <140309543+cristinaponcela@users.noreply.github.com> Date: Tue, 18 Aug 2026 17:10:35 +0200 Subject: [PATCH 08/10] Revert "fix(ai): anthropic fallback usage (#8308)" (#8313) This reverts commit a6c6f801802a7083609541dbd49eec203804d8fc. --- packages/ai/CHANGELOG.md | 1 - packages/ai/src/api/anthropic-messages.ts | 11 ++--------- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index e15dab0cf..b4de62a00 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -13,7 +13,6 @@ ### Fixed -- Fixed Anthropic server-side fallback responses being priced with the requested model instead of the returned fallback model ([#8285](https://github.com/earendil-works/pi/issues/8285)). - Fixed Azure OpenAI Responses ignoring `toolChoice` in provider-specific stream requests. - Added `deepseek-v4-pro-0813` to the Qwen Token Plan Individual catalog ([#8194](https://github.com/earendil-works/pi/issues/8194)). - Fixed Amazon Bedrock `after_provider_response`/`onResponse` to forward the raw response headers instead of only the synthesized request id header ([#8234](https://github.com/earendil-works/pi/issues/8234)). diff --git a/packages/ai/src/api/anthropic-messages.ts b/packages/ai/src/api/anthropic-messages.ts index f96d94052..f0e89a558 100644 --- a/packages/ai/src/api/anthropic-messages.ts +++ b/packages/ai/src/api/anthropic-messages.ts @@ -8,7 +8,6 @@ import type { RefusalStopDetails, } from "@anthropic-ai/sdk/resources/messages.js"; import { calculateCost } from "../models.ts"; -import { ANTHROPIC_MODELS } from "../providers/anthropic.models.ts"; import type { AnthropicMessagesCompat, AnthropicRefusalFallback, @@ -541,7 +540,6 @@ export const stream: StreamFunction<"anthropic-messages", AnthropicOptions> = ( try { let client: Anthropic; let isOAuth: boolean; - let usageModel = model; if (options?.client) { client = options.client; @@ -604,11 +602,6 @@ export const stream: StreamFunction<"anthropic-messages", AnthropicOptions> = ( if (event.type === "message_start") { output.responseId = event.message.id; output.model = event.message.model; - usageModel = - model.provider === "anthropic" - ? ((ANTHROPIC_MODELS as Record | undefined>)[output.model] ?? - model) - : model; // Capture initial token usage from message_start event // This ensures we have input token counts even if the stream is aborted early output.usage.input = event.message.usage.input_tokens || 0; @@ -619,7 +612,7 @@ export const stream: StreamFunction<"anthropic-messages", AnthropicOptions> = ( // Anthropic doesn't provide total_tokens, compute from components output.usage.totalTokens = output.usage.input + output.usage.output + output.usage.cacheRead + output.usage.cacheWrite; - calculateCost(usageModel, output.usage); + calculateCost(model, output.usage); } else if (event.type === "content_block_start") { if (event.content_block.type === "text") { const block: Block = { @@ -776,7 +769,7 @@ export const stream: StreamFunction<"anthropic-messages", AnthropicOptions> = ( // Anthropic doesn't provide total_tokens, compute from components output.usage.totalTokens = output.usage.input + output.usage.output + output.usage.cacheRead + output.usage.cacheWrite; - calculateCost(usageModel, output.usage); + calculateCost(model, output.usage); } } From 4809c2abcaa86257337aa9a44801f4af91144dbc Mon Sep 17 00:00:00 2001 From: Cristina Poncela Cubeiro <140309543+cristinaponcela@users.noreply.github.com> Date: Wed, 19 Aug 2026 09:11:45 +0200 Subject: [PATCH 09/10] fix(ai): anthropic fallback usage (#8319) * fix: anthropic fallback usage * small cleanup --- packages/ai/CHANGELOG.md | 1 + packages/ai/scripts/generate-models.ts | 27 ++++++++++++++++++- packages/ai/src/api/anthropic-messages.ts | 18 ++++++++++--- packages/ai/src/types.ts | 10 +++++-- .../src/core/compaction/compaction.ts | 4 +-- .../test/compaction-summary-reasoning.test.ts | 10 +++++-- 6 files changed, 60 insertions(+), 10 deletions(-) diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index b4de62a00..e15dab0cf 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -13,6 +13,7 @@ ### Fixed +- Fixed Anthropic server-side fallback responses being priced with the requested model instead of the returned fallback model ([#8285](https://github.com/earendil-works/pi/issues/8285)). - Fixed Azure OpenAI Responses ignoring `toolChoice` in provider-specific stream requests. - Added `deepseek-v4-pro-0813` to the Qwen Token Plan Individual catalog ([#8194](https://github.com/earendil-works/pi/issues/8194)). - Fixed Amazon Bedrock `after_provider_response`/`onResponse` to forward the raw response headers instead of only the synthesized request id header ([#8234](https://github.com/earendil-works/pi/issues/8234)). diff --git a/packages/ai/scripts/generate-models.ts b/packages/ai/scripts/generate-models.ts index 418b0a6d9..d376149dc 100644 --- a/packages/ai/scripts/generate-models.ts +++ b/packages/ai/scripts/generate-models.ts @@ -744,6 +744,30 @@ function applyAnthropicMessagesCompatMetadata(model: Model): void { } } +function isAnthropicFallbackMetadataModel(model: Model): model is Model<"anthropic-messages"> { + if (model.provider !== "anthropic" || model.api !== "anthropic-messages") return false; + return ( + model.id in ANTHROPIC_ALLOWED_FALLBACK_MODELS || + Object.values(ANTHROPIC_ALLOWED_FALLBACK_MODELS).some((fallbackModelIds) => fallbackModelIds.includes(model.id)) + ); +} + +function applyAnthropicFallbackCostMetadata(models: readonly Model<"anthropic-messages">[]): void { + const modelsById = new Map(models.map((model) => [model.id, model])); + for (const [modelId, fallbackModelIds] of Object.entries(ANTHROPIC_ALLOWED_FALLBACK_MODELS)) { + const model = modelsById.get(modelId); + if (!model?.compat?.allowedFallbackModels) continue; + + for (const fallbackModelId of fallbackModelIds) { + const fallback = model.compat.allowedFallbackModels.find((target) => target.model === fallbackModelId); + const fallbackModel = modelsById.get(fallbackModelId); + if (fallback && fallbackModel) { + fallback.cost = fallbackModel.cost; + } + } + } +} + function applyStrictToolCompatMetadata(model: Model): void { if ( (model.provider === "openai" || model.provider === "cloudflare-ai-gateway") && @@ -964,7 +988,7 @@ function getAnthropicMessagesCompat(provider: string, modelId: string): Anthropi if (provider === "anthropic") { const allowedFallbackModels = ANTHROPIC_ALLOWED_FALLBACK_MODELS[modelId]; if (allowedFallbackModels) { - compat.allowedFallbackModels = allowedFallbackModels; + compat.allowedFallbackModels = allowedFallbackModels.map((fallbackModel) => ({ model: fallbackModel })); } } if (provider === "xiaomi" || provider.startsWith("xiaomi-token-plan-")) { @@ -2785,6 +2809,7 @@ async function generateModels() { applyOpenAIToolSearchMetadata(model); applyOpenAIExplicitPromptCacheMetadata(model); } + applyAnthropicFallbackCostMetadata(allModels.filter(isAnthropicFallbackMetadataModel)); // Group by provider and deduplicate by model ID const providers: Record>> = {}; diff --git a/packages/ai/src/api/anthropic-messages.ts b/packages/ai/src/api/anthropic-messages.ts index f0e89a558..b224e52d2 100644 --- a/packages/ai/src/api/anthropic-messages.ts +++ b/packages/ai/src/api/anthropic-messages.ts @@ -540,6 +540,7 @@ export const stream: StreamFunction<"anthropic-messages", AnthropicOptions> = ( try { let client: Anthropic; let isOAuth: boolean; + let usageModel = model; if (options?.client) { client = options.client; @@ -602,6 +603,14 @@ export const stream: StreamFunction<"anthropic-messages", AnthropicOptions> = ( if (event.type === "message_start") { output.responseId = event.message.id; output.model = event.message.model; + const fallbackCost = + output.model === model.id + ? undefined + : ((Array.isArray(options?.refusalFallbacks) + ? options.refusalFallbacks.find((fallback) => fallback.model === output.model)?.cost + : undefined) ?? + model.compat?.allowedFallbackModels?.find((fallback) => fallback.model === output.model)?.cost); + usageModel = fallbackCost ? { ...model, id: output.model, cost: fallbackCost } : model; // Capture initial token usage from message_start event // This ensures we have input token counts even if the stream is aborted early output.usage.input = event.message.usage.input_tokens || 0; @@ -612,7 +621,7 @@ export const stream: StreamFunction<"anthropic-messages", AnthropicOptions> = ( // Anthropic doesn't provide total_tokens, compute from components output.usage.totalTokens = output.usage.input + output.usage.output + output.usage.cacheRead + output.usage.cacheWrite; - calculateCost(model, output.usage); + calculateCost(usageModel, output.usage); } else if (event.type === "content_block_start") { if (event.content_block.type === "text") { const block: Block = { @@ -769,7 +778,7 @@ export const stream: StreamFunction<"anthropic-messages", AnthropicOptions> = ( // Anthropic doesn't provide total_tokens, compute from components output.usage.totalTokens = output.usage.input + output.usage.output + output.usage.cacheRead + output.usage.cacheWrite; - calculateCost(model, output.usage); + calculateCost(usageModel, output.usage); } } @@ -1116,7 +1125,10 @@ function buildParams( } if (options?.refusalFallbacks !== undefined) { - params.fallbacks = options.refusalFallbacks; + params.fallbacks = + options.refusalFallbacks === "default" + ? "default" + : options.refusalFallbacks.map((fallback) => ({ model: fallback.model })); } return params; diff --git a/packages/ai/src/types.ts b/packages/ai/src/types.ts index e7df833d5..271873099 100644 --- a/packages/ai/src/types.ts +++ b/packages/ai/src/types.ts @@ -304,7 +304,13 @@ export interface ImagesOptions extends ProviderRequestOptions; -export type AnthropicRefusalFallback = "default" | readonly { model: string }[]; +export interface AnthropicRefusalFallbackTarget { + model: string; + /** Local pricing for this fallback target. Stripped before sending the provider request. @internal */ + cost?: ModelCost; +} + +export type AnthropicRefusalFallback = "default" | readonly AnthropicRefusalFallbackTarget[]; // Unified options with reasoning passed to streamSimple() and completeSimple() export interface SimpleStreamOptions extends StreamOptions { @@ -695,7 +701,7 @@ export interface AnthropicMessagesCompat { * When absent or empty, callers must omit `fallbacks`; Anthropic rejects the * field for models with no permitted fallback targets. */ - allowedFallbackModels?: string[]; + allowedFallbackModels?: AnthropicRefusalFallbackTarget[]; /** * Whether the provider supports deferred tools loaded by `tool_reference` * blocks in tool results. Default: true for first-party Anthropic models diff --git a/packages/coding-agent/src/core/compaction/compaction.ts b/packages/coding-agent/src/core/compaction/compaction.ts index a2a1c063f..7200da96f 100644 --- a/packages/coding-agent/src/core/compaction/compaction.ts +++ b/packages/coding-agent/src/core/compaction/compaction.ts @@ -33,7 +33,7 @@ import { serializeConversation, } from "./utils.ts"; -function getAnthropicSummarizationFallback(model: Model): readonly { model: string }[] | undefined { +function getAnthropicSummarizationFallback(model: Model): SimpleStreamOptions["refusalFallbacks"] { if (model.provider !== "anthropic" || model.api !== "anthropic-messages") { return undefined; } @@ -41,7 +41,7 @@ function getAnthropicSummarizationFallback(model: Model): readonly { model: const allowedFallbackModels = (model as Model<"anthropic-messages">).compat?.allowedFallbackModels; // Use the primary permitted fallback for now. If future Anthropic models expose // broader fallback behavior, this can become a user/config pick or a full chain. - return allowedFallbackModels && allowedFallbackModels.length > 0 ? [{ model: allowedFallbackModels[0] }] : undefined; + return allowedFallbackModels && allowedFallbackModels.length > 0 ? [allowedFallbackModels[0]] : undefined; } // ============================================================================ diff --git a/packages/coding-agent/test/compaction-summary-reasoning.test.ts b/packages/coding-agent/test/compaction-summary-reasoning.test.ts index 9bafc9d5d..462d48c7e 100644 --- a/packages/coding-agent/test/compaction-summary-reasoning.test.ts +++ b/packages/coding-agent/test/compaction-summary-reasoning.test.ts @@ -437,16 +437,22 @@ describe("generateSummary reasoning options", () => { }); it("sets Anthropic refusal fallback from model metadata", async () => { + const fallbackCost = { input: 5, output: 25, cacheRead: 0.5, cacheWrite: 6.25 }; await generateSummary( messages, - createModel(true, 8192, { allowedFallbackModels: ["claude-opus-4-8", "claude-opus-5"] }), + createModel(true, 8192, { + allowedFallbackModels: [ + { model: "claude-opus-4-8", cost: fallbackCost }, + { model: "claude-opus-5", cost: fallbackCost }, + ], + }), 2000, "test-key", ); expect(completeSimpleMock).toHaveBeenCalledTimes(1); expect(completeSimpleMock.mock.calls[0][2]).toMatchObject({ - refusalFallbacks: [{ model: "claude-opus-4-8" }], + refusalFallbacks: [{ model: "claude-opus-4-8", cost: fallbackCost }], }); }); From 55b0db4d3e9075345b6509a5b488250d79e026d8 Mon Sep 17 00:00:00 2001 From: Ramiz Wachtler Date: Wed, 19 Aug 2026 09:40:18 +0200 Subject: [PATCH 10/10] fix(ai): prevent copilot policy login rate limits (#8254) - fixes #7850 - fetch the account model catalog before policy updates - update only known, tool-capable, unconfigured models - retry throttled login requests within a bounded delay - preserve the existing non-retrying refresh behavior --- packages/ai/CHANGELOG.md | 1 + packages/ai/src/auth/oauth/github-copilot.ts | 196 ++++++++---- packages/ai/src/auth/oauth/kimi-coding.ts | 16 +- packages/ai/src/utils/sleep.ts | 14 + packages/ai/test/github-copilot-oauth.test.ts | 279 ++++++++++++------ 5 files changed, 337 insertions(+), 169 deletions(-) create mode 100644 packages/ai/src/utils/sleep.ts diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index e15dab0cf..324a0c6d8 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -38,6 +38,7 @@ ### Fixed - Fixed GitHub Copilot login triggering API rate limits while enabling model policies by limiting concurrent policy updates ([#6187](https://github.com/earendil-works/pi/issues/6187)). +- Fixed GitHub Copilot login still triggering API rate limits by updating only account models with unconfigured policies and honoring server retry delays ([#7850](https://github.com/earendil-works/pi/issues/7850)). - Fixed upstream request buffer limit failures to trigger automatic assistant retries. - Fixed OpenAI Responses function and custom tool calls to preserve namespaces during streaming, proxying, and replay ([#7709](https://github.com/earendil-works/pi/issues/7709)). - Fixed built-in and custom DeepSeek API models to send output limits through the supported `max_tokens` field. diff --git a/packages/ai/src/auth/oauth/github-copilot.ts b/packages/ai/src/auth/oauth/github-copilot.ts index 8bd634269..5a17d07f0 100644 --- a/packages/ai/src/auth/oauth/github-copilot.ts +++ b/packages/ai/src/auth/oauth/github-copilot.ts @@ -3,8 +3,9 @@ */ import { GITHUB_COPILOT_MODELS } from "../../providers/github-copilot.models.ts"; +import { sleep } from "../../utils/sleep.ts"; import type { OAuthAuth, OAuthCredential, ProviderAuthInteraction } from "../types.ts"; -import { abortableSleep, pollOAuthDeviceCodeFlow } from "./device-code.ts"; +import { pollOAuthDeviceCodeFlow } from "./device-code.ts"; const decode = (s: string) => atob(s); const CLIENT_ID = decode("SXYxLmI1MDdhMDhjODdlY2ZlOTg="); @@ -16,8 +17,6 @@ const COPILOT_HEADERS = { "Copilot-Integration-Id": "vscode-chat", } as const; const COPILOT_API_VERSION = "2026-06-01"; -const MAX_RETRY_AFTER_MS = 10_000; -const DEFAULT_RETRY_AFTER_MS = 1_000; type DeviceCodeResponse = { device_code: string; @@ -91,66 +90,108 @@ function asRecord(value: unknown): Record | undefined { return value && typeof value === "object" ? (value as Record) : undefined; } -function parseAvailableCopilotModelIds(raw: unknown, allowPolicyFallback: boolean): string[] { +function parseGitHubCopilotModelCatalog(raw: unknown, allowPolicyFallback: boolean) { const data = asRecord(raw)?.data; if (!Array.isArray(data)) { throw new Error("Invalid Copilot models response"); } - const pickerIds: string[] = []; - const policyEnabledIds: string[] = []; - for (const rawItem of data) { + const accountModels = data.flatMap((rawItem) => { const item = asRecord(rawItem); const id = item?.id; - if (!item || typeof id !== "string") continue; + if (!item || typeof id !== "string") return []; const capabilities = asRecord(item.capabilities); const supports = asRecord(capabilities?.supports); - if (supports?.tool_calls === false) continue; - const policy = asRecord(item.policy); - if (item.model_picker_enabled === true && policy?.state !== "disabled") pickerIds.push(id); - if (policy?.state === "enabled") policyEnabledIds.push(id); - } - return pickerIds.length > 0 || !allowPolicyFallback ? pickerIds : policyEnabledIds; + if (supports?.tool_calls === false) return []; + + return [ + { + id, + pickerEnabled: item.model_picker_enabled === true, + policyState: asRecord(item.policy)?.state, + }, + ]; + }); + const pickerModelIds = accountModels + .filter((model) => model.pickerEnabled && model.policyState !== "disabled") + .map((model) => model.id); + const usePolicyFallback = allowPolicyFallback && pickerModelIds.length === 0; + const availableModelIds = + pickerModelIds.length > 0 || !allowPolicyFallback + ? pickerModelIds + : accountModels.filter((model) => model.policyState === "enabled").map((model) => model.id); + const policyModelIds = accountModels + .filter( + (model) => + model.policyState === "unconfigured" && + Object.hasOwn(GITHUB_COPILOT_MODELS, model.id) && + (model.pickerEnabled || usePolicyFallback), + ) + .map((model) => model.id); + return { availableModelIds, policyModelIds }; } -async function fetchAvailableGitHubCopilotModelIds( +async function fetchWithRateLimitRetry( + url: string, + init: RequestInit, + signal: AbortSignal, + retryPolicy: { maxRetries: number; maxElapsedMs: number }, +): Promise { + const retryBudgetSignal = + retryPolicy.maxRetries > 0 && retryPolicy.maxElapsedMs > 0 + ? AbortSignal.timeout(retryPolicy.maxElapsedMs) + : undefined; + const requestSignal = retryBudgetSignal ? AbortSignal.any([signal, retryBudgetSignal]) : signal; + const retryDeadline = retryBudgetSignal ? Date.now() + retryPolicy.maxElapsedMs : undefined; + for (let retry = 0; ; retry++) { + const response = await fetch(url, { + ...init, + signal: AbortSignal.any([requestSignal, AbortSignal.timeout(5000)]), + }); + if (response.status !== 429 || retry === retryPolicy.maxRetries) return response; + + const retryAfter = response.headers.get("retry-after"); + let delayMs = 500 * 2 ** retry; + if (retryAfter) { + const seconds = Number.parseFloat(retryAfter); + delayMs = Number.isNaN(seconds) ? Date.parse(retryAfter) - Date.now() : seconds * 1000; + if (!Number.isFinite(delayMs)) return response; + } + delayMs = Math.max(0, delayMs); + if (retryDeadline !== undefined && delayMs >= retryDeadline - Date.now()) return response; + await response.body?.cancel(); + await sleep(delayMs, requestSignal); + } +} + +async function fetchGitHubCopilotModels( copilotToken: string, enterpriseDomain: string | undefined, signal: AbortSignal, -): Promise { + retryPolicy: { maxRetries: number; maxElapsedMs: number }, +) { const baseUrl = getGitHubCopilotBaseUrl(copilotToken, enterpriseDomain); // Some Individual accounts return false for every picker flag despite explicit enabled policies. // Limit the fallback to that endpoint so other account types keep strict picker semantics. const allowPolicyFallback = baseUrl === "https://api.individual.githubcopilot.com"; - const request = () => - fetch(`${baseUrl}/models`, { + const response = await fetchWithRateLimitRetry( + `${baseUrl}/models`, + { headers: { Accept: "application/json", Authorization: `Bearer ${copilotToken}`, ...COPILOT_HEADERS, "X-GitHub-Api-Version": COPILOT_API_VERSION, }, - signal: AbortSignal.any([signal, AbortSignal.timeout(5000)]), - }); - - // The login-time policy updates can drain the Copilot API rate-limit bucket, in which case - // this request is rejected with 429. Honor Retry-After and retry once instead of failing. - let response = await request(); - if (response.status === 429) { - const retryAfterSeconds = Number(response.headers.get("retry-after")); - const waitMs = - Number.isFinite(retryAfterSeconds) && retryAfterSeconds > 0 - ? Math.min(retryAfterSeconds * 1000, MAX_RETRY_AFTER_MS) - : DEFAULT_RETRY_AFTER_MS; - await abortableSleep(waitMs, signal, "Login cancelled"); - response = await request(); - } + }, + signal, + retryPolicy, + ); if (!response.ok) { - const text = await response.text(); - throw new Error(`${response.status} ${response.statusText}: ${text}`); + throw new Error(`${response.status} ${response.statusText}: ${await response.text()}`); } - return parseAvailableCopilotModelIds(await response.json(), allowPolicyFallback); + return parseGitHubCopilotModelCatalog(await response.json(), allowPolicyFallback); } async function fetchJson(url: string, init: RequestInit): Promise { @@ -315,9 +356,13 @@ async function refreshGitHubCopilotToken( signal: AbortSignal, ): Promise { const credentials = await refreshGitHubCopilotAccessToken(refreshToken, enterpriseDomain, signal); + const { availableModelIds } = await fetchGitHubCopilotModels(credentials.access, enterpriseDomain, signal, { + maxRetries: 0, + maxElapsedMs: 0, + }); return { ...credentials, - availableModelIds: await fetchAvailableGitHubCopilotModelIds(credentials.access, enterpriseDomain, signal), + availableModelIds, }; } @@ -334,38 +379,56 @@ async function enableGitHubCopilotModel( const baseUrl = getGitHubCopilotBaseUrl(token, enterpriseDomain); const url = `${baseUrl}/models/${modelId}/policy`; + let response: Response; try { - const response = await fetch(url, { - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${token}`, - ...COPILOT_HEADERS, - "openai-intent": "chat-policy", - "x-interaction-type": "chat-policy", + response = await fetchWithRateLimitRetry( + url, + { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${token}`, + ...COPILOT_HEADERS, + "openai-intent": "chat-policy", + "x-interaction-type": "chat-policy", + }, + body: JSON.stringify({ state: "enabled" }), }, - body: JSON.stringify({ state: "enabled" }), signal, - }); - return response.ok; + { maxRetries: 2, maxElapsedMs: 5000 }, + ); } catch (error) { if (signal.aborted) throw error; return false; } + if (response.status === 429) { + throw new Error(`${response.status} ${response.statusText}: ${await response.text()}`); + } + return response.ok; } /** - * Enable all known GitHub Copilot models that may require policy acceptance. - * Called after successful login to ensure all models are available. + * Enable the requested GitHub Copilot models and return the successful IDs. + * Policy updates are best effort; exhausted rate limiting stops the batch. */ -async function enableAllGitHubCopilotModels( +async function enableGitHubCopilotModels( token: string, + modelIds: readonly string[], enterpriseDomain: string | undefined, signal: AbortSignal, -): Promise { - for (const model of Object.values(GITHUB_COPILOT_MODELS)) { - await enableGitHubCopilotModel(token, model.id, enterpriseDomain, signal); +): Promise { + const enabledModelIds: string[] = []; + for (const modelId of modelIds) { + try { + if (await enableGitHubCopilotModel(token, modelId, enterpriseDomain, signal)) { + enabledModelIds.push(modelId); + } + } catch (error) { + if (signal.aborted) throw error; + break; + } } + return enabledModelIds; } async function loginGitHubCopilot(interaction: ProviderAuthInteraction): Promise { @@ -396,15 +459,28 @@ async function loginGitHubCopilot(interaction: ProviderAuthInteraction): Promise enterpriseDomain ?? undefined, interaction.signal, ); - interaction.notify({ type: "progress", message: "Enabling models..." }); - await enableAllGitHubCopilotModels(credentials.access, enterpriseDomain ?? undefined, interaction.signal); - return { - ...credentials, - availableModelIds: await fetchAvailableGitHubCopilotModelIds( + const models = await fetchGitHubCopilotModels( + credentials.access, + enterpriseDomain ?? undefined, + interaction.signal, + { + maxRetries: 2, + maxElapsedMs: 5000, + }, + ); + let enabledModelIds: string[] = []; + if (models.policyModelIds.length > 0) { + interaction.notify({ type: "progress", message: "Enabling models..." }); + enabledModelIds = await enableGitHubCopilotModels( credentials.access, + models.policyModelIds, enterpriseDomain ?? undefined, interaction.signal, - ), + ); + } + return { + ...credentials, + availableModelIds: [...new Set([...models.availableModelIds, ...enabledModelIds])], }; } diff --git a/packages/ai/src/auth/oauth/kimi-coding.ts b/packages/ai/src/auth/oauth/kimi-coding.ts index 349b5a141..6c5bd4953 100644 --- a/packages/ai/src/auth/oauth/kimi-coding.ts +++ b/packages/ai/src/auth/oauth/kimi-coding.ts @@ -7,6 +7,7 @@ */ import { getProviderEnvValue } from "../../utils/provider-env.ts"; +import { sleep } from "../../utils/sleep.ts"; import type { OAuthAuth, OAuthCredential, ProviderAuthInteraction } from "../types.ts"; import { pollOAuthDeviceCodeFlow } from "./device-code.ts"; @@ -206,21 +207,6 @@ async function pollForToken( }); } -function sleep(ms: number, signal: AbortSignal): Promise { - return new Promise((resolve, reject) => { - signal.throwIfAborted(); - const onAbort = () => { - clearTimeout(timeout); - reject(signal.reason); - }; - const timeout = setTimeout(() => { - signal.removeEventListener("abort", onAbort); - resolve(); - }, ms); - signal.addEventListener("abort", onAbort, { once: true }); - }); -} - function isRetryableRefreshFailure(response: Response): boolean { return response.status === 429 || response.status >= 500; } diff --git a/packages/ai/src/utils/sleep.ts b/packages/ai/src/utils/sleep.ts new file mode 100644 index 000000000..67b97aef9 --- /dev/null +++ b/packages/ai/src/utils/sleep.ts @@ -0,0 +1,14 @@ +export function sleep(ms: number, signal: AbortSignal): Promise { + return new Promise((resolve, reject) => { + signal.throwIfAborted(); + const onAbort = () => { + clearTimeout(timeout); + reject(signal.reason); + }; + const timeout = setTimeout(() => { + signal.removeEventListener("abort", onAbort); + resolve(); + }, ms); + signal.addEventListener("abort", onAbort, { once: true }); + }); +} diff --git a/packages/ai/test/github-copilot-oauth.test.ts b/packages/ai/test/github-copilot-oauth.test.ts index 61ceea3e5..267216e4f 100644 --- a/packages/ai/test/github-copilot-oauth.test.ts +++ b/packages/ai/test/github-copilot-oauth.test.ts @@ -6,11 +6,15 @@ import { githubCopilotProvider } from "../src/providers/github-copilot.ts"; const neverAbortedSignal = new AbortController().signal; -function jsonResponse(body: unknown, status: number = 200): Response { +const testCopilotAccessToken = "tid=test;exp=9999999999;proxy-ep=proxy.individual.githubcopilot.com;"; +const testCopilotModelsUrl = "https://api.individual.githubcopilot.com/models"; + +function jsonResponse(body: unknown, status: number = 200, headers?: Record): Response { return new Response(JSON.stringify(body), { status, headers: { "Content-Type": "application/json", + ...headers, }, }); } @@ -28,6 +32,37 @@ function getUrl(input: unknown): string { throw new Error(`Unsupported fetch input: ${String(input)}`); } +function stubGitHubCopilotLoginFetch(options: { + models: () => Response; + policy?: (modelId: string) => Response; +}): void { + const fetchMock = vi.fn(async (input: string | URL | Request): Promise => { + const url = getUrl(input); + if (url.endsWith("/login/device/code")) { + return jsonResponse({ + device_code: "device-code", + user_code: "ABCD-EFGH", + verification_uri: "https://github.com/login/device", + interval: 1, + expires_in: 900, + }); + } + if (url.endsWith("/login/oauth/access_token")) { + return jsonResponse({ access_token: "ghu_refresh_token" }); + } + if (url.includes("/copilot_internal/v2/token")) { + return jsonResponse({ token: testCopilotAccessToken, expires_at: 9999999999 }); + } + if (url === testCopilotModelsUrl) return options.models(); + if (url.startsWith(`${testCopilotModelsUrl}/`) && url.endsWith("/policy")) { + if (!options.policy) throw new Error(`Unexpected policy request: ${url}`); + return options.policy(url.slice(`${testCopilotModelsUrl}/`.length, -"/policy".length)); + } + throw new Error(`Unexpected fetch URL: ${url}`); + }); + vi.stubGlobal("fetch", fetchMock); +} + function loginGitHubCopilotForTest(options: { onDeviceCode(info: { userCode: string; @@ -180,6 +215,37 @@ describe("GitHub Copilot OAuth device flow", () => { expect(credentials.availableModelIds).toEqual([]); }); + it("does not retry model catalog throttling during credential refresh", async () => { + let catalogRequestCount = 0; + vi.stubGlobal( + "fetch", + vi.fn(async (input: unknown): Promise => { + const url = getUrl(input); + if (url.includes("/copilot_internal/v2/token")) { + return jsonResponse({ token: testCopilotAccessToken, expires_at: 9999999999 }); + } + if (url === testCopilotModelsUrl) { + catalogRequestCount += 1; + return jsonResponse({ error: "too many requests" }, 429, { "Retry-After": "0" }); + } + throw new Error(`Unexpected fetch URL: ${url}`); + }), + ); + + await expect( + githubCopilotOAuth.refresh( + { + type: "oauth", + access: "old-access-token", + refresh: "ghu_refresh_token", + expires: 0, + }, + neverAbortedSignal, + ), + ).rejects.toThrow("429"); + expect(catalogRequestCount).toBe(1); + }); + it("reports device-code details through onDeviceCode", async () => { vi.useFakeTimers(); vi.setSystemTime(new Date("2026-03-09T00:00:00Z")); @@ -239,125 +305,150 @@ describe("GitHub Copilot OAuth device flow", () => { await loginPromise; }); - it("enables model policies sequentially during login", async () => { + it("updates only known, tool-capable, unconfigured account model policies", async () => { vi.useFakeTimers(); - let activePolicyRequests = 0; - let maxActivePolicyRequests = 0; - let policyRequestCount = 0; - const fetchMock = vi.fn(async (input: unknown): Promise => { - const url = getUrl(input); - - if (url.endsWith("/login/device/code")) { + let catalogRequestCount = 0; + const policyModelIds: string[] = []; + stubGitHubCopilotLoginFetch({ + models: () => { + catalogRequestCount += 1; return jsonResponse({ - device_code: "device-code", - user_code: "ABCD-EFGH", - verification_uri: "https://github.com/login/device", - interval: 1, - expires_in: 900, + data: [ + { + id: "gpt-4.1", + model_picker_enabled: true, + policy: { state: "enabled" }, + capabilities: { supports: { tool_calls: true } }, + }, + { + id: "claude-sonnet-4.5", + model_picker_enabled: true, + policy: { state: "unconfigured" }, + capabilities: { supports: { tool_calls: true } }, + }, + { + id: "remote-only-model", + model_picker_enabled: true, + policy: { state: "unconfigured" }, + capabilities: { supports: { tool_calls: true } }, + }, + { + id: "gpt-5.4", + model_picker_enabled: true, + policy: { state: "unconfigured" }, + capabilities: { supports: { tool_calls: false } }, + }, + ], }); - } - - if (url.endsWith("/login/oauth/access_token")) { - return jsonResponse({ access_token: "ghu_refresh_token" }); - } - - if (url.includes("/copilot_internal/v2/token")) { - return jsonResponse({ - token: "tid=test;exp=9999999999;proxy-ep=proxy.individual.githubcopilot.com;", - expires_at: 9999999999, - }); - } - - if (url.endsWith("/models")) { - return jsonResponse({ data: [] }); - } - - if (url.includes("/models/") && url.endsWith("/policy")) { - policyRequestCount += 1; - activePolicyRequests += 1; - maxActivePolicyRequests = Math.max(maxActivePolicyRequests, activePolicyRequests); - await new Promise((resolve) => setTimeout(resolve, 10)); - activePolicyRequests -= 1; + }, + policy: (modelId) => { + policyModelIds.push(modelId); return new Response("", { status: 200 }); - } - - throw new Error(`Unexpected fetch URL: ${url}`); + }, }); - vi.stubGlobal("fetch", fetchMock); - const loginPromise = loginGitHubCopilotForTest({ onDeviceCode: () => {}, onPrompt: async () => "", }); - - await vi.advanceTimersByTimeAsync(0); - await vi.advanceTimersByTimeAsync(1000); await vi.advanceTimersByTimeAsync(1000); await loginPromise; - expect(policyRequestCount).toBeGreaterThan(1); - expect(maxActivePolicyRequests).toBe(1); + expect(catalogRequestCount).toBe(1); + expect(policyModelIds).toEqual(["claude-sonnet-4.5"]); }); - it("retries GET /models once after a 429, honoring Retry-After", async () => { + it("retries a throttled policy update after Retry-After", async () => { vi.useFakeTimers(); - let modelsRequestCount = 0; - const fetchMock = vi.fn(async (input: unknown): Promise => { - const url = getUrl(input); - - if (url.endsWith("/login/device/code")) { - return jsonResponse({ - device_code: "device-code", - user_code: "ABCD-EFGH", - verification_uri: "https://github.com/login/device", - interval: 1, - expires_in: 900, - }); - } - - if (url.endsWith("/login/oauth/access_token")) { - return jsonResponse({ access_token: "ghu_refresh_token" }); - } - - if (url.includes("/copilot_internal/v2/token")) { - return jsonResponse({ - token: "tid=test;exp=9999999999;proxy-ep=proxy.individual.githubcopilot.com;", - expires_at: 9999999999, - }); - } - - if (url.endsWith("/models")) { - modelsRequestCount += 1; - if (modelsRequestCount === 1) { - return new Response("too many requests", { status: 429, headers: { "retry-after": "1" } }); - } - return jsonResponse({ data: [{ id: "gpt-5.4", model_picker_enabled: true }] }); - } - - if (url.includes("/models/") && url.endsWith("/policy")) { - return new Response("", { status: 200 }); - } - - throw new Error(`Unexpected fetch URL: ${url}`); + let policyRequestCount = 0; + stubGitHubCopilotLoginFetch({ + models: () => + jsonResponse({ + data: [{ id: "claude-sonnet-4.5", model_picker_enabled: true, policy: { state: "unconfigured" } }], + }), + policy: () => { + policyRequestCount += 1; + return policyRequestCount === 1 + ? jsonResponse({ error: "too many requests" }, 429, { "Retry-After": "1" }) + : new Response("", { status: 200 }); + }, }); - vi.stubGlobal("fetch", fetchMock); - const loginPromise = loginGitHubCopilotForTest({ onDeviceCode: () => {}, onPrompt: async () => "", }); + await vi.advanceTimersByTimeAsync(1000); + expect(policyRequestCount).toBe(1); + await vi.advanceTimersByTimeAsync(999); + expect(policyRequestCount).toBe(1); + await vi.advanceTimersByTimeAsync(1); + await loginPromise; - await vi.advanceTimersByTimeAsync(0); - await vi.advanceTimersByTimeAsync(1000); // device poll - await vi.advanceTimersByTimeAsync(1000); // Retry-After wait - const credentials = await loginPromise; + expect(policyRequestCount).toBe(2); + }); - expect(modelsRequestCount).toBe(2); - expect(credentials.availableModelIds).toEqual(["gpt-5.4"]); + it("continues policy updates after a transport failure", async () => { + vi.useFakeTimers(); + + const modelIds = ["gpt-4.1", "claude-sonnet-4.5"]; + const policyModelIds: string[] = []; + stubGitHubCopilotLoginFetch({ + models: () => + jsonResponse({ + data: modelIds.map((id) => ({ id, model_picker_enabled: true, policy: { state: "unconfigured" } })), + }), + policy: (modelId) => { + policyModelIds.push(modelId); + if (policyModelIds.length === 1) throw new Error("fetch failed"); + return new Response("", { status: 200 }); + }, + }); + + const loginPromise = loginGitHubCopilotForTest({ + onDeviceCode: () => {}, + onPrompt: async () => "", + }); + await vi.advanceTimersByTimeAsync(1000); + await loginPromise; + + expect(policyModelIds).toEqual(modelIds); + }); + + it("stops policy updates and persists authentication when the retry delay exceeds the login budget", async () => { + vi.useFakeTimers(); + + const policyModelIds: string[] = []; + stubGitHubCopilotLoginFetch({ + models: () => + jsonResponse({ + data: [ + { id: "gpt-4.1", model_picker_enabled: true, policy: { state: "unconfigured" } }, + { id: "claude-sonnet-4.5", model_picker_enabled: true, policy: { state: "unconfigured" } }, + ], + }), + policy: (modelId) => { + policyModelIds.push(modelId); + return jsonResponse({ error: "too many requests" }, 429, { "Retry-After": "5" }); + }, + }); + + const store = new InMemoryCredentialStore(); + const models = createModels({ credentials: store }); + models.setProvider(githubCopilotProvider()); + const loginPromise = models.login("github-copilot", "oauth", { + signal: neverAbortedSignal, + prompt: async () => "", + notify: () => {}, + }); + + await vi.advanceTimersByTimeAsync(1000); + const credential = await loginPromise; + expect(credential).toMatchObject({ type: "oauth", access: testCopilotAccessToken }); + expect(policyModelIds).toEqual(["gpt-4.1"]); + expect(await store.read("github-copilot")).toEqual(credential); }); it("rejects a non-http(s) verification_uri before it reaches onDeviceCode", async () => {