Merge remote-tracking branch 'origin/main' into dev

This commit is contained in:
David Brailovsky 2026-08-19 09:51:14 +02:00
commit 01001d7ad5
34 changed files with 1508 additions and 343 deletions

View file

@ -13,6 +13,7 @@
### Fixed
- Fixed Anthropic server-side fallback responses being priced with the requested model instead of the returned fallback model ([#8285](https://github.com/earendil-works/pi/issues/8285)).
- Fixed Azure OpenAI Responses ignoring `toolChoice` in provider-specific stream requests.
- Added `deepseek-v4-pro-0813` to the Qwen Token Plan Individual catalog ([#8194](https://github.com/earendil-works/pi/issues/8194)).
- Fixed Amazon Bedrock `after_provider_response`/`onResponse` to forward the raw response headers instead of only the synthesized request id header ([#8234](https://github.com/earendil-works/pi/issues/8234)).
@ -37,6 +38,7 @@
### Fixed
- Fixed GitHub Copilot login triggering API rate limits while enabling model policies by limiting concurrent policy updates ([#6187](https://github.com/earendil-works/pi/issues/6187)).
- Fixed GitHub Copilot login still triggering API rate limits by updating only account models with unconfigured policies and honoring server retry delays ([#7850](https://github.com/earendil-works/pi/issues/7850)).
- Fixed upstream request buffer limit failures to trigger automatic assistant retries.
- Fixed OpenAI Responses function and custom tool calls to preserve namespaces during streaming, proxying, and replay ([#7709](https://github.com/earendil-works/pi/issues/7709)).
- Fixed built-in and custom DeepSeek API models to send output limits through the supported `max_tokens` field.

View file

@ -1209,8 +1209,10 @@ interface OpenAICompletionsCompat {
requiresThinkingAsText?: boolean; // Whether thinking blocks must be converted to text (default: false)
requiresReasoningContentOnAssistantMessages?: boolean; // Whether all replayed assistant messages must include empty reasoning_content when reasoning is enabled (default: auto-detected for DeepSeek)
thinkingFormat?: 'openai' | 'openrouter' | 'deepseek' | 'together' | 'baseten' | 'zai' | 'qwen' | 'chat-template' | 'qwen-chat-template' | 'string-thinking' | 'ant-ling'; // Format for reasoning param: 'openai' uses reasoning_effort, 'openrouter' uses reasoning: { effort }, 'deepseek' uses thinking: { type } plus reasoning_effort when supported, 'together' uses reasoning: { enabled } plus reasoning_effort when supported, 'baseten' uses configurable chat_template_args plus reasoning_effort when supported, 'zai' uses thinking: { type }, 'qwen' uses enable_thinking, 'chat-template' uses configurable chat_template_kwargs, 'qwen-chat-template' uses chat_template_kwargs.enable_thinking and preserve_thinking, 'string-thinking' uses top-level thinking, 'ant-ling' uses reasoning: { effort } only for mapped efforts (default: openai)
chatTemplateKwargs?: Record<string, string | number | boolean | null | { '$var': 'thinking.enabled' | 'thinking.effort'; omitWhenOff?: boolean }>; // chat_template_kwargs values; use $var for pi-controlled thinking values
chatTemplateArgs?: Record<string, string | number | boolean | null | { '$var': 'thinking.enabled' | 'thinking.effort'; omitWhenOff?: boolean }>; // chat_template_args values for thinkingFormat: 'baseten'; use $var for pi-controlled thinking values
chatTemplateKwargs?: Record<string, string | number | boolean | null | { '$var': 'thinking.enabled' | 'thinking.effort' | 'thinking.budget'; omitWhenOff?: boolean }>; // chat_template_kwargs values; use $var for pi-controlled thinking values
chatTemplateArgs?: Record<string, string | number | boolean | null | { '$var': 'thinking.enabled' | 'thinking.effort' | 'thinking.budget'; omitWhenOff?: boolean }>; // chat_template_args values for thinkingFormat: 'baseten'; use $var for pi-controlled thinking values
thinkingTokenBudgetField?: 'thinking_token_budget' | 'thinking_budget' | 'thinking_budget_tokens'; // Top-level field that caps reasoning tokens from thinkingBudgets (vLLM / Qwen / llama.cpp). Off by default.
supportsThinkingTokenBudget?: boolean; // Alias for thinkingTokenBudgetField: 'thinking_token_budget' (vLLM). Prefer thinkingTokenBudgetField. Default: false.
cacheControlFormat?: 'anthropic'; // Anthropic-style cache_control on system prompt, last tool, and last user/assistant text content
openRouterRouting?: OpenRouterRouting; // OpenRouter routing preferences (default: {})
vercelGatewayRouting?: VercelGatewayRouting; // Vercel AI Gateway routing preferences (default: {})

View file

@ -594,7 +594,12 @@ const OPENAI_COMPLETIONS_DEFAULT_COMPAT = {
supportsOpenAIGrammarTools: false,
sendSessionAffinityHeaders: false,
supportsLongCacheRetention: true,
} satisfies Required<Omit<OpenAICompletionsCompat, "cacheControlFormat" | "deferredToolsMode">> & {
} satisfies Required<
Omit<
OpenAICompletionsCompat,
"cacheControlFormat" | "deferredToolsMode" | "supportsThinkingTokenBudget" | "thinkingTokenBudgetField"
>
> & {
cacheControlFormat?: OpenAICompletionsCompat["cacheControlFormat"];
deferredToolsMode?: OpenAICompletionsCompat["deferredToolsMode"];
};
@ -739,6 +744,30 @@ function applyAnthropicMessagesCompatMetadata(model: Model<Api>): void {
}
}
function isAnthropicFallbackMetadataModel(model: Model<Api>): model is Model<"anthropic-messages"> {
if (model.provider !== "anthropic" || model.api !== "anthropic-messages") return false;
return (
model.id in ANTHROPIC_ALLOWED_FALLBACK_MODELS ||
Object.values(ANTHROPIC_ALLOWED_FALLBACK_MODELS).some((fallbackModelIds) => fallbackModelIds.includes(model.id))
);
}
function applyAnthropicFallbackCostMetadata(models: readonly Model<"anthropic-messages">[]): void {
const modelsById = new Map(models.map((model) => [model.id, model]));
for (const [modelId, fallbackModelIds] of Object.entries(ANTHROPIC_ALLOWED_FALLBACK_MODELS)) {
const model = modelsById.get(modelId);
if (!model?.compat?.allowedFallbackModels) continue;
for (const fallbackModelId of fallbackModelIds) {
const fallback = model.compat.allowedFallbackModels.find((target) => target.model === fallbackModelId);
const fallbackModel = modelsById.get(fallbackModelId);
if (fallback && fallbackModel) {
fallback.cost = fallbackModel.cost;
}
}
}
}
function applyStrictToolCompatMetadata(model: Model<Api>): void {
if (
(model.provider === "openai" || model.provider === "cloudflare-ai-gateway") &&
@ -959,7 +988,7 @@ function getAnthropicMessagesCompat(provider: string, modelId: string): Anthropi
if (provider === "anthropic") {
const allowedFallbackModels = ANTHROPIC_ALLOWED_FALLBACK_MODELS[modelId];
if (allowedFallbackModels) {
compat.allowedFallbackModels = allowedFallbackModels;
compat.allowedFallbackModels = allowedFallbackModels.map((fallbackModel) => ({ model: fallbackModel }));
}
}
if (provider === "xiaomi" || provider.startsWith("xiaomi-token-plan-")) {
@ -2780,6 +2809,7 @@ async function generateModels() {
applyOpenAIToolSearchMetadata(model);
applyOpenAIExplicitPromptCacheMetadata(model);
}
applyAnthropicFallbackCostMetadata(allModels.filter(isAnthropicFallbackMetadataModel));
// Group by provider and deduplicate by model ID
const providers: Record<string, Record<string, Model<any>>> = {};

View file

@ -540,6 +540,7 @@ export const stream: StreamFunction<"anthropic-messages", AnthropicOptions> = (
try {
let client: Anthropic;
let isOAuth: boolean;
let usageModel = model;
if (options?.client) {
client = options.client;
@ -602,6 +603,14 @@ export const stream: StreamFunction<"anthropic-messages", AnthropicOptions> = (
if (event.type === "message_start") {
output.responseId = event.message.id;
output.model = event.message.model;
const fallbackCost =
output.model === model.id
? undefined
: ((Array.isArray(options?.refusalFallbacks)
? options.refusalFallbacks.find((fallback) => fallback.model === output.model)?.cost
: undefined) ??
model.compat?.allowedFallbackModels?.find((fallback) => fallback.model === output.model)?.cost);
usageModel = fallbackCost ? { ...model, id: output.model, cost: fallbackCost } : model;
// Capture initial token usage from message_start event
// This ensures we have input token counts even if the stream is aborted early
output.usage.input = event.message.usage.input_tokens || 0;
@ -612,7 +621,7 @@ export const stream: StreamFunction<"anthropic-messages", AnthropicOptions> = (
// Anthropic doesn't provide total_tokens, compute from components
output.usage.totalTokens =
output.usage.input + output.usage.output + output.usage.cacheRead + output.usage.cacheWrite;
calculateCost(model, output.usage);
calculateCost(usageModel, output.usage);
} else if (event.type === "content_block_start") {
if (event.content_block.type === "text") {
const block: Block = {
@ -769,7 +778,7 @@ export const stream: StreamFunction<"anthropic-messages", AnthropicOptions> = (
// Anthropic doesn't provide total_tokens, compute from components
output.usage.totalTokens =
output.usage.input + output.usage.output + output.usage.cacheRead + output.usage.cacheWrite;
calculateCost(model, output.usage);
calculateCost(usageModel, output.usage);
}
}
@ -1116,7 +1125,10 @@ function buildParams(
}
if (options?.refusalFallbacks !== undefined) {
params.fallbacks = options.refusalFallbacks;
params.fallbacks =
options.refusalFallbacks === "default"
? "default"
: options.refusalFallbacks.map((fallback) => ({ model: fallback.model }));
}
return params;

View file

@ -30,6 +30,7 @@ import type {
TextContent,
ThinkingBudgets,
ThinkingContent,
ThinkingTokenBudgetField,
Tool,
ToolCall,
ToolResultMessage,
@ -54,7 +55,7 @@ import {
} from "./constrained-sampling.ts";
import { buildCopilotDynamicHeaders, hasCopilotVisionInput } from "./github-copilot-headers.ts";
import { clampOpenAIPromptCacheKey } from "./openai-prompt-cache.ts";
import { buildBaseOptions, clampReasoning, MIN_ANSWER_TOKENS } from "./simple-options.ts";
import { buildBaseOptions, clampThinkingBudgetToAnswerRoom, thinkingBudgetForLevel } from "./simple-options.ts";
import { transformMessages } from "./transform-messages.ts";
/**
@ -144,7 +145,7 @@ function isEncryptedReasoningDetail(detail: unknown): detail is OpenAIEncryptedR
export interface OpenAICompletionsOptions extends StreamOptions {
toolChoice?: OpenAI.Chat.Completions.ChatCompletionToolChoiceOption;
reasoningEffort?: "minimal" | "low" | "medium" | "high" | "xhigh" | "max";
/** Token budgets per thinking level. Only used when `compat.supportsThinkingTokenBudget` is set. */
/** Token budgets per thinking level. Used when `compat.thinkingTokenBudgetField` or `compat.supportsThinkingTokenBudget` is set, or by `{ "$var": "thinking.budget" }`. */
thinkingBudgets?: ThinkingBudgets;
}
@ -159,11 +160,12 @@ interface OpenAICompatCacheControl {
type ResolvedOpenAICompletionsCompat = Omit<
Required<OpenAICompletionsCompat>,
"cacheControlFormat" | "deferredToolsMode" | "supportsThinkingTokenBudget"
"cacheControlFormat" | "deferredToolsMode" | "supportsThinkingTokenBudget" | "thinkingTokenBudgetField"
> & {
cacheControlFormat?: OpenAICompletionsCompat["cacheControlFormat"];
deferredToolsMode?: OpenAICompletionsCompat["deferredToolsMode"];
supportsThinkingTokenBudget?: OpenAICompletionsCompat["supportsThinkingTokenBudget"];
thinkingTokenBudgetField?: OpenAICompletionsCompat["thinkingTokenBudgetField"];
};
type ResolvedChatTemplateKwargValue = string | number | boolean | null;
@ -752,6 +754,9 @@ function buildParams(
params.tool_choice = options.toolChoice;
}
const thinkingTokenBudgetField = resolveThinkingTokenBudgetField(compat);
const thinkingBudget = resolveClampedThinkingBudget(model, options, params);
if (compat.thinkingFormat === "zai" && model.reasoning) {
const zaiParams = params as Omit<typeof params, "reasoning_effort"> & {
thinking?: { type: "enabled" | "disabled"; clear_thinking?: boolean };
@ -779,7 +784,7 @@ function buildParams(
preserve_thinking: true,
};
} else if (compat.thinkingFormat === "chat-template" && model.reasoning) {
const chatTemplateKwargs = buildChatTemplateValues(model, options, compat.chatTemplateKwargs);
const chatTemplateKwargs = buildChatTemplateValues(model, options, compat.chatTemplateKwargs, thinkingBudget);
if (chatTemplateKwargs) {
(params as any).chat_template_kwargs = chatTemplateKwargs;
}
@ -788,7 +793,7 @@ function buildParams(
chat_template_args?: Record<string, ResolvedChatTemplateKwargValue>;
reasoning_effort?: string;
};
const chatTemplateArgs = buildChatTemplateValues(model, options, compat.chatTemplateArgs);
const chatTemplateArgs = buildChatTemplateValues(model, options, compat.chatTemplateArgs, thinkingBudget);
if (chatTemplateArgs) {
basetenParams.chat_template_args = chatTemplateArgs;
}
@ -851,25 +856,12 @@ function buildParams(
}
}
// vLLM caps reasoning with a top-level thinking_token_budget. Independent of
// thinkingFormat: the same server can serve zai, qwen or chat-template models.
// Reasoning and the answer share max_tokens here, so an uncapped reasoning
// phase can consume the whole response and leave no answer and no tool call.
if (compat.supportsThinkingTokenBudget && options?.reasoningEffort && model.reasoning) {
const level = clampReasoning(options.reasoningEffort)!;
const budgets: ThinkingBudgets = {
minimal: 1024,
low: 2048,
medium: 8192,
high: 16384,
...options.thinkingBudgets,
};
const ceiling = (params as { max_tokens?: number }).max_tokens ?? params.max_completion_tokens ?? model.maxTokens;
// Always leave room for the answer, otherwise the budget recreates the bug it prevents.
const budget = Math.min(budgets[level]!, Math.max(0, ceiling - MIN_ANSWER_TOKENS));
if (budget > 0) {
(params as { thinking_token_budget?: number }).thinking_token_budget = budget;
}
// Cap reasoning with a top-level budget field. Independent of thinkingFormat: the
// same server can serve zai, qwen or chat-template models. Reasoning and the answer
// share max_tokens here, so an uncapped reasoning phase can consume the whole
// response and leave no answer and no tool call.
if (thinkingTokenBudgetField && thinkingBudget !== undefined) {
Object.assign(params, { [thinkingTokenBudgetField]: thinkingBudget });
}
// OpenRouter provider routing preferences
@ -896,15 +888,38 @@ function buildParams(
return params;
}
function resolveThinkingTokenBudgetField(
compat: Pick<OpenAICompletionsCompat, "thinkingTokenBudgetField" | "supportsThinkingTokenBudget">,
): ThinkingTokenBudgetField | undefined {
if (compat.thinkingTokenBudgetField) return compat.thinkingTokenBudgetField;
if (compat.supportsThinkingTokenBudget) return "thinking_token_budget";
return undefined;
}
function resolveClampedThinkingBudget(
model: Model<"openai-completions">,
options: OpenAICompletionsOptions | undefined,
params: { max_tokens?: number | null; max_completion_tokens?: number | null },
): number | undefined {
if (!options?.reasoningEffort || !model.reasoning) return undefined;
const ceiling = params.max_tokens ?? params.max_completion_tokens ?? model.maxTokens;
const budget = clampThinkingBudgetToAnswerRoom(
thinkingBudgetForLevel(options.reasoningEffort, options.thinkingBudgets),
ceiling,
);
return budget > 0 ? budget : undefined;
}
function buildChatTemplateValues(
model: Model<"openai-completions">,
options: OpenAICompletionsOptions | undefined,
values: Record<string, ChatTemplateKwargValue>,
thinkingBudget?: number,
): Record<string, ResolvedChatTemplateKwargValue> | undefined {
const resolvedValues: Record<string, ResolvedChatTemplateKwargValue> = {};
for (const [key, value] of Object.entries(values)) {
const resolved = resolveChatTemplateKwargValue(model, options, value);
const resolved = resolveChatTemplateKwargValue(model, options, value, thinkingBudget);
if (resolved !== undefined) {
resolvedValues[key] = resolved;
}
@ -917,6 +932,7 @@ function resolveChatTemplateKwargValue(
model: Model<"openai-completions">,
options: OpenAICompletionsOptions | undefined,
value: ChatTemplateKwargValue,
thinkingBudget?: number,
): ResolvedChatTemplateKwargValue | undefined {
if (typeof value !== "object" || value === null) {
return value;
@ -929,6 +945,9 @@ function resolveChatTemplateKwargValue(
if (value.$var === "thinking.enabled") {
return !!reasoningEffort;
}
if (value.$var === "thinking.budget") {
return thinkingBudget;
}
const mappedValue = reasoningEffort ? model.thinkingLevelMap?.[reasoningEffort] : model.thinkingLevelMap?.off;
return mappedValue === undefined ? reasoningEffort : typeof mappedValue === "string" ? mappedValue : undefined;
@ -1532,6 +1551,7 @@ function detectCompat(model: Model<"openai-completions">): ResolvedOpenAIComplet
chatTemplateArgs: {},
zaiToolStream: false,
supportsThinkingTokenBudget: false,
thinkingTokenBudgetField: undefined,
supportsStrictMode: !isMoonshot && !isTogether && !isCloudflareAiGateway && !isNvidia,
supportsOpenAIGrammarTools: false,
cacheControlFormat,
@ -1577,6 +1597,7 @@ function getCompat(model: Model<"openai-completions">): ResolvedOpenAICompletion
chatTemplateArgs: model.compat.chatTemplateArgs ?? detected.chatTemplateArgs,
zaiToolStream: model.compat.zaiToolStream ?? detected.zaiToolStream,
supportsThinkingTokenBudget: model.compat.supportsThinkingTokenBudget ?? detected.supportsThinkingTokenBudget,
thinkingTokenBudgetField: model.compat.thinkingTokenBudgetField ?? detected.thinkingTokenBudgetField,
supportsStrictMode: model.compat.supportsStrictMode ?? detected.supportsStrictMode,
supportsOpenAIGrammarTools: model.compat.supportsOpenAIGrammarTools ?? detected.supportsOpenAIGrammarTools,
cacheControlFormat: model.compat.cacheControlFormat ?? detected.cacheControlFormat,

View file

@ -54,10 +54,28 @@ export function buildBaseOptions(
/** Tokens always left for the answer when a thinking budget shares the response ceiling. */
export const MIN_ANSWER_TOKENS = 1024;
export const DEFAULT_THINKING_BUDGETS: ThinkingBudgets = {
minimal: 1024,
low: 2048,
medium: 8192,
high: 16384,
};
export function clampReasoning(effort: ThinkingLevel | undefined): Exclude<ThinkingLevel, "xhigh" | "max"> | undefined {
return effort === "xhigh" || effort === "max" ? "high" : effort;
}
export function thinkingBudgetForLevel(reasoningLevel: ThinkingLevel, customBudgets?: ThinkingBudgets): number {
const budgets = { ...DEFAULT_THINKING_BUDGETS, ...customBudgets };
const level = clampReasoning(reasoningLevel)!;
return budgets[level]!;
}
/** Cap a thinking budget so at least MIN_ANSWER_TOKENS remain under a shared response ceiling. */
export function clampThinkingBudgetToAnswerRoom(thinkingBudget: number, ceiling: number): number {
return Math.min(thinkingBudget, Math.max(0, ceiling - MIN_ANSWER_TOKENS));
}
export function adjustMaxTokensForThinking(
// Undefined means no explicit caller cap. Use the model cap and fit thinking inside it.
baseMaxTokens: number | undefined,
@ -65,21 +83,12 @@ export function adjustMaxTokensForThinking(
reasoningLevel: ThinkingLevel,
customBudgets?: ThinkingBudgets,
): { maxTokens: number; thinkingBudget: number } {
const defaultBudgets: ThinkingBudgets = {
minimal: 1024,
low: 2048,
medium: 8192,
high: 16384,
};
const budgets = { ...defaultBudgets, ...customBudgets };
const level = clampReasoning(reasoningLevel)!;
let thinkingBudget = budgets[level]!;
let thinkingBudget = thinkingBudgetForLevel(reasoningLevel, customBudgets);
const maxTokens =
baseMaxTokens === undefined ? modelMaxTokens : Math.min(baseMaxTokens + thinkingBudget, modelMaxTokens);
if (maxTokens <= thinkingBudget) {
thinkingBudget = Math.max(0, maxTokens - MIN_ANSWER_TOKENS);
thinkingBudget = clampThinkingBudgetToAnswerRoom(thinkingBudget, maxTokens);
}
return { maxTokens, thinkingBudget };

View file

@ -3,8 +3,9 @@
*/
import { GITHUB_COPILOT_MODELS } from "../../providers/github-copilot.models.ts";
import { sleep } from "../../utils/sleep.ts";
import type { OAuthAuth, OAuthCredential, ProviderAuthInteraction } from "../types.ts";
import { abortableSleep, pollOAuthDeviceCodeFlow } from "./device-code.ts";
import { pollOAuthDeviceCodeFlow } from "./device-code.ts";
const decode = (s: string) => atob(s);
const CLIENT_ID = decode("SXYxLmI1MDdhMDhjODdlY2ZlOTg=");
@ -16,8 +17,6 @@ const COPILOT_HEADERS = {
"Copilot-Integration-Id": "vscode-chat",
} as const;
const COPILOT_API_VERSION = "2026-06-01";
const MAX_RETRY_AFTER_MS = 10_000;
const DEFAULT_RETRY_AFTER_MS = 1_000;
type DeviceCodeResponse = {
device_code: string;
@ -91,66 +90,108 @@ function asRecord(value: unknown): Record<string, unknown> | undefined {
return value && typeof value === "object" ? (value as Record<string, unknown>) : undefined;
}
function parseAvailableCopilotModelIds(raw: unknown, allowPolicyFallback: boolean): string[] {
function parseGitHubCopilotModelCatalog(raw: unknown, allowPolicyFallback: boolean) {
const data = asRecord(raw)?.data;
if (!Array.isArray(data)) {
throw new Error("Invalid Copilot models response");
}
const pickerIds: string[] = [];
const policyEnabledIds: string[] = [];
for (const rawItem of data) {
const accountModels = data.flatMap((rawItem) => {
const item = asRecord(rawItem);
const id = item?.id;
if (!item || typeof id !== "string") continue;
if (!item || typeof id !== "string") return [];
const capabilities = asRecord(item.capabilities);
const supports = asRecord(capabilities?.supports);
if (supports?.tool_calls === false) continue;
const policy = asRecord(item.policy);
if (item.model_picker_enabled === true && policy?.state !== "disabled") pickerIds.push(id);
if (policy?.state === "enabled") policyEnabledIds.push(id);
}
return pickerIds.length > 0 || !allowPolicyFallback ? pickerIds : policyEnabledIds;
if (supports?.tool_calls === false) return [];
return [
{
id,
pickerEnabled: item.model_picker_enabled === true,
policyState: asRecord(item.policy)?.state,
},
];
});
const pickerModelIds = accountModels
.filter((model) => model.pickerEnabled && model.policyState !== "disabled")
.map((model) => model.id);
const usePolicyFallback = allowPolicyFallback && pickerModelIds.length === 0;
const availableModelIds =
pickerModelIds.length > 0 || !allowPolicyFallback
? pickerModelIds
: accountModels.filter((model) => model.policyState === "enabled").map((model) => model.id);
const policyModelIds = accountModels
.filter(
(model) =>
model.policyState === "unconfigured" &&
Object.hasOwn(GITHUB_COPILOT_MODELS, model.id) &&
(model.pickerEnabled || usePolicyFallback),
)
.map((model) => model.id);
return { availableModelIds, policyModelIds };
}
async function fetchAvailableGitHubCopilotModelIds(
async function fetchWithRateLimitRetry(
url: string,
init: RequestInit,
signal: AbortSignal,
retryPolicy: { maxRetries: number; maxElapsedMs: number },
): Promise<Response> {
const retryBudgetSignal =
retryPolicy.maxRetries > 0 && retryPolicy.maxElapsedMs > 0
? AbortSignal.timeout(retryPolicy.maxElapsedMs)
: undefined;
const requestSignal = retryBudgetSignal ? AbortSignal.any([signal, retryBudgetSignal]) : signal;
const retryDeadline = retryBudgetSignal ? Date.now() + retryPolicy.maxElapsedMs : undefined;
for (let retry = 0; ; retry++) {
const response = await fetch(url, {
...init,
signal: AbortSignal.any([requestSignal, AbortSignal.timeout(5000)]),
});
if (response.status !== 429 || retry === retryPolicy.maxRetries) return response;
const retryAfter = response.headers.get("retry-after");
let delayMs = 500 * 2 ** retry;
if (retryAfter) {
const seconds = Number.parseFloat(retryAfter);
delayMs = Number.isNaN(seconds) ? Date.parse(retryAfter) - Date.now() : seconds * 1000;
if (!Number.isFinite(delayMs)) return response;
}
delayMs = Math.max(0, delayMs);
if (retryDeadline !== undefined && delayMs >= retryDeadline - Date.now()) return response;
await response.body?.cancel();
await sleep(delayMs, requestSignal);
}
}
async function fetchGitHubCopilotModels(
copilotToken: string,
enterpriseDomain: string | undefined,
signal: AbortSignal,
): Promise<string[]> {
retryPolicy: { maxRetries: number; maxElapsedMs: number },
) {
const baseUrl = getGitHubCopilotBaseUrl(copilotToken, enterpriseDomain);
// Some Individual accounts return false for every picker flag despite explicit enabled policies.
// Limit the fallback to that endpoint so other account types keep strict picker semantics.
const allowPolicyFallback = baseUrl === "https://api.individual.githubcopilot.com";
const request = () =>
fetch(`${baseUrl}/models`, {
const response = await fetchWithRateLimitRetry(
`${baseUrl}/models`,
{
headers: {
Accept: "application/json",
Authorization: `Bearer ${copilotToken}`,
...COPILOT_HEADERS,
"X-GitHub-Api-Version": COPILOT_API_VERSION,
},
signal: AbortSignal.any([signal, AbortSignal.timeout(5000)]),
});
// The login-time policy updates can drain the Copilot API rate-limit bucket, in which case
// this request is rejected with 429. Honor Retry-After and retry once instead of failing.
let response = await request();
if (response.status === 429) {
const retryAfterSeconds = Number(response.headers.get("retry-after"));
const waitMs =
Number.isFinite(retryAfterSeconds) && retryAfterSeconds > 0
? Math.min(retryAfterSeconds * 1000, MAX_RETRY_AFTER_MS)
: DEFAULT_RETRY_AFTER_MS;
await abortableSleep(waitMs, signal, "Login cancelled");
response = await request();
}
},
signal,
retryPolicy,
);
if (!response.ok) {
const text = await response.text();
throw new Error(`${response.status} ${response.statusText}: ${text}`);
throw new Error(`${response.status} ${response.statusText}: ${await response.text()}`);
}
return parseAvailableCopilotModelIds(await response.json(), allowPolicyFallback);
return parseGitHubCopilotModelCatalog(await response.json(), allowPolicyFallback);
}
async function fetchJson(url: string, init: RequestInit): Promise<unknown> {
@ -315,9 +356,13 @@ async function refreshGitHubCopilotToken(
signal: AbortSignal,
): Promise<OAuthCredential> {
const credentials = await refreshGitHubCopilotAccessToken(refreshToken, enterpriseDomain, signal);
const { availableModelIds } = await fetchGitHubCopilotModels(credentials.access, enterpriseDomain, signal, {
maxRetries: 0,
maxElapsedMs: 0,
});
return {
...credentials,
availableModelIds: await fetchAvailableGitHubCopilotModelIds(credentials.access, enterpriseDomain, signal),
availableModelIds,
};
}
@ -334,38 +379,56 @@ async function enableGitHubCopilotModel(
const baseUrl = getGitHubCopilotBaseUrl(token, enterpriseDomain);
const url = `${baseUrl}/models/${modelId}/policy`;
let response: Response;
try {
const response = await fetch(url, {
method: "POST",
headers: {
"Content-Type": "application/json",
Authorization: `Bearer ${token}`,
...COPILOT_HEADERS,
"openai-intent": "chat-policy",
"x-interaction-type": "chat-policy",
response = await fetchWithRateLimitRetry(
url,
{
method: "POST",
headers: {
"Content-Type": "application/json",
Authorization: `Bearer ${token}`,
...COPILOT_HEADERS,
"openai-intent": "chat-policy",
"x-interaction-type": "chat-policy",
},
body: JSON.stringify({ state: "enabled" }),
},
body: JSON.stringify({ state: "enabled" }),
signal,
});
return response.ok;
{ maxRetries: 2, maxElapsedMs: 5000 },
);
} catch (error) {
if (signal.aborted) throw error;
return false;
}
if (response.status === 429) {
throw new Error(`${response.status} ${response.statusText}: ${await response.text()}`);
}
return response.ok;
}
/**
* Enable all known GitHub Copilot models that may require policy acceptance.
* Called after successful login to ensure all models are available.
* Enable the requested GitHub Copilot models and return the successful IDs.
* Policy updates are best effort; exhausted rate limiting stops the batch.
*/
async function enableAllGitHubCopilotModels(
async function enableGitHubCopilotModels(
token: string,
modelIds: readonly string[],
enterpriseDomain: string | undefined,
signal: AbortSignal,
): Promise<void> {
for (const model of Object.values(GITHUB_COPILOT_MODELS)) {
await enableGitHubCopilotModel(token, model.id, enterpriseDomain, signal);
): Promise<string[]> {
const enabledModelIds: string[] = [];
for (const modelId of modelIds) {
try {
if (await enableGitHubCopilotModel(token, modelId, enterpriseDomain, signal)) {
enabledModelIds.push(modelId);
}
} catch (error) {
if (signal.aborted) throw error;
break;
}
}
return enabledModelIds;
}
async function loginGitHubCopilot(interaction: ProviderAuthInteraction): Promise<OAuthCredential> {
@ -396,15 +459,28 @@ async function loginGitHubCopilot(interaction: ProviderAuthInteraction): Promise
enterpriseDomain ?? undefined,
interaction.signal,
);
interaction.notify({ type: "progress", message: "Enabling models..." });
await enableAllGitHubCopilotModels(credentials.access, enterpriseDomain ?? undefined, interaction.signal);
return {
...credentials,
availableModelIds: await fetchAvailableGitHubCopilotModelIds(
const models = await fetchGitHubCopilotModels(
credentials.access,
enterpriseDomain ?? undefined,
interaction.signal,
{
maxRetries: 2,
maxElapsedMs: 5000,
},
);
let enabledModelIds: string[] = [];
if (models.policyModelIds.length > 0) {
interaction.notify({ type: "progress", message: "Enabling models..." });
enabledModelIds = await enableGitHubCopilotModels(
credentials.access,
models.policyModelIds,
enterpriseDomain ?? undefined,
interaction.signal,
),
);
}
return {
...credentials,
availableModelIds: [...new Set([...models.availableModelIds, ...enabledModelIds])],
};
}

View file

@ -7,6 +7,7 @@
*/
import { getProviderEnvValue } from "../../utils/provider-env.ts";
import { sleep } from "../../utils/sleep.ts";
import type { OAuthAuth, OAuthCredential, ProviderAuthInteraction } from "../types.ts";
import { pollOAuthDeviceCodeFlow } from "./device-code.ts";
@ -206,21 +207,6 @@ async function pollForToken(
});
}
function sleep(ms: number, signal: AbortSignal): Promise<void> {
return new Promise((resolve, reject) => {
signal.throwIfAborted();
const onAbort = () => {
clearTimeout(timeout);
reject(signal.reason);
};
const timeout = setTimeout(() => {
signal.removeEventListener("abort", onAbort);
resolve();
}, ms);
signal.addEventListener("abort", onAbort, { once: true });
});
}
function isRetryableRefreshFailure(response: Response): boolean {
return response.status === 429 || response.status >= 500;
}

View file

@ -38,6 +38,7 @@ export * from "./session-resources.ts";
export * from "./types.ts";
export * from "./utils/assistant-message-frame.ts";
export * from "./utils/diagnostics.ts";
export { estimateContextTokens } from "./utils/estimate.ts";
export * from "./utils/event-stream.ts";
export * from "./utils/json-parse.ts";
export * from "./utils/overflow.ts";

View file

@ -89,10 +89,13 @@ export type ChatTemplateKwargValue =
| boolean
| null
| {
$var: "thinking.enabled" | "thinking.effort";
$var: "thinking.enabled" | "thinking.effort" | "thinking.budget";
omitWhenOff?: boolean;
};
/** Top-level request field used to cap reasoning tokens on OpenAI-compatible servers. */
export type ThinkingTokenBudgetField = "thinking_token_budget" | "thinking_budget" | "thinking_budget_tokens";
/** Token budgets for each thinking level (token-based providers only) */
export interface ThinkingBudgets {
minimal?: number;
@ -301,7 +304,13 @@ export interface ImagesOptions extends ProviderRequestOptions<ImagesModel<Images
export type ProviderImagesOptions = ImagesOptions & Record<string, unknown>;
export type AnthropicRefusalFallback = "default" | readonly { model: string }[];
export interface AnthropicRefusalFallbackTarget {
model: string;
/** Local pricing for this fallback target. Stripped before sending the provider request. @internal */
cost?: ModelCost;
}
export type AnthropicRefusalFallback = "default" | readonly AnthropicRefusalFallbackTarget[];
// Unified options with reasoning passed to streamSimple() and completeSimple()
export interface SimpleStreamOptions extends StreamOptions {
@ -583,9 +592,9 @@ export interface OpenAICompletionsCompat {
| "qwen-chat-template"
| "string-thinking"
| "ant-ling";
/** Kwargs to send as `chat_template_kwargs` when `thinkingFormat` is `chat-template`. Use `{ "$var": "thinking.enabled" }` or `{ "$var": "thinking.effort" }` for pi-controlled thinking values. */
/** Kwargs to send as `chat_template_kwargs` when `thinkingFormat` is `chat-template`. Use `{ "$var": "thinking.enabled" }`, `{ "$var": "thinking.effort" }`, or `{ "$var": "thinking.budget" }` for pi-controlled thinking values. */
chatTemplateKwargs?: Record<string, ChatTemplateKwargValue>;
/** Arguments to send as `chat_template_args` when `thinkingFormat` is `baseten`. Use `{ "$var": "thinking.enabled" }` or `{ "$var": "thinking.effort" }` for pi-controlled thinking values. */
/** Arguments to send as `chat_template_args` when `thinkingFormat` is `baseten`. Use `{ "$var": "thinking.enabled" }`, `{ "$var": "thinking.effort" }`, or `{ "$var": "thinking.budget" }` for pi-controlled thinking values. */
chatTemplateArgs?: Record<string, ChatTemplateKwargValue>;
/** OpenRouter-compatible routing preferences sent as the `provider` request field. */
openRouterRouting?: OpenRouterRouting;
@ -593,7 +602,15 @@ export interface OpenAICompletionsCompat {
vercelGatewayRouting?: VercelGatewayRouting;
/** Whether z.ai supports top-level `tool_stream: true` for streaming tool call deltas. Default: false. */
zaiToolStream?: boolean;
/** Whether the provider supports top-level `thinking_token_budget` to cap reasoning tokens (vLLM). Reasoning and the answer share `max_tokens` on these endpoints, so without a budget a reasoning-heavy turn can consume the whole response and emit no answer. Default: false. */
/**
* Top-level request field used to cap reasoning tokens from `thinkingBudgets`.
* Reasoning and the answer share `max_tokens` on these endpoints, so without a budget a
* reasoning-heavy turn can consume the whole response and emit no answer.
* `"thinking_token_budget"` is vLLM, `"thinking_budget"` is Qwen/DashScope/SGLang,
* `"thinking_budget_tokens"` is llama.cpp. Off by default; not set on the generated catalog.
*/
thinkingTokenBudgetField?: ThinkingTokenBudgetField;
/** Alias for `thinkingTokenBudgetField: "thinking_token_budget"` (vLLM). Prefer `thinkingTokenBudgetField`. Default: false. */
supportsThinkingTokenBudget?: boolean;
/** Whether the provider supports OpenAI custom tools with Lark/regex grammar formats. When false, grammar-constrained tools fall back to normal function tools. Default: false; the generated model catalog enables it for capable models. */
supportsOpenAIGrammarTools?: boolean;
@ -684,7 +701,7 @@ export interface AnthropicMessagesCompat {
* When absent or empty, callers must omit `fallbacks`; Anthropic rejects the
* field for models with no permitted fallback targets.
*/
allowedFallbackModels?: string[];
allowedFallbackModels?: AnthropicRefusalFallbackTarget[];
/**
* Whether the provider supports deferred tools loaded by `tool_reference`
* blocks in tool results. Default: true for first-party Anthropic models

View file

@ -0,0 +1,14 @@
export function sleep(ms: number, signal: AbortSignal): Promise<void> {
return new Promise((resolve, reject) => {
signal.throwIfAborted();
const onAbort = () => {
clearTimeout(timeout);
reject(signal.reason);
};
const timeout = setTimeout(() => {
signal.removeEventListener("abort", onAbort);
resolve();
}, ms);
signal.addEventListener("abort", onAbort, { once: true });
});
}

View file

@ -6,11 +6,15 @@ import { githubCopilotProvider } from "../src/providers/github-copilot.ts";
const neverAbortedSignal = new AbortController().signal;
function jsonResponse(body: unknown, status: number = 200): Response {
const testCopilotAccessToken = "tid=test;exp=9999999999;proxy-ep=proxy.individual.githubcopilot.com;";
const testCopilotModelsUrl = "https://api.individual.githubcopilot.com/models";
function jsonResponse(body: unknown, status: number = 200, headers?: Record<string, string>): Response {
return new Response(JSON.stringify(body), {
status,
headers: {
"Content-Type": "application/json",
...headers,
},
});
}
@ -28,6 +32,37 @@ function getUrl(input: unknown): string {
throw new Error(`Unsupported fetch input: ${String(input)}`);
}
function stubGitHubCopilotLoginFetch(options: {
models: () => Response;
policy?: (modelId: string) => Response;
}): void {
const fetchMock = vi.fn(async (input: string | URL | Request): Promise<Response> => {
const url = getUrl(input);
if (url.endsWith("/login/device/code")) {
return jsonResponse({
device_code: "device-code",
user_code: "ABCD-EFGH",
verification_uri: "https://github.com/login/device",
interval: 1,
expires_in: 900,
});
}
if (url.endsWith("/login/oauth/access_token")) {
return jsonResponse({ access_token: "ghu_refresh_token" });
}
if (url.includes("/copilot_internal/v2/token")) {
return jsonResponse({ token: testCopilotAccessToken, expires_at: 9999999999 });
}
if (url === testCopilotModelsUrl) return options.models();
if (url.startsWith(`${testCopilotModelsUrl}/`) && url.endsWith("/policy")) {
if (!options.policy) throw new Error(`Unexpected policy request: ${url}`);
return options.policy(url.slice(`${testCopilotModelsUrl}/`.length, -"/policy".length));
}
throw new Error(`Unexpected fetch URL: ${url}`);
});
vi.stubGlobal("fetch", fetchMock);
}
function loginGitHubCopilotForTest(options: {
onDeviceCode(info: {
userCode: string;
@ -180,6 +215,37 @@ describe("GitHub Copilot OAuth device flow", () => {
expect(credentials.availableModelIds).toEqual([]);
});
it("does not retry model catalog throttling during credential refresh", async () => {
let catalogRequestCount = 0;
vi.stubGlobal(
"fetch",
vi.fn(async (input: unknown): Promise<Response> => {
const url = getUrl(input);
if (url.includes("/copilot_internal/v2/token")) {
return jsonResponse({ token: testCopilotAccessToken, expires_at: 9999999999 });
}
if (url === testCopilotModelsUrl) {
catalogRequestCount += 1;
return jsonResponse({ error: "too many requests" }, 429, { "Retry-After": "0" });
}
throw new Error(`Unexpected fetch URL: ${url}`);
}),
);
await expect(
githubCopilotOAuth.refresh(
{
type: "oauth",
access: "old-access-token",
refresh: "ghu_refresh_token",
expires: 0,
},
neverAbortedSignal,
),
).rejects.toThrow("429");
expect(catalogRequestCount).toBe(1);
});
it("reports device-code details through onDeviceCode", async () => {
vi.useFakeTimers();
vi.setSystemTime(new Date("2026-03-09T00:00:00Z"));
@ -239,125 +305,150 @@ describe("GitHub Copilot OAuth device flow", () => {
await loginPromise;
});
it("enables model policies sequentially during login", async () => {
it("updates only known, tool-capable, unconfigured account model policies", async () => {
vi.useFakeTimers();
let activePolicyRequests = 0;
let maxActivePolicyRequests = 0;
let policyRequestCount = 0;
const fetchMock = vi.fn(async (input: unknown): Promise<Response> => {
const url = getUrl(input);
if (url.endsWith("/login/device/code")) {
let catalogRequestCount = 0;
const policyModelIds: string[] = [];
stubGitHubCopilotLoginFetch({
models: () => {
catalogRequestCount += 1;
return jsonResponse({
device_code: "device-code",
user_code: "ABCD-EFGH",
verification_uri: "https://github.com/login/device",
interval: 1,
expires_in: 900,
data: [
{
id: "gpt-4.1",
model_picker_enabled: true,
policy: { state: "enabled" },
capabilities: { supports: { tool_calls: true } },
},
{
id: "claude-sonnet-4.5",
model_picker_enabled: true,
policy: { state: "unconfigured" },
capabilities: { supports: { tool_calls: true } },
},
{
id: "remote-only-model",
model_picker_enabled: true,
policy: { state: "unconfigured" },
capabilities: { supports: { tool_calls: true } },
},
{
id: "gpt-5.4",
model_picker_enabled: true,
policy: { state: "unconfigured" },
capabilities: { supports: { tool_calls: false } },
},
],
});
}
if (url.endsWith("/login/oauth/access_token")) {
return jsonResponse({ access_token: "ghu_refresh_token" });
}
if (url.includes("/copilot_internal/v2/token")) {
return jsonResponse({
token: "tid=test;exp=9999999999;proxy-ep=proxy.individual.githubcopilot.com;",
expires_at: 9999999999,
});
}
if (url.endsWith("/models")) {
return jsonResponse({ data: [] });
}
if (url.includes("/models/") && url.endsWith("/policy")) {
policyRequestCount += 1;
activePolicyRequests += 1;
maxActivePolicyRequests = Math.max(maxActivePolicyRequests, activePolicyRequests);
await new Promise<void>((resolve) => setTimeout(resolve, 10));
activePolicyRequests -= 1;
},
policy: (modelId) => {
policyModelIds.push(modelId);
return new Response("", { status: 200 });
}
throw new Error(`Unexpected fetch URL: ${url}`);
},
});
vi.stubGlobal("fetch", fetchMock);
const loginPromise = loginGitHubCopilotForTest({
onDeviceCode: () => {},
onPrompt: async () => "",
});
await vi.advanceTimersByTimeAsync(0);
await vi.advanceTimersByTimeAsync(1000);
await vi.advanceTimersByTimeAsync(1000);
await loginPromise;
expect(policyRequestCount).toBeGreaterThan(1);
expect(maxActivePolicyRequests).toBe(1);
expect(catalogRequestCount).toBe(1);
expect(policyModelIds).toEqual(["claude-sonnet-4.5"]);
});
it("retries GET /models once after a 429, honoring Retry-After", async () => {
it("retries a throttled policy update after Retry-After", async () => {
vi.useFakeTimers();
let modelsRequestCount = 0;
const fetchMock = vi.fn(async (input: unknown): Promise<Response> => {
const url = getUrl(input);
if (url.endsWith("/login/device/code")) {
return jsonResponse({
device_code: "device-code",
user_code: "ABCD-EFGH",
verification_uri: "https://github.com/login/device",
interval: 1,
expires_in: 900,
});
}
if (url.endsWith("/login/oauth/access_token")) {
return jsonResponse({ access_token: "ghu_refresh_token" });
}
if (url.includes("/copilot_internal/v2/token")) {
return jsonResponse({
token: "tid=test;exp=9999999999;proxy-ep=proxy.individual.githubcopilot.com;",
expires_at: 9999999999,
});
}
if (url.endsWith("/models")) {
modelsRequestCount += 1;
if (modelsRequestCount === 1) {
return new Response("too many requests", { status: 429, headers: { "retry-after": "1" } });
}
return jsonResponse({ data: [{ id: "gpt-5.4", model_picker_enabled: true }] });
}
if (url.includes("/models/") && url.endsWith("/policy")) {
return new Response("", { status: 200 });
}
throw new Error(`Unexpected fetch URL: ${url}`);
let policyRequestCount = 0;
stubGitHubCopilotLoginFetch({
models: () =>
jsonResponse({
data: [{ id: "claude-sonnet-4.5", model_picker_enabled: true, policy: { state: "unconfigured" } }],
}),
policy: () => {
policyRequestCount += 1;
return policyRequestCount === 1
? jsonResponse({ error: "too many requests" }, 429, { "Retry-After": "1" })
: new Response("", { status: 200 });
},
});
vi.stubGlobal("fetch", fetchMock);
const loginPromise = loginGitHubCopilotForTest({
onDeviceCode: () => {},
onPrompt: async () => "",
});
await vi.advanceTimersByTimeAsync(1000);
expect(policyRequestCount).toBe(1);
await vi.advanceTimersByTimeAsync(999);
expect(policyRequestCount).toBe(1);
await vi.advanceTimersByTimeAsync(1);
await loginPromise;
await vi.advanceTimersByTimeAsync(0);
await vi.advanceTimersByTimeAsync(1000); // device poll
await vi.advanceTimersByTimeAsync(1000); // Retry-After wait
const credentials = await loginPromise;
expect(policyRequestCount).toBe(2);
});
expect(modelsRequestCount).toBe(2);
expect(credentials.availableModelIds).toEqual(["gpt-5.4"]);
it("continues policy updates after a transport failure", async () => {
vi.useFakeTimers();
const modelIds = ["gpt-4.1", "claude-sonnet-4.5"];
const policyModelIds: string[] = [];
stubGitHubCopilotLoginFetch({
models: () =>
jsonResponse({
data: modelIds.map((id) => ({ id, model_picker_enabled: true, policy: { state: "unconfigured" } })),
}),
policy: (modelId) => {
policyModelIds.push(modelId);
if (policyModelIds.length === 1) throw new Error("fetch failed");
return new Response("", { status: 200 });
},
});
const loginPromise = loginGitHubCopilotForTest({
onDeviceCode: () => {},
onPrompt: async () => "",
});
await vi.advanceTimersByTimeAsync(1000);
await loginPromise;
expect(policyModelIds).toEqual(modelIds);
});
it("stops policy updates and persists authentication when the retry delay exceeds the login budget", async () => {
vi.useFakeTimers();
const policyModelIds: string[] = [];
stubGitHubCopilotLoginFetch({
models: () =>
jsonResponse({
data: [
{ id: "gpt-4.1", model_picker_enabled: true, policy: { state: "unconfigured" } },
{ id: "claude-sonnet-4.5", model_picker_enabled: true, policy: { state: "unconfigured" } },
],
}),
policy: (modelId) => {
policyModelIds.push(modelId);
return jsonResponse({ error: "too many requests" }, 429, { "Retry-After": "5" });
},
});
const store = new InMemoryCredentialStore();
const models = createModels({ credentials: store });
models.setProvider(githubCopilotProvider());
const loginPromise = models.login("github-copilot", "oauth", {
signal: neverAbortedSignal,
prompt: async () => "",
notify: () => {},
});
await vi.advanceTimersByTimeAsync(1000);
const credential = await loginPromise;
expect(credential).toMatchObject({ type: "oauth", access: testCopilotAccessToken });
expect(policyModelIds).toEqual(["gpt-4.1"]);
expect(await store.read("github-copilot")).toEqual(credential);
});
it("rejects a non-http(s) verification_uri before it reaches onDeviceCode", async () => {

View file

@ -39,15 +39,20 @@ const compat = {
chatTemplateArgs: {},
zaiToolStream: false,
supportsThinkingTokenBudget: false,
thinkingTokenBudgetField: undefined,
supportsStrictMode: true,
supportsOpenAIGrammarTools: false,
cacheControlFormat: undefined,
sendSessionAffinityHeaders: false,
sessionAffinityFormat: "openai",
supportsLongCacheRetention: true,
} satisfies Omit<Required<OpenAICompletionsCompat>, "cacheControlFormat" | "deferredToolsMode"> & {
} satisfies Omit<
Required<OpenAICompletionsCompat>,
"cacheControlFormat" | "deferredToolsMode" | "thinkingTokenBudgetField"
> & {
cacheControlFormat?: OpenAICompletionsCompat["cacheControlFormat"];
deferredToolsMode?: OpenAICompletionsCompat["deferredToolsMode"];
thinkingTokenBudgetField?: OpenAICompletionsCompat["thinkingTokenBudgetField"];
};
function buildModel(baseUrl = "http://127.0.0.1:1"): Model<"openai-completions"> {

View file

@ -41,21 +41,35 @@ vi.mock("openai", () => {
return { default: FakeOpenAI };
});
// vLLM-served reasoning model: reasoning and the answer share max_tokens.
const vllmModel: Model<"openai-completions"> = {
id: "zai-org/glm-5.2",
name: "GLM 5.2 (local vLLM)",
api: "openai-completions",
provider: "local-vllm",
baseUrl: "http://localhost:8000/v1",
reasoning: true,
input: ["text"],
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
contextWindow: 262144,
maxTokens: 16384,
compat: { thinkingFormat: "zai", supportsThinkingTokenBudget: true },
type CapturedParams = {
thinking_token_budget?: number;
thinking_budget?: number;
thinking_budget_tokens?: number;
thinking?: unknown;
chat_template_kwargs?: Record<string, unknown>;
};
function vllmModel(
compat: Model<"openai-completions">["compat"] = {
thinkingFormat: "zai",
supportsThinkingTokenBudget: true,
},
): Model<"openai-completions"> {
return {
id: "zai-org/glm-5.2",
name: "GLM 5.2 (local vLLM)",
api: "openai-completions",
provider: "local-vllm",
baseUrl: "http://localhost:8000/v1",
reasoning: true,
input: ["text"],
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
contextWindow: 262144,
maxTokens: 16384,
compat,
};
}
async function capture(
model: Model<"openai-completions">,
options?: {
@ -63,7 +77,7 @@ async function capture(
thinkingBudgets?: ThinkingBudgets;
maxTokens?: number;
},
): Promise<{ thinking_token_budget?: number; thinking?: unknown }> {
): Promise<CapturedParams> {
let payload: unknown;
await streamSimple(
@ -80,45 +94,109 @@ async function capture(
},
).result();
return (payload ?? mockState.lastParams) as { thinking_token_budget?: number; thinking?: unknown };
return (payload ?? mockState.lastParams) as CapturedParams;
}
describe("openai-completions thinking_token_budget", () => {
describe("openai-completions thinking token budget", () => {
beforeEach(() => {
mockState.lastParams = undefined;
});
it("sends the configured budget for the requested level", async () => {
const params = await capture(vllmModel, { reasoning: "medium", thinkingBudgets: { medium: 4096 } });
const params = await capture(vllmModel(), { reasoning: "medium", thinkingBudgets: { medium: 4096 } });
expect(params.thinking_token_budget).toBe(4096);
});
it("omits the budget when the compat flag is not set", async () => {
const model = { ...vllmModel, compat: { thinkingFormat: "zai" } } as Model<"openai-completions">;
const params = await capture(model, { reasoning: "medium", thinkingBudgets: { medium: 4096 } });
it("omits the budget when neither the field nor the alias is set", async () => {
const params = await capture(vllmModel({ thinkingFormat: "zai" }), {
reasoning: "medium",
thinkingBudgets: { medium: 4096 },
});
expect(params.thinking_token_budget).toBeUndefined();
expect(params.thinking_budget).toBeUndefined();
expect(params.thinking_budget_tokens).toBeUndefined();
});
it("omits the budget when thinking is off", async () => {
const params = await capture(vllmModel, { reasoning: undefined, thinkingBudgets: { high: 8192 } });
const params = await capture(vllmModel(), { reasoning: undefined, thinkingBudgets: { high: 8192 } });
expect(params.thinking_token_budget).toBeUndefined();
});
it("clamps xhigh and max to the high budget", async () => {
const xhigh = await capture(vllmModel, { reasoning: "xhigh", thinkingBudgets: { high: 8192 } });
const max = await capture(vllmModel, { reasoning: "max", thinkingBudgets: { high: 8192 } });
const xhigh = await capture(vllmModel(), { reasoning: "xhigh", thinkingBudgets: { high: 8192 } });
const max = await capture(vllmModel(), { reasoning: "max", thinkingBudgets: { high: 8192 } });
expect(xhigh.thinking_token_budget).toBe(8192);
expect(max.thinking_token_budget).toBe(8192);
});
it("leaves room for the answer when the budget meets the response ceiling", async () => {
// Default high budget (16384) equals the model ceiling, which would leave no answer.
const params = await capture(vllmModel, { reasoning: "high" });
const params = await capture(vllmModel(), { reasoning: "high" });
expect(params.thinking_token_budget).toBe(16384 - 1024);
});
it("uses the caller max_tokens as the ceiling when it is lower than the model cap", async () => {
const params = await capture(vllmModel, { reasoning: "high", thinkingBudgets: { high: 8192 }, maxTokens: 4096 });
const params = await capture(vllmModel(), {
reasoning: "high",
thinkingBudgets: { high: 8192 },
maxTokens: 4096,
});
expect(params.thinking_token_budget).toBe(4096 - 1024);
});
it.each(["thinking_budget", "thinking_budget_tokens"] as const)(
"sends %s when thinkingTokenBudgetField is set",
async (field) => {
const params = await capture(vllmModel({ thinkingFormat: "qwen", thinkingTokenBudgetField: field }), {
reasoning: "medium",
thinkingBudgets: { medium: 4096 },
});
expect(params[field]).toBe(4096);
expect(params.thinking_token_budget).toBeUndefined();
},
);
it("lets thinkingTokenBudgetField win over the boolean alias", async () => {
const params = await capture(
vllmModel({
thinkingFormat: "zai",
supportsThinkingTokenBudget: true,
thinkingTokenBudgetField: "thinking_budget",
}),
{ reasoning: "medium", thinkingBudgets: { medium: 4096 } },
);
expect(params.thinking_budget).toBe(4096);
expect(params.thinking_token_budget).toBeUndefined();
});
it("puts the clamped budget in chat_template_kwargs when $var is thinking.budget", async () => {
const params = await capture(
vllmModel({
thinkingFormat: "chat-template",
chatTemplateKwargs: {
enable_thinking: { $var: "thinking.enabled" },
thinking_budget: { $var: "thinking.budget" },
},
}),
{ reasoning: "high" },
);
expect(params.chat_template_kwargs).toEqual({
enable_thinking: true,
thinking_budget: 16384 - 1024,
});
expect(params.thinking_token_budget).toBeUndefined();
});
it("omits thinking.budget from chat_template_kwargs when thinking is off", async () => {
const params = await capture(
vllmModel({
thinkingFormat: "chat-template",
chatTemplateKwargs: {
enable_thinking: { $var: "thinking.enabled" },
thinking_budget: { $var: "thinking.budget" },
},
}),
{ reasoning: undefined },
);
expect(params.chat_template_kwargs).toEqual({ enable_thinking: false });
});
});

View file

@ -19,8 +19,9 @@ const emptyUsage: Usage = {
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
};
const compat: Omit<Required<OpenAICompletionsCompat>, "deferredToolsMode"> & {
const compat: Omit<Required<OpenAICompletionsCompat>, "deferredToolsMode" | "thinkingTokenBudgetField"> & {
deferredToolsMode?: OpenAICompletionsCompat["deferredToolsMode"];
thinkingTokenBudgetField?: OpenAICompletionsCompat["thinkingTokenBudgetField"];
} = {
supportsStore: true,
supportsDeveloperRole: true,
@ -39,6 +40,7 @@ const compat: Omit<Required<OpenAICompletionsCompat>, "deferredToolsMode"> & {
chatTemplateArgs: {},
zaiToolStream: false,
supportsThinkingTokenBudget: false,
thinkingTokenBudgetField: undefined,
supportsStrictMode: true,
supportsOpenAIGrammarTools: false,
cacheControlFormat: "anthropic",

View file

@ -2,6 +2,10 @@
## [Unreleased]
### Added
- Added transcript usage notices for compaction and branch summaries when cache miss notices are enabled.
### Fixed
- Fixed the subagent example repeatedly prompting before running project-local agents in trusted repositories ([#8261](https://github.com/earendil-works/pi/issues/8261)).

View file

@ -281,7 +281,9 @@ pi.on("session_before_compact", async (event, ctx) => {
const { preparation, branchEntries, customInstructions, reason, willRetry, signal } = event;
// preparation.messagesToSummarize - messages to summarize
// preparation.sourceMessages - optional active-context history prefix, including any previous summary
// preparation.turnPrefixMessages - split turn prefix (if isSplitTurn)
// preparation.turnPrefixSourceMessages - optional active-context prefix through the split turn
// preparation.previousSummary - previous compaction summary
// preparation.fileOps - extracted file operations
// preparation.tokensBefore - context tokens before compaction

View file

@ -752,8 +752,10 @@ interface ProviderModelConfig {
requiresThinkingAsText?: boolean;
requiresReasoningContentOnAssistantMessages?: boolean;
thinkingFormat?: "openai" | "openrouter" | "deepseek" | "together" | "baseten" | "zai" | "qwen" | "chat-template" | "qwen-chat-template" | "string-thinking" | "ant-ling";
chatTemplateKwargs?: Record<string, string | number | boolean | null | { "$var": "thinking.enabled" | "thinking.effort"; omitWhenOff?: boolean }>;
chatTemplateArgs?: Record<string, string | number | boolean | null | { "$var": "thinking.enabled" | "thinking.effort"; omitWhenOff?: boolean }>;
chatTemplateKwargs?: Record<string, string | number | boolean | null | { "$var": "thinking.enabled" | "thinking.effort" | "thinking.budget"; omitWhenOff?: boolean }>;
chatTemplateArgs?: Record<string, string | number | boolean | null | { "$var": "thinking.enabled" | "thinking.effort" | "thinking.budget"; omitWhenOff?: boolean }>;
thinkingTokenBudgetField?: "thinking_token_budget" | "thinking_budget" | "thinking_budget_tokens";
supportsThinkingTokenBudget?: boolean;
cacheControlFormat?: "anthropic";
sessionAffinityFormat?: "openai" | "openai-nosession" | "openrouter";
sendSessionAffinityHeaders?: boolean;
@ -771,4 +773,5 @@ interface ProviderModelConfig {
```
`openrouter` sends `reasoning: { effort }`. `deepseek` sends `thinking: { type: "enabled" | "disabled" }` and `reasoning_effort` when enabled. `together` sends `reasoning: { enabled }` and also `reasoning_effort` when `supportsReasoningEffort` is enabled. `qwen` is for DashScope-style top-level `enable_thinking`. Use `qwen-chat-template` for local Qwen-compatible servers that read `chat_template_kwargs.enable_thinking` and need `preserve_thinking`. Use `chat-template` for configurable `chat_template_kwargs`, for example DeepSeek V3.x behind vLLM with `chatTemplateKwargs: { "thinking": { "$var": "thinking.enabled" } }`. Use `thinkingFormat: "baseten"` with `chatTemplateArgs` when the provider expects toggle values under `chat_template_args` and optionally supports top-level `reasoning_effort`.
`thinkingTokenBudgetField` sends a clamped per-level thinking budget as a top-level request field (`thinking_token_budget` on vLLM, `thinking_budget` on Qwen/SGLang, `thinking_budget_tokens` on llama.cpp). `supportsThinkingTokenBudget: true` is an alias for the vLLM field name. Do not combine it with `reasoning_effort` on DashScope Qwen models.
`cacheControlFormat: "anthropic"` applies Anthropic-style `cache_control` markers to the system prompt, last tool definition, and last user, assistant, or tool-result text content.

View file

@ -254,6 +254,8 @@ Current behavior:
Only OpenAI-compatible APIs apply it (`openai-completions`, `openai-responses`, `azure-openai-responses`); other APIs ignore it. Keys override pi's named request fields (for example a `temperature` key here beats the request-level temperature), so prefer it as the single source of sampling truth for a model. In `modelOverrides`, `samplingParams` merges per key with the base model's value.
A constant thinking-token cap can go here too, but it will not follow `thinkingBudgets` or leave room for the answer. Prefer `compat.thinkingTokenBudgetField` (or the `supportsThinkingTokenBudget` alias) for that.
### Thinking Level Map
Use `thinkingLevelMap` on a model to describe model-specific thinking controls. Keys are pi thinking levels: `off`, `minimal`, `low`, `medium`, `high`, `xhigh`, `max`. Maps may contain holes; for example, a model can expose `high` and `max` without exposing `xhigh`.
@ -467,8 +469,10 @@ For providers with partial OpenAI compatibility, use the `compat` field.
| `requiresThinkingAsText` | Convert thinking blocks to plain text |
| `requiresReasoningContentOnAssistantMessages` | Include empty `reasoning_content` on all replayed assistant messages when reasoning is enabled |
| `thinkingFormat` | Use `reasoning_effort`, `openrouter`, `deepseek`, `together`, `baseten`, `zai`, `qwen`, `chat-template`, or `qwen-chat-template` thinking parameters |
| `chatTemplateKwargs` | `chat_template_kwargs` values for `thinkingFormat: "chat-template"`; use `{ "$var": "thinking.enabled" }` or `{ "$var": "thinking.effort" }` for pi-controlled thinking values |
| `chatTemplateArgs` | `chat_template_args` values for `thinkingFormat: "baseten"`; use `{ "$var": "thinking.enabled" }` or `{ "$var": "thinking.effort" }` for pi-controlled thinking values |
| `chatTemplateKwargs` | `chat_template_kwargs` values for `thinkingFormat: "chat-template"`; use `{ "$var": "thinking.enabled" }`, `{ "$var": "thinking.effort" }`, or `{ "$var": "thinking.budget" }` for pi-controlled thinking values |
| `chatTemplateArgs` | `chat_template_args` values for `thinkingFormat: "baseten"`; use `{ "$var": "thinking.enabled" }`, `{ "$var": "thinking.effort" }`, or `{ "$var": "thinking.budget" }` for pi-controlled thinking values |
| `thinkingTokenBudgetField` | Top-level request field used to cap reasoning tokens from `thinkingBudgets`, clamped so at least 1024 tokens remain for the answer. `"thinking_token_budget"` (vLLM), `"thinking_budget"` (Qwen/DashScope/SGLang), `"thinking_budget_tokens"` (llama.cpp). Off by default; not set on the generated catalog. |
| `supportsThinkingTokenBudget` | Alias for `thinkingTokenBudgetField: "thinking_token_budget"` (vLLM). Prefer `thinkingTokenBudgetField`. Default: `false`. |
| `cacheControlFormat` | Use Anthropic-style `cache_control` markers on the system prompt, last tool definition, and last user, assistant, or tool-result text content. Currently only `anthropic` is supported. |
| `sendSessionAffinityHeaders` | For `openai-completions`, send session-affinity headers from the session id when caching is enabled. Default: `false`. |
| `sessionAffinityFormat` | For `openai-completions` and `openai-responses`, the session-affinity header format: `openai` sends `session_id`/`x-client-request-id` (completions also `x-session-affinity`), `openai-nosession` omits the underscore-containing `session_id` header, `openrouter` sends `x-session-id`. Does not affect the `prompt_cache_key` body param. Default: auto-detected. |
@ -481,6 +485,8 @@ For providers with partial OpenAI compatibility, use the `compat` field.
`openrouter` uses `reasoning: { effort }`. `together` uses `reasoning: { enabled }` and also `reasoning_effort` when `supportsReasoningEffort` is enabled. `qwen` uses top-level `enable_thinking`. Use `qwen-chat-template` for local Qwen-compatible servers that require `chat_template_kwargs.enable_thinking` and `preserve_thinking`. Use `chat-template` for vLLM/Hugging Face chat templates that need configurable `chat_template_kwargs`, such as `chatTemplateKwargs: { "thinking": { "$var": "thinking.enabled" } }` for DeepSeek V3.x templates. Use `thinkingFormat: "baseten"` with `chatTemplateArgs` for providers that expose toggle controls through `chat_template_args` and optionally support top-level `reasoning_effort`.
`thinkingTokenBudgetField` is independent of `thinkingFormat`. Do not enable it on the generated Qwen catalog: those models already send `reasoning_effort`, and DashScope rejects `thinking_budget` together with `reasoning_effort`.
`cacheControlFormat: "anthropic"` is for OpenAI-compatible providers that expose Anthropic-style prompt caching through `cache_control` markers on text content and tool definitions.
Example:

View file

@ -31,8 +31,8 @@ Use `/trust` in interactive mode to save a project trust decision for future ses
| `defaultModel` | string | - | Default model ID |
| `defaultThinkingLevel` | string | - | `"off"`, `"minimal"`, `"low"`, `"medium"`, `"high"`, `"xhigh"`, `"max"` |
| `hideThinkingBlock` | boolean | `false` | Hide thinking blocks in output |
| `showCacheMissNotices` | boolean | `false` | Show transcript notices for significant prompt-cache misses |
| `thinkingBudgets` | object | - | Custom token budgets per thinking level |
| `showCacheMissNotices` | boolean | `false` | Show transcript notices for significant prompt-cache misses and compaction or branch-summary usage |
| `thinkingBudgets` | object | - | Custom token budgets per thinking level. Anthropic, Google, and Bedrock use these natively. OpenAI-compatible models use them when `compat.thinkingTokenBudgetField` (or `supportsThinkingTokenBudget`) is set. |
#### thinkingBudgets

View file

@ -54,6 +54,7 @@ import { normalizeToolResultImages } from "../utils/tool-result-images.ts";
import { formatNoApiKeyFoundMessage, formatNoModelSelectedMessage } from "./auth-guidance.ts";
import { type BashResult, executeBashWithOperations } from "./bash-executor.ts";
import {
type CompactionPreparation,
type CompactionResult,
calculateContextTokens,
collectEntriesForBranchSummary,
@ -1789,6 +1790,33 @@ export class AgentSession {
// Compaction
// =========================================================================
/** Generate Pi's built-in compaction summary for manual and automatic compaction. */
private async _runDefaultCompaction(
preparation: CompactionPreparation,
requestModel: Model<any>,
apiKey: string | undefined,
headers: Record<string, string> | undefined,
customInstructions: string | undefined,
signal: AbortSignal,
env: Record<string, string> | undefined,
reason: "manual" | "threshold" | "overflow",
): Promise<CompactionResult> {
return compact(
preparation,
requestModel,
apiKey,
headers,
customInstructions,
signal,
this.thinkingLevel,
this.agent.streamFunction,
env,
this.settingsManager.getRetrySettings(),
this._summarizationRetryCallbacks({ source: "compaction", reason }),
undefined, // cacheFriendly
);
}
/**
* Manually compact the session context.
*
@ -1868,19 +1896,15 @@ export class AgentSession {
details = extensionCompaction.details;
} else {
// Shared default summary generator, also used by automatic compaction.
const result = await compact(
const result = await this._runDefaultCompaction(
preparation,
requestModel,
apiKey,
headers,
customInstructions,
this._compactionAbortController.signal,
this.thinkingLevel,
this.agent.streamFunction,
env,
this.settingsManager.getRetrySettings(),
this._summarizationRetryCallbacks({ source: "compaction", reason: "manual" }),
undefined, // sessionId
"manual",
);
summary = result.summary;
firstKeptEntryId = result.firstKeptEntryId;
@ -2181,19 +2205,15 @@ export class AgentSession {
details = extensionCompaction.details;
} else {
// Shared default summary generator, also used by manual compaction.
const compactResult = await compact(
const compactResult = await this._runDefaultCompaction(
preparation,
requestModel,
apiKey,
headers,
undefined,
this._autoCompactionAbortController.signal,
this.thinkingLevel,
this.agent.streamFunction,
env,
this.settingsManager.getRetrySettings(),
this._summarizationRetryCallbacks({ source: "compaction", reason }),
undefined, // sessionId
reason,
);
summary = compactResult.summary;
firstKeptEntryId = compactResult.firstKeptEntryId;

View file

@ -6,7 +6,14 @@
*/
import type { AgentMessage, StreamFn, ThinkingLevel } from "@earendil-works/pi-agent-core";
import { contentText, type RetryCallbacks, type RetryPolicy, retryAssistantCall, uuidv7 } from "@earendil-works/pi-ai";
import {
contentText,
estimateContextTokens as estimateProviderContextTokens,
type RetryCallbacks,
type RetryPolicy,
retryAssistantCall,
uuidv7,
} from "@earendil-works/pi-ai";
import type { AssistantMessage, Context, Model, SimpleStreamOptions, Usage } from "@earendil-works/pi-ai/compat";
import { completeSimple } from "@earendil-works/pi-ai/compat";
import { convertToLlm } from "../messages.ts";
@ -26,7 +33,7 @@ import {
serializeConversation,
} from "./utils.ts";
function getAnthropicSummarizationFallback(model: Model<any>): readonly { model: string }[] | undefined {
function getAnthropicSummarizationFallback(model: Model<any>): SimpleStreamOptions["refusalFallbacks"] {
if (model.provider !== "anthropic" || model.api !== "anthropic-messages") {
return undefined;
}
@ -34,7 +41,7 @@ function getAnthropicSummarizationFallback(model: Model<any>): readonly { model:
const allowedFallbackModels = (model as Model<"anthropic-messages">).compat?.allowedFallbackModels;
// Use the primary permitted fallback for now. If future Anthropic models expose
// broader fallback behavior, this can become a user/config pick or a full chain.
return allowedFallbackModels && allowedFallbackModels.length > 0 ? [{ model: allowedFallbackModels[0] }] : undefined;
return allowedFallbackModels && allowedFallbackModels.length > 0 ? [allowedFallbackModels[0]] : undefined;
}
// ============================================================================
@ -95,6 +102,27 @@ function getMessageFromEntryForCompaction(entry: SessionEntry): AgentMessage | u
return sessionEntryToContextMessages(entry)[0];
}
/** Build an active-context prefix, placing the previous compaction summary before its retained messages. */
function collectSourceMessages(
entries: SessionEntry[],
startIndex: number,
endIndex: number,
previousCompactionIndex: number,
): AgentMessage[] {
const messages: AgentMessage[] = [];
if (previousCompactionIndex >= 0) {
messages.push(...sessionEntryToContextMessages(entries[previousCompactionIndex]));
}
for (let i = startIndex; i < endIndex; i++) {
// The latest compaction was moved to the front above. Keep older compaction
// entries because buildSessionContext retains them in the active provider prefix.
if (i !== previousCompactionIndex) {
messages.push(...sessionEntryToContextMessages(entries[i]));
}
}
return messages;
}
/** Result from compact() - SessionManager adds uuid/parentUuid when saving */
export interface CompactionResult<T = unknown> {
summary: string;
@ -140,6 +168,19 @@ export interface CompactionSettings {
keepRecentTokens: number;
}
/** Active provider contexts and request settings used to preserve cacheable compaction prefixes. */
export interface CacheFriendlySummaryOptions {
/** Exact provider context prefix containing the history to summarize. */
sourceContext?: Context;
/** Exact provider context prefix containing a split turn's prefix. */
turnPrefixSourceContext?: Context;
/** Provider request settings copied from the active agent request path. */
requestOptions?: Pick<
SimpleStreamOptions,
"sessionId" | "onPayload" | "onResponse" | "transport" | "thinkingBudgets" | "maxRetryDelayMs"
>;
}
export const DEFAULT_COMPACTION_SETTINGS: CompactionSettings = {
enabled: true,
reserveTokens: 16384,
@ -508,9 +549,7 @@ Use this EXACT format:
Keep each section concise. Preserve exact file paths, function names, and error messages.`;
const UPDATE_SUMMARIZATION_PROMPT = `The messages above are NEW conversation messages to incorporate into the existing summary provided in <previous-summary> tags.
Update the existing structured summary with new information. RULES:
const UPDATE_SUMMARIZATION_INSTRUCTIONS = `Update the existing structured summary with new information. RULES:
- PRESERVE all existing information from the previous summary
- ADD new progress, decisions, and context from the new messages
- UPDATE the Progress section: move items from "In Progress" to "Done" when completed
@ -547,6 +586,14 @@ Use this EXACT format:
Keep each section concise. Preserve exact file paths, function names, and error messages.`;
const UPDATE_SUMMARIZATION_PROMPT = `The messages above are NEW conversation messages to incorporate into the existing summary provided in <previous-summary> tags.
${UPDATE_SUMMARIZATION_INSTRUCTIONS}`;
const SOURCE_CONTEXT_UPDATE_SUMMARIZATION_PROMPT = `The messages above contain an existing structured summary of earlier conversation history followed by NEW conversation messages.
${UPDATE_SUMMARIZATION_INSTRUCTIONS}`;
function createSummarizationOptions(
model: Model<any>,
maxTokens: number,
@ -555,9 +602,18 @@ function createSummarizationOptions(
env: Record<string, string> | undefined,
signal: AbortSignal | undefined,
thinkingLevel: ThinkingLevel | undefined,
sessionId: string | undefined,
requestOptions: CacheFriendlySummaryOptions["requestOptions"] | undefined,
cacheRetention: SimpleStreamOptions["cacheRetention"] | undefined,
): SimpleStreamOptions {
const options: SimpleStreamOptions = { maxTokens, signal, apiKey, headers, env, sessionId };
const options: SimpleStreamOptions = {
...requestOptions,
maxTokens,
signal,
apiKey,
headers,
env,
cacheRetention,
};
const refusalFallbacks = getAnthropicSummarizationFallback(model);
if (refusalFallbacks) {
options.refusalFallbacks = refusalFallbacks;
@ -583,12 +639,15 @@ export async function completeSummarization(
retry?: RetryPolicy,
callbacks?: RetryCallbacks,
): Promise<AssistantMessage> {
// Avoid cache writes for one-off summaries. Reuse caller-supplied routing when available;
// callers without a session ID, including branch summaries, receive a fresh routing ID.
// Standalone one-off summaries default to no prompt caching. Cache-friendly callers
// explicitly request short retention so providers can reuse the active prefix.
// Callers without a session ID, including standalone branch summaries, receive a fresh routing ID.
const requestOptions: SimpleStreamOptions = {
...options,
cacheRetention: "none",
cacheRetention: options.cacheRetention ?? "none",
sessionId: options.sessionId ?? uuidv7(),
// Anthropic invalidates the messages cache when tool_choice changes. Its 20-content-block lookup
// can still reuse an earlier user-message checkpoint, but long tool-heavy turns may need reprocessing.
toolChoice: "none",
};
const produce = async (): Promise<AssistantMessage> =>
@ -616,7 +675,7 @@ export async function generateSummary(
env?: Record<string, string>,
retry?: RetryPolicy,
callbacks?: RetryCallbacks,
sessionId?: string,
cacheFriendly?: Pick<CacheFriendlySummaryOptions, "sourceContext" | "requestOptions">,
): Promise<string> {
return (
await generateSummaryWithUsage(
@ -633,11 +692,47 @@ export async function generateSummary(
env,
retry,
callbacks,
sessionId,
cacheFriendly,
)
).text;
}
/** Build a standalone summary request or append its instruction to an existing provider context. */
function buildSummarizationContext(promptText: string, sourceContext?: Context): Context {
const instructionMessage = {
role: "user" as const,
content: [{ type: "text" as const, text: promptText }],
timestamp: Date.now(),
};
if (sourceContext) {
return {
...sourceContext,
messages: [...sourceContext.messages, instructionMessage],
};
}
return {
systemPrompt: SUMMARIZATION_SYSTEM_PROMPT,
messages: [instructionMessage],
};
}
/**
* Extra room for provider framing and tokenizer variance omitted by the heuristic context estimate.
* This matches the 4096-token margin used when normal simple requests clamp maxTokens to their context window.
*/
const CACHE_FRIENDLY_CONTEXT_SAFETY_TOKENS = 4096;
/** Whether the source context leaves room for the requested summary output and provider safety margin. */
function cacheFriendlyContextFits(model: Model<any>, context: Context, maxTokens: number): boolean {
return (
model.contextWindow <= 0 ||
estimateProviderContextTokens(context).tokens + maxTokens + CACHE_FRIENDLY_CONTEXT_SAFETY_TOKENS <=
model.contextWindow
);
}
/** Generate or update a conversation summary and return its provider usage. */
export async function generateSummaryWithUsage(
currentMessages: AgentMessage[],
@ -653,39 +748,55 @@ export async function generateSummaryWithUsage(
env?: Record<string, string>,
retry?: RetryPolicy,
callbacks?: RetryCallbacks,
sessionId?: string,
cacheFriendly?: Pick<CacheFriendlySummaryOptions, "sourceContext" | "requestOptions">,
): Promise<{ text: string; usage: Usage }> {
const maxTokens = Math.min(
Math.floor(0.8 * reserveTokens),
model.maxTokens > 0 ? model.maxTokens : Number.POSITIVE_INFINITY,
);
// Provider-visible history prefix to reuse instead of serializing messages into a standalone prompt.
let sourceContext = cacheFriendly?.sourceContext;
// Use update prompt if we have a previous summary, otherwise initial prompt
let basePrompt = previousSummary ? UPDATE_SUMMARIZATION_PROMPT : SUMMARIZATION_PROMPT;
// Cache-friendly source contexts already contain the previous compaction summary,
// but still need iterative-update instructions so prior information is preserved.
let basePrompt = previousSummary
? sourceContext
? SOURCE_CONTEXT_UPDATE_SUMMARIZATION_PROMPT
: UPDATE_SUMMARIZATION_PROMPT
: SUMMARIZATION_PROMPT;
if (customInstructions) {
basePrompt = `${basePrompt}\n\nAdditional focus: ${customInstructions}`;
}
// Serialize conversation to text so model doesn't try to continue it
// Convert to LLM messages first (handles custom types like bashExecution, custom, etc.)
const llmMessages = convertToLlm(currentMessages);
const conversationText = serializeConversation(llmMessages);
// A valid prefix can still be too large to leave the intended summary budget,
// especially during overflow recovery or after switching to a smaller-context model.
// In that case, use standalone serialization, which truncates large tool results.
if (
sourceContext &&
!cacheFriendlyContextFits(model, buildSummarizationContext(basePrompt, sourceContext), maxTokens)
) {
sourceContext = undefined;
basePrompt = previousSummary ? UPDATE_SUMMARIZATION_PROMPT : SUMMARIZATION_PROMPT;
if (customInstructions) {
basePrompt = `${basePrompt}\n\nAdditional focus: ${customInstructions}`;
}
}
// Build the prompt with conversation wrapped in tags
let promptText = `<conversation>\n${conversationText}\n</conversation>\n\n`;
if (previousSummary) {
// Source contexts already contain the conversation. Standalone requests serialize it
// so the model treats those messages as data rather than continuing the conversation.
let promptText = "";
if (!sourceContext) {
const llmMessages = convertToLlm(currentMessages);
const conversationText = serializeConversation(llmMessages);
promptText = `<conversation>\n${conversationText}\n</conversation>\n\n`;
}
if (previousSummary && !sourceContext) {
promptText += `<previous-summary>\n${previousSummary}\n</previous-summary>\n\n`;
}
promptText += basePrompt;
const summarizationMessages = [
{
role: "user" as const,
content: [{ type: "text" as const, text: promptText }],
timestamp: Date.now(),
},
];
const sourceRequestOptions = sourceContext ? cacheFriendly?.requestOptions : undefined;
const sourceCacheRetention = sourceContext ? "short" : undefined;
const completionOptions = createSummarizationOptions(
model,
maxTokens,
@ -694,12 +805,13 @@ export async function generateSummaryWithUsage(
env,
signal,
thinkingLevel,
sessionId,
sourceRequestOptions,
sourceCacheRetention,
);
const response = await completeSummarization(
model,
{ systemPrompt: SUMMARIZATION_SYSTEM_PROMPT, messages: summarizationMessages },
buildSummarizationContext(promptText, sourceContext),
completionOptions,
streamFn,
retry,
@ -727,8 +839,15 @@ export interface CompactionPreparation {
firstKeptEntryId: string;
/** Messages that will be summarized and discarded */
messagesToSummarize: AgentMessage[];
/**
* Active-context prefix for the history summary.
* Includes the previous compaction summary before messages retained by that compaction.
*/
sourceMessages?: AgentMessage[];
/** Messages that will be turned into turn prefix summary (if splitting) */
turnPrefixMessages: AgentMessage[];
/** Active-context prefix through the split-turn prefix, or empty when not splitting. */
turnPrefixSourceMessages?: AgentMessage[];
/** Whether this is a split turn (cut point in middle of turn) */
isSplitTurn: boolean;
tokensBefore: number;
@ -786,6 +905,8 @@ export function prepareCompaction(
if (msg) messagesToSummarize.push(msg);
}
const sourceMessages = collectSourceMessages(pathEntries, boundaryStart, historyEnd, prevCompactionIndex);
// Messages for turn prefix summary (if splitting a turn)
const turnPrefixMessages: AgentMessage[] = [];
if (cutPoint.isSplitTurn) {
@ -794,6 +915,9 @@ export function prepareCompaction(
if (msg) turnPrefixMessages.push(msg);
}
}
const turnPrefixSourceMessages = cutPoint.isSplitTurn
? collectSourceMessages(pathEntries, boundaryStart, cutPoint.firstKeptEntryIndex, prevCompactionIndex)
: [];
if (messagesToSummarize.length === 0 && turnPrefixMessages.length === 0) {
return undefined;
@ -812,7 +936,9 @@ export function prepareCompaction(
return {
firstKeptEntryId,
messagesToSummarize,
sourceMessages,
turnPrefixMessages,
turnPrefixSourceMessages,
isSplitTurn: cutPoint.isSplitTurn,
tokensBefore,
previousSummary,
@ -840,13 +966,30 @@ Summarize the prefix to provide context for the retained suffix:
Be concise. Focus on what's needed to understand the kept suffix.`;
const SOURCE_CONTEXT_TURN_PREFIX_SUMMARIZATION_PROMPT = `The final turn in the source conversation was too large to keep in full. Its SUFFIX (recent work) is retained.
The source conversation may also contain complete earlier turns for background. Summarize only the final, incomplete turn. It begins with the last user-role request before this instruction. Do not summarize earlier turns except for details needed to understand this final turn's prefix.
Summarize the prefix to provide context for the retained suffix:
## Original Request
[What did the user ask for in this turn?]
## Early Progress
- [Key decisions and work done in the prefix]
## Context for Suffix
- [Information needed to understand the retained recent work]
Be concise. Focus on what's needed to understand the kept suffix.`;
/**
* Generate summaries for compaction using prepared data.
* Returns CompactionResult - SessionManager adds uuid/parentUuid when saving.
*
* @param preparation - Pre-calculated preparation from prepareCompaction()
* @param customInstructions - Optional custom focus for the summary
* @param sessionId - Optional routing session ID forwarded without enabling prompt caching
* @param cacheFriendly - Active provider contexts and request settings for cache-friendly summarization
*/
export async function compact(
preparation: CompactionPreparation,
@ -860,7 +1003,7 @@ export async function compact(
env?: Record<string, string>,
retry?: RetryPolicy,
callbacks?: RetryCallbacks,
sessionId?: string,
cacheFriendly?: CacheFriendlySummaryOptions,
): Promise<CompactionResult> {
const {
firstKeptEntryId,
@ -895,7 +1038,10 @@ export async function compact(
env,
retry,
callbacks,
sessionId,
{
sourceContext: cacheFriendly?.sourceContext,
requestOptions: cacheFriendly?.requestOptions,
},
);
historyText = historyResult.text;
historyUsage = historyResult.usage;
@ -912,7 +1058,10 @@ export async function compact(
streamFn,
retry,
callbacks,
sessionId,
{
sourceContext: cacheFriendly?.turnPrefixSourceContext,
requestOptions: cacheFriendly?.requestOptions,
},
);
// Merge into single summary
summary = `${historyText}\n\n---\n\n**Turn Context (split turn):**\n\n${turnPrefixResult.text}`;
@ -933,7 +1082,10 @@ export async function compact(
env,
retry,
callbacks,
sessionId,
{
sourceContext: cacheFriendly?.sourceContext,
requestOptions: cacheFriendly?.requestOptions,
},
);
summary = result.text;
summaryUsage = result.usage;
@ -971,27 +1123,51 @@ async function generateTurnPrefixSummary(
streamFn?: StreamFn,
retry?: RetryPolicy,
callbacks?: RetryCallbacks,
sessionId?: string,
cacheFriendly?: Pick<CacheFriendlySummaryOptions, "sourceContext" | "requestOptions">,
): Promise<{ text: string; usage: Usage }> {
const maxTokens = Math.min(
Math.floor(0.5 * reserveTokens),
model.maxTokens > 0 ? model.maxTokens : Number.POSITIVE_INFINITY,
); // Smaller budget for turn prefix
const llmMessages = convertToLlm(messages);
const conversationText = serializeConversation(llmMessages);
const promptText = `<conversation>\n${conversationText}\n</conversation>\n\n${TURN_PREFIX_SUMMARIZATION_PROMPT}`;
const summarizationMessages = [
{
role: "user" as const,
content: [{ type: "text" as const, text: promptText }],
timestamp: Date.now(),
},
];
// Reuse the provider-visible split-turn prefix only when it leaves room for the summary;
// otherwise serialize and truncate the messages in a standalone prompt.
let sourceContext = cacheFriendly?.sourceContext;
if (
sourceContext &&
!cacheFriendlyContextFits(
model,
buildSummarizationContext(SOURCE_CONTEXT_TURN_PREFIX_SUMMARIZATION_PROMPT, sourceContext),
maxTokens,
)
) {
sourceContext = undefined;
}
let promptText: string;
if (sourceContext) {
promptText = SOURCE_CONTEXT_TURN_PREFIX_SUMMARIZATION_PROMPT;
} else {
const llmMessages = convertToLlm(messages);
const conversationText = serializeConversation(llmMessages);
promptText = `<conversation>\n${conversationText}\n</conversation>\n\n${TURN_PREFIX_SUMMARIZATION_PROMPT}`;
}
const sourceRequestOptions = sourceContext ? cacheFriendly?.requestOptions : undefined;
const sourceCacheRetention = sourceContext ? "short" : undefined;
const response = await completeSummarization(
model,
{ systemPrompt: SUMMARIZATION_SYSTEM_PROMPT, messages: summarizationMessages },
createSummarizationOptions(model, maxTokens, apiKey, headers, env, signal, thinkingLevel, sessionId),
buildSummarizationContext(promptText, sourceContext),
createSummarizationOptions(
model,
maxTokens,
apiKey,
headers,
env,
signal,
thinkingLevel,
sourceRequestOptions,
sourceCacheRetention,
),
streamFn,
retry,
callbacks,

View file

@ -46,7 +46,7 @@ import type {
ToolDefinition,
} from "./types.ts";
/** Modules available to extensions via virtualModules (for compiled Bun binary) */
/** Modules available to extensions via virtualModules (for compiled binaries) */
const VIRTUAL_MODULES: Record<string, unknown> = {
typebox: _bundledTypebox,
"typebox/compile": _bundledTypeboxCompile,
@ -75,11 +75,14 @@ const VIRTUAL_MODULES: Record<string, unknown> = {
const require = createRequire(import.meta.url);
const isNodeSeaBinary =
("sea" in process.features && process.features.sea === true) ||
process.getBuiltinModule("node:sea")?.isSea() === true;
const isTypeScriptSourceRuntime = !isBunBinary && path.extname(fileURLToPath(import.meta.url)) === ".ts";
/**
* Get aliases for jiti (used in built Node.js mode).
* In Bun binary mode, virtualModules is used instead.
* In compiled binary mode, virtualModules is used instead.
*/
let _aliases: Record<string, string> | null = null;
@ -448,9 +451,9 @@ async function loadExtensionModule(extensionPath: string, cacheToken?: Extension
const jiti = createJiti(import.meta.url, {
moduleCache: false,
// Bun uses modules embedded in the executable. Source TypeScript reuses the
// host-resolved modules and root tsconfig paths. Built Node uses dist aliases.
...(isBunBinary
// Compiled binaries use modules embedded in the executable. Source TypeScript
// reuses host modules and root tsconfig paths. Built Node uses dist aliases.
...(isBunBinary || isNodeSeaBinary
? { virtualModules: VIRTUAL_MODULES, tryNative: false }
: isTypeScriptSourceRuntime
? { virtualModules: VIRTUAL_MODULES, tsconfigPaths: true }

View file

@ -100,7 +100,7 @@ export interface Settings {
branchSummary?: BranchSummarySettings;
retry?: RetrySettings;
hideThinkingBlock?: boolean;
showCacheMissNotices?: boolean; // default: false - show transcript notices for significant prompt-cache misses
showCacheMissNotices?: boolean; // default: false - show prompt-cache miss and compaction cost notices
externalEditor?: string; // Command for Ctrl+G external editor; takes precedence over VISUAL/EDITOR
shellPath?: string; // Custom shell path (e.g., for Cygwin users on Windows); supports leading ~ expansion
quietStartup?: boolean;

View file

@ -28,6 +28,7 @@ export { readStoredCredential } from "./core/auth-storage.ts";
export {
type BranchPreparation,
type BranchSummaryResult,
type CacheFriendlySummaryOptions,
type CollectEntriesResult,
type CompactionResult,
type CutPointResult,

View file

@ -549,7 +549,7 @@ export class SettingsSelectorComponent extends Container {
{
id: "cache-miss-notices",
label: "Cache miss notices",
description: "Show transcript notices for significant prompt-cache misses",
description: "Show transcript notices for significant prompt-cache misses and compaction costs",
currentValue: config.showCacheMissNotices ? "true" : "false",
values: ["true", "false"],
},

View file

@ -9,7 +9,7 @@ import * as os from "node:os";
import * as path from "node:path";
import type { AgentMessage } from "@earendil-works/pi-agent-core";
import type { AuthEvent, AuthPrompt } from "@earendil-works/pi-ai";
import type { AssistantMessage, ImageContent, Message, Model } from "@earendil-works/pi-ai/compat";
import type { AssistantMessage, ImageContent, Message, Model, Usage } from "@earendil-works/pi-ai/compat";
import type {
AutocompleteItem,
AutocompleteProvider,
@ -204,12 +204,22 @@ type CompactionQueuedMessage = {
mode: "steer" | "followUp";
};
type RenderSessionItem = AgentMessage | Extract<SessionEntry, { type: "custom" }>;
type CompactionCostNotice = {
type: "compaction_cost";
kind: "compaction" | "branch_summary";
usage: Usage;
};
type RenderSessionItem = AgentMessage | Extract<SessionEntry, { type: "custom" }> | CompactionCostNotice;
function isCustomSessionEntry(item: RenderSessionItem): item is Extract<SessionEntry, { type: "custom" }> {
return "type" in item && item.type === "custom";
}
function isCompactionCostNotice(item: RenderSessionItem): item is CompactionCostNotice {
return "type" in item && item.type === "compaction_cost";
}
const DEAD_TERMINAL_ERROR_CODES = new Set(["EIO", "EPIPE", "ENOTCONN"]);
function isDeadTerminalError(error: unknown): boolean {
@ -3349,8 +3359,13 @@ export class InteractiveMode {
this.showStatus("Auto-compaction cancelled");
}
} else if (event.result) {
const entries = this.sessionManager.buildContextEntries();
if (entries[0]?.type !== "compaction") {
throw new Error("Completed compaction is missing from the session context");
}
this.chatContainer.clear();
this.rebuildChatFromMessages();
// The latest compaction is prepended for model context; append it below at its chronological position.
this.renderSessionEntries(entries.slice(1));
this.addMessageToChat(
createCompactionSummaryMessage(
event.result.summary,
@ -3358,6 +3373,13 @@ export class InteractiveMode {
new Date().toISOString(),
),
);
if (event.result.usage) {
this.addCompactionCostNotice({
type: "compaction_cost",
kind: "compaction",
usage: event.result.usage,
});
}
this.footer.invalidate();
} else if (event.errorMessage) {
if (event.reason === "manual") {
@ -3629,6 +3651,10 @@ export class InteractiveMode {
this.addCustomEntryToChat(item);
continue;
}
if (isCompactionCostNotice(item)) {
this.addCompactionCostNotice(item);
continue;
}
const message = item;
// Assistant messages need special handling for tool calls
@ -3706,11 +3732,32 @@ export class InteractiveMode {
if (entry.type === "custom") {
return [entry];
}
return sessionEntryToContextMessages(entry);
const messages = sessionEntryToContextMessages(entry);
if ((entry.type === "compaction" || entry.type === "branch_summary") && entry.usage && messages.length > 0) {
return [...messages, { type: "compaction_cost", kind: entry.type, usage: entry.usage }];
}
return messages;
});
this.renderSessionItems(items, options);
}
/**
* Render billing usage for a compaction or branch summary. The notice is derived
* from persisted summary usage and is not stored as a separate session entry.
*/
private addCompactionCostNotice(notice: CompactionCostNotice): void {
if (!this.settingsManager.getShowCacheMissNotices()) return;
const { usage } = notice;
const tokens = usage.input + usage.output + usage.cacheRead + usage.cacheWrite;
const cost = usage.cost.total >= 0.01 ? ` (~$${usage.cost.total.toFixed(2)})` : "";
const label = notice.kind === "compaction" ? "Compaction" : "Branch summary";
this.chatContainer.addChild(new Spacer(1));
this.chatContainer.addChild(
new Text(theme.fg("warning", `${label}: ${formatTokens(tokens)} tokens billed${cost}`), 1, 0),
);
}
/**
* Show a transcript notice when a completed assistant message paid for a
* significant cache miss. Only states observable facts: the miss itself,

View file

@ -1,5 +1,5 @@
import type { AgentMessage } from "@earendil-works/pi-agent-core";
import type { AssistantMessage, Model } from "@earendil-works/pi-ai";
import type { AssistantMessage, Context, Model } from "@earendil-works/pi-ai";
import { beforeEach, describe, expect, it, vi } from "vitest";
import {
type CompactionPreparation,
@ -115,7 +115,7 @@ describe("generateSummary reasoning options", () => {
expect(sessionIds[0]).not.toBe(sessionIds[1]);
});
it("honors a caller-supplied routing session without prompt caching", async () => {
it("honors caller-supplied cache retention and routing", async () => {
await completeSummarization(
createModel(false),
{ systemPrompt: "Summarize", messages: [] },
@ -124,7 +124,249 @@ describe("generateSummary reasoning options", () => {
expect(completeSimpleMock.mock.calls[0][2]).toMatchObject({
sessionId: "current-routing-session",
cacheRetention: "none",
cacheRetention: "long",
toolChoice: "none",
});
});
it("preserves the standalone split-turn summary prompt", async () => {
const preparation: CompactionPreparation = {
firstKeptEntryId: "entry-keep",
messagesToSummarize: [],
turnPrefixMessages: messages,
isSplitTurn: true,
tokensBefore: 100,
fileOps: { read: new Set(), written: new Set(), edited: new Set() },
settings: { enabled: true, reserveTokens: 2000, keepRecentTokens: 20 },
};
await compact(preparation, createModel(false), "test-key");
const requestContext = completeSimpleMock.mock.calls[0][1] as Context;
const prompt = JSON.stringify(requestContext.messages);
expect(prompt).toContain("This is the PREFIX of a turn that was too large to keep");
expect(prompt).toContain("<conversation>");
});
it("appends instructions to a cache-friendly source context", async () => {
const sourceContext: Context = {
systemPrompt: "You are a coding agent.",
messages: [{ role: "user", content: "Previous summary and original request", timestamp: 1 }],
tools: [],
};
const onPayload = async (payload: unknown) => payload;
const onResponse = async () => {};
const preparation: CompactionPreparation = {
firstKeptEntryId: "entry-keep",
messagesToSummarize: messages,
turnPrefixMessages: [],
isSplitTurn: false,
tokensBefore: 100,
previousSummary: "Previous summary",
fileOps: { read: new Set(), written: new Set(), edited: new Set() },
settings: { enabled: true, reserveTokens: 2000, keepRecentTokens: 20 },
};
await compact(
preparation,
createModel(false),
"test-key",
undefined,
undefined,
undefined,
undefined,
undefined,
undefined,
undefined,
undefined,
{
sourceContext,
requestOptions: {
sessionId: "routing-session",
onPayload,
onResponse,
transport: "websocket",
thinkingBudgets: { low: 1234 },
maxRetryDelayMs: 4321,
},
},
);
const requestContext = completeSimpleMock.mock.calls[0][1] as Context;
expect(requestContext.systemPrompt).toBe(sourceContext.systemPrompt);
expect(requestContext.tools).toBe(sourceContext.tools);
expect(requestContext.messages.slice(0, -1)).toEqual(sourceContext.messages);
const instruction = JSON.stringify(requestContext.messages.at(-1));
expect(instruction).toContain("existing structured summary of earlier conversation history");
expect(instruction).toContain("PRESERVE all existing information from the previous summary");
expect(instruction).not.toContain("<conversation>");
expect(instruction).not.toContain("<previous-summary>");
expect(completeSimpleMock.mock.calls[0][2]).toMatchObject({
cacheRetention: "short",
sessionId: "routing-session",
toolChoice: "none",
transport: "websocket",
thinkingBudgets: { low: 1234 },
maxRetryDelayMs: 4321,
});
expect(completeSimpleMock.mock.calls[0][2]?.onPayload).toBe(onPayload);
expect(completeSimpleMock.mock.calls[0][2]?.onResponse).toBe(onResponse);
expect(sourceContext.messages).toHaveLength(1);
});
it("falls back to standalone summarization when a cache-friendly source cannot leave the summary budget", async () => {
const oversizedToolResult: AgentMessage = {
role: "toolResult",
toolCallId: "tool-call-1",
toolName: "read",
content: [{ type: "text", text: "x".repeat(800_000) }],
isError: false,
timestamp: 1,
};
const sourceContext: Context = {
systemPrompt: "You are a coding agent.",
messages: [oversizedToolResult],
tools: [],
};
const preparation: CompactionPreparation = {
firstKeptEntryId: "entry-keep",
messagesToSummarize: [oversizedToolResult],
turnPrefixMessages: [],
isSplitTurn: false,
tokensBefore: 250_000,
fileOps: { read: new Set(), written: new Set(), edited: new Set() },
settings: { enabled: true, reserveTokens: 2000, keepRecentTokens: 20 },
};
await compact(
preparation,
createModel(false),
"test-key",
undefined,
undefined,
undefined,
undefined,
undefined,
undefined,
undefined,
undefined,
{ sourceContext, requestOptions: { sessionId: "routing-session" } },
);
const requestContext = completeSimpleMock.mock.calls[0][1] as Context;
const prompt = JSON.stringify(requestContext.messages);
expect(requestContext.systemPrompt).not.toBe(sourceContext.systemPrompt);
expect(requestContext.tools).toBeUndefined();
expect(prompt).toContain("<conversation>");
expect(prompt).toContain("more characters truncated");
expect(prompt).not.toContain("x".repeat(3000));
expect(completeSimpleMock.mock.calls[0][2]).toMatchObject({ cacheRetention: "none", toolChoice: "none" });
expect(completeSimpleMock.mock.calls[0][2]?.sessionId).not.toBe("routing-session");
});
it("falls back to standalone summarization for an oversized split-turn source", async () => {
const splitUser = { role: "user" as const, content: "Large final request", timestamp: 1 };
const oversizedToolResult: AgentMessage = {
role: "toolResult",
toolCallId: "tool-call-1",
toolName: "read",
content: [{ type: "text", text: "x".repeat(800_000) }],
isError: false,
timestamp: 2,
};
const turnPrefixSourceContext: Context = {
systemPrompt: "You are a coding agent.",
messages: [splitUser, oversizedToolResult],
tools: [],
};
const preparation: CompactionPreparation = {
firstKeptEntryId: "entry-keep",
messagesToSummarize: [],
turnPrefixMessages: [splitUser, oversizedToolResult],
isSplitTurn: true,
tokensBefore: 250_000,
fileOps: { read: new Set(), written: new Set(), edited: new Set() },
settings: { enabled: true, reserveTokens: 2000, keepRecentTokens: 20 },
};
await compact(
preparation,
createModel(false),
"test-key",
undefined,
undefined,
undefined,
undefined,
undefined,
undefined,
undefined,
undefined,
{ turnPrefixSourceContext, requestOptions: { sessionId: "routing-session" } },
);
const requestContext = completeSimpleMock.mock.calls[0][1] as Context;
const prompt = JSON.stringify(requestContext.messages);
expect(requestContext.systemPrompt).not.toBe(turnPrefixSourceContext.systemPrompt);
expect(requestContext.tools).toBeUndefined();
expect(prompt).toContain("This is the PREFIX of a turn that was too large to keep");
expect(prompt).toContain("more characters truncated");
expect(prompt).not.toContain("source conversation may also contain complete earlier turns");
expect(completeSimpleMock.mock.calls[0][2]).toMatchObject({ cacheRetention: "none", toolChoice: "none" });
expect(completeSimpleMock.mock.calls[0][2]?.sessionId).not.toBe("routing-session");
});
it("limits a cache-friendly split-turn summary to the final incomplete turn", async () => {
const earlierUser = { role: "user" as const, content: "Earlier request", timestamp: 1 };
const earlierAssistant: AssistantMessage = {
...mockSummaryResponse,
content: [{ type: "text", text: "Earlier work completed" }],
timestamp: 2,
};
const splitUser = { role: "user" as const, content: "Large final request", timestamp: 3 };
const earlyAssistant: AssistantMessage = {
...mockSummaryResponse,
content: [{ type: "text", text: "Early work in final turn" }],
timestamp: 4,
};
const turnPrefixSourceContext: Context = {
systemPrompt: "You are a coding agent.",
messages: [earlierUser, earlierAssistant, splitUser, earlyAssistant],
tools: [],
};
const preparation: CompactionPreparation = {
firstKeptEntryId: "entry-keep",
messagesToSummarize: [],
turnPrefixMessages: [splitUser, earlyAssistant],
isSplitTurn: true,
tokensBefore: 100,
fileOps: { read: new Set(), written: new Set(), edited: new Set() },
settings: { enabled: true, reserveTokens: 2000, keepRecentTokens: 20 },
};
await compact(
preparation,
createModel(false),
"test-key",
undefined,
undefined,
undefined,
undefined,
undefined,
undefined,
undefined,
undefined,
{ turnPrefixSourceContext },
);
const requestContext = completeSimpleMock.mock.calls[0][1] as Context;
expect(requestContext.messages.slice(0, -1)).toEqual(turnPrefixSourceContext.messages);
const instruction = JSON.stringify(requestContext.messages.at(-1));
expect(instruction).toContain("Summarize only the final, incomplete turn");
expect(instruction).toContain("last user-role request before this instruction");
expect(instruction).toContain("Do not summarize earlier turns");
expect(instruction).not.toContain("Earlier request");
expect(completeSimpleMock.mock.calls[0][2]).toMatchObject({
cacheRetention: "short",
toolChoice: "none",
});
});
@ -195,16 +437,22 @@ describe("generateSummary reasoning options", () => {
});
it("sets Anthropic refusal fallback from model metadata", async () => {
const fallbackCost = { input: 5, output: 25, cacheRead: 0.5, cacheWrite: 6.25 };
await generateSummary(
messages,
createModel(true, 8192, { allowedFallbackModels: ["claude-opus-4-8", "claude-opus-5"] }),
createModel(true, 8192, {
allowedFallbackModels: [
{ model: "claude-opus-4-8", cost: fallbackCost },
{ model: "claude-opus-5", cost: fallbackCost },
],
}),
2000,
"test-key",
);
expect(completeSimpleMock).toHaveBeenCalledTimes(1);
expect(completeSimpleMock.mock.calls[0][2]).toMatchObject({
refusalFallbacks: [{ model: "claude-opus-4-8" }],
refusalFallbacks: [{ model: "claude-opus-4-8", cost: fallbackCost }],
});
});

View file

@ -5,6 +5,7 @@ import { readFileSync } from "fs";
import { join } from "path";
import { beforeEach, describe, expect, it } from "vitest";
import {
type CompactionPreparation,
type CompactionSettings,
calculateContextTokens,
compact,
@ -150,6 +151,17 @@ function createCustomMessageEntry(content: string): CustomMessageEntry {
return entry;
}
function requireSourceMessages(
preparation: CompactionPreparation | undefined,
): asserts preparation is CompactionPreparation & {
sourceMessages: AgentMessage[];
turnPrefixSourceMessages: AgentMessage[];
} {
if (!preparation?.sourceMessages || !preparation.turnPrefixSourceMessages) {
throw new Error("Expected compaction source messages");
}
}
function extractText(messages: AgentMessage[]): string {
return messages
.map((message) => {
@ -461,6 +473,44 @@ describe("buildSessionContext", () => {
});
});
describe("prepareCompaction source messages", () => {
it("provides the active history prefix for a normal compaction", () => {
const u1 = createMessageEntry(createUserMessage("old user ".repeat(20)));
const a1 = createMessageEntry(createAssistantMessage("old assistant ".repeat(20)));
const u2 = createMessageEntry(createUserMessage("recent user"));
const a2 = createMessageEntry(createAssistantMessage("ok"));
const settings: CompactionSettings = { ...DEFAULT_COMPACTION_SETTINGS, keepRecentTokens: 3 };
const preparation = prepareCompaction([u1, a1, u2, a2], settings);
expect(preparation).toBeDefined();
requireSourceMessages(preparation);
expect(preparation.isSplitTurn).toBe(false);
expect(preparation.sourceMessages).toEqual(preparation.messagesToSummarize);
expect(extractText(preparation.sourceMessages)).toContain("old user");
expect(extractText(preparation.sourceMessages)).not.toContain("recent user");
expect(preparation.turnPrefixSourceMessages).toEqual([]);
});
it("provides the active prefix through a split turn", () => {
const user = createMessageEntry(createUserMessage("large request ".repeat(20)));
const earlyAssistant = createMessageEntry(createAssistantMessage("early work ".repeat(20)));
const keptAssistant = createMessageEntry(createAssistantMessage("kept"));
const settings: CompactionSettings = { ...DEFAULT_COMPACTION_SETTINGS, keepRecentTokens: 1 };
const preparation = prepareCompaction([user, earlyAssistant, keptAssistant], settings);
expect(preparation).toBeDefined();
requireSourceMessages(preparation);
expect(preparation.isSplitTurn).toBe(true);
expect(preparation.messagesToSummarize).toEqual([]);
expect(preparation.sourceMessages).toEqual([]);
expect(preparation.turnPrefixSourceMessages).toEqual(preparation.turnPrefixMessages);
expect(extractText(preparation.turnPrefixSourceMessages)).toContain("large request");
expect(extractText(preparation.turnPrefixSourceMessages)).not.toContain("kept");
});
});
describe("prepareCompaction with previous compaction", () => {
it("should skip repeated compactions when kept messages still fit", () => {
const u1 = createMessageEntry(createUserMessage("user msg 1 (summarized by compaction1)"));
@ -479,6 +529,33 @@ describe("prepareCompaction with previous compaction", () => {
expect(preparation).toBeUndefined();
});
it("preserves retained older summaries in the active source prefix", () => {
const u1 = createMessageEntry(createUserMessage("user 1"));
const a1 = createMessageEntry(createAssistantMessage("assistant 1"));
const u2 = createMessageEntry(createUserMessage("user 2 ".repeat(20)));
const a2 = createMessageEntry(createAssistantMessage("assistant 2 ".repeat(20)));
const compaction1 = createCompactionEntry("First summary", u1.id);
const u3 = createMessageEntry(createUserMessage("user 3 ".repeat(20)));
const a3 = createMessageEntry(createAssistantMessage("assistant 3 ".repeat(20)));
const compaction2 = createCompactionEntry("Second summary", u2.id);
const u4 = createMessageEntry(createUserMessage("user 4 ".repeat(20)));
const a4 = createMessageEntry(createAssistantMessage("kept"));
const settings: CompactionSettings = { ...DEFAULT_COMPACTION_SETTINGS, keepRecentTokens: 1 };
const pathEntries = [u1, a1, u2, a2, compaction1, u3, a3, compaction2, u4, a4];
const preparation = prepareCompaction(pathEntries, settings);
expect(preparation).toBeDefined();
requireSourceMessages(preparation);
const activeMessages = buildSessionContext(pathEntries).messages;
expect(preparation.sourceMessages).toEqual(activeMessages.slice(0, preparation.sourceMessages.length));
expect(preparation.turnPrefixSourceMessages).toEqual(
activeMessages.slice(0, preparation.turnPrefixSourceMessages.length),
);
const sourceSummaries = preparation.sourceMessages.filter((message) => message.role === "compactionSummary");
expect(sourceSummaries.map((message) => message.summary)).toEqual(["Second summary", "First summary"]);
});
it("should re-summarize previously kept messages when the recent window moves past them", () => {
const u1 = createMessageEntry(createUserMessage("user msg 1 (summarized by compaction1)".repeat(4)));
const a1 = createMessageEntry(createAssistantMessage("assistant msg 1".repeat(4)));
@ -497,11 +574,15 @@ describe("prepareCompaction with previous compaction", () => {
const preparation = prepareCompaction([u1, a1, u2, a2, u3, a3, compaction1, u4, a4], settings);
expect(preparation).toBeDefined();
const summarizedText = extractText(preparation!.messagesToSummarize);
requireSourceMessages(preparation);
const summarizedText = extractText(preparation.messagesToSummarize);
expect(summarizedText).toContain("user msg 2 - kept by compaction1");
expect(summarizedText).toContain("user msg 3 - kept by compaction1");
expect(summarizedText).not.toContain("First summary");
expect(preparation!.previousSummary).toBe("First summary");
expect(preparation.previousSummary).toBe("First summary");
expect(preparation.sourceMessages[0]?.role).toBe("compactionSummary");
expect(extractText(preparation.sourceMessages)).toContain("First summary");
expect(preparation.sourceMessages.slice(1)).toEqual(preparation.messagesToSummarize);
});
});

View file

@ -1,8 +1,140 @@
import type { Usage } from "@earendil-works/pi-ai";
import { Container } from "@earendil-works/pi-tui";
import { describe, expect, test, vi } from "vitest";
import type { SessionEntry } from "../src/core/session-manager.ts";
import { InteractiveMode } from "../src/modes/interactive/interactive-mode.ts";
import { initTheme } from "../src/modes/interactive/theme/theme.ts";
import { stripAnsi } from "../src/utils/ansi.ts";
describe("InteractiveMode compaction events", () => {
test("rebuilds chat and appends a synthetic compaction summary at the bottom", async () => {
test("uses the cache miss notice setting for compaction and branch summary costs", () => {
const usage: Usage = {
input: 10,
output: 20,
cacheRead: 30,
cacheWrite: 40,
totalTokens: 100,
cost: { input: 0.01, output: 0.02, cacheRead: 0.03, cacheWrite: 0.065, total: 0.125 },
};
const addCompactionCostNotice = Reflect.get(InteractiveMode.prototype, "addCompactionCostNotice") as (
this: { chatContainer: Container; settingsManager: { getShowCacheMissNotices(): boolean } },
notice: {
type: "compaction_cost";
kind: "compaction" | "branch_summary";
usage: Usage;
},
) => void;
initTheme("dark");
const enabled = {
chatContainer: new Container(),
settingsManager: { getShowCacheMissNotices: () => true },
};
addCompactionCostNotice.call(enabled, { type: "compaction_cost", kind: "compaction", usage });
addCompactionCostNotice.call(enabled, {
type: "compaction_cost",
kind: "branch_summary",
usage,
});
const output = stripAnsi(enabled.chatContainer.render(120).join("\n"));
expect(output).toContain("Compaction: 100 tokens billed (~$0.13)");
expect(output).toContain("Branch summary: 100 tokens billed (~$0.13)");
const disabled = {
chatContainer: new Container(),
settingsManager: { getShowCacheMissNotices: () => false },
};
addCompactionCostNotice.call(disabled, { type: "compaction_cost", kind: "compaction", usage });
expect(disabled.chatContainer.children).toHaveLength(0);
});
test("renders each compaction cost after its summary", () => {
const currentUsage: Usage = {
input: 10,
output: 20,
cacheRead: 30,
cacheWrite: 40,
totalTokens: 100,
cost: { input: 0.01, output: 0.02, cacheRead: 0.03, cacheWrite: 0.04, total: 0.1 },
};
const previousUsage: Usage = {
input: 1,
output: 2,
cacheRead: 3,
cacheWrite: 4,
totalTokens: 10,
cost: { input: 0.001, output: 0.002, cacheRead: 0.003, cacheWrite: 0.004, total: 0.01 },
};
const entries: SessionEntry[] = [
{
type: "compaction",
id: "current",
parentId: "previous",
timestamp: "2025-01-02T00:00:00Z",
summary: "current summary",
firstKeptEntryId: "kept",
tokensBefore: 200,
usage: currentUsage,
},
{
type: "compaction",
id: "previous",
parentId: null,
timestamp: "2025-01-01T00:00:00Z",
summary: "previous summary",
firstKeptEntryId: "kept",
tokensBefore: 100,
usage: previousUsage,
},
];
const fakeThis = { renderSessionItems: vi.fn() };
const renderSessionEntries = Reflect.get(InteractiveMode.prototype, "renderSessionEntries") as (
this: typeof fakeThis,
entries: SessionEntry[],
) => void;
renderSessionEntries.call(fakeThis, entries);
expect(fakeThis.renderSessionItems).toHaveBeenCalledWith(
[
expect.objectContaining({ role: "compactionSummary", summary: "current summary" }),
{ type: "compaction_cost", kind: "compaction", usage: currentUsage },
expect.objectContaining({ role: "compactionSummary", summary: "previous summary" }),
{ type: "compaction_cost", kind: "compaction", usage: previousUsage },
],
{},
);
});
test("renders retained entries and appends the latest summary cost at the bottom", async () => {
const usage: Usage = {
input: 10,
output: 20,
cacheRead: 30,
cacheWrite: 40,
totalTokens: 100,
cost: { input: 0.01, output: 0.02, cacheRead: 0.03, cacheWrite: 0.065, total: 0.125 },
};
const latestCompaction: SessionEntry = {
type: "compaction",
id: "latest",
parentId: "previous",
timestamp: "2025-01-02T00:00:00Z",
summary: "summary",
firstKeptEntryId: "kept",
tokensBefore: 123,
usage,
};
const previousCompaction: SessionEntry = {
type: "compaction",
id: "previous",
parentId: null,
timestamp: "2025-01-01T00:00:00Z",
summary: "previous summary",
firstKeptEntryId: "kept",
tokensBefore: 100,
usage,
};
const fakeThis = {
isInitialized: true,
footer: { invalidate: vi.fn() },
@ -11,8 +143,10 @@ describe("InteractiveMode compaction events", () => {
defaultEditor: {},
statusContainer: { clear: vi.fn() },
chatContainer: { clear: vi.fn() },
rebuildChatFromMessages: vi.fn(),
sessionManager: { buildContextEntries: vi.fn().mockReturnValue([latestCompaction, previousCompaction]) },
renderSessionEntries: vi.fn(),
addMessageToChat: vi.fn(),
addCompactionCostNotice: vi.fn(),
showError: vi.fn(),
showStatus: vi.fn(),
clearStatusIndicator: vi.fn(),
@ -26,7 +160,7 @@ describe("InteractiveMode compaction events", () => {
event: {
type: "compaction_end";
reason: "manual" | "threshold" | "overflow";
result: { tokensBefore: number; summary: string } | undefined;
result: { tokensBefore: number; summary: string; usage?: Usage } | undefined;
aborted: boolean;
willRetry: boolean;
errorMessage?: string;
@ -39,13 +173,14 @@ describe("InteractiveMode compaction events", () => {
result: {
tokensBefore: 123,
summary: "summary",
usage,
},
aborted: false,
willRetry: false,
});
expect(fakeThis.chatContainer.clear).toHaveBeenCalledTimes(1);
expect(fakeThis.rebuildChatFromMessages).toHaveBeenCalledTimes(1);
expect(fakeThis.renderSessionEntries).toHaveBeenCalledWith([previousCompaction]);
expect(fakeThis.addMessageToChat).toHaveBeenCalledTimes(1);
expect(fakeThis.addMessageToChat).toHaveBeenCalledWith(
expect.objectContaining({
@ -54,6 +189,11 @@ describe("InteractiveMode compaction events", () => {
summary: "summary",
}),
);
expect(fakeThis.addCompactionCostNotice).toHaveBeenCalledWith({
type: "compaction_cost",
kind: "compaction",
usage,
});
expect(fakeThis.flushCompactionQueue).toHaveBeenCalledWith({ willRetry: false });
});

View file

@ -1,8 +1,11 @@
import type { AgentMessage } from "@earendil-works/pi-agent-core";
import {
type AssistantMessage,
type Context,
createAssistantMessageEventStream,
fauxAssistantMessage,
type Model,
type SimpleStreamOptions,
} from "@earendil-works/pi-ai";
import { afterEach, describe, expect, it, vi } from "vitest";
import { estimateTokens } from "../../src/core/compaction/index.ts";
@ -47,10 +50,15 @@ function createAssistant(
};
}
function useSummaryStreamFn(harness: Harness, summary: string): () => number {
function useSummaryStreamFn(
harness: Harness,
summary: string,
onRequest?: (context: Context, options: SimpleStreamOptions | undefined) => void,
): () => number {
let callCount = 0;
harness.session.agent.streamFunction = (model) => {
harness.session.agent.streamFunction = (model, context, options) => {
callCount++;
onRequest?.(context, options);
const stream = createAssistantMessageEventStream();
queueMicrotask(() => {
const message: AssistantMessage = {
@ -246,6 +254,34 @@ describe("AgentSession compaction characterization", () => {
expect(harness.faux.state.callCount).toBe(1);
});
it("uses the standalone compaction request context", async () => {
const harness = await createHarness({ settings: { compaction: { keepRecentTokens: 1 } } });
harnesses.push(harness);
seedCompactableSession(harness);
const transformContext = vi.fn(async (messages: AgentMessage[]) => messages);
harness.session.agent.transformContext = transformContext;
harness.session.agent.sessionId = "active-routing-session";
harness.session.agent.transport = "websocket";
let requestContext: Context | undefined;
let requestOptions: SimpleStreamOptions | undefined;
useSummaryStreamFn(harness, "standalone summary", (context, options) => {
requestContext = context;
requestOptions = options;
});
await harness.session.compact();
expect(transformContext).not.toHaveBeenCalled();
expect(requestContext?.systemPrompt).not.toBe(harness.session.agent.state.systemPrompt);
expect(requestContext?.tools).toBeUndefined();
expect(JSON.stringify(requestContext?.messages)).toContain("<conversation>");
expect(requestOptions).toMatchObject({ cacheRetention: "none" });
expect(requestOptions?.sessionId).not.toBe("active-routing-session");
expect(requestOptions?.transport).toBeUndefined();
});
it("persists usage from pi-generated manual compaction", async () => {
const harness = await createHarness({ withConfiguredAuth: false });
harnesses.push(harness);

View file

@ -0,0 +1,50 @@
import { afterAll, describe, expect, it, vi } from "vitest";
const state = vi.hoisted(() => {
const originalGetBuiltinModule = Object.getOwnPropertyDescriptor(process, "getBuiltinModule");
const getBuiltinModule = process.getBuiltinModule.bind(process);
Object.defineProperty(process, "getBuiltinModule", {
configurable: true,
value: (id: string) => (id === "node:sea" ? { isSea: () => true } : getBuiltinModule(id)),
});
return {
originalGetBuiltinModule,
createJiti: vi.fn((_id: unknown, _options: unknown) => ({
import: vi.fn(async () => () => {}),
})),
};
});
vi.mock("jiti/static", () => ({ createJiti: state.createJiti }));
import { loadExtensions } from "../../../src/core/extensions/loader.ts";
interface JitiOptionsProbe {
alias?: unknown;
tryNative?: boolean;
virtualModules?: Record<string, unknown>;
}
afterAll(() => {
if (state.originalGetBuiltinModule) {
Object.defineProperty(process, "getBuiltinModule", state.originalGetBuiltinModule);
}
});
describe("Node SEA extension loading", () => {
it("uses bundled virtual modules instead of filesystem aliases", async () => {
const result = await loadExtensions(["/extension.ts"], "/");
expect(result.errors).toEqual([]);
expect(result.extensions).toHaveLength(1);
expect(state.createJiti).toHaveBeenCalledOnce();
const options = state.createJiti.mock.calls[0][1] as JitiOptionsProbe;
// Source TypeScript also uses virtual modules, so tryNative: false is what
// proves the compiled-binary branch took precedence over the source branch.
expect(options.tryNative).toBe(false);
expect(options.alias).toBeUndefined();
expect(options.virtualModules?.typebox).toBeDefined();
expect(options.virtualModules?.["@earendil-works/pi-coding-agent"]).toBeDefined();
});
});

View file

@ -5,6 +5,7 @@
### Fixed
- Fixed duplicate fullscreen right-click paste in VS Code-based terminals on Windows ([#8186](https://github.com/earendil-works/pi/issues/8186)).
- Fixed padded text exceeding narrow terminal widths ([#8252](https://github.com/earendil-works/pi/issues/8252)).
## [0.84.2] - 2026-08-14

View file

@ -60,15 +60,16 @@ export class Text implements Component {
// Replace tabs with 3 spaces
const normalizedText = this.text.replace(/\t/g, " ");
// Calculate content width (subtract left/right margins)
const contentWidth = Math.max(1, width - this.paddingX * 2);
// Reduce margins when necessary so content and padding fit within the available width.
const paddingX = Math.min(this.paddingX, Math.max(0, Math.floor((width - 1) / 2)));
const contentWidth = Math.max(1, width - paddingX * 2);
// Wrap text (this preserves ANSI codes but does NOT pad)
const wrappedLines = wrapTextWithAnsi(normalizedText, contentWidth);
// Add margins and background to each line
const leftMargin = " ".repeat(this.paddingX);
const rightMargin = " ".repeat(this.paddingX);
const leftMargin = " ".repeat(paddingX);
const rightMargin = " ".repeat(paddingX);
const contentLines: string[] = [];
for (const line of wrappedLines) {