From caa6967b62122bad12e4a2af287932b9e881bdf0 Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Tue, 20 Oct 2026 09:54:23 +0200 Subject: [PATCH] fix(ai): keep the prompt cache across tool changes on Responses (#11142) Models with additional_tools declare every tool in additional_tools items, starting with one at the head of the input, like Codex. A later item replaces same-name definitions from earlier items but cannot replace a top-level tool or remove a tool, so removals are announced as text and later updates restate the tools that are still unavailable. Tool search and Kimi tool messages move the top-level tool list once per removal or redeclared name and keep later loads anchored, instead of resending the current tool list on every load after the first removal. fixes #10940 --- packages/ai/CHANGELOG.md | 2 + packages/ai/README.md | 2 +- packages/ai/src/api/azure-openai-responses.ts | 16 +- packages/ai/src/api/openai-codex-responses.ts | 16 +- packages/ai/src/api/openai-completions.ts | 12 +- .../ai/src/api/openai-responses-shared.ts | 76 +++++- packages/ai/src/api/openai-responses.ts | 12 +- packages/ai/src/utils/transcript.ts | 58 ++-- .../ai/test/openai-responses-compat.test.ts | 3 +- .../ai/test/transcript-tool-changes.test.ts | 250 +++++++++++++++--- 10 files changed, 368 insertions(+), 79 deletions(-) diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index c83a7c931..d8b177257 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -5,6 +5,7 @@ ### Breaking Changes - Renamed provider request option `telemetryContext` to `telemetry?: TelemetrySpan`, without a compatibility alias. Pass a parent span, not an adapter. +- `resolveTranscriptTools()` now takes a `TranscriptToolSupport` (`"none"`, `"additions"`, or `"changes"`) instead of a boolean, and `TranscriptTools.anchorsAdditions` is replaced by `foldedSystemMessages`, the number of leading system messages folded into `requestTools` ([#10940](https://github.com/earendil-works/pi/issues/10940)) ### Added @@ -16,6 +17,7 @@ - Fixed whitespace-only tool output (for example a bare `\r\n` from bash) being sent as-is to OpenAI Completions, OpenAI Responses, and Google providers; it now uses the same placeholder as empty output, since some providers reject whitespace-only tool content with a 400 ([#8720](https://github.com/earendil-works/pi/issues/8720)) - Fixed automatic retry not triggering for gateway upstream failures such as `upstream_error: Upstream request failed`, `upstream_stream_read_error`, and `Upstream HTTP/2 stream failed` ([#10920](https://github.com/earendil-works/pi/issues/10920)) +- Fixed OpenAI Responses deferred tool loads rewriting the top-level tool list, and losing the prompt cache, on every load after a tool removal. Models with `additional_tools` now declare all tools in `additional_tools` items, starting with one at the head of the input, so removals (announced as text) and redefinitions no longer rewrite the prefix. Tool search and Kimi tool messages rewrite the top-level tools once per removal or redeclaration and keep later loads anchored ([#10940](https://github.com/earendil-works/pi/issues/10940)) ## [1.3.0] - 2026-10-15 diff --git a/packages/ai/README.md b/packages/ai/README.md index c0db958ab..cc958bc40 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -1604,7 +1604,7 @@ getCurrentTools(messages); // [] A custom `Provider` or `ProviderStreams` implementation reads the prompt and tools the same way from `context.messages`; `context.systemPrompt` and `context.tools` do not exist at that layer. -Models that accept system messages mid-conversation (`supportsMidConvoSystemMessages` in the model's compat settings, set by the generated catalog for verified models) receive each later system message in place, so the cached prefix stays intact; section changes are framed by name for the model. Every other model receives `collapseSystemMessages(transcript)`: the replayed prompt and current tools as the leading system message, with later system messages dropped. Anthropic models that also set `supportsMidConvoToolChanges` send tool changes as native blocks (`inline-tools-2026-09-15` beta): the top-level tool list holds the initial tools plus a stable deferred placeholder and never changes, later tools are defined by value in `tool_addition` blocks (a new definition under an existing name replaces the old one), and removals are `tool_removal` references, so tool changes do not invalidate the prompt cache. That needs at least one initial tool; otherwise the current tool list is sent at the top level with the system text only. OpenAI Responses models with `supportsAdditionalTools` or `supportsToolSearch` anchor additive tool changes at their message; everything else sends the current tool list at the top level. +Models that accept system messages mid-conversation (`supportsMidConvoSystemMessages` in the model's compat settings, set by the generated catalog for verified models) receive each later system message in place, so the cached prefix stays intact; section changes are framed by name for the model. Every other model receives `collapseSystemMessages(transcript)`: the replayed prompt and current tools as the leading system message, with later system messages dropped. Anthropic models that also set `supportsMidConvoToolChanges` send tool changes as native blocks (`inline-tools-2026-09-15` beta): the top-level tool list holds the initial tools plus a stable deferred placeholder and never changes, later tools are defined by value in `tool_addition` blocks (a new definition under an existing name replaces the old one), and removals are `tool_removal` references, so tool changes do not invalidate the prompt cache. That needs at least one initial tool; otherwise the current tool list is sent at the top level with the system text only. OpenAI Responses models with `supportsAdditionalTools` declare every tool in `additional_tools` items, like Codex: the initial tools in one at the head of the input and later additions and redefinitions at their message, with no top-level tool list. A later item replaces same-name definitions from earlier items but cannot replace a top-level tool or remove any tool, so removals are announced as text, later updates restate the tools that are still unavailable, and calls to removed tools fail at execution. OpenAI Responses models with only `supportsToolSearch`, and Kimi models with `supportsMidConvoToolAdditions`, anchor new tools at their message; a removal or a redeclared name moves the tool state up to that point into the top-level list once. Everything else sends the current tool list at the top level. ## Context Serialization diff --git a/packages/ai/src/api/azure-openai-responses.ts b/packages/ai/src/api/azure-openai-responses.ts index 9b07bc9da..c65d4a236 100644 --- a/packages/ai/src/api/azure-openai-responses.ts +++ b/packages/ai/src/api/azure-openai-responses.ts @@ -11,7 +11,12 @@ import { getDeclaredTools, resolveTranscript, resolveTranscriptTools } from "../ import { type AzureEndpointOptions, resolveAzureConfig, resolveDeploymentName } from "./azure-openai-config.ts"; import { createGrammarToolInputProperties } from "./constrained-sampling.ts"; import { clampOpenAIPromptCacheKey } from "./openai-prompt-cache.ts"; -import { convertResponsesMessages, convertResponsesTools, processResponsesStream } from "./openai-responses-shared.ts"; +import { + convertResponsesMessages, + convertResponsesTools, + getResponsesToolSupport, + processResponsesStream, +} from "./openai-responses-shared.ts"; import { buildBaseOptions, resolveSamplingParams } from "./simple-options.ts"; const AZURE_TOOL_CALL_PROVIDERS = new Set(["openai", "openai-codex", "opencode", "azure"]); @@ -183,7 +188,14 @@ function buildParams( ) { const supportsAdditionalTools = model.compat?.supportsAdditionalTools ?? false; const supportsToolSearch = model.compat?.supportsToolSearch ?? false; - const transcriptTools = resolveTranscriptTools(context.messages, supportsAdditionalTools || supportsToolSearch); + const transcriptTools = resolveTranscriptTools( + context.messages, + getResponsesToolSupport({ + supportsMidConvoSystemMessages: model.compat?.supportsMidConvoSystemMessages ?? false, + supportsAdditionalTools, + supportsToolSearch, + }), + ); const messages = convertResponsesMessages(model, context, AZURE_TOOL_CALL_PROVIDERS, { grammarToolInputProperties, supportsMidConvoSystemMessages: model.compat?.supportsMidConvoSystemMessages ?? false, diff --git a/packages/ai/src/api/openai-codex-responses.ts b/packages/ai/src/api/openai-codex-responses.ts index 0245a0244..5ce24e068 100644 --- a/packages/ai/src/api/openai-codex-responses.ts +++ b/packages/ai/src/api/openai-codex-responses.ts @@ -42,7 +42,12 @@ import { import { uuidv7 } from "../utils/uuid.ts"; import { createGrammarToolInputProperties } from "./constrained-sampling.ts"; import { clampOpenAIPromptCacheKey } from "./openai-prompt-cache.ts"; -import { convertResponsesMessages, convertResponsesTools, processResponsesStream } from "./openai-responses-shared.ts"; +import { + convertResponsesMessages, + convertResponsesTools, + getResponsesToolSupport, + processResponsesStream, +} from "./openai-responses-shared.ts"; import { buildBaseOptions } from "./simple-options.ts"; // ============================================================================ @@ -539,7 +544,14 @@ function buildRequestBody( const supportsOpenAIGrammarTools = model.compat?.supportsOpenAIGrammarTools ?? false; const supportsAdditionalTools = model.compat?.supportsAdditionalTools ?? false; const supportsToolSearch = model.compat?.supportsToolSearch ?? false; - const transcriptTools = resolveTranscriptTools(context.messages, supportsAdditionalTools || supportsToolSearch); + const transcriptTools = resolveTranscriptTools( + context.messages, + getResponsesToolSupport({ + supportsMidConvoSystemMessages: model.compat?.supportsMidConvoSystemMessages ?? false, + supportsAdditionalTools, + supportsToolSearch, + }), + ); const messages = convertResponsesMessages(model, context, CODEX_TOOL_CALL_PROVIDERS, { includeSystemPrompt: false, grammarToolInputProperties, diff --git a/packages/ai/src/api/openai-completions.ts b/packages/ai/src/api/openai-completions.ts index b8310c496..8686373c9 100644 --- a/packages/ai/src/api/openai-completions.ts +++ b/packages/ai/src/api/openai-completions.ts @@ -814,7 +814,9 @@ function buildParams( ) { const transcriptTools = resolveTranscriptTools( context.messages, - compat.supportsMidConvoSystemMessages === true && compat.supportsMidConvoToolAdditions === true, + compat.supportsMidConvoSystemMessages === true && compat.supportsMidConvoToolAdditions === true + ? "additions" + : "none", ); const messages = convertMessages(model, context, compat, { grammarToolInputProperties, @@ -1230,11 +1232,14 @@ export function convertMessages( const transformedMessages = transformMessages(normalizedContext.messages, model, (id) => normalizeToolCallId(id)); const transcriptTools = resolveTranscriptTools( normalizedContext.messages, - compat.supportsMidConvoSystemMessages === true && compat.supportsMidConvoToolAdditions === true, + compat.supportsMidConvoSystemMessages === true && compat.supportsMidConvoToolAdditions === true + ? "additions" + : "none", ); const instructionRole = model.reasoning && compat.supportsDeveloperRole ? "developer" : "system"; let lastRole: string | null = null; + let systemIndex = 0; for (let i = 0; i < transformedMessages.length; i++) { const msg = transformedMessages[i]; @@ -1248,7 +1253,8 @@ export function convertMessages( } if (msg.role === "system") { - const addedTools = i > 0 && transcriptTools.anchorsAdditions ? (msg.toolsAdded ?? []) : []; + // transformMessages keeps system messages in order, so their position matches the transcript. + const addedTools = systemIndex++ >= transcriptTools.foldedSystemMessages ? (msg.toolsAdded ?? []) : []; if (addedTools.length > 0) { const kimiToolMessage: KimiToolSystemMessageParam = { role: "system", diff --git a/packages/ai/src/api/openai-responses-shared.ts b/packages/ai/src/api/openai-responses-shared.ts index 14cb28a77..2c4fb8504 100644 --- a/packages/ai/src/api/openai-responses-shared.ts +++ b/packages/ai/src/api/openai-responses-shared.ts @@ -36,7 +36,7 @@ import { shortHash } from "../utils/hash.ts"; import { parseStreamingJson, parseToolCallArguments } from "../utils/json-parse.ts"; import { sanitizeSurrogates } from "../utils/sanitize-unicode.ts"; import { getSystemMessageText, renderSystemMessageUpdate } from "../utils/text.ts"; -import { resolveTranscript, resolveTranscriptTools } from "../utils/transcript.ts"; +import { resolveTranscript, resolveTranscriptTools, type TranscriptToolSupport } from "../utils/transcript.ts"; import { appendGrammarToolInputJsonDelta, type GrammarToolInputJsonBuffer, @@ -153,6 +153,41 @@ export interface ConvertResponsesToolsOptions { // Message conversion // ============================================================================= +/** + * How a Responses transport expresses tool changes. + * + * `additional_tools` items accumulate: a later item adds tools and replaces same-name + * definitions from earlier items, but it cannot replace a top-level tool and cannot remove any + * tool (verified with strict schemas on the OpenAI, Codex, and Copilot backends, October 2026). + * Like Codex, every tool is therefore declared in `additional_tools` items, starting with one + * at the head of the input, and removals are announced as text. Synthetic tool search can + * only add tools. Without mid-conversation system messages there is nothing to anchor. + */ +export function getResponsesToolSupport(options: { + supportsMidConvoSystemMessages?: boolean; + supportsAdditionalTools?: boolean; + supportsToolSearch?: boolean; +}): TranscriptToolSupport { + if (options.supportsMidConvoSystemMessages && options.supportsAdditionalTools) return "changes"; + return options.supportsToolSearch ? "additions" : "none"; +} + +/** + * Precedes every additional_tools item after the first. Without it, models read the latest item as + * the complete tool set once a removal was announced; Codex precedes its incremental tool batches + * with the same guidance. Restating the tools that are still unavailable keeps smaller models + * (gpt-5.6-luna, gpt-6-luna) from calling them again after later loads (verified October 2026). + */ +function renderIncrementalToolsNotice(unavailable: ReadonlySet): string { + const notice = + "This is an incremental tools update. Previously declared tools remain available unless explicitly marked unavailable. A tool declared again here is available again, and its latest definition replaces the earlier one."; + return unavailable.size > 0 ? `${notice}\nStill unavailable: ${[...unavailable].join(", ")}.` : notice; +} + +function renderToolRemovalNotice(names: readonly string[]): string { + return `The following tools are no longer available. Do not call them:\n${names.map((name) => `- ${name}`).join("\n")}`; +} + export function convertResponsesMessages( model: Model, context: TranscriptContext, @@ -188,22 +223,29 @@ export function convertResponsesMessages( }; const transformedMessages = transformMessages(normalizedContext.messages, model, normalizeToolCallId); - const transcriptTools = resolveTranscriptTools( - normalizedContext.messages, - (options?.supportsAdditionalTools ?? false) || (options?.supportsToolSearch ?? false), - ); + const toolSupport = getResponsesToolSupport(options ?? {}); + const transcriptTools = resolveTranscriptTools(normalizedContext.messages, toolSupport); + let declaredAdditionalTools = false; + /** Tools announced as removed and not declared again since. */ + const unavailableTools = new Set(); const appendSystemToolAdditions = (message: SystemMessage, seed: string): void => { - const tools = transcriptTools.anchorsAdditions ? (message.toolsAdded ?? []) : []; + const tools = message.toolsAdded ?? []; + for (const tool of message.toolsRemoved ?? []) unavailableTools.add(tool.name); + for (const tool of tools) unavailableTools.delete(tool.name); if (tools.length === 0) return; - if (options?.supportsAdditionalTools) { + if (toolSupport === "changes") { + if (declaredAdditionalTools) { + messages.push({ role: instructionRole, content: renderIncrementalToolsNotice(unavailableTools) }); + } + declaredAdditionalTools = true; messages.push({ type: "additional_tools", role: "developer", - tools: convertResponsesTools(tools, options.toolOptions), + tools: convertResponsesTools(tools, options?.toolOptions), } satisfies ResponseInputItem); return; } - if (!options?.supportsToolSearch) return; + if (toolSupport !== "additions") return; const names = tools.map((tool) => tool.name); const callId = `pi_tool_load_${shortHash(`${seed}:${names.join(",")}`)}`; messages.push({ @@ -218,7 +260,7 @@ export function convertResponsesMessages( call_id: callId, execution: "client", status: "completed", - tools: convertResponsesTools(tools, { ...options.toolOptions, toolSearchResult: true }), + tools: convertResponsesTools(tools, { ...options?.toolOptions, toolSearchResult: true }), } satisfies ResponseToolSearchOutputItemParam); }; const includeInitialSystemMessage = options?.includeSystemPrompt ?? true; @@ -233,12 +275,22 @@ export function convertResponsesMessages( let msgIndex = 0; let sourceIndex = 0; + let systemIndex = 0; for (const msg of transformedMessages) { const isLeadingSystemMessage = sourceIndex++ === 0 && msg.role === "system"; if (msg.role === "system") { - if (!isLeadingSystemMessage) appendSystemToolAdditions(msg, `system:${msgIndex}`); + // transformMessages keeps system messages in order, so their position matches the transcript. + const anchorsToolChanges = systemIndex++ >= transcriptTools.foldedSystemMessages; + if (anchorsToolChanges) appendSystemToolAdditions(msg, `system:${msgIndex}`); if (!isLeadingSystemMessage || includeInitialSystemMessage) { - const text = isLeadingSystemMessage ? getSystemMessageText(msg) : renderSystemMessageUpdate(msg); + const parts = [isLeadingSystemMessage ? getSystemMessageText(msg) : renderSystemMessageUpdate(msg)]; + if (anchorsToolChanges) { + // A redefinition is a removal plus an addition of the same name; only true removals are announced. + const added = new Set((msg.toolsAdded ?? []).map((tool) => tool.name)); + const removed = (msg.toolsRemoved ?? []).map((tool) => tool.name).filter((name) => !added.has(name)); + if (removed.length > 0) parts.push(renderToolRemovalNotice(removed)); + } + const text = parts.filter((part) => part.length > 0).join("\n\n"); if (text.length > 0) { messages.push({ role: instructionRole, content: sanitizeSurrogates(text) }); } diff --git a/packages/ai/src/api/openai-responses.ts b/packages/ai/src/api/openai-responses.ts index dd8cf0d66..697abbeb8 100644 --- a/packages/ai/src/api/openai-responses.ts +++ b/packages/ai/src/api/openai-responses.ts @@ -25,7 +25,12 @@ import { getDeclaredTools, resolveTranscript, resolveTranscriptTools } from "../ import { createGrammarToolInputProperties } from "./constrained-sampling.ts"; import { buildCopilotDynamicHeaders, hasCopilotVisionInput } from "./github-copilot-headers.ts"; import { clampOpenAIPromptCacheKey } from "./openai-prompt-cache.ts"; -import { convertResponsesMessages, convertResponsesTools, processResponsesStream } from "./openai-responses-shared.ts"; +import { + convertResponsesMessages, + convertResponsesTools, + getResponsesToolSupport, + processResponsesStream, +} from "./openai-responses-shared.ts"; import { buildBaseOptions, resolveSamplingParams } from "./simple-options.ts"; const OPENAI_TOOL_CALL_PROVIDERS = new Set(["openai", "openai-codex", "opencode"]); @@ -336,10 +341,7 @@ function buildParams( grammarToolInputProperties: ReadonlyMap, configurationUpdateEffort: string | undefined, ) { - const transcriptTools = resolveTranscriptTools( - context.messages, - compat.supportsAdditionalTools || compat.supportsToolSearch, - ); + const transcriptTools = resolveTranscriptTools(context.messages, getResponsesToolSupport(compat)); const messages = convertResponsesMessages(model, context, OPENAI_TOOL_CALL_PROVIDERS, { grammarToolInputProperties, supportsMidConvoSystemMessages: compat.supportsMidConvoSystemMessages, diff --git a/packages/ai/src/utils/transcript.ts b/packages/ai/src/utils/transcript.ts index 1212696cc..5e359dd0b 100644 --- a/packages/ai/src/utils/transcript.ts +++ b/packages/ai/src/utils/transcript.ts @@ -210,28 +210,56 @@ export function hasNonAdditiveToolChanges(messages: TranscriptMessages): boolean return false; } +/** + * How a transport can express tool changes after the first request. + * + * - `"none"`: only the top-level tool list exists; it always holds the current tools. + * - `"additions"`: tools can be added at a later message, but not removed or redefined. + * - `"changes"`: every tool, including the initial ones, is declared in place, and a later + * declaration under the same name replaces the earlier one. Removals are not expressible on + * the wire, so the transport announces them as text and the removed tools stay declared. + */ +export type TranscriptToolSupport = "none" | "additions" | "changes"; + export interface TranscriptTools { /** Tools sent in the top-level request field. */ requestTools: Tool[]; /** - * Whether later system messages carry their own `toolsAdded` as in-place additions. - * When false, `requestTools` already holds the complete current tool set. + * Number of leading system messages whose tool changes `requestTools` already holds. Every + * later system message carries its own tool changes in place. */ - anchorsAdditions: boolean; + foldedSystemMessages: number; } /** - * Split tool declarations between the top-level request field and in-place additions. - * Transports that can anchor additions at a system message keep the initial tools at the - * top and load later ones where they appear; that only works when no tool was removed or - * redeclared, so everything else sends the current tool list. + * Split tool declarations between the top-level request field and in-place changes. + * + * With `"changes"`, the top-level list stays empty and every system message, including the + * leading one, declares its tools in place: an in-place declaration can replace an earlier + * in-place one, but not a top-level one. With `"additions"`, a removal or redefinition cannot be + * expressed in place, so the top-level list becomes the tool state after the last such change; + * later additions stay anchored, and the cached prefix is invalidated once per removal instead + * of on every later addition. With `"none"`, the top-level list is the current tool set. */ -export function resolveTranscriptTools(messages: TranscriptMessages, supportsToolAdditions: boolean): TranscriptTools { - const anchorsAdditions = supportsToolAdditions && !hasNonAdditiveToolChanges(messages); - return { - requestTools: anchorsAdditions - ? (getInitialSystemMessage(messages)?.toolsAdded ?? []) - : getCurrentTools(messages), - anchorsAdditions, - }; +export function resolveTranscriptTools(messages: TranscriptMessages, support: TranscriptToolSupport): TranscriptTools { + const systemMessages = messages.filter(isSystemMessage); + if (support === "none") { + return { requestTools: getCurrentTools(systemMessages), foldedSystemMessages: systemMessages.length }; + } + if (support === "changes") return { requestTools: [], foldedSystemMessages: 0 }; + let folded = getInitialSystemMessage(messages) ? 1 : 0; + if (support === "additions") { + // A name declared before counts as a redeclaration even after its removal: Kimi K3 ignores a + // removed tool that a later tool message declares again (verified October 2026). + const declared = new Set(); + for (let index = 0; index < systemMessages.length; index++) { + const message = systemMessages[index]; + const added = message.toolsAdded ?? []; + if ((message.toolsRemoved?.length ?? 0) > 0 || added.some((tool) => declared.has(tool.name))) { + folded = Math.max(folded, index + 1); + } + for (const tool of added) declared.add(tool.name); + } + } + return { requestTools: getCurrentTools(systemMessages.slice(0, folded)), foldedSystemMessages: folded }; } diff --git a/packages/ai/test/openai-responses-compat.test.ts b/packages/ai/test/openai-responses-compat.test.ts index 31aa59c02..8295cb7a8 100644 --- a/packages/ai/test/openai-responses-compat.test.ts +++ b/packages/ai/test/openai-responses-compat.test.ts @@ -148,9 +148,10 @@ describe("openai-responses provider defaults", () => { if (event.type === "done" || event.type === "error") break; } + // gpt-5.4 declares its tools in additional_tools items; tool_choice applies to them. expect(capturedPayload).toMatchObject({ tool_choice: "required", - tools: [expect.objectContaining({ name: "ping" })], + input: [{ type: "additional_tools", tools: [expect.objectContaining({ name: "ping" })] }, expect.anything()], }); }); diff --git a/packages/ai/test/transcript-tool-changes.test.ts b/packages/ai/test/transcript-tool-changes.test.ts index 1a1a99b39..5e5f935a8 100644 --- a/packages/ai/test/transcript-tool-changes.test.ts +++ b/packages/ai/test/transcript-tool-changes.test.ts @@ -1,7 +1,8 @@ import { Type } from "typebox"; import { describe, expect, test } from "vitest"; import { getModel, streamSimple } from "../src/compat.ts"; -import type { Api, Context, Model, Tool } from "../src/types.ts"; +import type { Api, Context, Message, Model, SystemMessage, Tool } from "../src/types.ts"; +import { resolveTranscriptTools } from "../src/utils/transcript.ts"; class PayloadCaptured extends Error {} @@ -34,6 +35,60 @@ const modelBase = { const baseTool = tool("base_tool"); const lateTool = tool("late_tool"); +const dropTool = tool("drop_tool"); + +/** + * Capture one request after each step: initial tools, removal of drop_tool, then two loads. + * Returns the payloads after the removal and after each load, plus the one before the removal. + */ +async function captureToolHistory(model: Model): Promise { + const steps: Message[][] = [ + [{ role: "user", content: "first", timestamp: 1 }], + [ + { role: "system", content: "", toolsRemoved: [{ name: "drop_tool" }], timestamp: 2 }, + { role: "user", content: "after removal", timestamp: 3 }, + ], + [ + { role: "system", content: "", toolsAdded: [lateTool], timestamp: 4 }, + { role: "user", content: "after first load", timestamp: 5 }, + ], + [ + { role: "system", content: "", toolsAdded: [tool("later_tool")], timestamp: 6 }, + { role: "user", content: "after second load", timestamp: 7 }, + ], + ]; + const messages: Message[] = [ + { role: "system", content: "base prompt", toolsAdded: [baseTool, dropTool], timestamp: 0 }, + ]; + const payloads: T[] = []; + for (const step of steps) { + messages.push(...step); + payloads.push(await capturePayload(model, { messages: [...messages] })); + } + return payloads; +} + +/** Every request's message list starts with the previous request's message list. */ +function expectGrowingPrefix(inputs: readonly unknown[][]): void { + for (let index = 1; index < inputs.length; index++) { + const previous = inputs[index - 1] ?? []; + expect(inputs[index]?.slice(0, previous.length)).toEqual(previous); + } +} + +type ResponsesPayload = { + tools?: Array<{ name: string; description?: string }>; + input: Array<{ type?: string; role?: string; content?: string; tools?: Array<{ name: string }> }>; +}; + +const additionalToolsModel: Model<"openai-responses"> = { + ...modelBase, + id: "gpt-5.4", + name: "GPT-5.4", + api: "openai-responses", + provider: "openai", + compat: { supportsMidConvoSystemMessages: true, supportsAdditionalTools: true }, +}; const context: Context = { messages: [ { @@ -232,29 +287,23 @@ describe("transcript system messages", () => { expect(payload.messages.map((message) => message.role)).toEqual(["user"]); }); - test("anchors OpenAI additions at their developer message", async () => { - const model: Model<"openai-responses"> = { - ...modelBase, - id: "gpt-5.4", - name: "GPT-5.4", - api: "openai-responses", - provider: "openai", - compat: { supportsMidConvoSystemMessages: true, supportsAdditionalTools: true }, - }; - const payload = await capturePayload<{ - tools?: Array<{ name: string }>; - input: Array<{ type?: string; role?: string; content?: string; tools?: Array<{ name: string }> }>; - }>(model, additionContext); + test("declares every OpenAI tool in additional_tools at its developer message", async () => { + const payload = await capturePayload(additionalToolsModel, additionContext); - expect(payload.tools?.map((value) => value.name)).toEqual(["base_tool"]); - expect(payload.input.find((item) => item.type === "additional_tools")?.tools?.map((value) => value.name)).toEqual( - ["late_tool"], - ); + // A later additional_tools item cannot replace a top-level tool, so no tool is declared there. + expect(payload.tools).toBeUndefined(); expect( - payload.input - .filter((item) => item.role === "developer" && item.type === undefined) - .map((item) => item.content), - ).toEqual(["base prompt", "updated guidance"]); + payload.input.filter((item) => item.type === "additional_tools").map((item) => item.tools?.[0]?.name), + ).toEqual(["base_tool", "late_tool"]); + // The incremental notice keeps models from reading a later item as the complete tool set. + expect(payload.input.map((item) => item.type ?? item.content)).toEqual([ + "additional_tools", + "base prompt", + expect.any(Array), + expect.stringContaining("incremental tools update"), + "additional_tools", + "updated guidance", + ]); }); test("maps system-message additions into synthetic tool search", async () => { @@ -297,23 +346,131 @@ describe("transcript system messages", () => { expect(payload.input[0]?.content).toBe("base prompt\n\nupdated guidance\n\n\nnew rules\n"); }); - test("falls back to the complete current tool state when removals are unsupported", async () => { - const model: Model<"openai-responses"> = { - ...modelBase, - id: "gpt-5.4", - name: "GPT-5.4", - api: "openai-responses", - provider: "openai", - compat: { supportsMidConvoSystemMessages: true, supportsAdditionalTools: true }, - }; - const payload = await capturePayload<{ - tools?: Array<{ name: string }>; - input: Array<{ type?: string; role?: string }>; - }>(model, context); + test("announces OpenAI removals as text", async () => { + const payload = await capturePayload(additionalToolsModel, context); - expect(payload.tools?.map((value) => value.name)).toEqual(["late_tool"]); - expect(payload.input.some((item) => item.type === "additional_tools")).toBe(false); - expect(payload.input.filter((item) => item.role === "developer")).toHaveLength(2); + expect(payload.tools).toBeUndefined(); + expect( + payload.input.filter((item) => item.type === "additional_tools").map((item) => item.tools?.[0]?.name), + ).toEqual(["base_tool", "late_tool"]); + const update = payload.input.at(-1)?.content; + expect(update).toContain("updated guidance"); + expect(update).toContain("The following tools are no longer available. Do not call them:\n- base_tool"); + }); + + test("redefines an OpenAI tool with additional_tools instead of announcing a removal", async () => { + const payload = await capturePayload(additionalToolsModel, { + messages: [ + { role: "system", content: "base prompt", toolsAdded: [baseTool], timestamp: 0 }, + { role: "user", content: "before", timestamp: 1 }, + { + role: "system", + content: "", + toolsRemoved: [{ name: "base_tool" }], + toolsAdded: [{ ...baseTool, description: "changed" }], + timestamp: 2, + }, + ], + }); + + expect(payload.tools).toBeUndefined(); + expect(payload.input).toMatchObject([ + { type: "additional_tools", tools: [{ name: "base_tool", description: "base_tool tool" }] }, + { role: "developer", content: "base prompt" }, + { role: "user" }, + { role: "developer", content: expect.stringContaining("latest definition replaces the earlier one") }, + { type: "additional_tools", tools: [{ name: "base_tool", description: "changed" }] }, + ]); + expect(payload.input).toHaveLength(5); + }); + + // #10940: deferred loads after a removal must not rewrite the cached prefix. + test("keeps the OpenAI request prefix across a removal and later loads", async () => { + const payloads = await captureToolHistory(additionalToolsModel); + + for (const payload of payloads) expect(payload.tools).toBeUndefined(); + expectGrowingPrefix(payloads.map((payload) => payload.input)); + const last = payloads.at(-1); + expect( + last?.input.filter((item) => item.type === "additional_tools").map((item) => item.tools?.map((t) => t.name)), + ).toEqual([["base_tool", "drop_tool"], ["late_tool"], ["later_tool"]]); + expect( + last?.input.some((item) => item.content?.includes("no longer available. Do not call them:\n- drop_tool")), + ).toBe(true); + // Each later load restates the tools that are still unavailable. + expect( + last?.input.filter((item) => item.content?.includes("incremental tools update")).map((item) => item.content), + ).toEqual([ + expect.stringContaining("\nStill unavailable: drop_tool."), + expect.stringContaining("\nStill unavailable: drop_tool."), + ]); + }); + + test("stops listing a tool as unavailable once it is declared again", async () => { + const payload = await capturePayload(additionalToolsModel, { + messages: [ + { role: "system", content: "base prompt", toolsAdded: [baseTool, dropTool], timestamp: 0 }, + { role: "system", content: "", toolsRemoved: [{ name: "drop_tool" }], timestamp: 1 }, + { role: "system", content: "", toolsAdded: [dropTool], timestamp: 2 }, + { role: "user", content: "go", timestamp: 3 }, + ], + }); + + const notice = payload.input.find((item) => item.content?.includes("incremental tools update"))?.content; + expect(notice).toBeDefined(); + expect(notice).not.toContain("Still unavailable"); + expect(payload.input.at(-2)).toMatchObject({ type: "additional_tools", tools: [{ name: "drop_tool" }] }); + }); + + // #10940: synthetic tool search cannot remove tools, so a removal rewrites the prefix once. + test("keeps the OpenAI tool search prefix across loads after a removal", async () => { + const model: Model<"openai-responses"> = { + ...additionalToolsModel, + compat: { supportsMidConvoSystemMessages: true, supportsToolSearch: true }, + }; + const [beforeRemoval, ...afterRemoval] = await captureToolHistory(model); + + expect(beforeRemoval?.tools?.map((value) => value.name)).toEqual(["base_tool", "drop_tool"]); + for (const payload of afterRemoval) expect(payload.tools?.map((value) => value.name)).toEqual(["base_tool"]); + expectGrowingPrefix(afterRemoval.map((payload) => payload.input)); + expect( + afterRemoval + .at(-1) + ?.input.filter((item) => item.type === "tool_search_output") + .map((item) => item.tools?.[0]?.name), + ).toEqual(["late_tool", "later_tool"]); + }); + + // #10940: removals and redeclarations move the top-level boundary once; plain additions stay anchored. + test("resolves the top-level tool boundary per transport", () => { + const messages = [ + { role: "system", content: "base prompt", toolsAdded: [baseTool, dropTool], timestamp: 0 }, + { role: "system", content: "", toolsRemoved: [{ name: "drop_tool" }], timestamp: 1 }, + { role: "system", content: "", toolsAdded: [dropTool], timestamp: 2 }, + { role: "system", content: "", toolsAdded: [{ ...lateTool, description: "x" }], timestamp: 3 }, + { role: "system", content: "", toolsAdded: [{ ...lateTool, description: "y" }], timestamp: 4 }, + ] satisfies SystemMessage[]; + + expect(resolveTranscriptTools(messages.slice(0, 2), "additions")).toEqual({ + requestTools: [baseTool], + foldedSystemMessages: 2, + }); + // Kimi K3 ignores a removed tool that a later tool message declares again, so the re-added + // drop_tool moves the boundary; the new late_tool after it stays anchored. + expect(resolveTranscriptTools(messages.slice(0, 4), "additions")).toEqual({ + requestTools: [baseTool, dropTool], + foldedSystemMessages: 3, + }); + // The redefinition of late_tool moves the boundary to its message. + expect(resolveTranscriptTools(messages, "additions")).toEqual({ + requestTools: [baseTool, dropTool, { ...lateTool, description: "y" }], + foldedSystemMessages: 5, + }); + expect(resolveTranscriptTools(messages, "changes")).toEqual({ requestTools: [], foldedSystemMessages: 0 }); + expect(resolveTranscriptTools(messages, "none")).toEqual({ + requestTools: [baseTool, dropTool, { ...lateTool, description: "y" }], + foldedSystemMessages: 5, + }); }); test("anchors Kimi additions in tool-bearing system messages", async () => { @@ -344,6 +501,23 @@ describe("transcript system messages", () => { messages: Array<{ role: string; tools?: Array<{ function?: { name: string } }> }>; }; + // #10940: Kimi tool messages cannot remove tools, so a removal rewrites the prefix once. + test("keeps the Kimi prefix across loads after a removal", async () => { + const [beforeRemoval, ...afterRemoval] = await captureToolHistory(getModel("kimi-coding", "k3")); + + expect(beforeRemoval?.tools?.map((value) => value.function?.name)).toEqual(["base_tool", "drop_tool"]); + for (const payload of afterRemoval) { + expect(payload.tools?.map((value) => value.function?.name)).toEqual(["base_tool"]); + } + expectGrowingPrefix(afterRemoval.map((payload) => payload.messages)); + expect( + afterRemoval + .at(-1) + ?.messages.filter((message) => message.tools) + .map((message) => message.tools?.[0]?.function?.name), + ).toEqual(["late_tool", "later_tool"]); + }); + // #10720: Kimi Coding uses Chat Completions so K3-style models keep added tools out of the top-level list. test.each(["k3", "k3-256k", "kimi-for-coding"] as const)("anchors additions for Kimi Coding %s", async (modelId) => { const payload = await capturePayload(getModel("kimi-coding", modelId), additionContext);