feat(ai): define Anthropic mid-conversation tools inline

Switch native Anthropic tool changes to the inline-tools-2026-09-15 beta. Later tools are defined by value in tool_addition blocks, so the top-level tool list stays fixed and same-name redefinitions no longer fall back to resending the full tool list. The deferred placeholder is kept.

Upgrade @anthropic-ai/sdk to 0.129.0 and deprecate hasToolRedefinitions().
This commit is contained in:
Armin Ronacher 2026-10-02 12:30:17 +02:00
parent 3874b3e989
commit b271b0a524
10 changed files with 136 additions and 97 deletions

8
package-lock.json generated
View file

@ -49,9 +49,9 @@
}
},
"node_modules/@anthropic-ai/sdk": {
"version": "0.124.0",
"resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.124.0.tgz",
"integrity": "sha512-cN5O8i9UVxHeOQAzj/XjshWXG8KiibJDw9OGpH2Z/eR3n/RBxdoLxDJOcfqAJWvjaMDFfHTBADU04hWRJVkDyA==",
"version": "0.129.0",
"resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.129.0.tgz",
"integrity": "sha512-MH7LB20kNpGLUpPTe2OG5XvL/KhX8HUO9XyBaFjgFk6TLS4Xjt0VPIuMKywe7POKKPslX+QxSlVok9e+8kG5Nw==",
"license": "MIT",
"dependencies": {
"json-schema-to-ts": "^3.1.1",
@ -5311,7 +5311,7 @@
"version": "1.0.0",
"license": "MIT",
"dependencies": {
"@anthropic-ai/sdk": "0.124.0",
"@anthropic-ai/sdk": "0.129.0",
"@aws-sdk/client-bedrock-runtime": "3.1127.0",
"@earendil-works/pi-telemetry": "^1.0.0",
"@google/genai": "2.21.0",

View file

@ -2,6 +2,11 @@
## [Unreleased]
### Changed
- Anthropic models with native mid-conversation tool changes now use the `inline-tools-2026-09-15` beta: later tools are defined by value in `tool_addition` blocks instead of being appended to the top-level tool list, and redefining a tool under the same name no longer falls back to resending the full tool list, so the prompt cache survives it. Upgraded `@anthropic-ai/sdk` to 0.129.0.
- Deprecated `hasToolRedefinitions()`; no built-in transport needs it anymore.
### Fixed
- Fixed "Selected model is at capacity" provider errors ending the turn instead of being retried ([#10278](https://github.com/earendil-works/pi/issues/10278))

View file

@ -1587,7 +1587,7 @@ getCurrentTools(messages); // []
A custom `Provider` or `ProviderStreams` implementation reads the prompt and tools the same way from `context.messages`; `context.systemPrompt` and `context.tools` do not exist at that layer.
Models that accept system messages mid-conversation (`supportsMidConvoSystemMessages` in the model's compat settings, set by the generated catalog for verified models) receive each later system message in place, so the cached prefix stays intact; section changes are framed by name for the model. Every other model receives `collapseSystemMessages(transcript)`: the replayed prompt and current tools as the leading system message, with later system messages dropped. Anthropic models that also set `supportsMidConvoToolChanges` send tool changes as native `tool_addition`/`tool_removal` blocks: the initial tools stay active at the top level, every later declaration is sent with `defer_loading` (plus a stable deferred placeholder from the first request, which keeps Anthropic's deferred-tool scaffolding in the cached prefix), and removed tools stay declared, so tool changes do not invalidate the prompt cache. That needs at least one initial tool and no same-name redefinition; otherwise the current tool list is sent at the top level with the system text only. OpenAI Responses models with `supportsAdditionalTools` or `supportsToolSearch` anchor additive tool changes at their message; everything else sends the current tool list at the top level.
Models that accept system messages mid-conversation (`supportsMidConvoSystemMessages` in the model's compat settings, set by the generated catalog for verified models) receive each later system message in place, so the cached prefix stays intact; section changes are framed by name for the model. Every other model receives `collapseSystemMessages(transcript)`: the replayed prompt and current tools as the leading system message, with later system messages dropped. Anthropic models that also set `supportsMidConvoToolChanges` send tool changes as native blocks (`inline-tools-2026-09-15` beta): the top-level tool list holds the initial tools plus a stable deferred placeholder and never changes, later tools are defined by value in `tool_addition` blocks (a new definition under an existing name replaces the old one), and removals are `tool_removal` references, so tool changes do not invalidate the prompt cache. That needs at least one initial tool; otherwise the current tool list is sent at the top level with the system text only. OpenAI Responses models with `supportsAdditionalTools` or `supportsToolSearch` anchor additive tool changes at their message; everything else sends the current tool list at the top level.
## Context Serialization

View file

@ -67,7 +67,7 @@
"prepublishOnly": "npm run clean && npm run build"
},
"dependencies": {
"@anthropic-ai/sdk": "0.124.0",
"@anthropic-ai/sdk": "0.129.0",
"@aws-sdk/client-bedrock-runtime": "3.1127.0",
"@earendil-works/pi-telemetry": "^1.0.0",
"@google/genai": "2.21.0",

View file

@ -1,7 +1,7 @@
import Anthropic, { type ClientOptions } from "@anthropic-ai/sdk";
import type {
BetaInputTransformation,
BetaStopReason,
BetaThinkingDroppedInputTransformation,
BetaTool,
BetaCacheControlEphemeral as CacheControlEphemeral,
BetaContentBlockParam as ContentBlockParam,
@ -48,9 +48,7 @@ import { sanitizeSurrogates } from "../utils/sanitize-unicode.ts";
import { getSystemMessageText, renderSystemMessageUpdate } from "../utils/text.ts";
import {
getCurrentTools,
getDeclaredTools,
getInitialSystemMessage,
hasToolRedefinitions,
resolveTranscript,
type TranscriptContext,
} from "../utils/transcript.ts";
@ -194,14 +192,14 @@ const INTERLEAVED_THINKING_BETA = "interleaved-thinking-2025-05-14";
const SERVER_SIDE_FALLBACK_BETA = "server-side-fallback-2026-07-01";
const MID_CONVERSATION_OUTPUT_CONFIG_BETA = "mid-conversation-output-config-2026-07-01";
const THINKING_BINDING_CONTROLS_BETA = "thinking-binding-controls-2026-08-01";
const MID_CONVERSATION_TOOL_CHANGES_BETA = "mid-conversation-tool-changes-2026-07-01";
const INLINE_TOOLS_BETA = "inline-tools-2026-09-15";
/**
* Stable deferred tool declared whenever native tool changes are in use. Anthropic adds
* hidden prompt scaffolding as soon as any tool has `defer_loading`; declaring this
* placeholder from the first request keeps that scaffolding in the cached prefix, so the
* first real late tool does not invalidate the cache (measured: full miss without it).
* It is never activated and the model cannot see it.
* hidden prompt scaffolding for mid-conversation tool changes; declaring this placeholder
* from the first request keeps that scaffolding in the cached prefix, so the first tool
* change does not invalidate the cache (measured: full miss without it). It is never
* activated and the model cannot see it.
*/
const DEFERRED_TOOL_PLACEHOLDER: BetaTool = {
name: "__pi_deferred_placeholder__",
@ -604,7 +602,7 @@ export const stream: StreamFunction<"anthropic-messages", AnthropicOptions> = (
let client: Anthropic;
let isOAuth: boolean;
let usageModel = model;
let inputTransformations: BetaThinkingDroppedInputTransformation[] | undefined;
let inputTransformations: BetaInputTransformation[] | undefined;
if (options?.client) {
client = options.client;
@ -1119,7 +1117,7 @@ function getBetaFeatures(
if (model.compat?.supportsMidConvoEffort === true) {
features.push(MID_CONVERSATION_OUTPUT_CONFIG_BETA, THINKING_BINDING_CONTROLS_BETA);
}
if (nativeToolChanges) features.push(MID_CONVERSATION_TOOL_CHANGES_BETA);
if (nativeToolChanges) features.push(INLINE_TOOLS_BETA);
return [...new Set(features)];
}
@ -1135,22 +1133,23 @@ function buildParams(
const initialSystemText = initialSystemMessage ? getSystemMessageText(initialSystemMessage) : "";
const transformedMessages = transformMessages(context.messages, model, normalizeToolCallId);
const conversationMessages = initialSystemMessage ? transformedMessages.slice(1) : transformedMessages;
// Native tool changes reference tools by name, so a redefined name cannot be expressed,
// and Anthropic rejects a tool list where every tool is deferred, so there must be an
// initial active tool to anchor the deferred ones. Otherwise the current tool list is sent.
// Native tool changes keep the request-level tool list fixed and define every later tool
// by value in a `tool_addition` block, which also expresses same-name redefinitions.
// Anthropic rejects a tool list where every tool is deferred, so there must be an initial
// active tool to anchor the placeholder. Otherwise the current tool list is sent.
const initialTools = initialSystemMessage?.toolsAdded ?? [];
const nativeToolChanges =
compat.supportsMidConvoSystemMessages &&
compat.supportsMidConvoToolChanges &&
initialTools.length > 0 &&
!hasToolRedefinitions(context.messages);
compat.supportsMidConvoSystemMessages && compat.supportsMidConvoToolChanges && initialTools.length > 0;
const converted = convertMessages(
conversationMessages,
isOAuthToken,
cacheControl,
compat.allowEmptySignature,
model.compat?.supportsMidConvoEffort === true ? model.provider : undefined,
nativeToolChanges,
nativeToolChanges
? (tools) =>
convertTools(tools, isOAuthToken, compat.supportsEagerToolInputStreaming, compat.supportsStrictTools)
: undefined,
);
const activeEffort = options?.effort ?? "high";
const betaFeatures = getBetaFeatures(model, context, isOAuthToken, nativeToolChanges, options);
@ -1204,12 +1203,10 @@ function buildParams(
const toolCacheControl = compat.supportsCacheControlOnTools ? cacheControl : undefined;
if (nativeToolChanges) {
// Initial tools stay active with the cache breakpoint on the last one. Every later
// declaration is deferred and only surfaced by its `tool_addition` block; removed
// tools stay declared and are withdrawn by `tool_removal`. The request-level list
// therefore only grows, keeping the cached prefix intact across tool changes.
const initialNames = new Set(initialTools.map((tool) => tool.name));
const laterTools = getDeclaredTools(context.messages).filter((tool) => !initialNames.has(tool.name));
// Initial tools stay active with the cache breakpoint on the last one, followed by the
// placeholder. The list never changes afterwards: later tools are defined by value in
// `tool_addition` blocks and withdrawn by `tool_removal`, so the cached prefix survives
// every tool change.
params.tools = [
...convertTools(
initialTools,
@ -1219,12 +1216,6 @@ function buildParams(
toolCacheControl,
),
DEFERRED_TOOL_PLACEHOLDER,
...convertTools(
laterTools,
isOAuthToken,
compat.supportsEagerToolInputStreaming,
compat.supportsStrictTools,
).map((tool) => ({ ...tool, defer_loading: true })),
];
} else {
const tools = getCurrentTools(context.messages);
@ -1320,7 +1311,8 @@ function convertMessages(
cacheControl?: CacheControlEphemeral,
allowEmptySignature = false,
managedProvider?: string,
nativeToolChanges = false,
/** Converts tool definitions for native `tool_addition` blocks; undefined when tool changes are not native. */
convertToolDefinitions?: (tools: Tool[]) => BetaTool[],
): ConvertedAnthropicMessages {
const params: MessageParam[] = [];
const assistantLevels = new Map<number, AnthropicEffort>();
@ -1344,18 +1336,19 @@ function convertMessages(
const text = renderSystemMessageUpdate(msg);
const blocks: ContentBlockParam[] = [];
if (text.length > 0) blocks.push({ type: "text", text: sanitizeSurrogates(text) });
if (nativeToolChanges) {
if (convertToolDefinitions) {
const added = msg.toolsAdded ?? [];
const redefined = new Set(added.map((tool) => tool.name));
for (const tool of msg.toolsRemoved ?? []) {
// A new definition under the same name replaces the old one, so no removal is needed.
if (redefined.has(tool.name)) continue;
blocks.push({
type: "tool_removal",
tool: { type: "tool_reference", name: isOAuthToken ? toClaudeCodeName(tool.name) : tool.name },
});
}
for (const tool of msg.toolsAdded ?? []) {
blocks.push({
type: "tool_addition",
tool: { type: "tool_reference", name: isOAuthToken ? toClaudeCodeName(tool.name) : tool.name },
});
for (const definition of convertToolDefinitions(added)) {
blocks.push({ type: "tool_addition", tool: { type: "tool_definition", definition } });
}
}
if (blocks.length > 0) pendingSystemMessages.push({ role: "system", content: blocks });

View file

@ -943,7 +943,7 @@ export interface AnthropicMessagesCompat {
supportsMidConvoEffort?: boolean;
/** Whether the exact model accepts system-role messages inside the conversation. When false, later system messages are folded into the top-level system prompt. Default: false. */
supportsMidConvoSystemMessages?: boolean;
/** Whether the exact model accepts mid-conversation `tool_addition` and `tool_removal` blocks. Requires `supportsMidConvoSystemMessages`. Default: false. */
/** Whether the exact model accepts mid-conversation `tool_addition` blocks with inline tool definitions (`inline-tools-2026-09-15`) and `tool_removal` blocks. Requires `supportsMidConvoSystemMessages`. Default: false. */
supportsMidConvoToolChanges?: boolean;
/**
* Models Anthropic accepts in `fallbacks` for server-side refusal fallback,

View file

@ -177,8 +177,11 @@ export function getDeclaredTools(messages: TranscriptMessages): Tool[] {
}
/**
* Whether a tool name was declared twice with different definitions. Transports that
* reference tools by name (Anthropic `tool_addition`/`tool_removal`) cannot express that.
* Whether a tool name was declared twice with different definitions. A transport that can
* only reference previously declared tools by name cannot replay such a history.
*
* @deprecated No built-in transport needs this anymore: Anthropic expresses redefinitions with
* inline `tool_definition` blocks. Kept for API compatibility and will be removed in a future release.
*/
export function hasToolRedefinitions(messages: TranscriptMessages): boolean {
const declared = new Map<string, Tool>();

View file

@ -71,40 +71,63 @@ const anthropicNativeModel: Model<"anthropic-messages"> = {
compat: { supportsMidConvoSystemMessages: true, supportsMidConvoToolChanges: true },
};
interface AnthropicToolDefinition {
name: string;
description?: string;
defer_loading?: boolean;
cache_control?: unknown;
}
interface AnthropicPayload {
betas?: string[];
system?: Array<{ text: string }>;
tools?: Array<{ name: string; defer_loading?: boolean; cache_control?: unknown }>;
messages: Array<{ role: string; content: Array<{ type: string; text?: string; tool?: { name: string } }> }>;
tools?: AnthropicToolDefinition[];
messages: Array<{
role: string;
content: Array<{
type: string;
text?: string;
cache_control?: unknown;
tool?: { type: string; name?: string; definition?: AnthropicToolDefinition };
}>;
}>;
}
describe("transcript system messages", () => {
test("sends Anthropic updates and tool changes in native system messages", async () => {
const payload = await capturePayload<AnthropicPayload>(anthropicNativeModel, context);
expect(payload.betas).toContain("mid-conversation-tool-changes-2026-07-01");
expect(payload.betas).toContain("inline-tools-2026-09-15");
expect(payload.betas).not.toContain("mid-conversation-tool-changes-2026-07-01");
expect(payload.system?.map((block) => block.text)).toEqual([
"base prompt\n\n<rules>\nold rules\n</rules>\n\n<docs>\nread docs\n</docs>",
]);
// Initial tools stay active and carry the cache breakpoint; the placeholder and every
// later declaration are deferred; the removed tool stays declared.
// The top-level list holds only the initial tools (with the cache breakpoint) and the
// deferred placeholder; later tools never appear there.
expect(payload.tools).toMatchObject([
{ name: "base_tool", cache_control: { type: "ephemeral" } },
{ name: "__pi_deferred_placeholder__", defer_loading: true },
{ name: "late_tool", defer_loading: true },
]);
expect(payload.tools).toHaveLength(2);
expect(payload.tools?.[0]?.defer_loading).toBeUndefined();
expect(payload.tools?.[1]?.cache_control).toBeUndefined();
expect(payload.tools?.[2]?.cache_control).toBeUndefined();
const update = payload.messages.at(-1);
expect(update).toMatchObject({
role: "system",
content: [
{ type: "text" },
{ type: "tool_removal", tool: { name: "base_tool" } },
{ type: "tool_addition", tool: { name: "late_tool" } },
{ type: "tool_removal", tool: { type: "tool_reference", name: "base_tool" } },
{
type: "tool_addition",
tool: { type: "tool_definition", definition: { name: "late_tool", description: "late_tool tool" } },
cache_control: { type: "ephemeral" },
},
],
});
// Cache control goes on the block, never on the inline definition; nothing is deferred.
const definition = update?.content[2]?.tool?.definition;
expect(definition?.cache_control).toBeUndefined();
expect(definition?.defer_loading).toBeUndefined();
expect(update?.content[0]?.text).toContain("updated guidance");
expect(update?.content[0]?.text).toContain("<rules>\nnew rules\n</rules>");
expect(update?.content[0]?.text).toContain('Removed system prompt section "docs"');
@ -116,39 +139,54 @@ describe("transcript system messages", () => {
expect(initial.tools?.map((tool) => tool.name)).toEqual(["base_tool", "__pi_deferred_placeholder__"]);
});
test("sends the current Anthropic tool list when native tool changes cannot express the history", async () => {
test("redefines an Anthropic tool by value under the same name", async () => {
const redefinedTool = { ...baseTool, description: "changed" };
const fallbackContexts: Context[] = [
// Same-name redefinition: blocks reference tools by name only.
{
messages: [
{ role: "system", content: "base prompt", toolsAdded: [baseTool], timestamp: 0 },
{
role: "system",
content: "updated guidance",
toolsRemoved: [{ name: "base_tool" }],
toolsAdded: [redefinedTool],
timestamp: 2,
},
],
},
// No initial tool: Anthropic rejects an all-deferred tool list.
{
messages: [
{ role: "system", content: "base prompt", timestamp: 0 },
{ role: "system", content: "updated guidance", toolsAdded: [redefinedTool], timestamp: 2 },
],
},
];
for (const fallbackContext of fallbackContexts) {
const payload = await capturePayload<AnthropicPayload>(anthropicNativeModel, fallbackContext);
expect(payload.betas ?? []).not.toContain("mid-conversation-tool-changes-2026-07-01");
expect(payload.tools).toMatchObject([
{ name: "base_tool", description: "changed", cache_control: { type: "ephemeral" } },
]);
expect(payload.tools?.[0]?.defer_loading).toBeUndefined();
expect(payload.messages.at(-1)?.content.map((block) => block.type)).toEqual(["text"]);
}
const payload = await capturePayload<AnthropicPayload>(anthropicNativeModel, {
messages: [
{ role: "system", content: "base prompt", toolsAdded: [baseTool], timestamp: 0 },
{ role: "user", content: "before", timestamp: 1 },
{
role: "system",
content: "",
toolsRemoved: [{ name: "base_tool" }],
toolsAdded: [redefinedTool],
timestamp: 2,
},
],
});
expect(payload.betas).toContain("inline-tools-2026-09-15");
// The top-level declaration keeps the original definition, so the cached prefix survives.
expect(payload.tools).toMatchObject([
{ name: "base_tool", description: "base_tool tool" },
{ name: "__pi_deferred_placeholder__" },
]);
// The new definition replaces the old one in place; no removal block is sent for it.
expect(payload.messages.at(-1)).toMatchObject({
role: "system",
content: [
{
type: "tool_addition",
tool: { type: "tool_definition", definition: { name: "base_tool", description: "changed" } },
},
],
});
expect(payload.messages.at(-1)?.content).toHaveLength(1);
});
test("sends the current Anthropic tool list without an initial tool", async () => {
// Anthropic rejects a tool list where every tool (here: the placeholder) is deferred.
const payload = await capturePayload<AnthropicPayload>(anthropicNativeModel, {
messages: [
{ role: "system", content: "base prompt", timestamp: 0 },
{ role: "user", content: "before", timestamp: 1 },
{ role: "system", content: "updated guidance", toolsAdded: [lateTool], timestamp: 2 },
],
});
expect(payload.betas ?? []).not.toContain("inline-tools-2026-09-15");
expect(payload.tools).toMatchObject([{ name: "late_tool", cache_control: { type: "ephemeral" } }]);
expect(payload.tools?.[0]?.defer_loading).toBeUndefined();
expect(payload.messages.at(-1)?.content.map((block) => block.type)).toEqual(["text"]);
});
test("folds Anthropic updates into the system prompt without native support", async () => {
@ -166,7 +204,7 @@ describe("transcript system messages", () => {
messages: Array<{ role: string }>;
}>(model, context);
expect(payload.betas ?? []).not.toContain("mid-conversation-tool-changes-2026-07-01");
expect(payload.betas ?? []).not.toContain("inline-tools-2026-09-15");
expect(payload.system?.map((block) => block.text)).toEqual([
"base prompt\n\nupdated guidance\n\n<rules>\nnew rules\n</rules>",
]);
@ -189,7 +227,7 @@ describe("transcript system messages", () => {
messages: Array<{ role: string }>;
}>(model, context);
expect(payload.betas ?? []).not.toContain("mid-conversation-tool-changes-2026-07-01");
expect(payload.betas ?? []).not.toContain("inline-tools-2026-09-15");
expect(payload.tools?.map((value) => value.name)).toEqual(["late_tool"]);
expect(payload.messages.map((message) => message.role)).toEqual(["user"]);
});

View file

@ -15,9 +15,9 @@
}
},
"node_modules/@anthropic-ai/sdk": {
"version": "0.124.0",
"resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.124.0.tgz",
"integrity": "sha512-cN5O8i9UVxHeOQAzj/XjshWXG8KiibJDw9OGpH2Z/eR3n/RBxdoLxDJOcfqAJWvjaMDFfHTBADU04hWRJVkDyA==",
"version": "0.129.0",
"resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.129.0.tgz",
"integrity": "sha512-MH7LB20kNpGLUpPTe2OG5XvL/KhX8HUO9XyBaFjgFk6TLS4Xjt0VPIuMKywe7POKKPslX+QxSlVok9e+8kG5Nw==",
"license": "MIT",
"dependencies": {
"json-schema-to-ts": "^3.1.1",
@ -413,7 +413,7 @@
"resolved": "https://registry.npmjs.org/@earendil-works/pi-ai/-/pi-ai-1.0.0.tgz",
"license": "MIT",
"dependencies": {
"@anthropic-ai/sdk": "0.124.0",
"@anthropic-ai/sdk": "0.129.0",
"@aws-sdk/client-bedrock-runtime": "3.1127.0",
"@earendil-works/pi-telemetry": "^1.0.0",
"@google/genai": "2.21.0",

View file

@ -41,9 +41,9 @@
}
},
"node_modules/@anthropic-ai/sdk": {
"version": "0.124.0",
"resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.124.0.tgz",
"integrity": "sha512-cN5O8i9UVxHeOQAzj/XjshWXG8KiibJDw9OGpH2Z/eR3n/RBxdoLxDJOcfqAJWvjaMDFfHTBADU04hWRJVkDyA==",
"version": "0.129.0",
"resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.129.0.tgz",
"integrity": "sha512-MH7LB20kNpGLUpPTe2OG5XvL/KhX8HUO9XyBaFjgFk6TLS4Xjt0VPIuMKywe7POKKPslX+QxSlVok9e+8kG5Nw==",
"license": "MIT",
"dependencies": {
"json-schema-to-ts": "^3.1.1",
@ -439,7 +439,7 @@
"resolved": "https://registry.npmjs.org/@earendil-works/pi-ai/-/pi-ai-1.0.0.tgz",
"license": "MIT",
"dependencies": {
"@anthropic-ai/sdk": "0.124.0",
"@anthropic-ai/sdk": "0.129.0",
"@aws-sdk/client-bedrock-runtime": "3.1127.0",
"@earendil-works/pi-telemetry": "^1.0.0",
"@google/genai": "2.21.0",