diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index cae191ec9..f72a763f8 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -11,6 +11,7 @@ - Fixed Mistral streams being aborted mid-generation (e.g. during long thinking) after the request timeout; the timeout now applies only to waiting for response headers ([#10609](https://github.com/earendil-works/pi/issues/10609)) - Fixed OpenRouter router models such as `openrouter/auto` showing a cost of -$1,000,000 per 1M tokens; OpenRouter's negative placeholder prices are now 0 +- Fixed Anthropic `strict: "prefer"` tools failing when the provider rejects the strict tool set for grammar size, tool count, or union limits; requests now retry once with preferred tools non-strict while keeping `strict: "require"` tools strict ([#10409](https://github.com/earendil-works/pi/issues/10409)) ## [1.1.0] - 2026-10-07 diff --git a/packages/ai/src/api/anthropic-messages.ts b/packages/ai/src/api/anthropic-messages.ts index 452a3ccd0..29e13db1d 100644 --- a/packages/ai/src/api/anthropic-messages.ts +++ b/packages/ai/src/api/anthropic-messages.ts @@ -636,24 +636,57 @@ export const stream: StreamFunction<"anthropic-messages", AnthropicOptions> = ( client = created.client; isOAuth = created.isOAuthToken; } - let params = buildParams(model, normalizedContext, isOAuth, options); - const nextParams = await options?.onPayload?.(params, model); - if (nextParams !== undefined) { - params = { ...(nextParams as MessageCreateParamsStreaming), stream: true }; - } const requestOptions = { ...(options?.signal ? { signal: options.signal } : {}), ...(options?.timeoutMs !== undefined ? { timeout: options.timeoutMs } : {}), maxRetries: 0, }; - const response = await retryProviderRequest( - () => client.beta.messages.create(params, requestOptions).asResponse(), - { - maxRetries: options?.maxRetries, - maxRetryDelayMs: options?.maxRetryDelayMs, - signal: options?.signal, - }, - ); + let disablePreferredStrictTools = false; + let response: Response; + for (;;) { + options?.signal?.throwIfAborted(); + const built = buildParams(model, normalizedContext, isOAuth, options, disablePreferredStrictTools); + let params = built.params; + const nextParams = await options?.onPayload?.(params, model); + if (nextParams !== undefined) { + params = { ...(nextParams as MessageCreateParamsStreaming), stream: true }; + } + try { + response = await retryProviderRequest( + () => client.beta.messages.create(params, requestOptions).asResponse(), + { + maxRetries: options?.maxRetries, + maxRetryDelayMs: options?.maxRetryDelayMs, + signal: options?.signal, + }, + ); + break; + } catch (error) { + // Only downgrade a rejected request, never an already-started stream. + if ( + options?.signal?.aborted || + disablePreferredStrictTools || + !built.hasPreferredStrictTools || + !(error instanceof Anthropic.APIError) || + error.status !== 400 || + error.type !== "invalid_request_error" || + ![ + "The compiled grammar is too large", + "Too many strict tools (", + "Schemas contains too many parameters with union types (", + ].some((message) => error.message.includes(message)) + ) { + throw error; + } + disablePreferredStrictTools = true; + appendAssistantMessageDiagnostic(output, { + type: "anthropic_strict_tools_fallback", + timestamp: Date.now(), + error: { message: error.message }, + details: { reason: "strict_tool_limits_exceeded" }, + }); + } + } await options?.onResponse?.({ status: response.status, headers: headersToRecord(response.headers) }, model); stream.push({ type: "start", partial: output }); @@ -1126,7 +1159,8 @@ function buildParams( context: TranscriptContext, isOAuthToken: boolean, options?: AnthropicOptions, -): MessageCreateParamsStreaming { + disablePreferredStrictTools = false, +): { params: MessageCreateParamsStreaming; hasPreferredStrictTools: boolean } { const { cacheControl } = getCacheControl(model, options?.cacheRetention, options?.env); const compat = getAnthropicCompat(model); const initialSystemMessage = getInitialSystemMessage(context.messages); @@ -1137,6 +1171,26 @@ function buildParams( // by value in a `tool_addition` block, which also expresses same-name redefinitions. // Anthropic rejects a tool list where every tool is deferred, so there must be an initial // active tool to anchor the placeholder. Otherwise the current tool list is sent. + let hasPreferredStrictTools = false; + const convertToolDefinitions = (tools: Tool[], toolCacheControl?: CacheControlEphemeral): BetaTool[] => { + const definitions = convertTools( + tools, + isOAuthToken, + compat.supportsEagerToolInputStreaming, + compat.supportsStrictTools, + toolCacheControl, + disablePreferredStrictTools, + ); + if ( + definitions.some((definition, index) => { + const config = tools[index].constrainedSampling; + return definition.strict === true && config && config.type === "json_schema" && config.strict === "prefer"; + }) + ) { + hasPreferredStrictTools = true; + } + return definitions; + }; const initialTools = initialSystemMessage?.toolsAdded ?? []; const nativeToolChanges = compat.supportsMidConvoSystemMessages && compat.supportsMidConvoToolChanges && initialTools.length > 0; @@ -1146,10 +1200,7 @@ function buildParams( cacheControl, compat.allowEmptySignature, model.compat?.supportsMidConvoEffort === true ? model.provider : undefined, - nativeToolChanges - ? (tools) => - convertTools(tools, isOAuthToken, compat.supportsEagerToolInputStreaming, compat.supportsStrictTools) - : undefined, + nativeToolChanges ? convertToolDefinitions : undefined, ); const activeEffort = options?.effort ?? "high"; const betaFeatures = getBetaFeatures(model, context, isOAuthToken, nativeToolChanges, options); @@ -1207,26 +1258,11 @@ function buildParams( // placeholder. The list never changes afterwards: later tools are defined by value in // `tool_addition` blocks and withdrawn by `tool_removal`, so the cached prefix survives // every tool change. - params.tools = [ - ...convertTools( - initialTools, - isOAuthToken, - compat.supportsEagerToolInputStreaming, - compat.supportsStrictTools, - toolCacheControl, - ), - DEFERRED_TOOL_PLACEHOLDER, - ]; + params.tools = [...convertToolDefinitions(initialTools, toolCacheControl), DEFERRED_TOOL_PLACEHOLDER]; } else { const tools = getCurrentTools(context.messages); if (tools.length > 0) { - params.tools = convertTools( - tools, - isOAuthToken, - compat.supportsEagerToolInputStreaming, - compat.supportsStrictTools, - toolCacheControl, - ); + params.tools = convertToolDefinitions(tools, toolCacheControl); } } @@ -1283,7 +1319,7 @@ function buildParams( params.fallbacks = allowedFallbackModels.map((fallback) => ({ model: fallback.model })); } - return params; + return { params, hasPreferredStrictTools }; } // Normalize tool call IDs to match Anthropic's required pattern and length @@ -1579,11 +1615,16 @@ function convertTools( supportsEagerToolInputStreaming: boolean, supportsStrictTools: boolean, cacheControl?: CacheControlEphemeral, + disablePreferredStrictTools = false, ): BetaTool[] { if (!tools) return []; return tools.map((tool, index) => { - const strict = resolveJsonSchemaStrictSampling(tool, supportsStrictTools, isAnthropicStrictUnsupportedKeyword); + const config = tool.constrainedSampling; + const strict = + disablePreferredStrictTools && config && config.type === "json_schema" && config.strict === "prefer" + ? undefined + : resolveJsonSchemaStrictSampling(tool, supportsStrictTools, isAnthropicStrictUnsupportedKeyword); const parameters = getJsonSchemaToolParameters(tool, strict); const schema = parameters as { properties?: unknown; required?: string[] }; const legacyInputSchema = { diff --git a/packages/ai/test/anthropic-strict-tool-fallback.test.ts b/packages/ai/test/anthropic-strict-tool-fallback.test.ts new file mode 100644 index 000000000..7aa4641ae --- /dev/null +++ b/packages/ai/test/anthropic-strict-tool-fallback.test.ts @@ -0,0 +1,288 @@ +import Anthropic from "@anthropic-ai/sdk"; +import type { BetaTool, MessageCreateParamsStreaming } from "@anthropic-ai/sdk/resources/beta/messages/messages.js"; +import { Type } from "typebox"; +import { describe, expect, it, vi } from "vitest"; +import { type AnthropicOptions, stream as streamAnthropic } from "../src/api/anthropic-messages.ts"; +import { stream as streamPiMessages } from "../src/api/pi-messages.ts"; +import type { Model, Tool, TranscriptContext } from "../src/types.ts"; +import { normalizeContext } from "../src/utils/transcript.ts"; + +const grammarError = + "The compiled grammar is too large, which would cause performance issues. Simplify your tool schemas or reduce the number of strict tools."; +const model: Model<"anthropic-messages"> = { + id: "claude-haiku-4-5", + name: "Test Claude", + api: "anthropic-messages", + provider: "anthropic", + baseUrl: "https://anthropic.example", + reasoning: false, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 200000, + maxTokens: 100, + compat: { supportsStrictTools: true }, +}; + +function tool(name: string, strict?: "prefer" | "require"): Tool { + return { + name, + description: "Look up a value", + parameters: Type.Object({ value: Type.String(), optional: Type.Optional(Type.Number()) }), + ...(strict ? { constrainedSampling: { type: "json_schema", strict } } : {}), + }; +} + +function context(tools: Tool[]): TranscriptContext { + return normalizeContext({ tools, messages: [{ role: "user", content: "Hello", timestamp: 1 }] }); +} + +function errorResponse(message = grammarError, status = 400, type = "invalid_request_error"): Response { + return Response.json({ type: "error", error: { type, message } }, { status }); +} + +function successResponse(): Response { + const events = [ + { + type: "message_start", + message: { id: "msg_test", model: model.id, usage: { input_tokens: 10, output_tokens: 0 } }, + }, + { type: "message_delta", delta: { stop_reason: "end_turn" }, usage: { output_tokens: 1 } }, + { type: "message_stop" }, + ]; + return new Response(events.map((event) => `event: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`).join(""), { + headers: { "content-type": "text/event-stream" }, + }); +} + +function harness(respond: (payload: MessageCreateParamsStreaming, attempt: number) => Response) { + const payloads: MessageCreateParamsStreaming[] = []; + const fetch: NonNullable = async (_input, init) => { + const payload = JSON.parse(init?.body as string) as MessageCreateParamsStreaming; + payloads.push(payload); + return respond(payload, payloads.length); + }; + return { payloads, options: { apiKey: "test-key", fetch, maxRetries: 0 } satisfies AnthropicOptions }; +} + +// Regression tests for https://github.com/earendil-works/pi/issues/10409. +describe("Anthropic preferred strict tool fallback", () => { + it.each([ + grammarError, + // Observed from the live API: 32 strict tools, then 20 tools with 24 optional fields each. + "Too many strict tools (32). The maximum number of strict tools supported is 20. Try reducing the number of tools marked as strict.", + "Schemas contains too many parameters with union types (480 parameters with type arrays or anyOf). This causes exponential compilation cost. Reduce the number of nullable or union-typed parameters (limit: 16 parameters with unions).", + ])("retries %s with original non-strict schemas and preserves required tools", async (message) => { + const preferred = tool("preferred", "prefer"); + const required = tool("required", "require"); + const ordinary = tool("ordinary"); + const transcript = context([preferred, required, ordinary]); + const original = JSON.stringify(transcript); + // Use the real SDK's HTTP error parsing, not a synthetic thrown Error. + const { payloads, options } = harness((_payload, attempt) => + attempt === 1 ? errorResponse(message) : successResponse(), + ); + const onPayload = vi.fn((payload: unknown) => ({ + ...(payload as MessageCreateParamsStreaming), + inference_geo: "us", + stream: false, + })); + const onResponse = vi.fn(); + const stream = streamAnthropic(model, transcript, { ...options, onPayload, onResponse, cacheRetention: "long" }); + const events: string[] = []; + for await (const event of stream) events.push(event.type); + const result = await stream.result(); + + expect(result.stopReason).toBe("stop"); + expect(events).toEqual(["start", "done"]); + expect(payloads).toHaveLength(2); + expect(onPayload).toHaveBeenCalledTimes(2); + expect(onResponse).toHaveBeenCalledTimes(1); + const first = payloads[0].tools as BetaTool[]; + const second = payloads[1].tools as BetaTool[]; + expect(first[0]).toMatchObject({ + strict: true, + input_schema: { + required: ["value", "optional"], + properties: { optional: { anyOf: [{ type: "number" }, { type: "null" }] } }, + }, + }); + expect(second[0].strict).toBeUndefined(); + expect(second[0].input_schema).toEqual({ + type: "object", + properties: { value: { type: "string" }, optional: { type: "number" } }, + required: ["value"], + }); + expect(second[1]).toEqual(first[1]); + expect(second[1].strict).toBe(true); + expect(second[2]).toEqual(first[2]); + expect(second[2].cache_control).toEqual({ type: "ephemeral", ttl: "1h" }); + expect(payloads.map((payload) => payload.stream)).toEqual([true, true]); + expect(payloads[1]).toMatchObject({ inference_geo: "us" }); + expect(JSON.stringify(transcript)).toBe(original); + expect(result.diagnostics?.[0]).toMatchObject({ + type: "anthropic_strict_tools_fallback", + details: { reason: "strict_tool_limits_exceeded" }, + }); + }); + + it("downgrades inline additions and same-name redefinitions without downgrading require", async () => { + const transcript = context([tool("lookup", "require")]); + transcript.messages.push( + { role: "system", content: "", toolsAdded: [tool("lookup", "prefer")], timestamp: 2 }, + { role: "system", content: "", toolsAdded: [tool("lookup", "require")], timestamp: 3 }, + ); + const { payloads, options } = harness((_payload, attempt) => + attempt === 1 ? errorResponse() : successResponse(), + ); + const result = await streamAnthropic( + { + ...model, + compat: { ...model.compat, supportsMidConvoSystemMessages: true, supportsMidConvoToolChanges: true }, + }, + transcript, + { ...options, apiKey: "sk-ant-oat-test" }, + ).result(); + expect(result.stopReason).toBe("stop"); + expect(payloads).toHaveLength(2); + expect(payloads[1].tools).toEqual(payloads[0].tools); + const additions = payloads.map((payload) => + payload.messages.flatMap((message) => + Array.isArray(message.content) + ? message.content.flatMap((block) => + block.type === "tool_addition" && block.tool.type === "tool_definition" + ? [block.tool.definition as BetaTool] + : [], + ) + : [], + ), + ); + expect(additions[0].map((definition) => definition.strict)).toEqual([true, true]); + expect(additions[1].map((definition) => definition.strict)).toEqual([undefined, true]); + expect(additions[1][0].input_schema.required).toEqual(["value"]); + expect(additions[1][1]).toEqual(additions[0][1]); + }); + + it.each([ + ["required", [tool("lookup", "require")]], + ["ordinary", [tool("lookup")]], + [ + "locally unsupported prefer", + [{ ...tool("lookup", "prefer"), parameters: Type.Object({ value: Type.Number({ minimum: 1 }) }) }], + ], + ])("does not retry a %s tool set", async (_name, tools) => { + const { payloads, options } = harness(() => errorResponse()); + const result = await streamAnthropic(model, context(tools), options).result(); + expect(result.stopReason).toBe("error"); + expect(result.errorMessage).toContain(grammarError); + expect(payloads).toHaveLength(1); + }); + + it.each([ + ["unrelated 400", "Invalid tool schema", 400, "invalid_request_error"], + ["wrong error type", grammarError, 400, "authentication_error"], + ["wrong HTTP status", grammarError, 500, "invalid_request_error"], + ])("does not downgrade on %s", async (_name, message, status, type) => { + const { payloads, options } = harness(() => errorResponse(message, status, type)); + const result = await streamAnthropic(model, context([tool("lookup", "prefer")]), options).result(); + expect(result.stopReason).toBe("error"); + expect(payloads).toHaveLength(1); + }); + + it("surfaces a second rejection rather than retrying again", async () => { + const { payloads, options } = harness((_payload, attempt) => + errorResponse(attempt === 1 ? grammarError : "Required grammar is still too large"), + ); + const result = await streamAnthropic( + model, + context([tool("lookup", "prefer"), tool("required", "require")]), + options, + ).result(); + expect(result.stopReason).toBe("error"); + expect(result.errorMessage).toContain("Required grammar is still too large"); + expect(payloads).toHaveLength(2); + expect((payloads[1].tools as BetaTool[])[1].strict).toBe(true); + }); + + it("does not downgrade a hook exception, even if it looks like the provider rejection", async () => { + const { payloads, options } = harness(() => successResponse()); + const result = await streamAnthropic(model, context([tool("lookup", "prefer")]), { + ...options, + onPayload: () => { + throw Anthropic.APIError.generate( + 400, + { error: { type: "invalid_request_error", message: grammarError } }, + grammarError, + new Headers(), + ); + }, + }).result(); + expect(result.stopReason).toBe("error"); + expect(payloads).toHaveLength(0); + expect(result.diagnostics).toBeUndefined(); + }); + + it("honors cancellation before the fallback attempt", async () => { + const controller = new AbortController(); + const { payloads, options } = harness(() => { + controller.abort(); + return errorResponse(); + }); + const result = await streamAnthropic(model, context([tool("lookup", "prefer")]), { + ...options, + signal: controller.signal, + }).result(); + expect(result.stopReason).toBe("aborted"); + expect(payloads).toHaveLength(1); + }); + + it("does not retry an SSE error after response headers", async () => { + const { payloads, options } = harness( + () => + new Response( + `event: error\ndata: ${JSON.stringify({ type: "error", error: { type: "invalid_request_error", message: grammarError } })}\n\n`, + { headers: { "content-type": "text/event-stream" } }, + ), + ); + const result = await streamAnthropic(model, context([tool("lookup", "prefer")]), options).result(); + expect(result.stopReason).toBe("error"); + expect(payloads).toHaveLength(1); + }); + + it("does not disable strict sampling on later requests", async () => { + const { payloads, options } = harness((_payload, attempt) => + attempt === 1 ? errorResponse() : successResponse(), + ); + const transcript = context([tool("lookup", "prefer")]); + await streamAnthropic(model, transcript, options).result(); + await streamAnthropic(model, transcript, options).result(); + expect(payloads.map((payload) => (payload.tools as BetaTool[])[0].strict)).toEqual([true, undefined, true]); + }); + + it("preserves prefer through pi-messages so the gateway's Anthropic adapter can fall back", async () => { + const { payloads, options } = harness((_payload, attempt) => + attempt === 1 ? errorResponse() : successResponse(), + ); + const transcript = context([tool("lookup", "prefer")]); + const gatewayModel: Model<"pi-messages"> = { + ...model, + api: "pi-messages", + provider: "radius", + baseUrl: "https://radius.example/v1", + compat: undefined, + }; + const result = await streamPiMessages(gatewayModel, transcript, { + apiKey: "radius-test", + fetch: async (_input, init) => { + const request = JSON.parse(init?.body as string) as { context: TranscriptContext }; + expect(request.context).toEqual(JSON.parse(JSON.stringify(transcript))); + const upstream = await streamAnthropic(model, request.context, options).result(); + return new Response( + `data: ${JSON.stringify({ type: "done", reason: upstream.stopReason, usage: upstream.usage })}\n\n`, + { headers: { "content-type": "text/event-stream" } }, + ); + }, + }).result(); + expect(result.stopReason).toBe("stop"); + expect(payloads.map((payload) => (payload.tools as BetaTool[])[0].strict)).toEqual([true, undefined]); + }); +});