diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index 9d1b49309..60baa1c16 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -29,6 +29,7 @@ - Fixed `pi-ai` CLI auth file being readable by other users. It is now written with mode `0600` - Fixed OpenCode DeepSeek V4.1 Flash sending `prompt_cache_retention`, which the OpenCode gateway rejects, with long cache retention - Fixed GPT-5.4 and GPT-5.5 failing with `Missing namespace for function_call` after switching from another model that called tools. On the OpenAI and OpenCode providers they now declare their tools in the top-level tool list instead of loading them through `additional_tools` or tool search ([#11415](https://github.com/earendil-works/pi/issues/11415)) +- Fixed Bedrock requests failing when `strict: "prefer"` tools exceed Claude's strict tool limits, such as more than 16 optional parameters across all strict tools; requests now retry once with preferred tools non-strict while keeping `strict: "require"` tools strict, as the Anthropic provider already does ([#11416](https://github.com/earendil-works/pi/issues/11416)) ## [1.4.0] - 2026-10-21 diff --git a/packages/ai/src/api/anthropic-messages.ts b/packages/ai/src/api/anthropic-messages.ts index 53764b406..9d6a8bdb7 100644 --- a/packages/ai/src/api/anthropic-messages.ts +++ b/packages/ai/src/api/anthropic-messages.ts @@ -59,6 +59,8 @@ import { import { getJsonSchemaToolParameters, + isStrictToolLimitErrorMessage, + prefersJsonSchemaStrictSampling, resolveJsonSchemaStrictSampling, type UnsupportedStrictSchemaKeywordCheck, } from "./constrained-sampling.ts"; @@ -674,11 +676,7 @@ export const stream: StreamFunction<"anthropic-messages", AnthropicOptions> = ( !(error instanceof Anthropic.APIError) || error.status !== 400 || error.type !== "invalid_request_error" || - ![ - "The compiled grammar is too large", - "Too many strict tools (", - "Schemas contains too many parameters with union types (", - ].some((message) => error.message.includes(message)) + !isStrictToolLimitErrorMessage(error.message) ) { throw error; } @@ -1185,10 +1183,9 @@ function buildParams( disablePreferredStrictTools, ); if ( - definitions.some((definition, index) => { - const config = tools[index].constrainedSampling; - return definition.strict === true && config && config.type === "json_schema" && config.strict === "prefer"; - }) + definitions.some( + (definition, index) => definition.strict === true && prefersJsonSchemaStrictSampling(tools[index]), + ) ) { hasPreferredStrictTools = true; } @@ -1630,9 +1627,8 @@ function convertTools( if (!tools) return []; return tools.map((tool, index) => { - const config = tool.constrainedSampling; const strict = - disablePreferredStrictTools && config && config.type === "json_schema" && config.strict === "prefer" + disablePreferredStrictTools && prefersJsonSchemaStrictSampling(tool) ? undefined : resolveJsonSchemaStrictSampling(tool, supportsStrictTools, isAnthropicStrictUnsupportedKeyword); const parameters = getJsonSchemaToolParameters(tool, strict); diff --git a/packages/ai/src/api/bedrock-converse-stream.ts b/packages/ai/src/api/bedrock-converse-stream.ts index 7a7740f06..0ed88be24 100644 --- a/packages/ai/src/api/bedrock-converse-stream.ts +++ b/packages/ai/src/api/bedrock-converse-stream.ts @@ -13,6 +13,7 @@ import { type ContentBlockStopEvent, ConversationRole, ConverseStreamCommand, + type ConverseStreamCommandOutput, type ConverseStreamMetadataEvent, ImageFormat, type Message, @@ -64,7 +65,12 @@ import { type TranscriptContext, withoutInitialSystemMessage, } from "../utils/transcript.ts"; -import { getJsonSchemaToolParameters, resolveJsonSchemaStrictSampling } from "./constrained-sampling.ts"; +import { + getJsonSchemaToolParameters, + isStrictToolLimitErrorMessage, + prefersJsonSchemaStrictSampling, + resolveJsonSchemaStrictSampling, +} from "./constrained-sampling.ts"; import { adjustMaxTokensForThinking, buildBaseOptions, @@ -269,29 +275,55 @@ export const stream: StreamFunction<"bedrock-converse-stream", BedrockOptions> = const inferenceMaxTokens = options.maxTokens ?? (isAnthropicClaudeModel(model) ? model.maxTokens : undefined); const initialSystemMessage = getInitialSystemMessage(normalizedContext.messages); const initialSystemPrompt = initialSystemMessage ? getSystemMessageText(initialSystemMessage) : undefined; - let commandInput = { - modelId: model.id, - messages: convertMessages(normalizedContext, model, cacheRetention, options.env), - system: buildSystemPrompt(initialSystemPrompt, model, cacheRetention, options.env), - inferenceConfig: { - ...(inferenceMaxTokens !== undefined && { maxTokens: inferenceMaxTokens }), - ...(options.temperature !== undefined && { temperature: options.temperature }), - }, - toolConfig: convertToolConfig( + let disablePreferredStrictTools = false; + let response: ConverseStreamCommandOutput; + for (;;) { + const tools = convertToolConfig( getCurrentTools(normalizedContext.messages), options.toolChoice, supportsStrictMode, - ), - additionalModelRequestFields: buildAdditionalModelRequestFields(model, options), - ...(options.requestMetadata !== undefined && { requestMetadata: options.requestMetadata }), - }; - const nextCommandInput = await options?.onPayload?.(commandInput, model); - if (nextCommandInput !== undefined) { - commandInput = nextCommandInput as typeof commandInput; + disablePreferredStrictTools, + ); + let commandInput = { + modelId: model.id, + messages: convertMessages(normalizedContext, model, cacheRetention, options.env), + system: buildSystemPrompt(initialSystemPrompt, model, cacheRetention, options.env), + inferenceConfig: { + ...(inferenceMaxTokens !== undefined && { maxTokens: inferenceMaxTokens }), + ...(options.temperature !== undefined && { temperature: options.temperature }), + }, + toolConfig: tools.toolConfig, + additionalModelRequestFields: buildAdditionalModelRequestFields(model, options), + ...(options.requestMetadata !== undefined && { requestMetadata: options.requestMetadata }), + }; + const nextCommandInput = await options?.onPayload?.(commandInput, model); + if (nextCommandInput !== undefined) { + commandInput = nextCommandInput as typeof commandInput; + } + try { + response = await client.send(new ConverseStreamCommand(commandInput), { abortSignal: options.signal }); + break; + } catch (error) { + // Bedrock checks the tool schemas before it starts the stream, so a retry never repeats streamed output. + if ( + options.signal?.aborted || + disablePreferredStrictTools || + !tools.hasPreferredStrictTools || + !(error instanceof BedrockRuntimeServiceException) || + error.name !== "ValidationException" || + !isStrictToolLimitErrorMessage(error.message) + ) { + throw error; + } + disablePreferredStrictTools = true; + appendAssistantMessageDiagnostic(output, { + type: "bedrock_strict_tools_fallback", + timestamp: Date.now(), + error: { message: error.message }, + details: { reason: "strict_tool_limits_exceeded" }, + }); + } } - const command = new ConverseStreamCommand(commandInput); - - const response = await client.send(command, { abortSignal: options.signal }); responseRequestId = normalizeDiagnosticValue(response.$metadata.requestId); if (!observedRawResponse && response.$metadata.httpStatusCode !== undefined) { const responseHeaders: Record = {}; @@ -1155,12 +1187,18 @@ function convertToolConfig( tools: Tool[] | undefined, toolChoice: BedrockOptions["toolChoice"], supportsStrictMode: boolean, -): ToolConfiguration | undefined { - if (!tools?.length) return undefined; - if (toolChoice === "none") return undefined; + disablePreferredStrictTools: boolean, +): { toolConfig: ToolConfiguration | undefined; hasPreferredStrictTools: boolean } { + if (!tools?.length || toolChoice === "none") return { toolConfig: undefined, hasPreferredStrictTools: false }; + let hasPreferredStrictTools = false; const bedrockTools: BedrockTool[] = tools.map((tool) => { - const strict = resolveJsonSchemaStrictSampling(tool, supportsStrictMode); + const preferred = prefersJsonSchemaStrictSampling(tool); + const strict = + disablePreferredStrictTools && preferred + ? undefined + : resolveJsonSchemaStrictSampling(tool, supportsStrictMode); + if (strict === true && preferred) hasPreferredStrictTools = true; return { toolSpec: { name: tool.name, @@ -1185,7 +1223,7 @@ function convertToolConfig( } } - return { tools: bedrockTools, toolChoice: bedrockToolChoice }; + return { toolConfig: { tools: bedrockTools, toolChoice: bedrockToolChoice }, hasPreferredStrictTools }; } function mapStopReason(reason: string | undefined): { stopReason: StopReason; errorMessage?: string } { diff --git a/packages/ai/src/api/constrained-sampling.ts b/packages/ai/src/api/constrained-sampling.ts index 62e03a105..adb729432 100644 --- a/packages/ai/src/api/constrained-sampling.ts +++ b/packages/ai/src/api/constrained-sampling.ts @@ -139,6 +139,27 @@ export function makeStrictJsonSchema( return cloned; } +/** Returns true for a tool that asks for strict JSON-schema sampling only when possible (`strict: "prefer"`). */ +export function prefersJsonSchemaStrictSampling(tool: Tool): boolean { + const config = tool.constrainedSampling; + return !!config && config.type === "json_schema" && config.strict === "prefer"; +} + +/** + * Claude rejects (on the Anthropic API and on Bedrock) a request whose strict tools exceed its limits with one of these + * messages. Observed limits: 20 strict tools, and 16 optional or union-typed parameters across all strict tools. + */ +const STRICT_TOOL_LIMIT_ERROR_MESSAGES = [ + "The compiled grammar is too large", + "Too many strict tools (", + "Schemas contains too many parameters with union types (", +]; + +/** Returns true when a provider error message says the request's strict tools exceed the provider's limits. */ +export function isStrictToolLimitErrorMessage(message: string): boolean { + return STRICT_TOOL_LIMIT_ERROR_MESSAGES.some((limitMessage) => message.includes(limitMessage)); +} + export function getJsonSchemaToolParameters(tool: Tool, strict: boolean | undefined): Tool["parameters"] { return (strict === true ? makeStrictJsonSchema(tool.parameters) : tool.parameters) as Tool["parameters"]; } diff --git a/packages/ai/test/bedrock-strict-tool-fallback.test.ts b/packages/ai/test/bedrock-strict-tool-fallback.test.ts new file mode 100644 index 000000000..c3853b461 --- /dev/null +++ b/packages/ai/test/bedrock-strict-tool-fallback.test.ts @@ -0,0 +1,173 @@ +import { Type } from "typebox"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const bedrockMock = vi.hoisted(() => ({ + inputs: [] as unknown[], + respond: undefined as ((attempt: number) => Promise) | undefined, +})); + +vi.mock("@aws-sdk/client-bedrock-runtime", () => { + class BedrockRuntimeServiceException extends Error { + constructor(options: { name: string; message: string }) { + super(options.message); + this.name = options.name; + } + } + + class BedrockRuntimeClient { + send(command: { input: unknown }): Promise { + bedrockMock.inputs.push(structuredClone(command.input)); + return bedrockMock.respond!(bedrockMock.inputs.length); + } + } + + class ConverseStreamCommand { + readonly input: unknown; + + constructor(input: unknown) { + this.input = input; + } + } + + return { + BedrockRuntimeClient, + BedrockRuntimeServiceException, + ConverseStreamCommand, + StopReason: { END_TURN: "end_turn" }, + CachePointType: { DEFAULT: "default" }, + CacheTTL: { ONE_HOUR: "ONE_HOUR" }, + ConversationRole: { ASSISTANT: "assistant", USER: "user" }, + ImageFormat: { JPEG: "jpeg", PNG: "png", GIF: "gif", WEBP: "webp" }, + ToolResultStatus: { ERROR: "error", SUCCESS: "success" }, + }; +}); + +import { BedrockRuntimeServiceException } from "@aws-sdk/client-bedrock-runtime"; +import { stream as streamBedrock } from "../src/api/bedrock-converse-stream.ts"; +import type { Model, Tool } from "../src/types.ts"; +import { normalizeContext } from "../src/utils/transcript.ts"; + +const model: Model<"bedrock-converse-stream"> = { + id: "us.anthropic.claude-opus-5-5", + name: "Claude Opus 5.5 (US)", + api: "bedrock-converse-stream", + provider: "amazon-bedrock", + baseUrl: "https://bedrock-runtime.us-east-1.amazonaws.com", + reasoning: false, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 200000, + maxTokens: 100, + compat: { supportsStrictMode: true }, +}; + +// Observed from live Bedrock with 20 optional parameters across strict tools. +const unionLimitMessage = + "The model returned the following errors: Schemas contains too many parameters with union types (20 parameters with type arrays or anyOf). This causes exponential compilation cost. Reduce the number of nullable or union-typed parameters (limit: 16 parameters with unions)."; + +type ToolSpecInput = { toolSpec: { name: string; strict?: boolean; inputSchema: { json: unknown } } }; +type CommandInput = { toolConfig: { tools: ToolSpecInput[] }; requestMetadata?: Record }; + +function tool(name: string, strict?: "prefer" | "require"): Tool { + return { + name, + description: "Look up a value", + parameters: Type.Object({ value: Type.String(), optional: Type.Optional(Type.Number()) }), + ...(strict ? { constrainedSampling: { type: "json_schema", strict } } : {}), + }; +} + +function sdkError(name: string, message: string): Error { + return new BedrockRuntimeServiceException({ name, message, $fault: "client", $metadata: { httpStatusCode: 400 } }); +} + +function successResponse(): Promise { + return Promise.resolve({ + $metadata: { httpStatusCode: 200 }, + stream: (async function* () { + yield { messageStart: { role: "assistant" } }; + yield { messageStop: { stopReason: "end_turn" } }; + })(), + }); +} + +function run(tools: Tool[], onPayload?: (payload: unknown) => unknown) { + return streamBedrock( + model, + normalizeContext({ tools, messages: [{ role: "user", content: "Hello", timestamp: 1 }] }), + { cacheRetention: "none", onPayload }, + ).result(); +} + +// Regression tests for https://github.com/earendil-works/pi/issues/11416. +describe("Bedrock preferred strict tool fallback", () => { + beforeEach(() => { + bedrockMock.inputs = []; + }); + + it("retries a strict tool limit rejection with preferred tools non-strict and keeps required tools strict", async () => { + bedrockMock.respond = (attempt) => + attempt === 1 ? Promise.reject(sdkError("ValidationException", unionLimitMessage)) : successResponse(); + const onPayload = vi.fn((payload: unknown) => ({ ...(payload as object), requestMetadata: { team: "a" } })); + + const result = await run([tool("preferred", "prefer"), tool("required", "require"), tool("ordinary")], onPayload); + + expect(result.stopReason).toBe("stop"); + expect(onPayload).toHaveBeenCalledTimes(2); + const [first, second] = bedrockMock.inputs as CommandInput[]; + expect(first.toolConfig.tools[0].toolSpec).toMatchObject({ + strict: true, + inputSchema: { + json: { + required: ["value", "optional"], + properties: { optional: { anyOf: [{ type: "number" }, { type: "null" }] } }, + }, + }, + }); + expect(second.toolConfig.tools[0].toolSpec.strict).toBeUndefined(); + expect(second.toolConfig.tools[0].toolSpec.inputSchema.json).toEqual({ + type: "object", + properties: { value: { type: "string" }, optional: { type: "number" } }, + required: ["value"], + }); + expect(second.toolConfig.tools[1]).toEqual(first.toolConfig.tools[1]); + expect(second.toolConfig.tools[1].toolSpec.strict).toBe(true); + expect(second.toolConfig.tools[2]).toEqual(first.toolConfig.tools[2]); + expect(second.requestMetadata).toEqual({ team: "a" }); + expect(result.diagnostics).toEqual([ + expect.objectContaining({ + type: "bedrock_strict_tools_fallback", + error: { message: unionLimitMessage }, + details: { reason: "strict_tool_limits_exceeded" }, + }), + ]); + }); + + it("does not retry without preferred strict tools", async () => { + bedrockMock.respond = () => Promise.reject(sdkError("ValidationException", unionLimitMessage)); + + const result = await run([tool("required", "require"), tool("ordinary")]); + + expect(result.stopReason).toBe("error"); + expect(result.errorMessage).toBe(`Validation error: ${unionLimitMessage}`); + expect(bedrockMock.inputs).toHaveLength(1); + }); + + it("does not retry other validation errors", async () => { + bedrockMock.respond = () => Promise.reject(sdkError("ValidationException", "messages: field required")); + + const result = await run([tool("preferred", "prefer")]); + + expect(result.stopReason).toBe("error"); + expect(bedrockMock.inputs).toHaveLength(1); + }); + + it("retries only once", async () => { + bedrockMock.respond = () => Promise.reject(sdkError("ValidationException", unionLimitMessage)); + + const result = await run([tool("preferred", "prefer")]); + + expect(result.stopReason).toBe("error"); + expect(bedrockMock.inputs).toHaveLength(2); + }); +});