diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index 5b0c5b9cb..0ee14a3d9 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -9,6 +9,7 @@ ### Fixed - Fixed context overflow detection for Z.AI CN endpoint `Prompt exceeds max length` errors ([#10208](https://github.com/earendil-works/pi/issues/10208)) +- Fixed Anthropic requests failing when a `strict: "prefer"` tool schema uses keywords Anthropic strict tool use rejects, such as `minimum`/`maximum`; such tools are now sent non-strict ([#9953](https://github.com/earendil-works/pi/issues/9953)) ## [0.99.1] - 2026-09-29 diff --git a/packages/ai/src/api/anthropic-messages.ts b/packages/ai/src/api/anthropic-messages.ts index 9f227a245..b0a4c915a 100644 --- a/packages/ai/src/api/anthropic-messages.ts +++ b/packages/ai/src/api/anthropic-messages.ts @@ -48,7 +48,11 @@ import { type TranscriptContext, } from "../utils/transcript.ts"; -import { getJsonSchemaToolParameters, resolveJsonSchemaStrictSampling } from "./constrained-sampling.ts"; +import { + getJsonSchemaToolParameters, + resolveJsonSchemaStrictSampling, + type UnsupportedStrictSchemaKeywordCheck, +} from "./constrained-sampling.ts"; import { buildCopilotDynamicHeaders, hasCopilotVisionInput } from "./github-copilot-headers.ts"; import { adjustMaxTokensForThinking, buildBaseOptions, clampMaxTokensToContext } from "./simple-options.ts"; import { transformMessages } from "./transform-messages.ts"; @@ -1462,6 +1466,41 @@ function shouldUseFineGrainedToolStreamingBeta( return getCurrentTools(context.messages).length > 0 && !getAnthropicCompat(model).supportsEagerToolInputStreaming; } +// Keywords Anthropic strict tool use rejects with a 400 for the whole request. +// https://platform.claude.com/docs/en/build-with-claude/structured-outputs#json-schema-limitations +const ANTHROPIC_STRICT_UNSUPPORTED_KEYWORDS = new Set([ + "minimum", + "maximum", + "exclusiveMinimum", + "exclusiveMaximum", + "multipleOf", + "maxItems", + "uniqueItems", + "minContains", + "maxContains", + "minProperties", + "maxProperties", +]); +const ANTHROPIC_STRICT_STRING_FORMATS = new Set([ + "date-time", + "time", + "date", + "duration", + "email", + "hostname", + "uri", + "ipv4", + "ipv6", + "uuid", +]); + +const isAnthropicStrictUnsupportedKeyword: UnsupportedStrictSchemaKeywordCheck = (key, value) => { + if (ANTHROPIC_STRICT_UNSUPPORTED_KEYWORDS.has(key)) return true; + if (key === "minItems") return value !== 0 && value !== 1; + if (key === "format") return typeof value !== "string" || !ANTHROPIC_STRICT_STRING_FORMATS.has(value); + return false; +}; + function convertTools( tools: Tool[], isOAuthToken: boolean, @@ -1472,7 +1511,7 @@ function convertTools( if (!tools) return []; return tools.map((tool, index) => { - const strict = resolveJsonSchemaStrictSampling(tool, supportsStrictTools); + const strict = resolveJsonSchemaStrictSampling(tool, supportsStrictTools, isAnthropicStrictUnsupportedKeyword); const parameters = getJsonSchemaToolParameters(tool, strict); const schema = parameters as { properties?: unknown; required?: string[] }; const legacyInputSchema = { diff --git a/packages/ai/src/api/constrained-sampling.ts b/packages/ai/src/api/constrained-sampling.ts index ff9c4f151..3f1cb262e 100644 --- a/packages/ai/src/api/constrained-sampling.ts +++ b/packages/ai/src/api/constrained-sampling.ts @@ -9,6 +9,9 @@ interface JsonSchemaObject { class UnsupportedStrictJsonSchemaError extends Error {} +/** Returns true when a provider's strict mode rejects this schema keyword with this value. */ +export type UnsupportedStrictSchemaKeywordCheck = (key: string, value: unknown) => boolean; + const UNSUPPORTED_STRICT_SCHEMA_KEYS = [ "$ref", "$defs", @@ -50,7 +53,7 @@ function schemaAllowsNull(schema: unknown): boolean { return Array.isArray(schema.anyOf) && schema.anyOf.some((variant) => schemaAllowsNull(variant)); } -function makeJsonSchemaNodeStrict(schema: unknown): void { +function makeJsonSchemaNodeStrict(schema: unknown, isUnsupportedKeyword?: UnsupportedStrictSchemaKeywordCheck): void { if (!isJsonSchemaObject(schema)) { throw new UnsupportedStrictJsonSchemaError("boolean schemas are unsupported"); } @@ -59,6 +62,13 @@ function makeJsonSchemaNodeStrict(schema: unknown): void { throw new UnsupportedStrictJsonSchemaError(`${key} schemas are unsupported`); } } + if (isUnsupportedKeyword) { + for (const [key, value] of Object.entries(schema)) { + if (isUnsupportedKeyword(key, value)) { + throw new UnsupportedStrictJsonSchemaError(`${key}: ${JSON.stringify(value)} is unsupported`); + } + } + } if (schema.anyOf !== undefined) { if (!Array.isArray(schema.anyOf) || schema.anyOf.length === 0) { @@ -68,7 +78,7 @@ function makeJsonSchemaNodeStrict(schema: unknown): void { if (isStructuredSchema(variant)) { throw new UnsupportedStrictJsonSchemaError("object and array unions are unsupported"); } - makeJsonSchemaNodeStrict(variant); + makeJsonSchemaNodeStrict(variant, isUnsupportedKeyword); } } @@ -76,7 +86,7 @@ function makeJsonSchemaNodeStrict(schema: unknown): void { if (Array.isArray(schema.items)) { throw new UnsupportedStrictJsonSchemaError("tuple schemas are unsupported"); } - makeJsonSchemaNodeStrict(schema.items); + makeJsonSchemaNodeStrict(schema.items, isUnsupportedKeyword); } const isObjectSchema = schema.type === "object"; @@ -104,7 +114,7 @@ function makeJsonSchemaNodeStrict(schema: unknown): void { throw new UnsupportedStrictJsonSchemaError("required contains an unknown property"); } for (const [key, property] of Object.entries(properties)) { - makeJsonSchemaNodeStrict(property); + makeJsonSchemaNodeStrict(property, isUnsupportedKeyword); if (!required.has(key) && !schemaAllowsNull(property)) { properties[key] = { anyOf: [property, { type: "null" }] }; } @@ -114,12 +124,15 @@ function makeJsonSchemaNodeStrict(schema: unknown): void { } /** Convert a tool schema to the strict subset expected by provider constrained sampling. */ -export function makeStrictJsonSchema(schema: Tool["parameters"]): Record { +export function makeStrictJsonSchema( + schema: Tool["parameters"], + isUnsupportedKeyword?: UnsupportedStrictSchemaKeywordCheck, +): Record { const cloned: unknown = structuredClone(schema); if (!isJsonSchemaObject(cloned)) { throw new UnsupportedStrictJsonSchemaError("root schema must have type object"); } - makeJsonSchemaNodeStrict(cloned); + makeJsonSchemaNodeStrict(cloned, isUnsupportedKeyword); if (cloned.type !== "object") { throw new UnsupportedStrictJsonSchemaError("root schema must have type object"); } @@ -205,13 +218,21 @@ function inferGrammarInputProperty(tool: Tool): string { return inputProperty; } -export function resolveJsonSchemaStrictSampling(tool: Tool, supportsStrictMode: boolean): boolean | undefined { +/** + * Decide whether a JSON-schema tool is sent in strict mode. `isUnsupportedKeyword` lets a provider + * reject extra keywords its strict mode does not accept, so "prefer" tools fall back to non-strict. + */ +export function resolveJsonSchemaStrictSampling( + tool: Tool, + supportsStrictMode: boolean, + isUnsupportedKeyword?: UnsupportedStrictSchemaKeywordCheck, +): boolean | undefined { const config = tool.constrainedSampling; if (!config || config.type !== "json_schema") return undefined; if (supportsStrictMode) { try { - makeStrictJsonSchema(tool.parameters); + makeStrictJsonSchema(tool.parameters, isUnsupportedKeyword); return true; } catch (error) { if (!(error instanceof UnsupportedStrictJsonSchemaError)) throw error; diff --git a/packages/ai/test/anthropic-eager-tool-input-compat.test.ts b/packages/ai/test/anthropic-eager-tool-input-compat.test.ts index a7892ded7..e04bb47e7 100644 --- a/packages/ai/test/anthropic-eager-tool-input-compat.test.ts +++ b/packages/ai/test/anthropic-eager-tool-input-compat.test.ts @@ -33,20 +33,6 @@ const tool: Tool = { parameters: Type.Object({ value: Type.String() }), }; -const schemaCompatibilityTool: Tool = { - ...tool, - parameters: Type.Object({ value: Type.String() }, { additionalProperties: false, title: "LookupInput" }), -}; - -const strictTool: Tool = { - ...tool, - parameters: Type.Object( - { value: Type.String(), optional: Type.Optional(Type.Number()) }, - { title: "StrictLookupInput" }, - ), - constrainedSampling: { type: "json_schema", strict: "prefer" }, -}; - function createContext(tools: Tool[] = [tool]): Context { return { messages: [{ role: "user", content: "Use the tool", timestamp: Date.now() }], @@ -117,14 +103,6 @@ function getFirstTool(body: Record): Record { return tools[0] as Record; } -function getFirstToolInputSchema(body: Record): Record { - const inputSchema = getFirstTool(body).input_schema; - if (typeof inputSchema !== "object" || inputSchema === null || Array.isArray(inputSchema)) { - throw new Error("Expected first tool input schema in request body"); - } - return inputSchema as Record; -} - describe("Anthropic eager tool input streaming compatibility", () => { it("sends per-tool eager_input_streaming by default", async () => { const request = await captureAnthropicRequest(undefined, createContext()); @@ -146,26 +124,4 @@ describe("Anthropic eager tool input streaming compatibility", () => { expect(request.body.tools).toBeUndefined(); expect(request.headers["anthropic-beta"]).toBeUndefined(); }); - - it("only sends the full input schema for strict JSON-schema tools", async () => { - const legacyRequest = await captureAnthropicRequest( - { supportsStrictTools: true }, - createContext([schemaCompatibilityTool]), - ); - const parameters = schemaCompatibilityTool.parameters as { properties?: unknown; required?: unknown }; - expect(getFirstToolInputSchema(legacyRequest.body)).toEqual({ - type: "object", - properties: parameters.properties, - required: parameters.required, - }); - - const strictRequest = await captureAnthropicRequest({ supportsStrictTools: true }, createContext([strictTool])); - expect(getFirstTool(strictRequest.body).strict).toBe(true); - expect(getFirstToolInputSchema(strictRequest.body)).toMatchObject({ - additionalProperties: false, - required: ["value", "optional"], - properties: { optional: { anyOf: [{ type: "number" }, { type: "null" }] } }, - title: "StrictLookupInput", - }); - }); }); diff --git a/packages/ai/test/anthropic-strict-tool-schema.test.ts b/packages/ai/test/anthropic-strict-tool-schema.test.ts new file mode 100644 index 000000000..587cde17f --- /dev/null +++ b/packages/ai/test/anthropic-strict-tool-schema.test.ts @@ -0,0 +1,124 @@ +import { Type } from "typebox"; +import { describe, expect, it } from "vitest"; +import { stream as streamAnthropic } from "../src/api/anthropic-messages.ts"; +import type { Model, Tool } from "../src/types.ts"; +import { normalizeContext } from "../src/utils/transcript.ts"; + +interface AnthropicToolPayload { + tools?: Array<{ strict?: boolean; input_schema: Record }>; +} + +class PayloadCaptured extends Error { + constructor() { + super("payload captured"); + this.name = "PayloadCaptured"; + } +} + +function createModel(): Model<"anthropic-messages"> { + return { + id: "claude-opus-4-8", + name: "Claude Opus 4.8", + api: "anthropic-messages", + provider: "test-anthropic", + baseUrl: "http://127.0.0.1:9", + reasoning: true, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 200000, + maxTokens: 32000, + compat: { forceAdaptiveThinking: true, supportsStrictTools: true }, + }; +} + +function createTool(parameters: Tool["parameters"], constrainedSampling?: Tool["constrainedSampling"]): Tool { + return { + name: "lookup", + description: "Look up a value", + parameters, + ...(constrainedSampling ? { constrainedSampling } : {}), + }; +} + +function createStrictTool(parameters: Tool["parameters"]): Tool { + return createTool(parameters, { type: "json_schema", strict: "prefer" }); +} + +async function captureFirstTool(tool: Tool): Promise[number]> { + let payload: AnthropicToolPayload | undefined; + const stream = streamAnthropic( + createModel(), + normalizeContext({ messages: [{ role: "user", content: "Use the tool", timestamp: Date.now() }], tools: [tool] }), + { + apiKey: "test-key", + cacheRetention: "none", + onPayload: (value) => { + payload = value as AnthropicToolPayload; + throw new PayloadCaptured(); + }, + }, + ); + await stream.result(); + + const firstTool = payload?.tools?.[0]; + if (!firstTool) { + throw new Error("Expected a tool in the captured Anthropic payload"); + } + return firstTool; +} + +describe("Anthropic strict tool schemas", () => { + it("only sends the full input schema for strict JSON-schema tools", async () => { + const legacyParameters = Type.Object( + { value: Type.String() }, + { additionalProperties: false, title: "LookupInput" }, + ); + const legacyTool = await captureFirstTool(createTool(legacyParameters)); + expect(legacyTool.strict).toBeUndefined(); + expect(legacyTool.input_schema).toEqual({ + type: "object", + properties: legacyParameters.properties, + required: legacyParameters.required, + }); + + const strictTool = await captureFirstTool( + createStrictTool( + Type.Object( + { value: Type.String(), optional: Type.Optional(Type.Number()) }, + { title: "StrictLookupInput" }, + ), + ), + ); + expect(strictTool.strict).toBe(true); + expect(strictTool.input_schema).toMatchObject({ + additionalProperties: false, + required: ["value", "optional"], + properties: { optional: { anyOf: [{ type: "number" }, { type: "null" }] } }, + title: "StrictLookupInput", + }); + }); + + // https://github.com/earendil-works/pi/issues/9953 + it("sends prefer tools non-strict when they use keywords Anthropic strict mode rejects", async () => { + const unsupportedParameters: Tool["parameters"][] = [ + Type.Object({ timeoutMs: Type.Optional(Type.Integer({ minimum: 1, maximum: 300000 })) }), + Type.Object({ options: Type.Object({ tags: Type.Array(Type.String(), { minItems: 2 }) }) }), + Type.Object({ expression: Type.String({ format: "regex" }) }), + ]; + for (const parameters of unsupportedParameters) { + const tool = await captureFirstTool(createStrictTool(parameters)); + expect(tool.strict).toBeUndefined(); + } + + const supportedTool = await captureFirstTool( + createStrictTool( + Type.Object({ + code: Type.String({ minLength: 1, maxLength: 1000, pattern: "^[a-z]+$" }), + url: Type.String({ format: "uri" }), + tags: Type.Array(Type.String(), { minItems: 1 }), + }), + ), + ); + expect(supportedTool.strict).toBe(true); + }); +});