mirror of
https://github.com/badlogic/pi-mono.git
synced 2026-11-05 23:37:25 +00:00
fix(ai): send Anthropic tools non-strict when schema has rejected keywords
Anthropic strict tool use rejects keywords like minimum/maximum with a 400 for the whole request, so one bounded-integer tool broke every turn. resolveJsonSchemaStrictSampling() now takes an optional per-provider keyword check. With Anthropic's check, "prefer" tools that hit it fall back to non-strict. Other providers are unchanged. fixes #9953
This commit is contained in:
parent
7ef68d4f2f
commit
295cc72b03
5 changed files with 195 additions and 54 deletions
|
|
@ -9,6 +9,7 @@
|
|||
### Fixed
|
||||
|
||||
- Fixed context overflow detection for Z.AI CN endpoint `Prompt exceeds max length` errors ([#10208](https://github.com/earendil-works/pi/issues/10208))
|
||||
- Fixed Anthropic requests failing when a `strict: "prefer"` tool schema uses keywords Anthropic strict tool use rejects, such as `minimum`/`maximum`; such tools are now sent non-strict ([#9953](https://github.com/earendil-works/pi/issues/9953))
|
||||
|
||||
## [0.99.1] - 2026-09-29
|
||||
|
||||
|
|
|
|||
|
|
@ -48,7 +48,11 @@ import {
|
|||
type TranscriptContext,
|
||||
} from "../utils/transcript.ts";
|
||||
|
||||
import { getJsonSchemaToolParameters, resolveJsonSchemaStrictSampling } from "./constrained-sampling.ts";
|
||||
import {
|
||||
getJsonSchemaToolParameters,
|
||||
resolveJsonSchemaStrictSampling,
|
||||
type UnsupportedStrictSchemaKeywordCheck,
|
||||
} from "./constrained-sampling.ts";
|
||||
import { buildCopilotDynamicHeaders, hasCopilotVisionInput } from "./github-copilot-headers.ts";
|
||||
import { adjustMaxTokensForThinking, buildBaseOptions, clampMaxTokensToContext } from "./simple-options.ts";
|
||||
import { transformMessages } from "./transform-messages.ts";
|
||||
|
|
@ -1462,6 +1466,41 @@ function shouldUseFineGrainedToolStreamingBeta(
|
|||
return getCurrentTools(context.messages).length > 0 && !getAnthropicCompat(model).supportsEagerToolInputStreaming;
|
||||
}
|
||||
|
||||
// Keywords Anthropic strict tool use rejects with a 400 for the whole request.
|
||||
// https://platform.claude.com/docs/en/build-with-claude/structured-outputs#json-schema-limitations
|
||||
const ANTHROPIC_STRICT_UNSUPPORTED_KEYWORDS = new Set([
|
||||
"minimum",
|
||||
"maximum",
|
||||
"exclusiveMinimum",
|
||||
"exclusiveMaximum",
|
||||
"multipleOf",
|
||||
"maxItems",
|
||||
"uniqueItems",
|
||||
"minContains",
|
||||
"maxContains",
|
||||
"minProperties",
|
||||
"maxProperties",
|
||||
]);
|
||||
const ANTHROPIC_STRICT_STRING_FORMATS = new Set([
|
||||
"date-time",
|
||||
"time",
|
||||
"date",
|
||||
"duration",
|
||||
"email",
|
||||
"hostname",
|
||||
"uri",
|
||||
"ipv4",
|
||||
"ipv6",
|
||||
"uuid",
|
||||
]);
|
||||
|
||||
const isAnthropicStrictUnsupportedKeyword: UnsupportedStrictSchemaKeywordCheck = (key, value) => {
|
||||
if (ANTHROPIC_STRICT_UNSUPPORTED_KEYWORDS.has(key)) return true;
|
||||
if (key === "minItems") return value !== 0 && value !== 1;
|
||||
if (key === "format") return typeof value !== "string" || !ANTHROPIC_STRICT_STRING_FORMATS.has(value);
|
||||
return false;
|
||||
};
|
||||
|
||||
function convertTools(
|
||||
tools: Tool[],
|
||||
isOAuthToken: boolean,
|
||||
|
|
@ -1472,7 +1511,7 @@ function convertTools(
|
|||
if (!tools) return [];
|
||||
|
||||
return tools.map((tool, index) => {
|
||||
const strict = resolveJsonSchemaStrictSampling(tool, supportsStrictTools);
|
||||
const strict = resolveJsonSchemaStrictSampling(tool, supportsStrictTools, isAnthropicStrictUnsupportedKeyword);
|
||||
const parameters = getJsonSchemaToolParameters(tool, strict);
|
||||
const schema = parameters as { properties?: unknown; required?: string[] };
|
||||
const legacyInputSchema = {
|
||||
|
|
|
|||
|
|
@ -9,6 +9,9 @@ interface JsonSchemaObject {
|
|||
|
||||
class UnsupportedStrictJsonSchemaError extends Error {}
|
||||
|
||||
/** Returns true when a provider's strict mode rejects this schema keyword with this value. */
|
||||
export type UnsupportedStrictSchemaKeywordCheck = (key: string, value: unknown) => boolean;
|
||||
|
||||
const UNSUPPORTED_STRICT_SCHEMA_KEYS = [
|
||||
"$ref",
|
||||
"$defs",
|
||||
|
|
@ -50,7 +53,7 @@ function schemaAllowsNull(schema: unknown): boolean {
|
|||
return Array.isArray(schema.anyOf) && schema.anyOf.some((variant) => schemaAllowsNull(variant));
|
||||
}
|
||||
|
||||
function makeJsonSchemaNodeStrict(schema: unknown): void {
|
||||
function makeJsonSchemaNodeStrict(schema: unknown, isUnsupportedKeyword?: UnsupportedStrictSchemaKeywordCheck): void {
|
||||
if (!isJsonSchemaObject(schema)) {
|
||||
throw new UnsupportedStrictJsonSchemaError("boolean schemas are unsupported");
|
||||
}
|
||||
|
|
@ -59,6 +62,13 @@ function makeJsonSchemaNodeStrict(schema: unknown): void {
|
|||
throw new UnsupportedStrictJsonSchemaError(`${key} schemas are unsupported`);
|
||||
}
|
||||
}
|
||||
if (isUnsupportedKeyword) {
|
||||
for (const [key, value] of Object.entries(schema)) {
|
||||
if (isUnsupportedKeyword(key, value)) {
|
||||
throw new UnsupportedStrictJsonSchemaError(`${key}: ${JSON.stringify(value)} is unsupported`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (schema.anyOf !== undefined) {
|
||||
if (!Array.isArray(schema.anyOf) || schema.anyOf.length === 0) {
|
||||
|
|
@ -68,7 +78,7 @@ function makeJsonSchemaNodeStrict(schema: unknown): void {
|
|||
if (isStructuredSchema(variant)) {
|
||||
throw new UnsupportedStrictJsonSchemaError("object and array unions are unsupported");
|
||||
}
|
||||
makeJsonSchemaNodeStrict(variant);
|
||||
makeJsonSchemaNodeStrict(variant, isUnsupportedKeyword);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -76,7 +86,7 @@ function makeJsonSchemaNodeStrict(schema: unknown): void {
|
|||
if (Array.isArray(schema.items)) {
|
||||
throw new UnsupportedStrictJsonSchemaError("tuple schemas are unsupported");
|
||||
}
|
||||
makeJsonSchemaNodeStrict(schema.items);
|
||||
makeJsonSchemaNodeStrict(schema.items, isUnsupportedKeyword);
|
||||
}
|
||||
|
||||
const isObjectSchema = schema.type === "object";
|
||||
|
|
@ -104,7 +114,7 @@ function makeJsonSchemaNodeStrict(schema: unknown): void {
|
|||
throw new UnsupportedStrictJsonSchemaError("required contains an unknown property");
|
||||
}
|
||||
for (const [key, property] of Object.entries(properties)) {
|
||||
makeJsonSchemaNodeStrict(property);
|
||||
makeJsonSchemaNodeStrict(property, isUnsupportedKeyword);
|
||||
if (!required.has(key) && !schemaAllowsNull(property)) {
|
||||
properties[key] = { anyOf: [property, { type: "null" }] };
|
||||
}
|
||||
|
|
@ -114,12 +124,15 @@ function makeJsonSchemaNodeStrict(schema: unknown): void {
|
|||
}
|
||||
|
||||
/** Convert a tool schema to the strict subset expected by provider constrained sampling. */
|
||||
export function makeStrictJsonSchema(schema: Tool["parameters"]): Record<string, unknown> {
|
||||
export function makeStrictJsonSchema(
|
||||
schema: Tool["parameters"],
|
||||
isUnsupportedKeyword?: UnsupportedStrictSchemaKeywordCheck,
|
||||
): Record<string, unknown> {
|
||||
const cloned: unknown = structuredClone(schema);
|
||||
if (!isJsonSchemaObject(cloned)) {
|
||||
throw new UnsupportedStrictJsonSchemaError("root schema must have type object");
|
||||
}
|
||||
makeJsonSchemaNodeStrict(cloned);
|
||||
makeJsonSchemaNodeStrict(cloned, isUnsupportedKeyword);
|
||||
if (cloned.type !== "object") {
|
||||
throw new UnsupportedStrictJsonSchemaError("root schema must have type object");
|
||||
}
|
||||
|
|
@ -205,13 +218,21 @@ function inferGrammarInputProperty(tool: Tool): string {
|
|||
return inputProperty;
|
||||
}
|
||||
|
||||
export function resolveJsonSchemaStrictSampling(tool: Tool, supportsStrictMode: boolean): boolean | undefined {
|
||||
/**
|
||||
* Decide whether a JSON-schema tool is sent in strict mode. `isUnsupportedKeyword` lets a provider
|
||||
* reject extra keywords its strict mode does not accept, so "prefer" tools fall back to non-strict.
|
||||
*/
|
||||
export function resolveJsonSchemaStrictSampling(
|
||||
tool: Tool,
|
||||
supportsStrictMode: boolean,
|
||||
isUnsupportedKeyword?: UnsupportedStrictSchemaKeywordCheck,
|
||||
): boolean | undefined {
|
||||
const config = tool.constrainedSampling;
|
||||
if (!config || config.type !== "json_schema") return undefined;
|
||||
|
||||
if (supportsStrictMode) {
|
||||
try {
|
||||
makeStrictJsonSchema(tool.parameters);
|
||||
makeStrictJsonSchema(tool.parameters, isUnsupportedKeyword);
|
||||
return true;
|
||||
} catch (error) {
|
||||
if (!(error instanceof UnsupportedStrictJsonSchemaError)) throw error;
|
||||
|
|
|
|||
|
|
@ -33,20 +33,6 @@ const tool: Tool = {
|
|||
parameters: Type.Object({ value: Type.String() }),
|
||||
};
|
||||
|
||||
const schemaCompatibilityTool: Tool = {
|
||||
...tool,
|
||||
parameters: Type.Object({ value: Type.String() }, { additionalProperties: false, title: "LookupInput" }),
|
||||
};
|
||||
|
||||
const strictTool: Tool = {
|
||||
...tool,
|
||||
parameters: Type.Object(
|
||||
{ value: Type.String(), optional: Type.Optional(Type.Number()) },
|
||||
{ title: "StrictLookupInput" },
|
||||
),
|
||||
constrainedSampling: { type: "json_schema", strict: "prefer" },
|
||||
};
|
||||
|
||||
function createContext(tools: Tool[] = [tool]): Context {
|
||||
return {
|
||||
messages: [{ role: "user", content: "Use the tool", timestamp: Date.now() }],
|
||||
|
|
@ -117,14 +103,6 @@ function getFirstTool(body: Record<string, unknown>): Record<string, unknown> {
|
|||
return tools[0] as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function getFirstToolInputSchema(body: Record<string, unknown>): Record<string, unknown> {
|
||||
const inputSchema = getFirstTool(body).input_schema;
|
||||
if (typeof inputSchema !== "object" || inputSchema === null || Array.isArray(inputSchema)) {
|
||||
throw new Error("Expected first tool input schema in request body");
|
||||
}
|
||||
return inputSchema as Record<string, unknown>;
|
||||
}
|
||||
|
||||
describe("Anthropic eager tool input streaming compatibility", () => {
|
||||
it("sends per-tool eager_input_streaming by default", async () => {
|
||||
const request = await captureAnthropicRequest(undefined, createContext());
|
||||
|
|
@ -146,26 +124,4 @@ describe("Anthropic eager tool input streaming compatibility", () => {
|
|||
expect(request.body.tools).toBeUndefined();
|
||||
expect(request.headers["anthropic-beta"]).toBeUndefined();
|
||||
});
|
||||
|
||||
it("only sends the full input schema for strict JSON-schema tools", async () => {
|
||||
const legacyRequest = await captureAnthropicRequest(
|
||||
{ supportsStrictTools: true },
|
||||
createContext([schemaCompatibilityTool]),
|
||||
);
|
||||
const parameters = schemaCompatibilityTool.parameters as { properties?: unknown; required?: unknown };
|
||||
expect(getFirstToolInputSchema(legacyRequest.body)).toEqual({
|
||||
type: "object",
|
||||
properties: parameters.properties,
|
||||
required: parameters.required,
|
||||
});
|
||||
|
||||
const strictRequest = await captureAnthropicRequest({ supportsStrictTools: true }, createContext([strictTool]));
|
||||
expect(getFirstTool(strictRequest.body).strict).toBe(true);
|
||||
expect(getFirstToolInputSchema(strictRequest.body)).toMatchObject({
|
||||
additionalProperties: false,
|
||||
required: ["value", "optional"],
|
||||
properties: { optional: { anyOf: [{ type: "number" }, { type: "null" }] } },
|
||||
title: "StrictLookupInput",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
124
packages/ai/test/anthropic-strict-tool-schema.test.ts
Normal file
124
packages/ai/test/anthropic-strict-tool-schema.test.ts
Normal file
|
|
@ -0,0 +1,124 @@
|
|||
import { Type } from "typebox";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { stream as streamAnthropic } from "../src/api/anthropic-messages.ts";
|
||||
import type { Model, Tool } from "../src/types.ts";
|
||||
import { normalizeContext } from "../src/utils/transcript.ts";
|
||||
|
||||
interface AnthropicToolPayload {
|
||||
tools?: Array<{ strict?: boolean; input_schema: Record<string, unknown> }>;
|
||||
}
|
||||
|
||||
class PayloadCaptured extends Error {
|
||||
constructor() {
|
||||
super("payload captured");
|
||||
this.name = "PayloadCaptured";
|
||||
}
|
||||
}
|
||||
|
||||
function createModel(): Model<"anthropic-messages"> {
|
||||
return {
|
||||
id: "claude-opus-4-8",
|
||||
name: "Claude Opus 4.8",
|
||||
api: "anthropic-messages",
|
||||
provider: "test-anthropic",
|
||||
baseUrl: "http://127.0.0.1:9",
|
||||
reasoning: true,
|
||||
input: ["text"],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 200000,
|
||||
maxTokens: 32000,
|
||||
compat: { forceAdaptiveThinking: true, supportsStrictTools: true },
|
||||
};
|
||||
}
|
||||
|
||||
function createTool(parameters: Tool["parameters"], constrainedSampling?: Tool["constrainedSampling"]): Tool {
|
||||
return {
|
||||
name: "lookup",
|
||||
description: "Look up a value",
|
||||
parameters,
|
||||
...(constrainedSampling ? { constrainedSampling } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function createStrictTool(parameters: Tool["parameters"]): Tool {
|
||||
return createTool(parameters, { type: "json_schema", strict: "prefer" });
|
||||
}
|
||||
|
||||
async function captureFirstTool(tool: Tool): Promise<NonNullable<AnthropicToolPayload["tools"]>[number]> {
|
||||
let payload: AnthropicToolPayload | undefined;
|
||||
const stream = streamAnthropic(
|
||||
createModel(),
|
||||
normalizeContext({ messages: [{ role: "user", content: "Use the tool", timestamp: Date.now() }], tools: [tool] }),
|
||||
{
|
||||
apiKey: "test-key",
|
||||
cacheRetention: "none",
|
||||
onPayload: (value) => {
|
||||
payload = value as AnthropicToolPayload;
|
||||
throw new PayloadCaptured();
|
||||
},
|
||||
},
|
||||
);
|
||||
await stream.result();
|
||||
|
||||
const firstTool = payload?.tools?.[0];
|
||||
if (!firstTool) {
|
||||
throw new Error("Expected a tool in the captured Anthropic payload");
|
||||
}
|
||||
return firstTool;
|
||||
}
|
||||
|
||||
describe("Anthropic strict tool schemas", () => {
|
||||
it("only sends the full input schema for strict JSON-schema tools", async () => {
|
||||
const legacyParameters = Type.Object(
|
||||
{ value: Type.String() },
|
||||
{ additionalProperties: false, title: "LookupInput" },
|
||||
);
|
||||
const legacyTool = await captureFirstTool(createTool(legacyParameters));
|
||||
expect(legacyTool.strict).toBeUndefined();
|
||||
expect(legacyTool.input_schema).toEqual({
|
||||
type: "object",
|
||||
properties: legacyParameters.properties,
|
||||
required: legacyParameters.required,
|
||||
});
|
||||
|
||||
const strictTool = await captureFirstTool(
|
||||
createStrictTool(
|
||||
Type.Object(
|
||||
{ value: Type.String(), optional: Type.Optional(Type.Number()) },
|
||||
{ title: "StrictLookupInput" },
|
||||
),
|
||||
),
|
||||
);
|
||||
expect(strictTool.strict).toBe(true);
|
||||
expect(strictTool.input_schema).toMatchObject({
|
||||
additionalProperties: false,
|
||||
required: ["value", "optional"],
|
||||
properties: { optional: { anyOf: [{ type: "number" }, { type: "null" }] } },
|
||||
title: "StrictLookupInput",
|
||||
});
|
||||
});
|
||||
|
||||
// https://github.com/earendil-works/pi/issues/9953
|
||||
it("sends prefer tools non-strict when they use keywords Anthropic strict mode rejects", async () => {
|
||||
const unsupportedParameters: Tool["parameters"][] = [
|
||||
Type.Object({ timeoutMs: Type.Optional(Type.Integer({ minimum: 1, maximum: 300000 })) }),
|
||||
Type.Object({ options: Type.Object({ tags: Type.Array(Type.String(), { minItems: 2 }) }) }),
|
||||
Type.Object({ expression: Type.String({ format: "regex" }) }),
|
||||
];
|
||||
for (const parameters of unsupportedParameters) {
|
||||
const tool = await captureFirstTool(createStrictTool(parameters));
|
||||
expect(tool.strict).toBeUndefined();
|
||||
}
|
||||
|
||||
const supportedTool = await captureFirstTool(
|
||||
createStrictTool(
|
||||
Type.Object({
|
||||
code: Type.String({ minLength: 1, maxLength: 1000, pattern: "^[a-z]+$" }),
|
||||
url: Type.String({ format: "uri" }),
|
||||
tags: Type.Array(Type.String(), { minItems: 1 }),
|
||||
}),
|
||||
),
|
||||
);
|
||||
expect(supportedTool.strict).toBe(true);
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue