From 0a4fad5489f303fe023e3948d473731d7b2f8acb Mon Sep 17 00:00:00 2001 From: David Brailovsky Date: Mon, 2 Nov 2026 22:43:27 +0100 Subject: [PATCH] fix(ai): leave room for OpenRouter's input overcount when clamping max tokens (#11712) OpenRouter checks input + max_tokens against the context window before routing, using its own tokenizer. For prose it counted up to 12% more input tokens than the provider. pi left only 4096 tokens spare, so models whose output limit is close to the window (e.g. DeepSeek flash) were rejected with a 400, which pi then treated as a context overflow and compacted. The default output limit now leaves 15% of the estimated input unused (at least 4096). When that would leave less than 32768 tokens of output, only the fixed 4096 margin is kept and output is capped at 32768. fixes #10497 --- packages/ai/CHANGELOG.md | 1 + packages/ai/src/api/simple-options.ts | 17 +++++- packages/ai/test/clamp-max-tokens.test.ts | 71 +++++++++++++++++++++++ 3 files changed, 87 insertions(+), 2 deletions(-) create mode 100644 packages/ai/test/clamp-max-tokens.test.ts diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index 1823bff41..b6e781ae5 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -6,6 +6,7 @@ - Fixed OpenCode Go Claude Haiku 5.5 failing with a 400 (`Invalid signature in thinking block ... bound to a different conversation`) after the tool list, system prompt, or earlier history changed, which left the session failing on every retry. OpenCode Go and Zen Claude 5.5 models, Opus 5, and Fable 5.1 now use per-message effort and send `thinking.block_binding: drop_block`, like the Anthropic provider ([#10954](https://github.com/earendil-works/pi/issues/10954)) - Fixed Anthropic non-strict tool schemas dropping root `$defs` and `definitions`, which left property `$ref`s pointing at nothing. Claude then produced wrong arguments for such tools (common in MCP servers), for example a JSON string instead of an object, and some Anthropic-compatible gateways rejected the request outright ([#9557](https://github.com/earendil-works/pi/issues/9557)) +- Fixed OpenRouter rejecting requests with a 400 (`This endpoint's maximum context length is ...`) for models whose output limit is close to the context window, such as DeepSeek flash. OpenRouter counts the input with its own tokenizer, which can be several percent higher than the provider's count, and pi left only 4,096 tokens spare. pi then treated the 400 as a context overflow and compacted the session, even at around 20% of the window. The default output limit now leaves 15% of the estimated input unused, and near a full window at most 32,768 output tokens are requested ([#10497](https://github.com/earendil-works/pi/issues/10497)) ## [1.5.0] - 2026-11-02 diff --git a/packages/ai/src/api/simple-options.ts b/packages/ai/src/api/simple-options.ts index 85bfa78f9..375c9e4be 100644 --- a/packages/ai/src/api/simple-options.ts +++ b/packages/ai/src/api/simple-options.ts @@ -13,12 +13,25 @@ import type { import { estimateContextTokens } from "../utils/estimate.ts"; const CONTEXT_SAFETY_TOKENS = 4096; +// OpenRouter counts the input with its own tokenizer and rejects input + max_tokens above the +// window before routing. It counted prose up to 12% above the provider's count (#10497). +const CONTEXT_SAFETY_RATIO = 0.15; +// The 15% margin is kept only while it leaves at least this much output. Otherwise (a nearly full +// large window, or any small window such as a 32k local model) only the fixed margin is kept, and +// output is capped at this size so an OpenRouter overcount still fits for a while before compaction. +const MIN_OUTPUT_WITH_RATIO_MARGIN = 32768; const MIN_MAX_TOKENS = 1; +/** Fit maxTokens into the context window, leaving room for the input being counted higher than estimated. */ export function clampMaxTokensToContext(model: Model, context: TranscriptContext, maxTokens: number): number { if (model.contextWindow <= 0) return Math.max(MIN_MAX_TOKENS, maxTokens); - const available = model.contextWindow - estimateContextTokens(context).tokens - CONTEXT_SAFETY_TOKENS; - return Math.min(maxTokens, Math.max(MIN_MAX_TOKENS, available)); + const input = estimateContextTokens(context).tokens; + const margin = Math.max(CONTEXT_SAFETY_TOKENS, Math.ceil(input * CONTEXT_SAFETY_RATIO)); + let room = model.contextWindow - input - margin; + if (room < MIN_OUTPUT_WITH_RATIO_MARGIN) { + room = Math.min(MIN_OUTPUT_WITH_RATIO_MARGIN, model.contextWindow - input - CONTEXT_SAFETY_TOKENS); + } + return Math.min(maxTokens, Math.max(MIN_MAX_TOKENS, room)); } export function resolveSamplingParams( diff --git a/packages/ai/test/clamp-max-tokens.test.ts b/packages/ai/test/clamp-max-tokens.test.ts new file mode 100644 index 000000000..ada7a22a4 --- /dev/null +++ b/packages/ai/test/clamp-max-tokens.test.ts @@ -0,0 +1,71 @@ +import { describe, expect, it } from "vitest"; +import { clampMaxTokensToContext } from "../src/api/simple-options.ts"; +import { getModel } from "../src/compat.ts"; +import type { AssistantMessage, Model } from "../src/types.ts"; +import { normalizeContext } from "../src/utils/transcript.ts"; + +const WINDOW = 1_048_576; + +function modelWith(maxTokens: number, contextWindow = WINDOW): Model<"openai-completions"> { + const { compat: _compat, ...base } = getModel("openai", "gpt-4o-mini")!; + return { ...base, api: "openai-completions", contextWindow, maxTokens }; +} + +// The last assistant usage anchors the estimate, so the estimated input is exactly `inputTokens`. +function contextWithInput(inputTokens: number) { + const assistant: AssistantMessage = { + role: "assistant", + content: [{ type: "text", text: "ok" }], + api: "openai-completions", + provider: "openai", + model: "gpt-4o-mini", + usage: { + input: inputTokens, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: inputTokens, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + stopReason: "stop", + timestamp: 2, + }; + return normalizeContext({ messages: [{ role: "user", content: "hi", timestamp: 1 }, assistant] }); +} + +describe("clampMaxTokensToContext", () => { + it("leaves 15% of the input unused so an OpenRouter overcount still fits (#10497)", () => { + // The failing request from #10497: OpenRouter counted 229,747 input tokens where pi estimated 220,815. + const maxTokens = clampMaxTokensToContext(modelWith(943_718), contextWithInput(220_815), 943_718); + expect(maxTokens).toBe(WINDOW - 220_815 - 33_123); + expect(229_747 + maxTokens).toBeLessThanOrEqual(WINDOW); + }); + + it("leaves at least 4096 tokens unused for small inputs", () => { + expect(clampMaxTokensToContext(modelWith(WINDOW), contextWithInput(10_000), WINDOW)).toBe(WINDOW - 10_000 - 4096); + }); + + it("keeps the model output limit when it already fits", () => { + expect(clampMaxTokensToContext(modelWith(64_000), contextWithInput(100_000), 64_000)).toBe(64_000); + }); + + it("sends up to 32768 tokens when the proportional margin leaves no room", () => { + expect(clampMaxTokensToContext(modelWith(943_718), contextWithInput(912_000), 943_718)).toBe(32_768); + }); + + it("uses the real room minus 4096 when less than 32768 tokens fit", () => { + expect(clampMaxTokensToContext(modelWith(943_718), contextWithInput(1_030_000), 943_718)).toBe( + WINDOW - 1_030_000 - 4096, + ); + }); + + it("keeps only the fixed 4096 margin for small windows", () => { + expect(clampMaxTokensToContext(modelWith(32_768, 32_768), contextWithInput(20_000), 32_768)).toBe( + 32_768 - 20_000 - 4096, + ); + }); + + it("sends 1 token when the input already exceeds the window", () => { + expect(clampMaxTokensToContext(modelWith(943_718), contextWithInput(1_050_000), 943_718)).toBe(1); + }); +});