From 650e7a6126d4f45a7bd4457b198fcdcc6fcf98c1 Mon Sep 17 00:00:00 2001 From: David Brailovsky Date: Tue, 25 Aug 2026 12:11:21 +0200 Subject: [PATCH] fix(ai): derive OpenRouter reasoning controls (#8614) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #8454 Co-authored-by: Petr Houška --- packages/ai/CHANGELOG.md | 1 + packages/ai/scripts/generate-models.ts | 29 ++++- .../scripts/openrouter-reasoning-options.ts | 23 ++++ .../test/openrouter-reasoning-options.test.ts | 109 ++++++++++++++++++ 4 files changed, 160 insertions(+), 2 deletions(-) create mode 100644 packages/ai/scripts/openrouter-reasoning-options.ts create mode 100644 packages/ai/test/openrouter-reasoning-options.test.ts diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index 6cc3e74da..2a035edf3 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -34,6 +34,7 @@ - Fixed Kimi OpenAI-compatible usage reporting so top-level `cached_tokens` count as cache reads instead of normal input tokens ([#8075](https://github.com/earendil-works/pi/issues/8075)). - Fixed Google Generative AI and Vertex AI custom models ignoring `thinkingLevelMap`, which dropped extended thinking controls ([#8135](https://github.com/earendil-works/pi/issues/8135)). - Fixed Xiaomi model catalog generation retaining shut-down MiMo V2 model names after models.dev marked them deprecated ([#8187](https://github.com/earendil-works/pi/issues/8187)). +- Fixed OpenRouter reasoning controls by deriving `off` support and available effort levels from OpenRouter's model metadata, preventing reasoning-mandatory models from receiving `effort: "none"` ([#8454](https://github.com/earendil-works/pi/issues/8454)). ## [0.84.2] - 2026-08-14 diff --git a/packages/ai/scripts/generate-models.ts b/packages/ai/scripts/generate-models.ts index 9bd3396ce..51b1f95af 100644 --- a/packages/ai/scripts/generate-models.ts +++ b/packages/ai/scripts/generate-models.ts @@ -4,6 +4,10 @@ import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, renameSy import { dirname, join, resolve } from "path"; import { fileURLToPath } from "url"; import { getEffortThinkingLevelMap, type ModelsDevReasoningOption } from "./models-dev-reasoning-options.ts"; +import { + getOpenRouterThinkingLevelMap, + type OpenRouterReasoningMetadata, +} from "./openrouter-reasoning-options.ts"; import { CLOUDFLARE_AI_GATEWAY_ANTHROPIC_BASE_URL, CLOUDFLARE_AI_GATEWAY_COMPAT_BASE_URL, @@ -127,6 +131,25 @@ interface NvidiaNimModelListItem { id: string; } +interface OpenRouterModelListItem { + id: string; + name: string; + supported_parameters?: string[]; + architecture?: { modality?: string }; + pricing?: { + prompt?: string; + completion?: string; + input_cache_read?: string; + input_cache_write?: string; + }; + top_provider?: { + context_length?: number; + max_completion_tokens?: number; + }; + context_length?: number; + reasoning?: OpenRouterReasoningMetadata; +} + interface AiGatewayModel { id: string; name?: string; @@ -1051,11 +1074,11 @@ async function fetchOpenRouterModels(): Promise[]> { console.log("Fetching models from OpenRouter API..."); const response = await fetch("https://openrouter.ai/api/v1/models"); if (!response.ok) throw new Error(`OpenRouter API returned ${response.status}`); - const data = await response.json(); + const data = (await response.json()) as { data?: OpenRouterModelListItem[] }; const models: Model[] = []; - for (const model of data.data) { + for (const model of data.data ?? []) { // Only include models that support tools if (!model.supported_parameters?.includes("tools")) continue; @@ -1078,6 +1101,7 @@ async function fetchOpenRouterModels(): Promise[]> { const cacheWriteCost = roundCost(parseFloat(model.pricing?.input_cache_write || "0") * 1_000_000); const contextWindow = model.top_provider?.context_length || model.context_length || 4096; + const thinkingLevelMap = getOpenRouterThinkingLevelMap(model.reasoning); const normalizedModel: Model = { id: modelKey, @@ -1086,6 +1110,7 @@ async function fetchOpenRouterModels(): Promise[]> { baseUrl: "https://openrouter.ai/api/v1", provider, reasoning: model.supported_parameters?.includes("reasoning") || false, + ...(thinkingLevelMap && { thinkingLevelMap }), input, cost: { input: inputCost, diff --git a/packages/ai/scripts/openrouter-reasoning-options.ts b/packages/ai/scripts/openrouter-reasoning-options.ts new file mode 100644 index 000000000..6fc4a5c6c --- /dev/null +++ b/packages/ai/scripts/openrouter-reasoning-options.ts @@ -0,0 +1,23 @@ +import type { ThinkingLevel, ThinkingLevelMap } from "../src/types.ts"; +import { getEffortThinkingLevelMap } from "./models-dev-reasoning-options.ts"; + +export interface OpenRouterReasoningMetadata { + mandatory?: boolean; + default_enabled?: boolean; + supported_efforts?: Array; + default_effort?: ThinkingLevel | "none"; +} + +/** Convert OpenRouter's reasoning metadata into Pi model capabilities. */ +export function getOpenRouterThinkingLevelMap( + reasoning: OpenRouterReasoningMetadata | undefined, +): ThinkingLevelMap | undefined { + if (!reasoning) return undefined; + if (!reasoning.supported_efforts?.length) return reasoning.mandatory === true ? { off: null } : undefined; + + // OpenRouter's supported_efforts uses the same effort values as models.dev reasoning_options, + // so both sources can share the same Pi thinking-level conversion. + const map = getEffortThinkingLevelMap([{ type: "effort", values: reasoning.supported_efforts }]); + if (!map) return reasoning.mandatory === true ? { off: null } : undefined; + return { ...map, off: reasoning.mandatory === true ? null : "none" }; +} diff --git a/packages/ai/test/openrouter-reasoning-options.test.ts b/packages/ai/test/openrouter-reasoning-options.test.ts new file mode 100644 index 000000000..20eb8ba7a --- /dev/null +++ b/packages/ai/test/openrouter-reasoning-options.test.ts @@ -0,0 +1,109 @@ +import { describe, expect, it } from "vitest"; +import { getOpenRouterThinkingLevelMap } from "../scripts/openrouter-reasoning-options.ts"; +import { streamSimple } from "../src/api/openai-completions.ts"; +import type { Context, Model, ThinkingLevelMap } from "../src/types.ts"; + +const context: Context = { + messages: [{ role: "user", content: "Hello", timestamp: 0 }], +}; + +function openRouterModel(thinkingLevelMap?: ThinkingLevelMap): Model<"openai-completions"> { + return { + id: "stealth/ox-alpha", + name: "Ox Alpha", + api: "openai-completions", + provider: "openrouter", + baseUrl: "https://example.invalid/v1", + reasoning: true, + thinkingLevelMap, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 128000, + maxTokens: 4096, + compat: { thinkingFormat: "openrouter" }, + }; +} + +async function capturePayload(model: Model<"openai-completions">, reasoning?: "low") { + let payload: { reasoning?: { effort?: string } } | undefined; + await streamSimple(model, context, { + apiKey: "test", + reasoning, + onPayload: (request) => { + payload = request as { reasoning?: { effort?: string } }; + throw new Error("payload captured"); + }, + }).result(); + if (!payload) throw new Error("OpenRouter payload was not captured"); + return payload; +} + +describe("getOpenRouterThinkingLevelMap", () => { + it("marks mandatory reasoning and unsupported efforts unavailable", () => { + expect( + getOpenRouterThinkingLevelMap({ + mandatory: true, + default_enabled: true, + supported_efforts: ["max", "high", "low"], + default_effort: "max", + }), + ).toEqual({ + off: null, + minimal: null, + low: "low", + medium: null, + high: "high", + xhigh: null, + max: "max", + }); + }); + + it("still marks off unavailable when OpenRouter omits effort metadata", () => { + expect(getOpenRouterThinkingLevelMap({ mandatory: true })).toEqual({ off: null }); + }); + + it("keeps off available while restricting optional models to supported efforts", () => { + expect( + getOpenRouterThinkingLevelMap({ + mandatory: false, + default_enabled: true, + supported_efforts: ["high", "low"], + }), + ).toEqual({ + off: "none", + minimal: null, + low: "low", + medium: null, + high: "high", + xhigh: null, + max: null, + }); + }); + + it("does not add metadata for optional models without effort controls", () => { + expect(getOpenRouterThinkingLevelMap({ mandatory: false })).toBeUndefined(); + }); +}); + +describe("OpenRouter mandatory reasoning payloads", () => { + const mandatoryMap = getOpenRouterThinkingLevelMap({ + mandatory: true, + supported_efforts: ["max", "high", "low"], + }); + + it("omits reasoning when a background call does not request it", async () => { + expect(await capturePayload(openRouterModel(mandatoryMap))).not.toHaveProperty("reasoning"); + }); + + it("still sends an explicitly selected supported effort", async () => { + expect(await capturePayload(openRouterModel(mandatoryMap), "low")).toMatchObject({ + reasoning: { effort: "low" }, + }); + }); + + it("continues to explicitly disable reasoning for optional models", async () => { + expect(await capturePayload(openRouterModel())).toMatchObject({ + reasoning: { effort: "none" }, + }); + }); +});