From b9497c8c1f2bfd922856c09d3f19117fea90ac74 Mon Sep 17 00:00:00 2001 From: David Brailovsky Date: Thu, 6 Aug 2026 11:30:04 +0200 Subject: [PATCH] fix(ai): correct Fireworks GLM prompt caching, closes #7676 --- packages/ai/CHANGELOG.md | 1 + packages/ai/scripts/generate-models.ts | 134 +++++++++++------- packages/ai/test/fireworks-models.test.ts | 25 ++++ .../openai-completions-prompt-cache.test.ts | 10 ++ 4 files changed, 115 insertions(+), 55 deletions(-) diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index 29ac5cecc..b63dee291 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -91,6 +91,7 @@ - Fixed newer provider refreshes being blocked by or overwritten by an older stalled generation, including persisted catalog publication. - Updated GPT-5.6 Terra and Luna pricing across OpenAI and passthrough model catalogs. - Fixed Fireworks Kimi K3 models to use the OpenAI-compatible API with native reasoning-effort levels and deferred tools ([#7199](https://github.com/earendil-works/pi/issues/7199), [#7230](https://github.com/earendil-works/pi/pull/7230) by [@XBeg9](https://github.com/XBeg9)). +- Fixed Fireworks GLM 5.2 models sending the unsupported `prompt_cache_retention` field when long cache retention is enabled, and enabled session affinity for automatic prompt caching ([#7676](https://github.com/earendil-works/pi/issues/7676)). - Updated Groq's Qwen reasoning override for the replacement `qwen/qwen3.6-27b` model. - Fixed the OpenCode Go provider display name. - Fixed provider error normalization treating arrays and class instances as structured response bodies instead of preserving their original errors ([#7205](https://github.com/earendil-works/pi/pull/7205) by [@erikogenvik](https://github.com/erikogenvik)). diff --git a/packages/ai/scripts/generate-models.ts b/packages/ai/scripts/generate-models.ts index 7dc057f53..754662bcb 100644 --- a/packages/ai/scripts/generate-models.ts +++ b/packages/ai/scripts/generate-models.ts @@ -1187,6 +1187,84 @@ function processBasetenModels(provider: ModelsDevProvider | undefined): Model[] { + if (!provider?.models) return []; + + const anthropicCompat: AnthropicMessagesCompat = { + sendSessionAffinityHeaders: true, + supportsEagerToolInputStreaming: false, + supportsCacheControlOnTools: false, + supportsLongCacheRetention: false, + }; + const openAICompat: OpenAICompletionsCompat = { + supportsStore: false, + supportsDeveloperRole: false, + sendSessionAffinityHeaders: true, + supportsLongCacheRetention: false, + }; + const kimiK3Compat: OpenAICompletionsCompat = { + ...openAICompat, + requiresReasoningContentOnAssistantMessages: true, + thinkingFormat: "openai", + deferredToolsMode: "kimi", + }; + const models: Model[] = []; + + for (const [modelId, model] of Object.entries(provider.models)) { + if (model.tool_call !== true) continue; + + const input: ("text" | "image")[] = model.modalities?.input?.includes("image") + ? ["text", "image"] + : ["text"]; + const common = { + id: modelId, + name: model.name || modelId, + provider: "fireworks", + reasoning: model.reasoning === true, + input, + cost: { + input: model.cost?.input || 0, + output: model.cost?.output || 0, + cacheRead: model.cost?.cache_read || 0, + cacheWrite: model.cost?.cache_write || 0, + }, + contextWindow: model.limit?.context || 4096, + maxTokens: model.limit?.output || 4096, + }; + + if (modelId.includes("glm-5p2")) { + models.push({ + ...common, + api: "openai-completions", + baseUrl: "https://api.fireworks.ai/inference/v1", + compat: openAICompat, + }); + } else if (modelId.includes("kimi-k3")) { + models.push({ + ...common, + api: "openai-completions", + baseUrl: "https://api.fireworks.ai/inference/v1", + compat: kimiK3Compat, + }); + } else { + models.push({ + ...common, + api: "anthropic-messages", + // Fireworks Anthropic-compatible API - SDK appends /v1/messages. + baseUrl: "https://api.fireworks.ai/inference", + // Fireworks prompt caching uses automatic prefix matching + session affinity. + // x-session-affinity routes requests to the same replica for cache hits. + // cache_control on tools and eager_input_streaming are not supported. + // See: https://docs.fireworks.ai/tools-sdks/anthropic-compatibility + compat: anthropicCompat, + }); + } + recordModelsDevReasoningOptions("fireworks", modelId, model); + } + + return models; +} + async function loadModelsDevData(): Promise[]> { try { console.log("Fetching models from models.dev API..."); @@ -1639,43 +1717,7 @@ async function loadModelsDevData(): Promise[]> { } } - // Process Fireworks models - if (data["fireworks-ai"]?.models) { - for (const [modelId, model] of Object.entries(data["fireworks-ai"].models)) { - const m = model as ModelsDevModel; - if (m.tool_call !== true) continue; - - models.push({ - id: modelId, - name: m.name || modelId, - api: "anthropic-messages", - provider: "fireworks", - // Fireworks Anthropic-compatible API - SDK appends /v1/messages - baseUrl: "https://api.fireworks.ai/inference", - reasoning: m.reasoning === true, - input: m.modalities?.input?.includes("image") ? ["text", "image"] : ["text"], - cost: { - input: m.cost?.input || 0, - output: m.cost?.output || 0, - cacheRead: m.cost?.cache_read || 0, - cacheWrite: m.cost?.cache_write || 0, - }, - contextWindow: m.limit?.context || 4096, - maxTokens: m.limit?.output || 4096, - // Fireworks prompt caching uses automatic prefix matching + session affinity. - // x-session-affinity routes requests to the same replica for cache hits. - // cache_control on tools and eager_input_streaming are not supported. - // See: https://docs.fireworks.ai/tools-sdks/anthropic-compatibility - compat: { - sendSessionAffinityHeaders: true, - supportsEagerToolInputStreaming: false, - supportsCacheControlOnTools: false, - supportsLongCacheRetention: false, - }, - }); - recordModelsDevReasoningOptions("fireworks", modelId, m); - } - } + models.push(...processFireworksModels(data["fireworks-ai"])); // Process NVIDIA NIM models if (data.nvidia?.models) { @@ -2269,24 +2311,6 @@ async function generateModels() { candidate.cost.output = 1.9; candidate.cost.cacheRead = 0.119; } - if (candidate.provider === "fireworks" && candidate.id.includes("glm-5p2")) { - candidate.api = "openai-completions"; - candidate.baseUrl = "https://api.fireworks.ai/inference/v1"; - candidate.compat = { supportsStore: false, supportsDeveloperRole: false }; - } - if (candidate.provider === "fireworks" && candidate.id.includes("kimi-k3")) { - candidate.api = "openai-completions"; - candidate.baseUrl = "https://api.fireworks.ai/inference/v1"; - candidate.compat = { - supportsStore: false, - supportsDeveloperRole: false, - requiresReasoningContentOnAssistantMessages: true, - thinkingFormat: "openai", - deferredToolsMode: "kimi", - sendSessionAffinityHeaders: true, - supportsLongCacheRetention: false, - }; - } } diff --git a/packages/ai/test/fireworks-models.test.ts b/packages/ai/test/fireworks-models.test.ts index 65ee6f87d..b37991b5b 100644 --- a/packages/ai/test/fireworks-models.test.ts +++ b/packages/ai/test/fireworks-models.test.ts @@ -58,6 +58,31 @@ describe("Fireworks models", () => { expect(fast.thinkingLevelMap).toEqual(base.thinkingLevelMap); }); + it.each(["accounts/fireworks/models/glm-5p2", "accounts/fireworks/routers/glm-5p2-fast"] as const)( + "omits unsupported long cache retention for %s", + async (modelId) => { + const model = getModel("fireworks", modelId); + let payload: Record | undefined; + const response = streamSimple( + model, + { messages: [{ role: "user", content: "test", timestamp: 0 }] }, + { + apiKey: "test-fireworks-key", + cacheRetention: "long", + sessionId: "test-fireworks-session", + onPayload: (value) => { + payload = value as Record; + throw new Error("payload captured"); + }, + }, + ); + await response.result(); + + expect(payload).toBeDefined(); + expect(payload?.prompt_cache_retention).toBeUndefined(); + }, + ); + it("routes Kimi K3 through the OpenAI-compatible API with native effort controls", async () => { const base = getModel("fireworks", "accounts/fireworks/models/kimi-k3"); const fast = getModel("fireworks", "accounts/fireworks/routers/kimi-k3-fast"); diff --git a/packages/ai/test/openai-completions-prompt-cache.test.ts b/packages/ai/test/openai-completions-prompt-cache.test.ts index b697ff0f4..28b255160 100644 --- a/packages/ai/test/openai-completions-prompt-cache.test.ts +++ b/packages/ai/test/openai-completions-prompt-cache.test.ts @@ -171,6 +171,16 @@ describe("openai-completions prompt caching", () => { expect(headers["x-session-affinity"]).toBe("session-affinity"); }); + it.each(["accounts/fireworks/models/glm-5p2", "accounts/fireworks/routers/glm-5p2-fast"] as const)( + "sends Fireworks session affinity for %s", + async (modelId) => { + const model = getModel("fireworks", modelId); + const { headers } = await captureRequest({ sessionId: "fireworks-session" }, model); + + expect(headers["x-session-affinity"]).toBe("fireworks-session"); + }, + ); + it("uses OpenAI no-session format when configured", async () => { const model = createModel({ compat: { sendSessionAffinityHeaders: true, sessionAffinityFormat: "openai-nosession" },