From 0f4d5e960a03b43708129ba89c182dfd0b51110b Mon Sep 17 00:00:00 2001 From: Danila Poyarkov Date: Wed, 4 Nov 2026 14:36:35 +0400 Subject: [PATCH] fix(ai): allow prompt cache keys for compatible proxies (#9442) --- packages/ai/src/api/azure-openai-responses.ts | 3 +- packages/ai/src/api/openai-completions.ts | 12 ++- packages/ai/src/api/openai-responses.ts | 4 +- packages/ai/src/providers/compat-schema.ts | 16 ++++ ...letions-non-null-assistant-content.test.ts | 5 +- .../openai-completions-prompt-cache.test.ts | 84 +++++++++++++++++++ ...penai-completions-thinking-as-text.test.ts | 2 +- ...nai-completions-tool-result-images.test.ts | 5 +- .../openai-responses-prompt-cache.test.ts | 72 ++++++++++++++++ .../coding-agent/docs/models-configuration.md | 2 + .../coding-agent/schemas/models.schema.json | 17 ++++ .../coding-agent/test/model-registry.test.ts | 37 ++++++++ 12 files changed, 252 insertions(+), 7 deletions(-) create mode 100644 packages/ai/test/openai-responses-prompt-cache.test.ts diff --git a/packages/ai/src/api/azure-openai-responses.ts b/packages/ai/src/api/azure-openai-responses.ts index 24e73fd05..ec98e3320 100644 --- a/packages/ai/src/api/azure-openai-responses.ts +++ b/packages/ai/src/api/azure-openai-responses.ts @@ -205,7 +205,8 @@ function buildParams( model: deploymentName, input: messages, stream: true, - prompt_cache_key: clampOpenAIPromptCacheKey(options?.sessionId), + prompt_cache_key: + model.compat?.promptCacheKeyMode === "disabled" ? undefined : clampOpenAIPromptCacheKey(options?.sessionId), store: false, }; diff --git a/packages/ai/src/api/openai-completions.ts b/packages/ai/src/api/openai-completions.ts index 155e9e5e5..30587eefd 100644 --- a/packages/ai/src/api/openai-completions.ts +++ b/packages/ai/src/api/openai-completions.ts @@ -177,12 +177,14 @@ type ResolvedOpenAICompletionsCompat = Omit< Required, | "cacheControlFormat" | "supportsThinkingTokenBudget" + | "promptCacheKeyMode" | "thinkingTokenBudgetField" | "supportsMidConvoSystemMessages" | "supportsMidConvoToolAdditions" | "vllmPriority" > & { cacheControlFormat?: OpenAICompletionsCompat["cacheControlFormat"]; + promptCacheKeyMode?: OpenAICompletionsCompat["promptCacheKeyMode"]; supportsThinkingTokenBudget?: OpenAICompletionsCompat["supportsThinkingTokenBudget"]; thinkingTokenBudgetField?: OpenAICompletionsCompat["thinkingTokenBudgetField"]; supportsMidConvoSystemMessages?: OpenAICompletionsCompat["supportsMidConvoSystemMessages"]; @@ -803,14 +805,19 @@ function buildParams( grammarToolInputProperties, }); const cacheControl = getCompatCacheControl(compat, cacheRetention); + const promptCacheKeyMode = compat.promptCacheKeyMode ?? "auto"; + const supportsPromptCacheKey = + promptCacheKeyMode === "enabled" || + (promptCacheKeyMode === "auto" && + (model.baseUrl.includes("api.openai.com") || + (cacheRetention === "long" && compat.supportsLongCacheRetention))); const params: OpenAI.Chat.Completions.ChatCompletionCreateParamsStreaming = { model: model.id, messages, stream: true, prompt_cache_key: - (model.baseUrl.includes("api.openai.com") && cacheRetention !== "none") || - (cacheRetention === "long" && compat.supportsLongCacheRetention) + cacheRetention !== "none" && supportsPromptCacheKey ? clampOpenAIPromptCacheKey(options?.sessionId) : undefined, prompt_cache_retention: cacheRetention === "long" && compat.supportsLongCacheRetention ? "24h" : undefined, @@ -1726,6 +1733,7 @@ function getCompat(model: Model<"openai-completions">): ResolvedOpenAICompletion sendSessionAffinityHeaders: model.compat.sendSessionAffinityHeaders ?? detected.sendSessionAffinityHeaders, sessionAffinityFormat: model.compat.sessionAffinityFormat ?? detected.sessionAffinityFormat, supportsLongCacheRetention: model.compat.supportsLongCacheRetention ?? detected.supportsLongCacheRetention, + promptCacheKeyMode: model.compat.promptCacheKeyMode, vllmPriority: model.compat.vllmPriority, }; } diff --git a/packages/ai/src/api/openai-responses.ts b/packages/ai/src/api/openai-responses.ts index 8d576a702..0ff9f8bfc 100644 --- a/packages/ai/src/api/openai-responses.ts +++ b/packages/ai/src/api/openai-responses.ts @@ -93,6 +93,7 @@ function getCompat(model: Model<"openai-responses">): Required) { + return opt(union([stringLiteral("auto"), stringLiteral("enabled"), stringLiteral("disabled")], options)); +} + const OpenAICompletionsCompatProperties = { supportsStore: opt( bool({ @@ -340,6 +344,10 @@ const OpenAICompletionsCompatProperties = { description: "Session-affinity header format. openai sends session_id, x-client-request-id, and x-session-affinity; openai-nosession sends x-client-request-id and x-session-affinity; openrouter sends x-session-id. Does not affect prompt_cache_key. Default: auto-detected.", }), + promptCacheKeyMode: optionalPromptCacheKeyMode({ + description: + "Controls the prompt_cache_key request field. auto sends it to api.openai.com, or with long cache retention when supportsLongCacheRetention is true; enabled sends it whenever caching is active, for compatible proxies; disabled never sends it. Default: auto.", + }), supportsLongCacheRetention: optionalCompatBoolean({ description: 'Whether the provider supports long prompt cache retention (prompt_cache_retention: "24h" or Anthropic-style cache_control.ttl: "1h", depending on format). Default: auto-detected from provider and URL.', @@ -372,6 +380,10 @@ const OpenAIResponsesCompatProperties = { description: "Session-affinity header format. openai sends session-id and x-client-request-id; openai-nosession sends x-client-request-id; openrouter sends x-session-id. Does not affect prompt_cache_key. Default: auto-detected.", }), + promptCacheKeyMode: optionalPromptCacheKeyMode({ + description: + "Controls the prompt_cache_key request field. auto and enabled send it whenever caching is active; disabled never sends it, for compatible proxies that reject it. Default: auto.", + }), supportsLongCacheRetention: optionalCompatBoolean({ description: 'Whether the provider supports long prompt cache retention. This uses prompt_cache_options.ttl: "30m" on GPT-5.6+ and prompt_cache_retention: "24h" on earlier models.', @@ -571,6 +583,10 @@ const ProviderCompatPropertyOverrides = { sessionAffinityFormat: optionalSessionAffinityFormat({ description: "Session-affinity header format. Defaults are API-specific or auto-detected.", }), + promptCacheKeyMode: optionalPromptCacheKeyMode({ + description: + "Controls the prompt_cache_key request field for openai-completions and openai-responses: auto keeps each API's default, enabled sends it whenever caching is active, disabled never sends it. Default: auto.", + }), supportsMidConvoEffort: optionalCompatBoolean({ description: "Whether thinking level changes are sent inside the conversation instead of changing the request's effort, so a change keeps the prompt cache. Used by the anthropic-messages and openai-responses APIs.", diff --git a/packages/ai/test/openai-completions-non-null-assistant-content.test.ts b/packages/ai/test/openai-completions-non-null-assistant-content.test.ts index 8cc592a32..96933cf1a 100644 --- a/packages/ai/test/openai-completions-non-null-assistant-content.test.ts +++ b/packages/ai/test/openai-completions-non-null-assistant-content.test.ts @@ -12,7 +12,10 @@ const emptyUsage: Usage = { cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, }; -const baseCompat: Omit, "thinkingTokenBudgetField" | "vllmPriority"> & { +const baseCompat: Omit< + Required, + "promptCacheKeyMode" | "thinkingTokenBudgetField" | "vllmPriority" +> & { thinkingTokenBudgetField?: OpenAICompletionsCompat["thinkingTokenBudgetField"]; } = { supportsStore: true, diff --git a/packages/ai/test/openai-completions-prompt-cache.test.ts b/packages/ai/test/openai-completions-prompt-cache.test.ts index 45f3df5fa..98ad31c79 100644 --- a/packages/ai/test/openai-completions-prompt-cache.test.ts +++ b/packages/ai/test/openai-completions-prompt-cache.test.ts @@ -151,6 +151,90 @@ describe("openai-completions prompt caching", () => { expect(payload?.prompt_cache_retention).toBeUndefined(); }); + it.each([undefined, "short", "long"] as const)( + "sends a stable cache key through an opted-in proxy with retention %s", + async (cacheRetention) => { + const model = createModel({ + baseUrl: "https://proxy.example.com/v1", + compat: { promptCacheKeyMode: "enabled", supportsLongCacheRetention: false }, + }); + const options = { cacheRetention, sessionId: "session-proxy" }; + const first = await captureRequest(options, model); + const repeated = await captureRequest(options, model); + const other = await captureRequest({ ...options, sessionId: "other-session" }, model); + + expect(first.payload?.prompt_cache_key).toBe("session-proxy"); + expect(repeated.payload?.prompt_cache_key).toBe(first.payload?.prompt_cache_key); + expect(other.payload?.prompt_cache_key).toBe("other-session"); + expect(first.payload?.prompt_cache_retention).toBeUndefined(); + expect(first.headers.session_id).toBeUndefined(); + }, + ); + + it.each([undefined, "auto"] as const)("preserves default proxy behavior with mode %s", async (mode) => { + const model = createModel({ + baseUrl: "https://proxy.example.com/v1", + compat: mode ? { promptCacheKeyMode: mode } : undefined, + }); + const short = await captureRequest({ sessionId: "session-proxy" }, model); + const long = await captureRequest({ cacheRetention: "long", sessionId: "session-proxy" }, model); + + expect(short.payload?.prompt_cache_key).toBeUndefined(); + expect(long.payload?.prompt_cache_key).toBe("session-proxy"); + expect(long.payload?.prompt_cache_retention).toBe("24h"); + }); + + it("respects disabled caching on an opted-in proxy", async () => { + process.env.PI_CACHE_RETENTION = "long"; + const model = createModel({ + baseUrl: "https://proxy.example.com/v1", + compat: { promptCacheKeyMode: "enabled" }, + }); + const { payload } = await captureRequest({ cacheRetention: "none", sessionId: "session-proxy" }, model); + + expect(payload?.prompt_cache_key).toBeUndefined(); + expect(payload?.prompt_cache_retention).toBeUndefined(); + }); + + it.each(["short", "long"] as const)("allows disabling cache keys with %s retention", async (cacheRetention) => { + const model = createModel({ compat: { promptCacheKeyMode: "disabled" } }); + const { payload } = await captureRequest({ cacheRetention, sessionId: "session-disabled" }, model); + + expect(payload?.prompt_cache_key).toBeUndefined(); + expect(payload?.prompt_cache_retention).toBe(cacheRetention === "long" ? "24h" : undefined); + }); + + it("lets a proxy disable cache keys independently of long retention", async () => { + const model = createModel({ + baseUrl: "https://proxy.example.com/v1", + compat: { promptCacheKeyMode: "disabled", supportsLongCacheRetention: true }, + }); + const { payload } = await captureRequest({ cacheRetention: "long", sessionId: "session-proxy" }, model); + + expect(payload?.prompt_cache_key).toBeUndefined(); + expect(payload?.prompt_cache_retention).toBe("24h"); + }); + + it("does not invent an identity for an opted-in proxy", async () => { + const model = createModel({ + baseUrl: "https://proxy.example.com/v1", + compat: { promptCacheKeyMode: "enabled" }, + }); + const { payload } = await captureRequest({}, model); + + expect(payload?.prompt_cache_key).toBeUndefined(); + }); + + it("clamps proxy cache keys to 64 Unicode characters", async () => { + const model = createModel({ + baseUrl: "https://proxy.example.com/v1", + compat: { promptCacheKeyMode: "enabled" }, + }); + const { payload } = await captureRequest({ sessionId: "𐐀".repeat(67) }, model); + + expect(payload?.prompt_cache_key).toBe("𐐀".repeat(64)); + }); + it("uses PI_CACHE_RETENTION for direct OpenAI requests", async () => { process.env.PI_CACHE_RETENTION = "long"; const { payload } = await captureRequest({ sessionId: "session-env" }); diff --git a/packages/ai/test/openai-completions-thinking-as-text.test.ts b/packages/ai/test/openai-completions-thinking-as-text.test.ts index a98c61612..80f76f53b 100644 --- a/packages/ai/test/openai-completions-thinking-as-text.test.ts +++ b/packages/ai/test/openai-completions-thinking-as-text.test.ts @@ -52,7 +52,7 @@ const compat = { supportsLongCacheRetention: true, } satisfies Omit< Required, - "cacheControlFormat" | "thinkingTokenBudgetField" | "vllmPriority" + "cacheControlFormat" | "promptCacheKeyMode" | "thinkingTokenBudgetField" | "vllmPriority" > & { cacheControlFormat?: OpenAICompletionsCompat["cacheControlFormat"]; thinkingTokenBudgetField?: OpenAICompletionsCompat["thinkingTokenBudgetField"]; diff --git a/packages/ai/test/openai-completions-tool-result-images.test.ts b/packages/ai/test/openai-completions-tool-result-images.test.ts index 6819ed045..f3d2dd7cb 100644 --- a/packages/ai/test/openai-completions-tool-result-images.test.ts +++ b/packages/ai/test/openai-completions-tool-result-images.test.ts @@ -12,7 +12,10 @@ const emptyUsage: Usage = { cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, }; -const compat: Omit, "thinkingTokenBudgetField" | "vllmPriority"> & { +const compat: Omit< + Required, + "promptCacheKeyMode" | "thinkingTokenBudgetField" | "vllmPriority" +> & { thinkingTokenBudgetField?: OpenAICompletionsCompat["thinkingTokenBudgetField"]; } = { supportsStore: true, diff --git a/packages/ai/test/openai-responses-prompt-cache.test.ts b/packages/ai/test/openai-responses-prompt-cache.test.ts new file mode 100644 index 000000000..c8722c4ab --- /dev/null +++ b/packages/ai/test/openai-responses-prompt-cache.test.ts @@ -0,0 +1,72 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { stream } from "../src/api/openai-responses.ts"; +import { normalizeContext } from "../src/compat.ts"; +import type { Model, OpenAIResponsesCompat } from "../src/types.ts"; + +const context = normalizeContext({ + messages: [{ role: "user", content: "hello", timestamp: 1 }], +}); + +function createModel(promptCacheKeyMode?: OpenAIResponsesCompat["promptCacheKeyMode"]): Model<"openai-responses"> { + return { + id: "test-model", + name: "Test Model", + api: "openai-responses", + provider: "test-proxy", + baseUrl: "https://proxy.example.com/v1", + compat: { promptCacheKeyMode }, + reasoning: false, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 128000, + maxTokens: 4096, + }; +} + +afterEach(() => { + vi.unstubAllGlobals(); +}); + +async function capturePayload( + promptCacheKeyMode?: OpenAIResponsesCompat["promptCacheKeyMode"], + cacheRetention?: "none" | "short" | "long", +): Promise> { + let payload: Record | undefined; + vi.stubGlobal( + "fetch", + vi.fn(async (_url: string | URL, init?: RequestInit) => { + payload = JSON.parse(String(init?.body)); + return new Response( + 'data: {"type":"response.completed","response":{"status":"completed","usage":{"input_tokens":1,"output_tokens":1,"total_tokens":2,"input_tokens_details":{"cached_tokens":0}}}}\n\n', + { status: 200, headers: { "content-type": "text/event-stream" } }, + ); + }), + ); + + await stream(createModel(promptCacheKeyMode), context, { + apiKey: "test-key", + sessionId: "session-123", + cacheRetention, + }).result(); + + if (!payload) throw new Error("Expected request payload"); + return payload; +} + +describe("OpenAI Responses prompt cache key policy", () => { + it("preserves the existing default behavior", async () => { + expect((await capturePayload()).prompt_cache_key).toBe("session-123"); + }); + + it("emits the key when enabled", async () => { + expect((await capturePayload("enabled")).prompt_cache_key).toBe("session-123"); + }); + + it("omits the key when disabled", async () => { + expect((await capturePayload("disabled")).prompt_cache_key).toBeUndefined(); + }); + + it("lets cacheRetention none override enabled mode", async () => { + expect((await capturePayload("enabled", "none")).prompt_cache_key).toBeUndefined(); + }); +}); diff --git a/packages/coding-agent/docs/models-configuration.md b/packages/coding-agent/docs/models-configuration.md index 2181fbc75..1cc988273 100644 --- a/packages/coding-agent/docs/models-configuration.md +++ b/packages/coding-agent/docs/models-configuration.md @@ -136,6 +136,7 @@ See [Describe model input and caching](models.md#describe-model-input-and-cachin | `cacheControlFormat` | `"anthropic"` | | Cache control convention for prompt caching. Anthropic applies cache_control markers to the system prompt, last tool definition, and last user, assistant, or tool-result text content. | | `sendSessionAffinityHeaders` | boolean | | Whether to send session-affinity data from options.sessionId. Default: true for OpenRouter endpoints, false otherwise. | | `sessionAffinityFormat` | `"openai" \| "openai-nosession" \| "openrouter"` | | Session-affinity header format. openai sends session_id, x-client-request-id, and x-session-affinity; openai-nosession sends x-client-request-id and x-session-affinity; openrouter sends x-session-id. Does not affect prompt_cache_key. Default: auto-detected. | +| `promptCacheKeyMode` | `"auto" \| "enabled" \| "disabled"` | | Controls the prompt_cache_key request field. auto sends it to api.openai.com, or with long cache retention when supportsLongCacheRetention is true; enabled sends it whenever caching is active, for compatible proxies; disabled never sends it. Default: auto. | | `supportsLongCacheRetention` | boolean | | Whether the provider supports long prompt cache retention (prompt_cache_retention: "24h" or Anthropic-style cache_control.ttl: "1h", depending on format). Default: auto-detected from provider and URL. | | `vllmPriority` | number | | vLLM scheduler priority sent as the top-level priority request field. Lower values are handled earlier and the server default is 0. Only meaningful with --scheduling-policy priority. Off by default and not set on the generated catalog. | @@ -148,6 +149,7 @@ Also applies to `azure-openai-responses` and `openai-codex-responses`. | `supportsDeveloperRole` | boolean | `true` | Whether the provider supports the developer role instead of system. | | `supportsMidConvoSystemMessages` | boolean | `false` | Whether the exact model accepts developer or system messages after the conversation has started. When false, later system messages are folded into the leading system message. The generated catalog enables this for verified models. | | `sessionAffinityFormat` | `"openai" \| "openai-nosession" \| "openrouter"` | | Session-affinity header format. openai sends session-id and x-client-request-id; openai-nosession sends x-client-request-id; openrouter sends x-session-id. Does not affect prompt_cache_key. Default: auto-detected. | +| `promptCacheKeyMode` | `"auto" \| "enabled" \| "disabled"` | | Controls the prompt_cache_key request field. auto and enabled send it whenever caching is active; disabled never sends it, for compatible proxies that reject it. Default: auto. | | `supportsLongCacheRetention` | boolean | `true` | Whether the provider supports long prompt cache retention. This uses prompt_cache_options.ttl: "30m" on GPT-5.6+ and prompt_cache_retention: "24h" on earlier models. | | `supportsStrictMode` | boolean | | Whether the provider supports strict JSON-schema function tools. Defaults are API-specific; generated OpenAI models enable it explicitly. | | `supportsOpenAIGrammarTools` | boolean | `false` | Whether to emit OpenAI custom tools with Lark or regex grammar formats. When false, grammar-constrained tools fall back to normal function tools. The generated catalog enables this for capable models. | diff --git a/packages/coding-agent/schemas/models.schema.json b/packages/coding-agent/schemas/models.schema.json index aea82ce5f..2f6fee4f3 100644 --- a/packages/coding-agent/schemas/models.schema.json +++ b/packages/coding-agent/schemas/models.schema.json @@ -1017,6 +1017,23 @@ ], "description": "Session-affinity header format. Defaults are API-specific or auto-detected." }, + "promptCacheKeyMode": { + "anyOf": [ + { + "type": "string", + "const": "auto" + }, + { + "type": "string", + "const": "enabled" + }, + { + "type": "string", + "const": "disabled" + } + ], + "description": "Controls the prompt_cache_key request field for openai-completions and openai-responses: auto keeps each API's default, enabled sends it whenever caching is active, disabled never sends it. Default: auto." + }, "supportsLongCacheRetention": { "type": "boolean", "description": "Whether the provider supports long prompt cache retention. Defaults are API-specific or auto-detected." diff --git a/packages/coding-agent/test/model-registry.test.ts b/packages/coding-agent/test/model-registry.test.ts index 67142f701..13586226f 100644 --- a/packages/coding-agent/test/model-registry.test.ts +++ b/packages/coding-agent/test/model-registry.test.ts @@ -549,6 +549,27 @@ describe("ModelRegistry", () => { expect(compat?.supportsEagerToolInputStreaming).toBe(false); }); + test("custom models inherit and override prompt cache key mode independently of long retention", async () => { + const provider: ModelsJsonProvider = { + baseUrl: "https://proxy.example.com/v1", + api: "openai-completions", + compat: { promptCacheKeyMode: "enabled", supportsLongCacheRetention: false }, + models: [{ id: "inherited" }, { id: "disabled", compat: { promptCacheKeyMode: "disabled" } }], + }; + writeRawModelsJson({ demo: provider }); + + const registry = await createModelRegistry(authStorage, modelsJsonPath); + expect(registry.getError()).toBeUndefined(); + expect(registry.find("demo", "inherited")?.compat).toMatchObject({ + promptCacheKeyMode: "enabled", + supportsLongCacheRetention: false, + }); + expect(registry.find("demo", "disabled")?.compat).toMatchObject({ + promptCacheKeyMode: "disabled", + supportsLongCacheRetention: false, + }); + }); + test("compat schema accepts long cache retention flag", async () => { writeRawModelsJson({ demo: { @@ -922,6 +943,22 @@ describe("ModelRegistry", () => { expect((opus?.compat as OpenAICompletionsCompat | undefined)?.supportsFinishReason).toBe(true); }); + test("built-in models inherit prompt cache key support unless overridden", async () => { + const provider: ModelsJsonProvider = { + baseUrl: "https://proxy.example.com/v1", + compat: { promptCacheKeyMode: "enabled" }, + modelOverrides: { + "gpt-4o-mini": { compat: { promptCacheKeyMode: "disabled" } }, + }, + }; + writeRawModelsJson({ openai: provider }); + + const registry = await createModelRegistry(authStorage, modelsJsonPath); + expect(registry.getError()).toBeUndefined(); + expect(registry.find("openai", "gpt-4o")?.compat).toMatchObject({ promptCacheKeyMode: "enabled" }); + expect(registry.find("openai", "gpt-4o-mini")?.compat).toMatchObject({ promptCacheKeyMode: "disabled" }); + }); + test("model override deep merges compat settings", async () => { writeRawModelsJson({ openrouter: {