mirror of
https://github.com/badlogic/pi-mono.git
synced 2026-11-05 23:37:25 +00:00
fix(ai): allow prompt cache keys for compatible proxies (#9442)
This commit is contained in:
parent
a3a1005053
commit
0f4d5e960a
12 changed files with 252 additions and 7 deletions
|
|
@ -136,6 +136,7 @@ See [Describe model input and caching](models.md#describe-model-input-and-cachin
|
|||
| `cacheControlFormat` | `"anthropic"` | | Cache control convention for prompt caching. Anthropic applies cache_control markers to the system prompt, last tool definition, and last user, assistant, or tool-result text content. |
|
||||
| `sendSessionAffinityHeaders` | boolean | | Whether to send session-affinity data from options.sessionId. Default: true for OpenRouter endpoints, false otherwise. |
|
||||
| `sessionAffinityFormat` | `"openai" \| "openai-nosession" \| "openrouter"` | | Session-affinity header format. openai sends session_id, x-client-request-id, and x-session-affinity; openai-nosession sends x-client-request-id and x-session-affinity; openrouter sends x-session-id. Does not affect prompt_cache_key. Default: auto-detected. |
|
||||
| `promptCacheKeyMode` | `"auto" \| "enabled" \| "disabled"` | | Controls the prompt_cache_key request field. auto sends it to api.openai.com, or with long cache retention when supportsLongCacheRetention is true; enabled sends it whenever caching is active, for compatible proxies; disabled never sends it. Default: auto. |
|
||||
| `supportsLongCacheRetention` | boolean | | Whether the provider supports long prompt cache retention (prompt_cache_retention: "24h" or Anthropic-style cache_control.ttl: "1h", depending on format). Default: auto-detected from provider and URL. |
|
||||
| `vllmPriority` | number | | vLLM scheduler priority sent as the top-level priority request field. Lower values are handled earlier and the server default is 0. Only meaningful with --scheduling-policy priority. Off by default and not set on the generated catalog. |
|
||||
|
||||
|
|
@ -148,6 +149,7 @@ Also applies to `azure-openai-responses` and `openai-codex-responses`.
|
|||
| `supportsDeveloperRole` | boolean | `true` | Whether the provider supports the developer role instead of system. |
|
||||
| `supportsMidConvoSystemMessages` | boolean | `false` | Whether the exact model accepts developer or system messages after the conversation has started. When false, later system messages are folded into the leading system message. The generated catalog enables this for verified models. |
|
||||
| `sessionAffinityFormat` | `"openai" \| "openai-nosession" \| "openrouter"` | | Session-affinity header format. openai sends session-id and x-client-request-id; openai-nosession sends x-client-request-id; openrouter sends x-session-id. Does not affect prompt_cache_key. Default: auto-detected. |
|
||||
| `promptCacheKeyMode` | `"auto" \| "enabled" \| "disabled"` | | Controls the prompt_cache_key request field. auto and enabled send it whenever caching is active; disabled never sends it, for compatible proxies that reject it. Default: auto. |
|
||||
| `supportsLongCacheRetention` | boolean | `true` | Whether the provider supports long prompt cache retention. This uses prompt_cache_options.ttl: "30m" on GPT-5.6+ and prompt_cache_retention: "24h" on earlier models. |
|
||||
| `supportsStrictMode` | boolean | | Whether the provider supports strict JSON-schema function tools. Defaults are API-specific; generated OpenAI models enable it explicitly. |
|
||||
| `supportsOpenAIGrammarTools` | boolean | `false` | Whether to emit OpenAI custom tools with Lark or regex grammar formats. When false, grammar-constrained tools fall back to normal function tools. The generated catalog enables this for capable models. |
|
||||
|
|
|
|||
|
|
@ -1017,6 +1017,23 @@
|
|||
],
|
||||
"description": "Session-affinity header format. Defaults are API-specific or auto-detected."
|
||||
},
|
||||
"promptCacheKeyMode": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string",
|
||||
"const": "auto"
|
||||
},
|
||||
{
|
||||
"type": "string",
|
||||
"const": "enabled"
|
||||
},
|
||||
{
|
||||
"type": "string",
|
||||
"const": "disabled"
|
||||
}
|
||||
],
|
||||
"description": "Controls the prompt_cache_key request field for openai-completions and openai-responses: auto keeps each API's default, enabled sends it whenever caching is active, disabled never sends it. Default: auto."
|
||||
},
|
||||
"supportsLongCacheRetention": {
|
||||
"type": "boolean",
|
||||
"description": "Whether the provider supports long prompt cache retention. Defaults are API-specific or auto-detected."
|
||||
|
|
|
|||
|
|
@ -549,6 +549,27 @@ describe("ModelRegistry", () => {
|
|||
expect(compat?.supportsEagerToolInputStreaming).toBe(false);
|
||||
});
|
||||
|
||||
test("custom models inherit and override prompt cache key mode independently of long retention", async () => {
|
||||
const provider: ModelsJsonProvider = {
|
||||
baseUrl: "https://proxy.example.com/v1",
|
||||
api: "openai-completions",
|
||||
compat: { promptCacheKeyMode: "enabled", supportsLongCacheRetention: false },
|
||||
models: [{ id: "inherited" }, { id: "disabled", compat: { promptCacheKeyMode: "disabled" } }],
|
||||
};
|
||||
writeRawModelsJson({ demo: provider });
|
||||
|
||||
const registry = await createModelRegistry(authStorage, modelsJsonPath);
|
||||
expect(registry.getError()).toBeUndefined();
|
||||
expect(registry.find("demo", "inherited")?.compat).toMatchObject({
|
||||
promptCacheKeyMode: "enabled",
|
||||
supportsLongCacheRetention: false,
|
||||
});
|
||||
expect(registry.find("demo", "disabled")?.compat).toMatchObject({
|
||||
promptCacheKeyMode: "disabled",
|
||||
supportsLongCacheRetention: false,
|
||||
});
|
||||
});
|
||||
|
||||
test("compat schema accepts long cache retention flag", async () => {
|
||||
writeRawModelsJson({
|
||||
demo: {
|
||||
|
|
@ -922,6 +943,22 @@ describe("ModelRegistry", () => {
|
|||
expect((opus?.compat as OpenAICompletionsCompat | undefined)?.supportsFinishReason).toBe(true);
|
||||
});
|
||||
|
||||
test("built-in models inherit prompt cache key support unless overridden", async () => {
|
||||
const provider: ModelsJsonProvider = {
|
||||
baseUrl: "https://proxy.example.com/v1",
|
||||
compat: { promptCacheKeyMode: "enabled" },
|
||||
modelOverrides: {
|
||||
"gpt-4o-mini": { compat: { promptCacheKeyMode: "disabled" } },
|
||||
},
|
||||
};
|
||||
writeRawModelsJson({ openai: provider });
|
||||
|
||||
const registry = await createModelRegistry(authStorage, modelsJsonPath);
|
||||
expect(registry.getError()).toBeUndefined();
|
||||
expect(registry.find("openai", "gpt-4o")?.compat).toMatchObject({ promptCacheKeyMode: "enabled" });
|
||||
expect(registry.find("openai", "gpt-4o-mini")?.compat).toMatchObject({ promptCacheKeyMode: "disabled" });
|
||||
});
|
||||
|
||||
test("model override deep merges compat settings", async () => {
|
||||
writeRawModelsJson({
|
||||
openrouter: {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue