diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index a57f01a99..f7510bf47 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Added + +- Added cache-preserving thinking level changes for GPT-6 models on the openai provider: with an API key, the level is sent as configuration_update input items instead of changing reasoning.effort, controlled by compat.supportsMidConvoEffort ([#10853](https://github.com/earendil-works/pi/issues/10853)) + ### Fixed - Fixed tool-call argument parsing dropping fields when a string contains an invalid `\u` escape such as `\usepackage` ([#10868](https://github.com/earendil-works/pi/issues/10868)) diff --git a/packages/ai/scripts/generate-models.ts b/packages/ai/scripts/generate-models.ts index ad4839e14..d6b998e12 100644 --- a/packages/ai/scripts/generate-models.ts +++ b/packages/ai/scripts/generate-models.ts @@ -388,6 +388,9 @@ const OPENAI_CODEX_ADDITIONAL_TOOLS_MODEL_IDS = new Set([ "gpt-6-luna", "gpt-6.1-sol", ]); +// OpenAI documents configuration_update for the GPT-6 family. +// https://developers.openai.com/api/docs/guides/reasoning +const OPENAI_MID_CONVO_EFFORT_MODEL_IDS = new Set(["gpt-6-astra", "gpt-6-sol", "gpt-6-luna", "gpt-6.1-sol"]); const OPENAI_LONG_CONTEXT_INPUT_THRESHOLD = 272000; const OPENAI_SHORT_CONTEXT_CAPPED_MODEL_IDS = new Set([ "gpt-5.4", @@ -976,6 +979,18 @@ function applyOpenAIExplicitPromptCacheMetadata(model: Model): void { }; } +// GPT-6 models accept `configuration_update` input items that change the reasoning effort +// without changing the cached prompt prefix. Not enabled for openai-codex (Codex CLI keeps it +// behind a flag that is off by default) or proxies. +function applyOpenAIMidConvoEffortMetadata(model: Model): void { + if (model.provider !== "openai" || model.api !== "openai-responses") return; + if (!OPENAI_MID_CONVO_EFFORT_MODEL_IDS.has(model.id)) return; + model.compat = { + ...(model.compat as OpenAIResponsesCompat | undefined), + supportsMidConvoEffort: true, + }; +} + // Anthropic ephemeral entries have a hard five-minute lifetime; `ttl: "1h"` // extends it to one hour. Only direct Anthropic is annotated so cache warming // does not assume equivalent behavior through proxies. @@ -3348,6 +3363,7 @@ async function generateModels() { applyOpenAICompletionsTranscriptMetadata(model); applyOpenAIResponsesTranscriptMetadata(model); applyOpenAIExplicitPromptCacheMetadata(model); + applyOpenAIMidConvoEffortMetadata(model); applyPromptCacheMetadata(model); applyImageInputMetadata(model); } diff --git a/packages/ai/src/api/openai-responses-shared.ts b/packages/ai/src/api/openai-responses-shared.ts index 59c9b5b42..31abd39c8 100644 --- a/packages/ai/src/api/openai-responses-shared.ts +++ b/packages/ai/src/api/openai-responses-shared.ts @@ -1,6 +1,7 @@ import type OpenAI from "openai"; import type { Tool as OpenAITool, + ResponseConfigurationUpdateItemParam, ResponseCreateParamsStreaming, ResponseInput, ResponseInputContent, @@ -129,6 +130,16 @@ export interface ConvertResponsesMessagesOptions { supportsAdditionalTools?: boolean; supportsToolSearch?: boolean; toolOptions?: ConvertResponsesToolsOptions; + /** + * Effort for the reply being generated. When set, the thinking level is sent as + * configuration_update items instead of a changing reasoning.effort, which would invalidate the + * prompt cache. Each item sets the effort from that point on: one goes before every earlier + * reply with the level recorded on it (`providerThinkingLevel`), and one with this effort goes + * at the end. Earlier input therefore never changes. OpenAI documents the item before user + * messages only. After tool results it was verified on gpt-6-luna (October 2026) to be accepted + * and to apply to the next reply. + */ + configurationUpdateEffort?: string; } export interface ConvertResponsesToolsOptions { @@ -211,6 +222,12 @@ export function convertResponsesMessages( } satisfies ResponseToolSearchOutputItemParam); }; const includeInitialSystemMessage = options?.includeSystemPrompt ?? true; + const pushConfigurationUpdate = (effort: string): void => { + messages.push({ + type: "configuration_update", + reasoning: { effort: effort as NonNullable }, + } satisfies ResponseConfigurationUpdateItemParam); + }; const compat = model.compat as { supportsDeveloperRole?: boolean } | undefined; const instructionRole = model.reasoning && compat?.supportsDeveloperRole !== false ? "developer" : "system"; @@ -328,6 +345,13 @@ export function convertResponsesMessages( } } if (output.length === 0) continue; + if ( + options?.configurationUpdateEffort !== undefined && + isSameModel && + msg.providerThinkingLevel !== undefined + ) { + pushConfigurationUpdate(msg.providerThinkingLevel); + } messages.push(...output); } else if (msg.role === "toolResult") { const [callId] = msg.toolCallId.split("|"); @@ -349,6 +373,7 @@ export function convertResponsesMessages( } if (!isLeadingSystemMessage) msgIndex++; } + if (options?.configurationUpdateEffort !== undefined) pushConfigurationUpdate(options.configurationUpdateEffort); return messages; } diff --git a/packages/ai/src/api/openai-responses.ts b/packages/ai/src/api/openai-responses.ts index 07e5624e9..dd8cf0d66 100644 --- a/packages/ai/src/api/openai-responses.ts +++ b/packages/ai/src/api/openai-responses.ts @@ -32,6 +32,8 @@ const OPENAI_TOOL_CALL_PROVIDERS = new Set(["openai", "openai-codex", "opencode" // OpenAI Responses rejects max_output_tokens below 16: https://github.com/earendil-works/pi/issues/6265 const OPENAI_RESPONSES_MIN_OUTPUT_TOKENS = 16; const CHATGPT_USAGE_URL = "https://chatgpt.com/settings/usage"; +// Fixed reasoning.effort while configuration_update items carry the thinking level. +const CONFIGURATION_UPDATE_REQUEST_EFFORT = "medium"; /** * OpenAI API keys start with `sk-`; a different credential sent directly to OpenAI @@ -91,9 +93,27 @@ function getCompat(model: Model<"openai-responses">): Required, + options: OpenAIResponsesOptions | undefined, + compat: Required, +): string | undefined { + if ( + !compat.supportsMidConvoEffort || + !options?.reasoningEffort || + // Sign in with ChatGPT rejects configuration_update items. + isChatGPTSignIn(model, options.apiKey) + ) { + return undefined; + } + return model.thinkingLevelMap?.[options.reasoningEffort] ?? options.reasoningEffort; +} + function getPromptCacheRetention( compat: Required, cacheRetention: CacheRetention, @@ -134,12 +154,15 @@ export const stream: StreamFunction<"openai-responses", OpenAIResponsesOptions> // Start async processing (async () => { + const compat = getCompat(model); + const configurationUpdateEffort = getConfigurationUpdateEffort(model, options, compat); const output: AssistantMessage = { role: "assistant", content: [], api: model.api as Api, provider: model.provider, model: model.id, + ...(configurationUpdateEffort === undefined ? {} : { providerThinkingLevel: configurationUpdateEffort }), usage: { input: 0, output: 0, @@ -157,7 +180,6 @@ export const stream: StreamFunction<"openai-responses", OpenAIResponsesOptions> const apiKey = getClientApiKey(model.provider, options?.apiKey, options?.headers); const cacheRetention = resolveCacheRetention(options?.cacheRetention, options?.env); const cacheSessionId = cacheRetention === "none" ? undefined : options?.sessionId; - const compat = getCompat(model); const grammarToolInputProperties = createGrammarToolInputProperties( getDeclaredTools(normalizedContext.messages), compat.supportsOpenAIGrammarTools, @@ -170,7 +192,14 @@ export const stream: StreamFunction<"openai-responses", OpenAIResponsesOptions> options?.fetch, cacheSessionId, ); - let params = buildParams(model, normalizedContext, options, compat, grammarToolInputProperties); + let params = buildParams( + model, + normalizedContext, + options, + compat, + grammarToolInputProperties, + configurationUpdateEffort, + ); const nextParams = await options?.onPayload?.(params, model); if (nextParams !== undefined) { params = nextParams as ResponseCreateParamsStreaming; @@ -303,11 +332,9 @@ function buildParams( model: Model<"openai-responses">, context: TranscriptContext, options: OpenAIResponsesOptions | undefined, - compat: Required = getCompat(model), - grammarToolInputProperties: ReadonlyMap = createGrammarToolInputProperties( - getDeclaredTools(context.messages), - compat.supportsOpenAIGrammarTools, - ), + compat: Required, + grammarToolInputProperties: ReadonlyMap, + configurationUpdateEffort: string | undefined, ) { const transcriptTools = resolveTranscriptTools( context.messages, @@ -322,6 +349,7 @@ function buildParams( supportsStrictMode: compat.supportsStrictMode, supportsOpenAIGrammarTools: compat.supportsOpenAIGrammarTools, }, + configurationUpdateEffort, }); const cacheRetention = resolveCacheRetention(options?.cacheRetention, options?.env); @@ -363,9 +391,10 @@ function buildParams( const reasoningEffort = options?.reasoningEffort ?? (options?.reasoningSummary ? "medium" : undefined); if (model.reasoning) { if (reasoningEffort) { - const effort = options?.reasoningEffort + const requestedEffort = options?.reasoningEffort ? (model.thinkingLevelMap?.[options.reasoningEffort] ?? options.reasoningEffort) : reasoningEffort; + const effort = configurationUpdateEffort === undefined ? requestedEffort : CONFIGURATION_UPDATE_REQUEST_EFFORT; params.reasoning = { effort: effort as NonNullable["effort"], summary: options?.reasoningSummary || "auto", diff --git a/packages/ai/src/providers/compat-schema.ts b/packages/ai/src/providers/compat-schema.ts index 4bea0c14f..9db22b80f 100644 --- a/packages/ai/src/providers/compat-schema.ts +++ b/packages/ai/src/providers/compat-schema.ts @@ -396,6 +396,13 @@ export const OpenAIResponsesCompatSchema = Type.Object( default: false, }), ), + supportsMidConvoEffort: Type.Optional( + Type.Boolean({ + description: + "Whether thinking level changes are sent as configuration_update input items instead of changing reasoning.effort, so a change keeps the prompt cache. Only used by the openai-responses API, and ignored with Sign in with ChatGPT on the openai provider.", + default: false, + }), + ), supportsMaxOutputTokens: Type.Optional( Type.Boolean({ description: "Whether the provider accepts max_output_tokens. Some Codex-protocol gateways reject it.", @@ -526,6 +533,11 @@ const ProviderCompatPropertyOverrides = { sessionAffinityFormat: optionalSessionAffinityFormat({ description: "Session-affinity header format. Defaults are API-specific or auto-detected.", }), + supportsMidConvoEffort: optionalCompatBoolean({ + description: + "Whether thinking level changes are sent inside the conversation instead of changing the request's effort, so a change keeps the prompt cache. Used by the anthropic-messages and openai-responses APIs.", + default: false, + }), supportsLongCacheRetention: optionalCompatBoolean({ description: "Whether the provider supports long prompt cache retention. Defaults are API-specific or auto-detected.", diff --git a/packages/ai/test/openai-responses-mid-convo-effort.test.ts b/packages/ai/test/openai-responses-mid-convo-effort.test.ts new file mode 100644 index 000000000..d6ef68d4c --- /dev/null +++ b/packages/ai/test/openai-responses-mid-convo-effort.test.ts @@ -0,0 +1,204 @@ +import { describe, expect, it } from "vitest"; +import { type OpenAIResponsesOptions, stream as streamOpenAIResponses } from "../src/api/openai-responses.ts"; +import type { AssistantMessage, Message, Model, ToolResultMessage } from "../src/types.ts"; +import { normalizeContext } from "../src/utils/transcript.ts"; + +interface Payload { + input: Array>; + reasoning?: { effort?: string }; +} + +const thinkingLevelMap = { + off: "none", + minimal: null, + low: "low", + medium: "medium", + high: "high", + xhigh: "xhigh", + max: "max", +} as const; + +function openAIModel(compat: Model<"openai-responses">["compat"] = { supportsMidConvoEffort: true }) { + return { + id: "gpt-6-luna", + name: "GPT-6 Luna", + api: "openai-responses", + provider: "openai", + baseUrl: "https://api.openai.com/v1", + reasoning: true, + thinkingLevelMap, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 272000, + maxTokens: 128000, + compat, + } satisfies Model<"openai-responses">; +} + +const user = (text: string): Message => ({ role: "user", content: text, timestamp: 0 }); + +const toolResult = (callId: string): ToolResultMessage => ({ + role: "toolResult", + toolCallId: callId, + toolName: "lookup", + content: [{ type: "text", text: "result" }], + isError: false, + timestamp: 0, +}); + +const update = (effort: string) => ({ type: "configuration_update", reasoning: { effort } }); + +function responseEvents(responseId: string, withToolCall: boolean): object[] { + const item = withToolCall + ? { type: "function_call", id: "fc_1", call_id: "call_1", name: "lookup", arguments: "{}", status: "completed" } + : { + type: "message", + id: `msg_${responseId}`, + role: "assistant", + status: "completed", + content: [{ type: "output_text", text: "answer", annotations: [] }], + }; + return [ + { type: "response.created", response: { id: responseId } }, + { type: "response.output_item.added", output_index: 0, item }, + { type: "response.output_item.done", output_index: 0, item }, + { type: "response.completed", response: { id: responseId, status: "completed" } }, + ]; +} + +async function runOpenAI( + messages: Message[], + reasoningEffort: OpenAIResponsesOptions["reasoningEffort"], + options: { apiKey?: string; model?: Model<"openai-responses">; withToolCall?: boolean } = {}, +): Promise<{ payload: Payload; message: AssistantMessage }> { + let payload: Payload | undefined; + const fetchMock = async (_url: string | URL | Request, init?: RequestInit): Promise => { + payload = JSON.parse(String(init?.body)) as Payload; + const body = responseEvents("resp_1", options.withToolCall ?? false) + .map((event) => `data: ${JSON.stringify(event)}\n\n`) + .join(""); + return new Response(body, { status: 200, headers: { "content-type": "text/event-stream" } }); + }; + const message = await streamOpenAIResponses(options.model ?? openAIModel(), normalizeContext({ messages }), { + apiKey: options.apiKey ?? "sk-test", + reasoningEffort, + fetch: fetchMock as typeof globalThis.fetch, + }).result(); + if (!payload) throw new Error("Request payload was not captured"); + expect(message.stopReason).not.toBe("error"); + return { payload, message }; +} + +describe("OpenAI Responses configuration_update", () => { + it("keeps reasoning.effort constant and only appends input when the level changes", async () => { + const history: Message[] = [user("one")]; + const first = await runOpenAI(history, "high"); + history.push(first.message, user("two")); + const second = await runOpenAI(history, "high"); + history.push(second.message, user("three")); + const third = await runOpenAI(history, "low"); + + expect(first.payload.input).toEqual([ + { role: "user", content: [{ type: "input_text", text: "one" }] }, + update("high"), + ]); + for (const { payload } of [first, second, third]) { + expect(payload.reasoning?.effort).toBe("medium"); + } + expect(second.payload.input.slice(0, first.payload.input.length)).toEqual(first.payload.input); + expect(third.payload.input.slice(0, second.payload.input.length)).toEqual(second.payload.input); + expect(third.payload.input.filter((item) => item.type === "configuration_update")).toEqual([ + update("high"), + update("high"), + update("low"), + ]); + expect(third.payload.input.at(-1)).toEqual(update("low")); + expect([first, second, third].map(({ message }) => message.providerThinkingLevel)).toEqual([ + "high", + "high", + "low", + ]); + }); + + it("applies a level change made during a tool loop to the next reply", async () => { + const history: Message[] = [user("one")]; + const toolCall = await runOpenAI(history, "high", { withToolCall: true }); + history.push(toolCall.message, toolResult("call_1|fc_1")); + const afterTool = await runOpenAI(history, "low"); + + expect(afterTool.payload.input.slice(0, toolCall.payload.input.length)).toEqual(toolCall.payload.input); + expect(afterTool.payload.input.slice(-2)).toEqual([ + expect.objectContaining({ type: "function_call_output", call_id: "call_1" }), + update("low"), + ]); + expect(afterTool.message.providerThinkingLevel).toBe("low"); + }); + + it("adds items only before replies recorded by this model", async () => { + const unstamped = (await runOpenAI([user("one")], "high", { model: openAIModel({}) })).message; + const otherProvider: AssistantMessage = { ...unstamped, provider: "opencode", providerThinkingLevel: "high" }; + const otherModel: AssistantMessage = { ...unstamped, model: "gpt-6-sol", providerThinkingLevel: "high" }; + const recorded: AssistantMessage = { ...unstamped, providerThinkingLevel: "high" }; + + const { payload } = await runOpenAI( + [ + user("one"), + unstamped, + user("two"), + otherProvider, + user("three"), + otherModel, + user("four"), + recorded, + user("five"), + ], + "low", + ); + + expect(payload.input.map((item) => item.type ?? item.role)).toEqual([ + "user", + "message", + "user", + "message", + "user", + "message", + "user", + "configuration_update", + "message", + "user", + "configuration_update", + ]); + expect(payload.input.filter((item) => item.type === "configuration_update")).toEqual([ + update("high"), + update("low"), + ]); + }); + + it.each([ + { + name: "Sign in with ChatGPT", + apiKey: "chatgpt-access-token", + model: openAIModel(), + level: "low", + effort: "low", + }, + { + name: "the compat flag turned off", + apiKey: "sk-test", + model: openAIModel({ supportsMidConvoEffort: false }), + level: "low", + effort: "low", + }, + { name: "thinking off", apiKey: "sk-test", model: openAIModel(), level: undefined, effort: "none" }, + ] as const)("sends the level as reasoning.effort with $name", async ({ apiKey, model, level, effort }) => { + const first = await runOpenAI([user("one")], "high", { apiKey, model }); + const { payload, message } = await runOpenAI([user("one"), first.message, user("two")], level, { + apiKey, + model, + }); + + expect(payload.reasoning?.effort).toBe(effort); + expect(payload.input.some((item) => item.type === "configuration_update")).toBe(false); + expect(message.providerThinkingLevel).toBeUndefined(); + }); +}); diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 462db34ff..ae39a9dd3 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Added + +- Changing the thinking level mid-session on OpenAI GPT-6 models (API key) now keeps the prompt cache ([#10853](https://github.com/earendil-works/pi/issues/10853)) + ### Fixed - Pinned internal Pi dependencies to exact release versions to prevent fresh installs from mixing incompatible packages ([#10865](https://github.com/earendil-works/pi/issues/10865)) diff --git a/packages/coding-agent/schemas/models.schema.json b/packages/coding-agent/schemas/models.schema.json index 40e27fa3e..3313f85bd 100644 --- a/packages/coding-agent/schemas/models.schema.json +++ b/packages/coding-agent/schemas/models.schema.json @@ -1003,6 +1003,11 @@ "description": "Whether the model accepts prompt_cache_options. Older OpenAI models reject the parameter.", "default": false }, + "supportsMidConvoEffort": { + "type": "boolean", + "description": "Whether thinking level changes are sent inside the conversation instead of changing the request's effort, so a change keeps the prompt cache. Used by the anthropic-messages and openai-responses APIs.", + "default": false + }, "supportsMaxOutputTokens": { "type": "boolean", "description": "Whether the provider accepts max_output_tokens. Some Codex-protocol gateways reject it.", @@ -1038,11 +1043,6 @@ "description": "Whether the provider supports Anthropic strict tool schemas. Generated Anthropic models enable it explicitly.", "default": false }, - "supportsMidConvoEffort": { - "type": "boolean", - "description": "Whether the exact model transport supports effort-only system messages and thinking binding controls.", - "default": false - }, "supportsMidConvoToolChanges": { "type": "boolean", "description": "Whether the exact model accepts mid-conversation tool_addition and tool_removal blocks. Requires supportsMidConvoSystemMessages.",