fix(ai): add supportsMaxOutputTokens compat flag for openai-responses (#8941)

Some OpenAI Responses-compatible gateways (e.g. Codex-protocol proxies)
reject the max_output_tokens parameter with a 400 BadRequest. The
openai-responses API always sent it when maxTokens was set, with no way
to opt out.

Add a supportsMaxOutputTokens compat flag (default: true) to
OpenAIResponsesCompat. When false, buildParams omits max_output_tokens
from the request body. Also register the flag in the coding-agent
model-config schema so custom providers can set it.

packages/ai/src/api/openai-responses.ts

@@ -75,6 +75,7 @@ function getCompat(model: Model<"openai-responses">): Required<OpenAIResponsesCo
 		supportsAdditionalTools: model.compat?.supportsAdditionalTools ?? false,
 		supportsToolSearch: model.compat?.supportsToolSearch ?? false,
 		supportsExplicitPromptCacheMode: model.compat?.supportsExplicitPromptCacheMode ?? false,
+		supportsMaxOutputTokens: model.compat?.supportsMaxOutputTokens ?? true,
 	};
 }
 
@@ -297,7 +298,7 @@ function buildParams(
 		store: false,
 	};
 
-	if (options?.maxTokens) {
+	if (options?.maxTokens && compat.supportsMaxOutputTokens) {
 		params.max_output_tokens = Math.max(options.maxTokens, OPENAI_RESPONSES_MIN_OUTPUT_TOKENS);
 	}
 

packages/ai/src/types.ts

@@ -642,6 +642,8 @@ export interface OpenAIResponsesCompat {
 	supportsToolSearch?: boolean;
 	/** Whether the model accepts `prompt_cache_options` (OpenAI GPT-5.6+ explicit prompt caching). Older OpenAI models reject the parameter. Default: false. */
 	supportsExplicitPromptCacheMode?: boolean;
+	/** Whether the provider accepts the `max_output_tokens` parameter. Some Codex-protocol gateways reject it. Default: true. */
+	supportsMaxOutputTokens?: boolean;
 }
 
 /** Compatibility settings for Anthropic Messages-compatible APIs. */

packages/ai/test/openai-responses-compat.test.ts

@@ -522,3 +522,78 @@ describe("openai-responses provider defaults", () => {
 		expect(result.usage.cost.total).toBe((model.cost.input + model.cost.output) * multiplier * tokenScale);
 	});
 });
+
+describe("openai-responses max_output_tokens compat", () => {
+	afterEach(() => {
+		vi.restoreAllMocks();
+	});
+
+	it("sends max_output_tokens by default", async () => {
+		let capturedPayload: { max_output_tokens?: number } | undefined;
+
+		vi.spyOn(globalThis, "fetch").mockResolvedValue(
+			new Response("data: [DONE]\n\n", {
+				status: 200,
+				headers: { "content-type": "text/event-stream" },
+			}),
+		);
+
+		const stream = streamOpenAIResponses(
+			getModel("openai", "gpt-5.4"),
+			{
+				systemPrompt: "sys",
+				messages: [{ role: "user", content: "hi", timestamp: Date.now() }],
+			},
+			{
+				apiKey: "test-key",
+				maxTokens: 1024,
+				onPayload: (payload) => {
+					capturedPayload = payload as { max_output_tokens?: number };
+				},
+			},
+		);
+
+		for await (const event of stream) {
+			if (event.type === "done" || event.type === "error") break;
+		}
+
+		expect(capturedPayload?.max_output_tokens).toBe(1024);
+	});
+
+	it("omits max_output_tokens when supportsMaxOutputTokens is false", async () => {
+		const baseModel = getModel("openai", "gpt-5.4");
+		const model: Model<"openai-responses"> = {
+			...baseModel,
+			compat: { ...baseModel.compat, supportsMaxOutputTokens: false },
+		};
+		let capturedPayload: { max_output_tokens?: number } | undefined;
+
+		vi.spyOn(globalThis, "fetch").mockResolvedValue(
+			new Response("data: [DONE]\n\n", {
+				status: 200,
+				headers: { "content-type": "text/event-stream" },
+			}),
+		);
+
+		const stream = streamOpenAIResponses(
+			model,
+			{
+				systemPrompt: "sys",
+				messages: [{ role: "user", content: "hi", timestamp: Date.now() }],
+			},
+			{
+				apiKey: "test-key",
+				maxTokens: 1024,
+				onPayload: (payload) => {
+					capturedPayload = payload as { max_output_tokens?: number };
+				},
+			},
+		);
+
+		for await (const event of stream) {
+			if (event.type === "done" || event.type === "error") break;
+		}
+
+		expect(capturedPayload?.max_output_tokens).toBeUndefined();
+	});
+});

packages/coding-agent/src/core/model-config.ts

@@ -121,6 +121,7 @@ const OpenAIResponsesCompatSchema = Type.Object({
 	supportsOpenAIGrammarTools: Type.Optional(Type.Boolean()),
 	supportsAdditionalTools: Type.Optional(Type.Boolean()),
 	supportsToolSearch: Type.Optional(Type.Boolean()),
+	supportsMaxOutputTokens: Type.Optional(Type.Boolean()),
 });
 
 const AnthropicMessagesCompatSchema = Type.Object({