diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index 474a52678..817ad386f 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Added + +- Added model image-input limit and cache-safe resize metadata to the generated catalog ([#9631](https://github.com/earendil-works/pi/issues/9631)). + ## [0.86.1] - 2026-09-20 ### Added diff --git a/packages/ai/scripts/generate-models.ts b/packages/ai/scripts/generate-models.ts index 54a2e331b..4de461449 100644 --- a/packages/ai/scripts/generate-models.ts +++ b/packages/ai/scripts/generate-models.ts @@ -400,6 +400,16 @@ const OPENAI_LONG_CONTEXT_PRICING_MODEL_IDS = new Set([ "gpt-6-astra", ]); +// Keep the generated default no less restrictive than coding-agent's historical +// image preprocessing. Provider limits can narrow this profile, but unknown +// providers retain the cache-safe 2000px / 4.5 MiB behavior. +const DEFAULT_IMAGE_RESIZE = { + maxWidth: 2000, + maxHeight: 2000, + maxBytes: 4.5 * 1024 * 1024, + jpegQuality: 80, +} as const; + function withOpenAiLongContextPricing(cost: Model["cost"]): Model["cost"] { return { ...cost, @@ -947,6 +957,34 @@ function applyPromptCacheMetadata(model: Model): void { // behavior; a documented TTL alone does not establish full cache loss. } +function applyImageInputMetadata(model: Model): void { + if (!model.input.includes("image")) return; + + const providerLimits: Model["inputLimits"] = + model.provider === "anthropic" + ? { + maxRequestBytes: 32 * 1024 * 1024, + images: { maxPerRequest: model.contextWindow === 200000 ? 100 : 600 }, + } + : model.provider === "amazon-bedrock" + ? { images: { maxPerMessage: 20 } } + : model.provider === "openai" + ? { maxRequestBytes: 512 * 1024 * 1024, images: { maxPerRequest: 1500 } } + : model.provider === "google" + ? { maxRequestBytes: 20 * 1024 * 1024, images: { maxPerRequest: 3600 } } + : undefined; + const configuredImages = model.inputLimits?.images; + model.inputLimits = { + ...providerLimits, + ...model.inputLimits, + images: { + ...providerLimits?.images, + ...configuredImages, + resize: { ...DEFAULT_IMAGE_RESIZE, ...configuredImages?.resize }, + }, + }; +} + function isGemma4Model(modelId: string): boolean { return /gemma-?4/.test(modelId.toLowerCase()); } @@ -3099,6 +3137,7 @@ async function generateModels() { applyOpenAIResponsesTranscriptMetadata(model); applyOpenAIExplicitPromptCacheMetadata(model); applyPromptCacheMetadata(model); + applyImageInputMetadata(model); } applyAnthropicAllowedFallbackModelMetadata(allModels.filter(isAnthropicFallbackMetadataModel)); diff --git a/packages/ai/src/providers/faux.ts b/packages/ai/src/providers/faux.ts index ae7a2b19c..a38c18774 100644 --- a/packages/ai/src/providers/faux.ts +++ b/packages/ai/src/providers/faux.ts @@ -43,6 +43,7 @@ export interface FauxModelDefinition { name?: string; reasoning?: boolean; input?: ("text" | "image")[]; + inputLimits?: Model["inputLimits"]; cost?: { input: number; output: number; cacheRead: number; cacheWrite: number }; contextWindow?: number; maxTokens?: number; @@ -480,6 +481,7 @@ export function createFauxCore(options: RegisterFauxProviderOptions) { baseUrl: DEFAULT_BASE_URL, reasoning: definition.reasoning ?? false, input: definition.input ?? ["text", "image"], + inputLimits: definition.inputLimits, cost: definition.cost ?? { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, contextWindow: definition.contextWindow ?? 128000, maxTokens: definition.maxTokens ?? 16384, diff --git a/packages/ai/src/types.ts b/packages/ai/src/types.ts index 2f28cc893..eb67adfcd 100644 --- a/packages/ai/src/types.ts +++ b/packages/ai/src/types.ts @@ -955,6 +955,29 @@ export interface ModelCost extends ModelCostRates { tiers?: ModelCostTier[]; } +export interface ModelImageResizeOptions { + maxWidth?: number; + maxHeight?: number; + /** Maximum base64-encoded payload size in bytes. */ + maxBytes?: number; + jpegQuality?: number; +} + +export interface ModelImageInputLimits { + /** Cache-safe resize profile applied before a new image enters conversation history. */ + resize?: ModelImageResizeOptions; + /** Maximum images accepted in one provider message. */ + maxPerMessage?: number; + /** Maximum images accepted across one provider request. */ + maxPerRequest?: number; +} + +export interface ModelInputLimits { + /** Maximum serialized provider request size in bytes. */ + maxRequestBytes?: number; + images?: ModelImageInputLimits; +} + // Model interface for the unified model system export interface Model { id: string; @@ -969,6 +992,8 @@ export interface Model { */ thinkingLevelMap?: ThinkingLevelMap; input: ("text" | "image")[]; + /** Provider input limits and cache-safe preprocessing metadata. */ + inputLimits?: ModelInputLimits; cost: ModelCost; /** Prompt cache lifetimes per retention tier. Unset when the provider's cache behavior is unknown. */ promptCache?: ModelPromptCache; diff --git a/packages/ai/test/providers.test.ts b/packages/ai/test/providers.test.ts index b6a960c38..3c7202f96 100644 --- a/packages/ai/test/providers.test.ts +++ b/packages/ai/test/providers.test.ts @@ -4,7 +4,13 @@ import { envApiKeyAuth } from "../src/auth/helpers.ts"; import type { AuthContext, AuthEvent } from "../src/auth/types.ts"; import { createModels, createProvider, getSupportedThinkingLevels } from "../src/models.ts"; import { InMemoryModelsStore } from "../src/models-store.ts"; -import { builtinModels, builtinProviders, getBuiltinModel } from "../src/providers/all.ts"; +import { + builtinModels, + builtinProviders, + getBuiltinModel, + getBuiltinModels, + getBuiltinProviders, +} from "../src/providers/all.ts"; import { amazonBedrockProvider } from "../src/providers/amazon-bedrock.ts"; import { anthropicProvider } from "../src/providers/anthropic.ts"; import { cloudflareAIGatewayProvider } from "../src/providers/cloudflare-ai-gateway.ts"; @@ -33,6 +39,13 @@ const neverAbortedSignal = new AbortController().signal; const context = normalizeContext({ messages: [{ role: "user", content: "hi", timestamp: Date.now() }] }); +const DEFAULT_IMAGE_RESIZE = { + maxWidth: 2000, + maxHeight: 2000, + maxBytes: 4.5 * 1024 * 1024, + jpegQuality: 80, +}; + describe("builtin providers", () => { it("builtinModels registers every builtin provider with models", async () => { const models = builtinModels(); @@ -68,6 +81,40 @@ describe("builtin providers", () => { expect(getBuiltinModel("anthropic", "claude-haiku-4-5").compat?.supportsStrictTools).toBe(true); }); + it("keeps the conservative resize profile on every vision model", () => { + const visionModels = getBuiltinProviders() + .flatMap((provider) => getBuiltinModels(provider)) + .filter((model) => model.input.includes("image")); + expect(visionModels.length).toBeGreaterThan(0); + for (const model of visionModels) { + expect(model.inputLimits?.images?.resize).toEqual(DEFAULT_IMAGE_RESIZE); + } + }); + + it("records known direct-provider image request limits", () => { + expect(getBuiltinModel("anthropic", "claude-haiku-4-5").inputLimits).toMatchObject({ + maxRequestBytes: 32 * 1024 * 1024, + images: { maxPerRequest: 100 }, + }); + expect(getBuiltinModel("anthropic", "claude-opus-5").inputLimits?.images?.maxPerRequest).toBe(600); + expect(getBuiltinModel("amazon-bedrock", "anthropic.claude-haiku-4-5-20251001-v1:0").inputLimits).toMatchObject({ + images: { maxPerMessage: 20 }, + }); + expect(getBuiltinModel("openai", "gpt-4o").inputLimits).toMatchObject({ + maxRequestBytes: 512 * 1024 * 1024, + images: { maxPerRequest: 1500 }, + }); + expect(getBuiltinModel("google", "gemini-2.5-flash").inputLimits).toMatchObject({ + maxRequestBytes: 20 * 1024 * 1024, + images: { maxPerRequest: 3600 }, + }); + }); + + it("does not infer image limits from gateway API compatibility", () => { + const openRouterModel = getBuiltinModels("openrouter").find((model) => model.input.includes("image")); + expect(openRouterModel?.inputLimits).toEqual({ images: { resize: DEFAULT_IMAGE_RESIZE } }); + }); + it("uses models.dev effort levels for Google thinking models", () => { // Regression test for https://github.com/earendil-works/pi/issues/9455 for (const provider of ["google", "google-vertex"] as const) { diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 2614163b6..f4a43ad60 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Added + +- Added per-model image resize profiles through `inputLimits.images.resize` in `models.json`, applied to file attachments, image reads, and tool-result images ([#9631](https://github.com/earendil-works/pi/issues/9631)). + ### Fixed - Fixed idle prompt-cache warming rebuilding expired caches when its timer or an extension decision is delayed. diff --git a/packages/coding-agent/docs/models.md b/packages/coding-agent/docs/models.md index b855ca8e4..58da1a031 100644 --- a/packages/coding-agent/docs/models.md +++ b/packages/coding-agent/docs/models.md @@ -205,6 +205,7 @@ If your command is slow, expensive, rate-limited, or should keep using a previou | `reasoning` | No | `false` | Supports extended thinking | | `thinkingLevelMap` | No | omitted | Maps pi thinking levels to provider values and marks unsupported levels (see below) | | `input` | No | `["text"]` | Input types: `["text"]` or `["text", "image"]` | +| `inputLimits` | No | omitted | Request limits and image preprocessing for this model (see below) | | `contextWindow` | No | `128000` | Context window size in tokens | | `maxTokens` | No | `16384` | Maximum output tokens | | `samplingParams` | No | omitted | Sampling parameters merged verbatim into every request body (see below) | @@ -238,6 +239,33 @@ Current behavior: - `/model`, `--list-models`, and the interactive footer display entries by model `id`. - The configured `name` is used for model matching and secondary model detail text. It does not replace the footer/status-bar model id. +### Image Input Limits + +Use `inputLimits.images.resize` to configure how new images are encoded before they enter conversation history: + +```json +{ + "id": "vision-model", + "input": ["text", "image"], + "inputLimits": { + "images": { + "resize": { + "maxWidth": 1568, + "maxHeight": 1568, + "maxBytes": 524288, + "jpegQuality": 75 + } + } + } +} +``` + +`maxBytes` is the maximum base64-encoded payload size. Omitted resize fields use pi's conservative defaults: 2000×2000, 4.5 MiB encoded, and JPEG quality 80. Built-in vision models carry that profile explicitly so unknown gateways never receive larger images than before. + +Pi applies the selected model's resize profile to `@file` attachments, the `read` tool, and images returned by tools. Images are encoded once before they enter history; changing models does not rewrite historical images or invalidate the cached conversation prefix. The `images.autoResize` setting can disable resizing globally. + +The catalog can also record `inputLimits.maxRequestBytes`, `images.maxPerMessage`, and `images.maxPerRequest`. These fields describe hard provider limits; this initial implementation does not yet rewrite or reject conversation history based on them. + ### Prompt Cache Lifetimes `promptCache` states how long the provider keeps a prompt cache entry alive for each retention tier pi can request (`short` is the default tier; `long` is used when `PI_CACHE_RETENTION=long`). Values are seconds and are estimates: providers publish ranges, so pick the conservative end. @@ -374,7 +402,7 @@ Use `modelOverrides` to customize built-in models and matching extension-registe } ``` -`modelOverrides` supports these fields per model: `name`, `reasoning`, `thinkingLevelMap`, `input`, `cost` (partial), `promptCache` (merged per tier), `contextWindow`, `maxTokens`, `samplingParams` (merged per key), `headers`, `compat`. +`modelOverrides` supports these fields per model: `name`, `reasoning`, `thinkingLevelMap`, `input`, `inputLimits` (deep-merged), `cost` (partial), `promptCache` (merged per tier), `contextWindow`, `maxTokens`, `samplingParams` (merged per key), `headers`, `compat`. Use a `promptCache` override to enable cache warming through a proxy whose backing cache you know, for example OpenRouter routed to Anthropic: diff --git a/packages/coding-agent/src/cli/file-processor.ts b/packages/coding-agent/src/cli/file-processor.ts index af3c1c100..d20913275 100644 --- a/packages/coding-agent/src/cli/file-processor.ts +++ b/packages/coding-agent/src/cli/file-processor.ts @@ -17,7 +17,7 @@ export interface ProcessedFiles { } export interface ProcessFileOptions { - /** Whether to auto-resize images to 2000x2000 max. Default: true */ + /** Whether to auto-resize images. Default: true */ autoResizeImages?: boolean; } diff --git a/packages/coding-agent/src/core/agent-session.ts b/packages/coding-agent/src/core/agent-session.ts index b0f4d1840..9845d9b0e 100644 --- a/packages/coding-agent/src/core/agent-session.ts +++ b/packages/coding-agent/src/core/agent-session.ts @@ -50,6 +50,7 @@ import { } from "@earendil-works/pi-ai/compat"; import { getThemeByName, theme } from "../modes/interactive/theme/theme.ts"; import { stripFrontmatter } from "../utils/frontmatter.ts"; +import { processImage } from "../utils/image-process.ts"; import { sleep } from "../utils/sleep.ts"; import { normalizeToolResultImages } from "../utils/tool-result-images.ts"; import { formatNoApiKeyFoundMessage, formatNoModelSelectedMessage } from "./auth-guidance.ts"; @@ -543,8 +544,10 @@ export class AgentSession { const content = hookResult?.content ?? result.content ?? []; // Runs after the extension hook so images injected or replaced by extensions are normalized too. + const resizeOptions = this.model?.inputLimits?.images?.resize; const normalizedContent = await normalizeToolResultImages(content, { autoResizeImages: this.settingsManager.getImageAutoResize(), + ...(resizeOptions ? { resizeOptions } : {}), }); if (!hookResult && normalizedContent === content) { @@ -1284,6 +1287,28 @@ export class AgentSession { return { text, images }; } + private async _normalizePromptImages( + images: ImageContent[] | undefined, + ): Promise<{ images: ImageContent[]; hints: string[] }> { + if (!images) return { images: [], hints: [] }; + + const normalizedImages: ImageContent[] = []; + const hints: string[] = []; + for (const image of images) { + const processed = await processImage(Buffer.from(image.data, "base64"), image.mimeType, { + autoResizeImages: this.settingsManager.getImageAutoResize(), + resizeOptions: this.model?.inputLimits?.images?.resize, + }); + if (!processed.ok) { + hints.push(processed.message); + continue; + } + normalizedImages.push({ type: "image", data: processed.data, mimeType: processed.mimeType }); + hints.push(...processed.hints); + } + return { images: normalizedImages, hints }; + } + /** * Send a prompt to the agent. * - Handles extension commands (registered via pi.registerCommand) immediately, even during streaming @@ -1383,27 +1408,8 @@ export class AgentSession { await this._checkCompaction(lastAssistant, false); } - // Build messages array (custom message if any, then user message) - messages = []; - - // Add user message - const userContent: (TextContent | ImageContent)[] = [{ type: "text", text: expandedText }]; - if (currentImages) { - userContent.push(...currentImages); - } - messages.push({ - role: "user", - content: userContent, - timestamp: Date.now(), - }); - - // Inject any pending "nextTurn" messages as context alongside the user message - for (const msg of this._pendingNextTurnMessages) { - messages.push(msg); - } - this._pendingNextTurnMessages = []; - - // Emit before_agent_start extension event + // Emit before_agent_start before normalizing images so extension-driven model + // selection determines the resize profile used for the request and history. const selectedToolsBefore = this._baseSystemPromptOptions.selectedTools; const result = await this._extensionRunner.emitBeforeAgentStart( expandedText, @@ -1417,6 +1423,27 @@ export class AgentSession { result.systemPromptOptions.selectedTools.length !== selectedToolsBefore.length || result.systemPromptOptions.selectedTools.some((name, index) => name !== selectedToolsBefore[index]); if (!handlerEditedTools) result.systemPromptOptions.selectedTools = this.getActiveToolNames(); + + const normalized = await this._normalizePromptImages(currentImages); + const userText = + normalized.hints.length > 0 ? `${expandedText}\n\n${normalized.hints.join("\n")}` : expandedText; + + // Build messages only after hooks and image normalization have completed. + messages = []; + const userContent: (TextContent | ImageContent)[] = [{ type: "text", text: userText }]; + userContent.push(...normalized.images); + messages.push({ + role: "user", + content: userContent, + timestamp: Date.now(), + }); + + // Inject any pending "nextTurn" messages as context alongside the user message + for (const msg of this._pendingNextTurnMessages) { + messages.push(msg); + } + this._pendingNextTurnMessages = []; + for (const msg of result.messages) { messages.push({ role: "custom", diff --git a/packages/coding-agent/src/core/extensions/types.ts b/packages/coding-agent/src/core/extensions/types.ts index 4ad88984d..c470a36c9 100644 --- a/packages/coding-agent/src/core/extensions/types.ts +++ b/packages/coding-agent/src/core/extensions/types.ts @@ -1603,6 +1603,8 @@ export interface ProviderModelConfig { thinkingLevelMap?: Model["thinkingLevelMap"]; /** Supported input types. */ input: ("text" | "image")[]; + /** Provider input limits and cache-safe image preprocessing metadata. */ + inputLimits?: Model["inputLimits"]; /** Per-million-token cost rates and optional request-wide input pricing tiers. */ cost: Model["cost"]; /** Best-effort prompt cache lifetime in seconds per retention tier. Unset disables cache warming. */ diff --git a/packages/coding-agent/src/core/model-config.ts b/packages/coding-agent/src/core/model-config.ts index ca3cf40de..e0df3473b 100644 --- a/packages/coding-agent/src/core/model-config.ts +++ b/packages/coding-agent/src/core/model-config.ts @@ -140,6 +140,22 @@ const ModelPromptCacheSchema = Type.Object({ short: Type.Optional(Type.Number({ exclusiveMinimum: 0 })), long: Type.Optional(Type.Number({ exclusiveMinimum: 0 })), }); +const ImageResizeSchema = Type.Object({ + maxWidth: Type.Optional(Type.Integer({ minimum: 1 })), + maxHeight: Type.Optional(Type.Integer({ minimum: 1 })), + maxBytes: Type.Optional(Type.Integer({ minimum: 1 })), + jpegQuality: Type.Optional(Type.Integer({ minimum: 1, maximum: 100 })), +}); +const ModelInputLimitsSchema = Type.Object({ + maxRequestBytes: Type.Optional(Type.Integer({ minimum: 1 })), + images: Type.Optional( + Type.Object({ + resize: Type.Optional(ImageResizeSchema), + maxPerMessage: Type.Optional(Type.Integer({ minimum: 1 })), + maxPerRequest: Type.Optional(Type.Integer({ minimum: 1 })), + }), + ), +}); const AnthropicMessagesCompatSchema = Type.Object({ supportsEagerToolInputStreaming: Type.Optional(Type.Boolean()), @@ -177,6 +193,7 @@ const ModelDefinitionSchema = Type.Object({ reasoning: Type.Optional(Type.Boolean()), thinkingLevelMap: Type.Optional(ThinkingLevelMapSchema), input: Type.Optional(Type.Array(Type.Union([Type.Literal("text"), Type.Literal("image")]))), + inputLimits: Type.Optional(ModelInputLimitsSchema), cost: Type.Optional(ModelCostSchema), promptCache: Type.Optional(ModelPromptCacheSchema), contextWindow: Type.Optional(Type.Number()), @@ -191,6 +208,7 @@ const ModelOverrideSchema = Type.Object({ reasoning: Type.Optional(Type.Boolean()), thinkingLevelMap: Type.Optional(ThinkingLevelMapSchema), input: Type.Optional(Type.Array(Type.Union([Type.Literal("text"), Type.Literal("image")]))), + inputLimits: Type.Optional(ModelInputLimitsSchema), cost: Type.Optional( Type.Object({ input: Type.Optional(Type.Number()), diff --git a/packages/coding-agent/src/core/provider-composer.ts b/packages/coding-agent/src/core/provider-composer.ts index 5b8eaf332..10e5f4b7b 100644 --- a/packages/coding-agent/src/core/provider-composer.ts +++ b/packages/coding-agent/src/core/provider-composer.ts @@ -64,6 +64,7 @@ export interface ProviderConfigInput { reasoning: boolean; thinkingLevelMap?: Model["thinkingLevelMap"]; input: ("text" | "image")[]; + inputLimits?: Model["inputLimits"]; cost: Model["cost"]; promptCache?: Model["promptCache"]; contextWindow: number; @@ -105,6 +106,26 @@ function mergeCompat( return merged; } +function mergeInputLimits( + base: Model["inputLimits"], + override: ModelsJsonModelOverride["inputLimits"], +): Model["inputLimits"] { + if (!override) return base; + return { + ...base, + ...override, + images: override.images + ? { + ...base?.images, + ...override.images, + resize: override.images.resize + ? { ...base?.images?.resize, ...override.images.resize } + : base?.images?.resize, + } + : base?.images, + }; +} + function applyModelOverride(model: Model, override: ModelsJsonModelOverride): Model { return { ...model, @@ -114,6 +135,7 @@ function applyModelOverride(model: Model, override: ModelsJsonModelOverride ? { ...model.thinkingLevelMap, ...override.thinkingLevelMap } : model.thinkingLevelMap, input: (override.input as ("text" | "image")[] | undefined) ?? model.input, + inputLimits: mergeInputLimits(model.inputLimits, override.inputLimits), cost: override.cost ? { input: override.cost.input ?? model.cost.input, @@ -162,6 +184,7 @@ function modelFromJson( reasoning: definition.reasoning ?? false, thinkingLevelMap: definition.thinkingLevelMap, input: (definition.input ?? ["text"]) as ("text" | "image")[], + inputLimits: definition.inputLimits, cost: definition.cost ?? { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, promptCache: definition.promptCache, contextWindow: definition.contextWindow ?? 128000, diff --git a/packages/coding-agent/src/core/tools/read.ts b/packages/coding-agent/src/core/tools/read.ts index a93142cc3..91034861b 100644 --- a/packages/coding-agent/src/core/tools/read.ts +++ b/packages/coding-agent/src/core/tools/read.ts @@ -1,5 +1,5 @@ import type { AgentTool } from "@earendil-works/pi-agent-core"; -import type { Api, ImageContent, Model, TextContent } from "@earendil-works/pi-ai"; +import type { Api, ImageContent, Model, ModelImageResizeOptions, TextContent } from "@earendil-works/pi-ai"; import { constants } from "fs"; import { access as fsAccess, readFile as fsReadFile } from "fs/promises"; import { type Static, Type } from "typebox"; @@ -48,8 +48,10 @@ const defaultReadOperations: ReadOperations = { }; export interface ReadToolOptions { - /** Whether to auto-resize images to 2000x2000 max. Default: true */ + /** Whether to auto-resize images. Default: true */ autoResizeImages?: boolean; + /** Fallback resize profile when the execution context has no model metadata. */ + resizeOptions?: ModelImageResizeOptions; /** Custom operations for file reading. Default: local filesystem */ operations?: ReadOperations; } @@ -66,6 +68,7 @@ export function createReadToolDefinition( options?: ReadToolOptions, ): ToolDefinition { const autoResizeImages = options?.autoResizeImages ?? true; + const fallbackResizeOptions = options?.resizeOptions; const ops = options?.operations ?? defaultReadOperations; return { name: "read", @@ -109,7 +112,10 @@ export function createReadToolDefinition( if (mimeType) { // Read image as binary. const buffer = await ops.readFile(absolutePath); - const processed = await processImage(buffer, mimeType, { autoResizeImages }); + const processed = await processImage(buffer, mimeType, { + autoResizeImages, + resizeOptions: ctx?.model?.inputLimits?.images?.resize ?? fallbackResizeOptions, + }); if (!processed.ok) { let textNote = `Read image file [${mimeType}]\n${processed.message}`; if (nonVisionImageNote) textNote += `\n${nonVisionImageNote}`; diff --git a/packages/coding-agent/src/main.ts b/packages/coding-agent/src/main.ts index 5272cba6d..195a1127e 100644 --- a/packages/coding-agent/src/main.ts +++ b/packages/coding-agent/src/main.ts @@ -209,7 +209,6 @@ async function runAuthCommand(args: string[]): Promise { async function prepareInitialMessage( parsed: Args, - autoResizeImages: boolean, stdinContent?: string, ): Promise<{ initialMessage?: string; @@ -219,7 +218,8 @@ async function prepareInitialMessage( return buildInitialMessage({ parsed, stdinContent }); } - const { text, images } = await processFileArguments(parsed.fileArgs, { autoResizeImages }); + // AgentSession resizes these after extension hooks select the request model. + const { text, images } = await processFileArguments(parsed.fileArgs, { autoResizeImages: false }); return buildInitialMessage({ parsed, fileText: text, @@ -880,11 +880,7 @@ export async function main(args: string[], options?: MainOptions) { } time("readPipedStdin"); - const { initialMessage, initialImages } = await prepareInitialMessage( - parsed, - settingsManager.getImageAutoResize(), - stdinContent, - ); + const { initialMessage, initialImages } = await prepareInitialMessage(parsed, stdinContent); time("prepareInitialMessage"); // pi reads user-authored themes, so it opts into full validation before any theme loads. setThemeJsonValidator(validateThemeJson); diff --git a/packages/coding-agent/src/utils/tool-result-images.ts b/packages/coding-agent/src/utils/tool-result-images.ts index ec2226c92..702d98772 100644 --- a/packages/coding-agent/src/utils/tool-result-images.ts +++ b/packages/coding-agent/src/utils/tool-result-images.ts @@ -1,4 +1,4 @@ -import type { ImageContent, TextContent } from "@earendil-works/pi-ai"; +import type { ImageContent, ModelImageResizeOptions, TextContent } from "@earendil-works/pi-ai"; import { processImage } from "./image-process.ts"; export type ToolResultContent = TextContent | ImageContent; @@ -6,6 +6,8 @@ export type ToolResultContent = TextContent | ImageContent; export interface NormalizeToolResultImagesOptions { /** Whether oversized images are resized to inline provider limits. Default: true */ autoResizeImages?: boolean; + /** Model-specific resize profile. Uses the conservative built-in defaults when omitted. */ + resizeOptions?: ModelImageResizeOptions; } /** @@ -37,7 +39,10 @@ export async function normalizeToolResultImages( continue; } - const processed = await processImage(Buffer.from(block.data, "base64"), block.mimeType, { autoResizeImages }); + const processed = await processImage(Buffer.from(block.data, "base64"), block.mimeType, { + autoResizeImages, + resizeOptions: options?.resizeOptions, + }); if (!processed.ok) { // Unlike `read`, keep the original block. The tool already produced this image and the // failure may just be an unavailable image backend, so passing it through preserves the diff --git a/packages/coding-agent/test/image-resize-callers.test.ts b/packages/coding-agent/test/image-resize-callers.test.ts index 2337ae8ca..874c3681b 100644 --- a/packages/coding-agent/test/image-resize-callers.test.ts +++ b/packages/coding-agent/test/image-resize-callers.test.ts @@ -1,6 +1,7 @@ import { mkdirSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import type { Api, Model } from "@earendil-works/pi-ai"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; vi.mock("../src/utils/image-resize.js", () => ({ @@ -9,7 +10,8 @@ vi.mock("../src/utils/image-resize.js", () => ({ })); import { processFileArguments } from "../src/cli/file-processor.ts"; -import { createReadTool } from "../src/core/tools/read.ts"; +import type { ExtensionContext } from "../src/core/extensions/types.ts"; +import { createReadTool, createReadToolDefinition } from "../src/core/tools/read.ts"; import { resizeImage } from "../src/utils/image-resize.ts"; const TINY_PNG_BASE64 = @@ -50,4 +52,44 @@ describe("image resize callers", () => { expect(result.images).toHaveLength(0); expect(result.text).toContain("Image omitted"); }); + + it("passes the current model resize profile to the read tool", async () => { + const imagePath = join(testDir, "test.png"); + writeFileSync(imagePath, Buffer.from(TINY_PNG_BASE64, "base64")); + const resize = { maxWidth: 1234, maxHeight: 1000, maxBytes: 500000, jpegQuality: 70 }; + const model: Model = { + id: "vision-model", + name: "Vision model", + api: "test", + provider: "test", + baseUrl: "https://example.com", + reasoning: false, + input: ["text", "image"], + inputLimits: { images: { resize } }, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 1000, + maxTokens: 100, + }; + const ctx = { cwd: testDir, model } as unknown as ExtensionContext; + + await createReadToolDefinition(testDir).execute( + "test-read-model-profile", + { path: imagePath }, + undefined, + undefined, + ctx, + ); + + expect(resizeImage).toHaveBeenCalledWith(expect.any(Uint8Array), "image/png", resize); + }); + + it("can defer resizing file attachments until prompt dispatch", async () => { + const imagePath = join(testDir, "test.png"); + writeFileSync(imagePath, Buffer.from(TINY_PNG_BASE64, "base64")); + + const result = await processFileArguments([imagePath], { autoResizeImages: false }); + + expect(result.images).toHaveLength(1); + expect(resizeImage).not.toHaveBeenCalled(); + }); }); diff --git a/packages/coding-agent/test/model-registry.test.ts b/packages/coding-agent/test/model-registry.test.ts index 525a37ec6..17ba7a182 100644 --- a/packages/coding-agent/test/model-registry.test.ts +++ b/packages/coding-agent/test/model-registry.test.ts @@ -815,6 +815,54 @@ describe("ModelRegistry", () => { expect(registry.find("anthropic", "claude-sonnet-4-6")?.promptCache).toEqual({ short: 300, long: 1800 }); }); + // Regression test for https://github.com/earendil-works/pi/issues/9631 + test("model override deep-merges image resize limits", async () => { + writeRawModelsJson({ + test: { + baseUrl: "https://example.com", + apiKey: "test-key", + api: "openai-completions", + models: [ + { + id: "vision-model", + input: ["text", "image"], + inputLimits: { + maxRequestBytes: 32 * 1024 * 1024, + images: { + maxPerRequest: 100, + resize: { + maxWidth: 2000, + maxHeight: 2000, + maxBytes: 4.5 * 1024 * 1024, + jpegQuality: 80, + }, + }, + }, + }, + ], + modelOverrides: { + "vision-model": { + inputLimits: { + images: { resize: { maxWidth: 1568, maxBytes: 524288, jpegQuality: 75 } }, + }, + }, + }, + }, + }); + + const registry = await createModelRegistry(authStorage, modelsJsonPath); + const model = registry.find("test", "vision-model"); + + expect(registry.getError()).toBeUndefined(); + expect(model?.inputLimits).toMatchObject({ + maxRequestBytes: 32 * 1024 * 1024, + images: { + maxPerRequest: 100, + resize: { maxWidth: 1568, maxHeight: 2000, maxBytes: 524288, jpegQuality: 75 }, + }, + }); + }); + test("model override with compat.openRouterRouting", async () => { writeRawModelsJson({ openrouter: { diff --git a/packages/coding-agent/test/suite/agent-session-prompt.test.ts b/packages/coding-agent/test/suite/agent-session-prompt.test.ts index 8e1a0ba33..7361383f3 100644 --- a/packages/coding-agent/test/suite/agent-session-prompt.test.ts +++ b/packages/coding-agent/test/suite/agent-session-prompt.test.ts @@ -4,18 +4,32 @@ import { join } from "node:path"; import type { AgentTool } from "@earendil-works/pi-agent-core"; import { fauxAssistantMessage, fauxToolCall, type Model } from "@earendil-works/pi-ai"; import { Type } from "typebox"; -import { afterEach, describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import type { ExtensionAPI, InputEvent } from "../../src/core/extensions/index.ts"; import type { PromptTemplate } from "../../src/core/prompt-templates.ts"; import { createSyntheticSourceInfo } from "../../src/core/source-info.ts"; import { createTestResourceLoader } from "../utilities.ts"; import { createHarness, getMessageText, type Harness } from "./harness.ts"; +const processImage = vi.hoisted(() => + vi.fn(async (_bytes: Uint8Array, mimeType: string) => ({ + ok: true as const, + data: Buffer.from("normalized").toString("base64"), + mimeType, + hints: [], + })), +); +vi.mock("../../src/utils/image-process.ts", () => ({ processImage })); + +const TINY_PNG_BASE64 = + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg=="; + describe("AgentSession prompt characterization", () => { const harnesses: Harness[] = []; const tempDirs: string[] = []; afterEach(() => { + processImage.mockClear(); while (harnesses.length > 0) { harnesses.pop()?.cleanup(); } @@ -145,6 +159,44 @@ describe("AgentSession prompt characterization", () => { expect(sawImage).toBe(true); }); + // Regression test for https://github.com/earendil-works/pi/issues/9631 + it("uses the model selected by before_agent_start for image normalization", async () => { + let strictModel: Model | undefined; + const harness = await createHarness({ + models: [{ id: "wide" }, { id: "strict" }], + extensionFactories: [ + (pi) => { + pi.on("before_agent_start", async () => { + if (!strictModel) throw new Error("Expected strict model"); + await pi.setModel(strictModel); + }); + }, + ], + }); + harnesses.push(harness); + strictModel = harness.getModel("strict"); + if (!strictModel) throw new Error("Expected strict model"); + const resizeOptions = { maxWidth: 1000, maxHeight: 1000, maxBytes: 500000, jpegQuality: 70 }; + strictModel.inputLimits = { images: { resize: resizeOptions } }; + harness.setResponses([fauxAssistantMessage("done")]); + + await harness.session.prompt("inspect", { + images: [{ type: "image", data: TINY_PNG_BASE64, mimeType: "image/png" }], + }); + + expect(harness.session.model?.id).toBe("strict"); + expect(processImage).toHaveBeenCalledWith(expect.any(Uint8Array), "image/png", { + autoResizeImages: true, + resizeOptions, + }); + const userMessage = harness.session.messages.find((message) => message.role === "user"); + expect(userMessage?.content).toContainEqual({ + type: "image", + data: Buffer.from("normalized").toString("base64"), + mimeType: "image/png", + }); + }); + it("expands skill commands before sending the prompt", async () => { const tempDir = join(tmpdir(), `pi-skill-${Date.now()}-${Math.random().toString(36).slice(2)}`); mkdirSync(tempDir, { recursive: true }); diff --git a/packages/coding-agent/test/suite/agent-session-tool-result-images.test.ts b/packages/coding-agent/test/suite/agent-session-tool-result-images.test.ts index 884ef2f26..8f8c8670a 100644 --- a/packages/coding-agent/test/suite/agent-session-tool-result-images.test.ts +++ b/packages/coding-agent/test/suite/agent-session-tool-result-images.test.ts @@ -34,12 +34,15 @@ describe("AgentSession tool result images", () => { } }); - it("passes images.autoResize to tool result normalization", async () => { + it("passes image settings and the current model profile to tool result normalization", async () => { + const resizeOptions = { maxWidth: 1200, maxHeight: 1000, maxBytes: 500000, jpegQuality: 70 }; const harness = await createHarness({ tools: [screenshotTool], settings: { images: { autoResize: false } }, }); harnesses.push(harness); + if (!harness.session.model) throw new Error("Expected a model"); + harness.session.model.inputLimits = { images: { resize: resizeOptions } }; harness.setResponses([ fauxAssistantMessage([fauxToolCall("screenshot", {})], { stopReason: "toolUse" }), fauxAssistantMessage("done"), @@ -47,6 +50,9 @@ describe("AgentSession tool result images", () => { await harness.session.prompt("take a screenshot"); - expect(normalizeToolResultImages).toHaveBeenCalledWith(expect.any(Array), { autoResizeImages: false }); + expect(normalizeToolResultImages).toHaveBeenCalledWith(expect.any(Array), { + autoResizeImages: false, + resizeOptions, + }); }); }); diff --git a/packages/coding-agent/test/suite/harness.ts b/packages/coding-agent/test/suite/harness.ts index a654811e6..22e8d7e43 100644 --- a/packages/coding-agent/test/suite/harness.ts +++ b/packages/coding-agent/test/suite/harness.ts @@ -131,6 +131,7 @@ export async function createHarness(options: HarnessOptions = {}): Promise