fix(coding-agent): pick the smaller of PNG and JPEG when resizing images (#11577)

When an image needs resizing, `resizeImageInProcess` builds a list of
encodings: PNG first, then JPEG at decreasing qualities. It returned the
first one under `maxBytes`. Since PNG was always first, it was used
whenever it fit, even when it was many times larger than the JPEG. The
doc comment says "Try both PNG and JPEG formats, pick the smaller one",
and the code worked that way before v0.62.0. Because of this, photos
were sent as multi-MB PNGs, which made requests large enough to fail or
time out.

This change compares PNG with JPEG at the configured quality at each
size step and tries the smaller one first. It tries the other one next,
and the lower JPEG qualities only after that. Screenshots and other
images where PNG is smaller still go out as PNG.

**Testing**
- Added a regression test to `test/image-processing.test.ts`: a
generated photo-like 1200×800 JPEG resized to 1000 px must come back as
`image/jpeg`. The test fails without the fix (PNG is returned) and
passes with it.
- `node ../../node_modules/vitest/dist/cli.js --run
test/image-processing.test.ts test/image-resize-callers.test.ts`: all
pass.
- `npm run check`: no issues in the changed files. It does report type
errors, all in `packages/ai/test` (model ID unions), which this change
does not touch.
- Reproduction with a photo-like 2400×1620 JPEG (460 KB base64):
  - before: `image/png`, 4,008,480 bytes base64, at 2000×1350
  - after: `image/jpeg`, 255,316 bytes base64, at 2000×1350

Fixes #11551

Requested by @davidbrai

Co-authored-by: triagent <triagent@localhost>

packages/coding-agent/src/utils/image-resize-core.ts

@@ -134,9 +134,11 @@ export async function resizeImageInProcess(
 		let currentHeight = targetHeight;
 
 		while (true) {
-			const candidates = tryEncodings(currentWidth, currentHeight, qualitySteps);
-			for (const candidate of candidates) {
-				if (candidate.encodedSize < opts.maxBytes) {
+			const [png, jpeg, ...lowerQualityJpegs] = tryEncodings(currentWidth, currentHeight, qualitySteps);
+			// Prefer the smaller of PNG and JPEG at the configured quality; lower JPEG quality only if neither fits.
+			const preferred = jpeg && jpeg.encodedSize < png.encodedSize ? [jpeg, png] : [png, jpeg];
+			for (const candidate of [...preferred, ...lowerQualityJpegs]) {
+				if (candidate && candidate.encodedSize < opts.maxBytes) {
 					return {
 						data: candidate.data,
 						mimeType: candidate.mimeType,

packages/coding-agent/test/image-processing.test.ts

@@ -5,6 +5,8 @@
 import { describe, expect, it } from "vitest";
 import { convertToPng, loadPngTranscoder } from "../src/utils/image-convert.ts";
 import { formatDimensionNote, resizeImage } from "../src/utils/image-resize.ts";
+import { resizeImageInProcess } from "../src/utils/image-resize-core.ts";
+import { loadPhoton } from "../src/utils/photon.ts";
 
 // Small 2x2 red PNG image (base64) - generated with ImageMagick
 const TINY_PNG =
@@ -25,6 +27,31 @@ const MEDIUM_PNG_100x100 =
 const LARGE_PNG_200x200 =
 	"iVBORw0KGgoAAAANSUhEUgAAAMgAAADIEAIAAAByquWKAAAAIGNIUk0AAHomAACAhAAA+gAAAIDoAAB1MAAA6mAAADqYAAAXcJy6UTwAAAAGYktHRP///////wlY99wAAAAHdElNRQfqAQ4AMzqU03BFAAAEEElEQVR42u3dvWqUURSF4XXMRAyIhSAWQvwnWtjaWusFx9bW1kKD/wELESxEiBjDjJdwysOC57mCXW32+zGcGbtdACpsdi+T8Xz1GABzm91xMl6sHgNgblzcT/Y+rh4DYG6cv0/2H64eA2BOEgI1fHQHaoyzbXIwVo8BMCcJgRqSEKgxft9Lrn5aPQbA3Ph1klw7Wj0GwJwkBGpYWECN8fMiub63egyAORcWUMPCAmqMH3eTG59XjwEwN76/S24+Wj0GwJwkBGpYWECN8e1fcmuzegyAORcWUMPCAmqM09vJ4dfVYwDMjS9vkzuPV48BMCcJgRqeSAZqjA/nyYP91WMAzElCoIYkBGqMk8Pk6HT1GABzm+3L5JIkBApIQqCGj+5AjfHmb/Lk8uoxAOYkIVBDEgI1LCygxmZ7nFyShEABFxZQw8ICaozXf5KnV1aPATDnwgJqWFhADQsLqOFnDUANFxZQw8ICaoxXZ8mzg9VjAMy5sIAaFhZQw3tYQA1/QgHUkIRADUkI1JCEQA1JCNSQhEANFxZQwzcsoIYkBGpIQqCGJARqSEKghiQEalhYQA1/QgHUcGEBNSwsoIYkBGq4sIAaFhZQw8ICaviGBdRwYQE1LCyghiQEariwgBoWFlDDe1hADS+OAjUkIVBDEgI1JCFQQxICNSQhUMOFBdTwDQuoIQmBGpIQqCEJgRqSEKghCYEaFhZQw4ujQA0XFlDDwgJqSEKghgsLqGFhATUsLKCGb1hADRcWUMPCAmpIQqCGCwuoYWEBNbyHBdTw4ihQQxICNSQhUEMSAjUkIVBDEgI1XFhADd+wgBqSEKghCYEakhCoIQmBGpIQqGFhATW8OArUcGEBNSwsoIYkBGq4sIAaFhZQw8ICaviGBdRwYQE1LCyghiQEariwgBoWFlDDe1hADS+OAjUkIVBDEgI1JCFQQxICNSQhUMOFBdTwDQuoIQmBGpIQqCEJgRqSEKghCYEa49lB8ups9RgAc14cBWpIQqCGhQXUkIRADRcWUMPCAmqMp1eS139WjwEw5xsWUEMSAjUsLKCGJARquLCAGhYWUGM8uZy8+bt6DIA5L44CNSQhUMMTyUCNcXSanByuHgNgThICNSQhUGM82E8+nK8eA2DOzxqAGpIQqDHuPE6+vF09BsDcOPyanN5ePQbAnCQEavgdFlBj3Nok3/6tHgNgzoujQA1JCNQYNx8l39+tHgNgbtz4nPy4u3oMgDlJCNSwsIAa4/pe8vNi9RgAc37WANSQhECNce0o+XWyegyAuXH1U/L73uoxAOYkIVDDwgJqjIORnG1XjwEw508ogBqSEKgx9h8m5+9XjwEwN/Y+Jhf3V48BMCcJgRpjPE+2x6vHAJgbSbLbrR4DYO4/GqiSgXN+ksgAAAAldEVYdGRhdGU6Y3JlYXRlADIwMjYtMDEtMTRUMDA6NTE6NTcrMDA6MDDpysx4AAAAJXRFWHRkYXRlOm1vZGlmeQAyMDI2LTAxLTE0VDAwOjUxOjU3KzAwOjAwmJd0xAAAACh0RVh0ZGF0ZTp0aW1lc3RhbXAAMjAyNi0wMS0xNFQwMDo1MTo1NyswMDowMM+CVRsAAAAASUVORK5CYII=";
 
+/** Photo-like JPEG: smooth gradients with light deterministic noise, so PNG compresses it poorly. */
+async function photoLikeJpeg(width: number, height: number): Promise<Uint8Array> {
+	const photon = await loadPhoton();
+	if (!photon) throw new Error("Photon not available");
+	const pixels = new Uint8Array(width * height * 4);
+	let seed = 1;
+	for (let y = 0; y < height; y++) {
+		for (let x = 0; x < width; x++) {
+			seed = (seed * 1103515245 + 12345) & 0x7fffffff;
+			const noise = seed % 6;
+			const i = (y * width + x) * 4;
+			pixels[i] = ((x / width) * 200 + noise) | 0;
+			pixels[i + 1] = ((y / height) * 200 + noise) | 0;
+			pixels[i + 2] = (128 + Math.sin(x / 30) * 60 + noise) | 0;
+			pixels[i + 3] = 255;
+		}
+	}
+	const image = new photon.PhotonImage(pixels, width, height);
+	try {
+		return image.get_bytes_jpeg(85);
+	} finally {
+		image.free();
+	}
+}
+
 function imageBytes(base64Data: string): Uint8Array {
 	return Buffer.from(base64Data, "base64");
 }
@@ -166,6 +193,18 @@ describe("resizeImage", () => {
 		expect(result).toBeNull();
 	});
 
+	// Regression test for #11551: PNG was returned whenever it fit, even when JPEG was much smaller.
+	it("should pick JPEG over a larger PNG when resizing a photo", async () => {
+		const input = await photoLikeJpeg(1200, 800);
+		const result = await resizeImageInProcess(input, "image/jpeg", { maxWidth: 1000, maxHeight: 1000 });
+
+		expect(result).not.toBeNull();
+		expect(result!.wasResized).toBe(true);
+		expect(result!.mimeType).toBe("image/jpeg");
+		expect(result!.width).toBe(1000);
+		expect(result!.data.length).toBeLessThan(Math.ceil(input.byteLength / 3) * 4 * 2);
+	});
+
 	it("should handle JPEG input", async () => {
 		const result = await resizeImage(imageBytes(TINY_JPEG), "image/jpeg", {
 			maxWidth: 100,