fix(tui): cache recently offscreen Kitty images

packages/tui/CHANGELOG.md

@@ -8,7 +8,7 @@
 - Fixed phantom alternate-screen text selection from unmatched mouse events when changing terminal pane focus.
 - Fixed spaces in searchable settings queries changing the selected value instead of filtering multi-word labels.
 - Fixed alternate-screen Kitty images crossing vertical layout clip boundaries and overlapping sticky regions while scrolling.
-- Fixed alternate-screen redraws retransmitting Kitty image data, dropping adjacent row content when reusing placements, rendering fixed-basis scroll content twice per frame, and scanning clipped transcript rows while painting.
+- Fixed alternate-screen redraws retransmitting Kitty image data when placements move or recently offscreen images return, dropping adjacent row content when reusing placements, rendering fixed-basis scroll content twice per frame, and scanning clipped transcript rows while painting.
 
 ### Added
 

packages/tui/src/terminal-image.ts

@@ -275,6 +275,8 @@ interface RegisteredKittyImageMetadata extends KittyImageMetadata {
 export interface KittyImagePlacement {
 	imageId: number;
 	transmissionGeneration: number;
+	transmissionBytes: number;
+	estimatedDecodedBytes: number;
 	sequence: string;
 	replacementLine: string;
 }
@@ -359,6 +361,8 @@ export function getKittyImagePlacement(line: string): KittyImagePlacement | unde
 	return {
 		imageId: metadata.imageId,
 		transmissionGeneration: metadata.transmissionGeneration,
+		transmissionBytes: transmissionEnd - match.index,
+		estimatedDecodedBytes: metadata.widthPx * metadata.heightPx * 4,
 		sequence,
 		replacementLine: `${line.slice(0, match.index)}${sequence}${line.slice(transmissionEnd)}`,
 	};

packages/tui/src/tui-alt-screen.ts

@@ -45,6 +45,15 @@ const END_SYNCHRONIZED_OUTPUT = "\x1b[?2026l";
 const OSC133_ZONE_PREFIX = /^(?:\x1b\]133;[ABC](?:\x07|\x1b\\))+/;
 const OSC133_PROMPT_START = /^\x1b\]133;A(?:\x07|\x1b\\)/;
 const PAGE_SCROLL_OVERLAP = 4;
+const MAX_CACHED_OFFSCREEN_KITTY_IMAGES = 16;
+const MAX_CACHED_OFFSCREEN_KITTY_TRANSMISSION_BYTES = 32 * 1024 * 1024;
+const MAX_CACHED_OFFSCREEN_KITTY_DECODED_BYTES = 64 * 1024 * 1024;
+
+interface CachedKittyImage {
+	transmissionGeneration: number;
+	transmissionBytes: number;
+	estimatedDecodedBytes: number;
+}
 
 interface SelectionPoint {
 	row: number;
@@ -100,7 +109,7 @@ export class TuiAltScreen extends TuiBase implements ViewportTUI {
 	private altScreenActive = false;
 	private imageProtocol: ImageProtocol = null;
 	private savedCapabilities?: TerminalCapabilities;
-	private readonly uploadedKittyImages = new Map<number, number>();
+	private readonly uploadedKittyImages = new Map<number, CachedKittyImage>();
 	private selectionAnchor?: SelectionPoint;
 	private selectionFocus?: SelectionPoint;
 	private selectionDragPointer?: { x: number; y: number };
@@ -232,26 +241,54 @@ export class TuiAltScreen extends TuiBase implements ViewportTUI {
 		return this.imageProtocol === "kitty" ? deleteAllKittyImages() : "";
 	}
 
-	private prepareKittyScreen(screen: string[]): { lines: string[]; staleImageDeletion: string } {
+	private prepareKittyScreen(screen: string[]): { lines: string[]; evictedImageDeletion: string } {
 		const visibleImageIds = new Set<number>();
 		const lines = screen.map((line) => {
 			const placement = getKittyImagePlacement(line);
 			if (!placement) return line;
 			visibleImageIds.add(placement.imageId);
-			if (this.uploadedKittyImages.get(placement.imageId) === placement.transmissionGeneration) {
-				return placement.replacementLine;
-			}
-			this.uploadedKittyImages.set(placement.imageId, placement.transmissionGeneration);
-			return line;
+
+			const cachedImage = this.uploadedKittyImages.get(placement.imageId);
+			const nextCachedImage = {
+				transmissionGeneration: placement.transmissionGeneration,
+				transmissionBytes: placement.transmissionBytes,
+				estimatedDecodedBytes: placement.estimatedDecodedBytes,
+			};
+			if (cachedImage) this.uploadedKittyImages.delete(placement.imageId);
+			this.uploadedKittyImages.set(placement.imageId, nextCachedImage);
+
+			return cachedImage?.transmissionGeneration === placement.transmissionGeneration
+				? placement.replacementLine
+				: line;
 		});
 
-		let staleImageDeletion = "";
-		for (const imageId of this.uploadedKittyImages.keys()) {
+		let cachedOffscreenImageCount = 0;
+		let cachedOffscreenTransmissionBytes = 0;
+		let cachedOffscreenDecodedBytes = 0;
+		for (const [imageId, cachedImage] of this.uploadedKittyImages) {
 			if (visibleImageIds.has(imageId)) continue;
-			staleImageDeletion += deleteKittyImage(imageId);
-			this.uploadedKittyImages.delete(imageId);
+			cachedOffscreenImageCount += 1;
+			cachedOffscreenTransmissionBytes += cachedImage.transmissionBytes;
+			cachedOffscreenDecodedBytes += cachedImage.estimatedDecodedBytes;
 		}
-		return { lines, staleImageDeletion };
+
+		let evictedImageDeletion = "";
+		for (const [imageId, cachedImage] of this.uploadedKittyImages) {
+			if (
+				cachedOffscreenImageCount <= MAX_CACHED_OFFSCREEN_KITTY_IMAGES &&
+				cachedOffscreenTransmissionBytes <= MAX_CACHED_OFFSCREEN_KITTY_TRANSMISSION_BYTES &&
+				cachedOffscreenDecodedBytes <= MAX_CACHED_OFFSCREEN_KITTY_DECODED_BYTES
+			) {
+				break;
+			}
+			if (visibleImageIds.has(imageId)) continue;
+			evictedImageDeletion += deleteKittyImage(imageId);
+			this.uploadedKittyImages.delete(imageId);
+			cachedOffscreenImageCount -= 1;
+			cachedOffscreenTransmissionBytes -= cachedImage.transmissionBytes;
+			cachedOffscreenDecodedBytes -= cachedImage.estimatedDecodedBytes;
+		}
+		return { lines, evictedImageDeletion };
 	}
 
 	protected override resetRenderState(): void {
@@ -808,7 +845,7 @@ export class TuiAltScreen extends TuiBase implements ViewportTUI {
 		const preparedKittyScreen =
 			redrawImages && this.imageProtocol === "kitty"
 				? this.prepareKittyScreen(screen)
-				: { lines: screen, staleImageDeletion: "" };
+				: { lines: screen, evictedImageDeletion: "" };
 
 		let buffer = BEGIN_SYNCHRONIZED_OUTPUT;
 		if (fullRedraw) {
@@ -822,7 +859,7 @@ export class TuiAltScreen extends TuiBase implements ViewportTUI {
 			if (this.imageProtocol === "iterm2") buffer += "\x1b[2J";
 			else if (this.imageProtocol === "kitty") buffer += deleteAllKittyPlacements();
 		}
-		buffer += preparedKittyScreen.staleImageDeletion;
+		buffer += preparedKittyScreen.evictedImageDeletion;
 
 		for (let row = 0; row < height; row++) {
 			if (!fullRedraw && !imagesNeedRedraw && screen[row] === this.previousScreen[row]) continue;

packages/tui/test/terminal-image.test.ts

@@ -450,6 +450,8 @@ describe("Kitty image cursor movement", () => {
 		const line = `left ${cropKittyImageLine(transmission, 2, 1)} right`;
 		const placement = getKittyImagePlacement(line);
 		assert.ok(placement);
+		assert.strictEqual(placement.transmissionBytes, line.length - "left ".length - " right".length);
+		assert.strictEqual(placement.estimatedDecodedBytes, 100 * 100 * 4);
 		assert.strictEqual(placement.sequence, "\x1b_Ga=p,q=2,C=1,c=3,i=42,y=66,h=34,r=1\x1b\\");
 		assert.strictEqual(placement.replacementLine, `left ${placement.sequence} right`);
 		assert.ok(!placement.replacementLine.includes("AAAA"));

packages/tui/test/tui-alt-screen.test.ts

@@ -461,6 +461,130 @@ describe("TuiAltScreen", () => {
 		}
 	});
 
+	it("retains recently offscreen Kitty images for placement-only reuse", async () => {
+		setCapabilities({ images: "kitty", trueColor: true, hyperlinks: true });
+		try {
+			const terminal = new RecordingTerminal(20, 1);
+			const tui = new TuiAltScreen(terminal);
+			const imageId = 321;
+			const imageLine = encodeKitty("AAAA", { columns: 2, rows: 1, imageId, moveCursor: false });
+			registerKittyImageMetadata({ imageId, columns: 2, rows: 1, widthPx: 100, heightPx: 50 });
+			tui.setLayoutRoot(
+				new ScrollView(
+					{
+						render: () => [imageLine, "after"],
+						invalidate: () => {},
+					},
+					{ primary: true },
+				),
+			);
+			tui.start();
+			await terminal.waitForRender();
+			assert.ok(terminal.events.some((event) => event.type === "write" && event.data.includes("\x1b_Ga=T")));
+
+			const eventCount = terminal.events.length;
+			tui.scrollBy(1);
+			await terminal.waitForRender();
+			tui.scrollBy(-1);
+			await terminal.waitForRender();
+			const reentryWrites = terminal.events
+				.slice(eventCount)
+				.filter((event): event is { type: "write"; data: string } => event.type === "write")
+				.map((event) => event.data)
+				.join("");
+			assert.ok(reentryWrites.includes("\x1b_Ga=p,q=2"));
+			assert.ok(!reentryWrites.includes("\x1b_Ga=T"));
+			assert.ok(!reentryWrites.includes(`\x1b_Ga=d,d=I,i=${imageId},q=2\x1b\\`));
+			tui.stop();
+		} finally {
+			resetCapabilitiesCache();
+		}
+	});
+
+	it("evicts the least recently visible Kitty image when the cache is full", async () => {
+		setCapabilities({ images: "kitty", trueColor: true, hyperlinks: true });
+		try {
+			const terminal = new RecordingTerminal(20, 1);
+			const tui = new TuiAltScreen(terminal);
+			const firstImageId = 500;
+			const imageLines = Array.from({ length: 18 }, (_, index) => {
+				const imageId = firstImageId + index;
+				registerKittyImageMetadata({ imageId, columns: 2, rows: 1, widthPx: 100, heightPx: 50 });
+				return encodeKitty("AAAA", { columns: 2, rows: 1, imageId, moveCursor: false });
+			});
+			tui.setLayoutRoot(
+				new ScrollView(
+					{
+						render: () => imageLines,
+						invalidate: () => {},
+					},
+					{ primary: true },
+				),
+			);
+			tui.start();
+			await terminal.waitForRender();
+			for (let index = 1; index < imageLines.length; index++) {
+				tui.scrollBy(1);
+				await terminal.waitForRender();
+			}
+			assert.ok(
+				terminal.events.some(
+					(event) => event.type === "write" && event.data.includes(`\x1b_Ga=d,d=I,i=${firstImageId},q=2\x1b\\`),
+				),
+			);
+
+			const eventCount = terminal.events.length;
+			tui.scrollToTop();
+			await terminal.waitForRender();
+			const reentryWrites = terminal.events
+				.slice(eventCount)
+				.filter((event): event is { type: "write"; data: string } => event.type === "write")
+				.map((event) => event.data)
+				.join("");
+			assert.ok(reentryWrites.includes("\x1b_Ga=T"));
+			tui.stop();
+		} finally {
+			resetCapabilitiesCache();
+		}
+	});
+
+	it("evicts offscreen Kitty images when decoded raster memory exceeds the cache quota", async () => {
+		setCapabilities({ images: "kitty", trueColor: true, hyperlinks: true });
+		try {
+			const terminal = new RecordingTerminal(20, 1);
+			const tui = new TuiAltScreen(terminal);
+			const firstImageId = 600;
+			const imageLines = Array.from({ length: 4 }, (_, index) => {
+				const imageId = firstImageId + index;
+				registerKittyImageMetadata({ imageId, columns: 2, rows: 1, widthPx: 3840, heightPx: 2160 });
+				return encodeKitty("AAAA", { columns: 2, rows: 1, imageId, moveCursor: false });
+			});
+			tui.setLayoutRoot(
+				new ScrollView(
+					{
+						render: () => imageLines,
+						invalidate: () => {},
+					},
+					{ primary: true },
+				),
+			);
+			tui.start();
+			await terminal.waitForRender();
+			for (let index = 1; index < imageLines.length; index++) {
+				tui.scrollBy(1);
+				await terminal.waitForRender();
+			}
+			assert.ok(
+				terminal.events.some(
+					(event) => event.type === "write" && event.data.includes(`\x1b_Ga=d,d=I,i=${firstImageId},q=2\x1b\\`),
+				),
+			);
+			tui.stop();
+		} finally {
+			resetCapabilitiesCache();
+		}
+	});
+
 	it("opens an OSC 8 hyperlink on click but not on drag", async () => {
 		const terminal = new RecordingTerminal(20, 3);
 		const openedUrls: string[] = [];