From a8ee03b8156c2232d67ad2cdb79683b4a5c8fdbe Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Mon, 3 Aug 2026 14:02:27 +0200 Subject: [PATCH] fix(tui): cache recently offscreen Kitty images --- packages/tui/CHANGELOG.md | 2 +- packages/tui/src/terminal-image.ts | 4 + packages/tui/src/tui-alt-screen.ts | 65 +++++++++--- packages/tui/test/terminal-image.test.ts | 2 + packages/tui/test/tui-alt-screen.test.ts | 124 +++++++++++++++++++++++ 5 files changed, 182 insertions(+), 15 deletions(-) diff --git a/packages/tui/CHANGELOG.md b/packages/tui/CHANGELOG.md index 59b03af13..f13551751 100644 --- a/packages/tui/CHANGELOG.md +++ b/packages/tui/CHANGELOG.md @@ -8,7 +8,7 @@ - Fixed phantom alternate-screen text selection from unmatched mouse events when changing terminal pane focus. - Fixed spaces in searchable settings queries changing the selected value instead of filtering multi-word labels. - Fixed alternate-screen Kitty images crossing vertical layout clip boundaries and overlapping sticky regions while scrolling. -- Fixed alternate-screen redraws retransmitting Kitty image data, dropping adjacent row content when reusing placements, rendering fixed-basis scroll content twice per frame, and scanning clipped transcript rows while painting. +- Fixed alternate-screen redraws retransmitting Kitty image data when placements move or recently offscreen images return, dropping adjacent row content when reusing placements, rendering fixed-basis scroll content twice per frame, and scanning clipped transcript rows while painting. ### Added diff --git a/packages/tui/src/terminal-image.ts b/packages/tui/src/terminal-image.ts index a5239a0e3..a9e2501cd 100644 --- a/packages/tui/src/terminal-image.ts +++ b/packages/tui/src/terminal-image.ts @@ -275,6 +275,8 @@ interface RegisteredKittyImageMetadata extends KittyImageMetadata { export interface KittyImagePlacement { imageId: number; transmissionGeneration: number; + transmissionBytes: number; + estimatedDecodedBytes: number; sequence: string; replacementLine: string; } @@ -359,6 +361,8 @@ export function getKittyImagePlacement(line: string): KittyImagePlacement | unde return { imageId: metadata.imageId, transmissionGeneration: metadata.transmissionGeneration, + transmissionBytes: transmissionEnd - match.index, + estimatedDecodedBytes: metadata.widthPx * metadata.heightPx * 4, sequence, replacementLine: `${line.slice(0, match.index)}${sequence}${line.slice(transmissionEnd)}`, }; diff --git a/packages/tui/src/tui-alt-screen.ts b/packages/tui/src/tui-alt-screen.ts index 0bb8c1c53..ed0c1b71e 100644 --- a/packages/tui/src/tui-alt-screen.ts +++ b/packages/tui/src/tui-alt-screen.ts @@ -45,6 +45,15 @@ const END_SYNCHRONIZED_OUTPUT = "\x1b[?2026l"; const OSC133_ZONE_PREFIX = /^(?:\x1b\]133;[ABC](?:\x07|\x1b\\))+/; const OSC133_PROMPT_START = /^\x1b\]133;A(?:\x07|\x1b\\)/; const PAGE_SCROLL_OVERLAP = 4; +const MAX_CACHED_OFFSCREEN_KITTY_IMAGES = 16; +const MAX_CACHED_OFFSCREEN_KITTY_TRANSMISSION_BYTES = 32 * 1024 * 1024; +const MAX_CACHED_OFFSCREEN_KITTY_DECODED_BYTES = 64 * 1024 * 1024; + +interface CachedKittyImage { + transmissionGeneration: number; + transmissionBytes: number; + estimatedDecodedBytes: number; +} interface SelectionPoint { row: number; @@ -100,7 +109,7 @@ export class TuiAltScreen extends TuiBase implements ViewportTUI { private altScreenActive = false; private imageProtocol: ImageProtocol = null; private savedCapabilities?: TerminalCapabilities; - private readonly uploadedKittyImages = new Map(); + private readonly uploadedKittyImages = new Map(); private selectionAnchor?: SelectionPoint; private selectionFocus?: SelectionPoint; private selectionDragPointer?: { x: number; y: number }; @@ -232,26 +241,54 @@ export class TuiAltScreen extends TuiBase implements ViewportTUI { return this.imageProtocol === "kitty" ? deleteAllKittyImages() : ""; } - private prepareKittyScreen(screen: string[]): { lines: string[]; staleImageDeletion: string } { + private prepareKittyScreen(screen: string[]): { lines: string[]; evictedImageDeletion: string } { const visibleImageIds = new Set(); const lines = screen.map((line) => { const placement = getKittyImagePlacement(line); if (!placement) return line; visibleImageIds.add(placement.imageId); - if (this.uploadedKittyImages.get(placement.imageId) === placement.transmissionGeneration) { - return placement.replacementLine; - } - this.uploadedKittyImages.set(placement.imageId, placement.transmissionGeneration); - return line; + + const cachedImage = this.uploadedKittyImages.get(placement.imageId); + const nextCachedImage = { + transmissionGeneration: placement.transmissionGeneration, + transmissionBytes: placement.transmissionBytes, + estimatedDecodedBytes: placement.estimatedDecodedBytes, + }; + if (cachedImage) this.uploadedKittyImages.delete(placement.imageId); + this.uploadedKittyImages.set(placement.imageId, nextCachedImage); + + return cachedImage?.transmissionGeneration === placement.transmissionGeneration + ? placement.replacementLine + : line; }); - let staleImageDeletion = ""; - for (const imageId of this.uploadedKittyImages.keys()) { + let cachedOffscreenImageCount = 0; + let cachedOffscreenTransmissionBytes = 0; + let cachedOffscreenDecodedBytes = 0; + for (const [imageId, cachedImage] of this.uploadedKittyImages) { if (visibleImageIds.has(imageId)) continue; - staleImageDeletion += deleteKittyImage(imageId); - this.uploadedKittyImages.delete(imageId); + cachedOffscreenImageCount += 1; + cachedOffscreenTransmissionBytes += cachedImage.transmissionBytes; + cachedOffscreenDecodedBytes += cachedImage.estimatedDecodedBytes; } - return { lines, staleImageDeletion }; + + let evictedImageDeletion = ""; + for (const [imageId, cachedImage] of this.uploadedKittyImages) { + if ( + cachedOffscreenImageCount <= MAX_CACHED_OFFSCREEN_KITTY_IMAGES && + cachedOffscreenTransmissionBytes <= MAX_CACHED_OFFSCREEN_KITTY_TRANSMISSION_BYTES && + cachedOffscreenDecodedBytes <= MAX_CACHED_OFFSCREEN_KITTY_DECODED_BYTES + ) { + break; + } + if (visibleImageIds.has(imageId)) continue; + evictedImageDeletion += deleteKittyImage(imageId); + this.uploadedKittyImages.delete(imageId); + cachedOffscreenImageCount -= 1; + cachedOffscreenTransmissionBytes -= cachedImage.transmissionBytes; + cachedOffscreenDecodedBytes -= cachedImage.estimatedDecodedBytes; + } + return { lines, evictedImageDeletion }; } protected override resetRenderState(): void { @@ -808,7 +845,7 @@ export class TuiAltScreen extends TuiBase implements ViewportTUI { const preparedKittyScreen = redrawImages && this.imageProtocol === "kitty" ? this.prepareKittyScreen(screen) - : { lines: screen, staleImageDeletion: "" }; + : { lines: screen, evictedImageDeletion: "" }; let buffer = BEGIN_SYNCHRONIZED_OUTPUT; if (fullRedraw) { @@ -822,7 +859,7 @@ export class TuiAltScreen extends TuiBase implements ViewportTUI { if (this.imageProtocol === "iterm2") buffer += "\x1b[2J"; else if (this.imageProtocol === "kitty") buffer += deleteAllKittyPlacements(); } - buffer += preparedKittyScreen.staleImageDeletion; + buffer += preparedKittyScreen.evictedImageDeletion; for (let row = 0; row < height; row++) { if (!fullRedraw && !imagesNeedRedraw && screen[row] === this.previousScreen[row]) continue; diff --git a/packages/tui/test/terminal-image.test.ts b/packages/tui/test/terminal-image.test.ts index 0a9df341f..34f2bb74d 100644 --- a/packages/tui/test/terminal-image.test.ts +++ b/packages/tui/test/terminal-image.test.ts @@ -450,6 +450,8 @@ describe("Kitty image cursor movement", () => { const line = `left ${cropKittyImageLine(transmission, 2, 1)} right`; const placement = getKittyImagePlacement(line); assert.ok(placement); + assert.strictEqual(placement.transmissionBytes, line.length - "left ".length - " right".length); + assert.strictEqual(placement.estimatedDecodedBytes, 100 * 100 * 4); assert.strictEqual(placement.sequence, "\x1b_Ga=p,q=2,C=1,c=3,i=42,y=66,h=34,r=1\x1b\\"); assert.strictEqual(placement.replacementLine, `left ${placement.sequence} right`); assert.ok(!placement.replacementLine.includes("AAAA")); diff --git a/packages/tui/test/tui-alt-screen.test.ts b/packages/tui/test/tui-alt-screen.test.ts index b65cf59e5..c508c01a4 100644 --- a/packages/tui/test/tui-alt-screen.test.ts +++ b/packages/tui/test/tui-alt-screen.test.ts @@ -461,6 +461,130 @@ describe("TuiAltScreen", () => { } }); + it("retains recently offscreen Kitty images for placement-only reuse", async () => { + setCapabilities({ images: "kitty", trueColor: true, hyperlinks: true }); + try { + const terminal = new RecordingTerminal(20, 1); + const tui = new TuiAltScreen(terminal); + const imageId = 321; + const imageLine = encodeKitty("AAAA", { columns: 2, rows: 1, imageId, moveCursor: false }); + registerKittyImageMetadata({ imageId, columns: 2, rows: 1, widthPx: 100, heightPx: 50 }); + tui.setLayoutRoot( + new ScrollView( + { + render: () => [imageLine, "after"], + invalidate: () => {}, + }, + { primary: true }, + ), + ); + tui.start(); + await terminal.waitForRender(); + assert.ok(terminal.events.some((event) => event.type === "write" && event.data.includes("\x1b_Ga=T"))); + + const eventCount = terminal.events.length; + tui.scrollBy(1); + await terminal.waitForRender(); + tui.scrollBy(-1); + await terminal.waitForRender(); + const reentryWrites = terminal.events + .slice(eventCount) + .filter((event): event is { type: "write"; data: string } => event.type === "write") + .map((event) => event.data) + .join(""); + assert.ok(reentryWrites.includes("\x1b_Ga=p,q=2")); + assert.ok(!reentryWrites.includes("\x1b_Ga=T")); + assert.ok(!reentryWrites.includes(`\x1b_Ga=d,d=I,i=${imageId},q=2\x1b\\`)); + tui.stop(); + } finally { + resetCapabilitiesCache(); + } + }); + + it("evicts the least recently visible Kitty image when the cache is full", async () => { + setCapabilities({ images: "kitty", trueColor: true, hyperlinks: true }); + try { + const terminal = new RecordingTerminal(20, 1); + const tui = new TuiAltScreen(terminal); + const firstImageId = 500; + const imageLines = Array.from({ length: 18 }, (_, index) => { + const imageId = firstImageId + index; + registerKittyImageMetadata({ imageId, columns: 2, rows: 1, widthPx: 100, heightPx: 50 }); + return encodeKitty("AAAA", { columns: 2, rows: 1, imageId, moveCursor: false }); + }); + tui.setLayoutRoot( + new ScrollView( + { + render: () => imageLines, + invalidate: () => {}, + }, + { primary: true }, + ), + ); + tui.start(); + await terminal.waitForRender(); + for (let index = 1; index < imageLines.length; index++) { + tui.scrollBy(1); + await terminal.waitForRender(); + } + assert.ok( + terminal.events.some( + (event) => event.type === "write" && event.data.includes(`\x1b_Ga=d,d=I,i=${firstImageId},q=2\x1b\\`), + ), + ); + + const eventCount = terminal.events.length; + tui.scrollToTop(); + await terminal.waitForRender(); + const reentryWrites = terminal.events + .slice(eventCount) + .filter((event): event is { type: "write"; data: string } => event.type === "write") + .map((event) => event.data) + .join(""); + assert.ok(reentryWrites.includes("\x1b_Ga=T")); + tui.stop(); + } finally { + resetCapabilitiesCache(); + } + }); + + it("evicts offscreen Kitty images when decoded raster memory exceeds the cache quota", async () => { + setCapabilities({ images: "kitty", trueColor: true, hyperlinks: true }); + try { + const terminal = new RecordingTerminal(20, 1); + const tui = new TuiAltScreen(terminal); + const firstImageId = 600; + const imageLines = Array.from({ length: 4 }, (_, index) => { + const imageId = firstImageId + index; + registerKittyImageMetadata({ imageId, columns: 2, rows: 1, widthPx: 3840, heightPx: 2160 }); + return encodeKitty("AAAA", { columns: 2, rows: 1, imageId, moveCursor: false }); + }); + tui.setLayoutRoot( + new ScrollView( + { + render: () => imageLines, + invalidate: () => {}, + }, + { primary: true }, + ), + ); + tui.start(); + await terminal.waitForRender(); + for (let index = 1; index < imageLines.length; index++) { + tui.scrollBy(1); + await terminal.waitForRender(); + } + assert.ok( + terminal.events.some( + (event) => event.type === "write" && event.data.includes(`\x1b_Ga=d,d=I,i=${firstImageId},q=2\x1b\\`), + ), + ); + tui.stop(); + } finally { + resetCapabilitiesCache(); + } + }); + it("opens an OSC 8 hyperlink on click but not on drag", async () => { const terminal = new RecordingTerminal(20, 3); const openedUrls: string[] = [];