mirror of
https://github.com/badlogic/pi-mono.git
synced 2026-11-05 23:37:25 +00:00
9e870f6f00
Several places truncate text by UTF-16 code units with `slice()`. When the cut lands between the two halves of a surrogate pair (an emoji or other character outside the Basic Multilingual Plane), the result keeps a lone high or low surrogate. That is invalid UTF-16. Example: a grep match line with 499 ASCII characters followed by `😀` is cut at 500 units, so the output ends in `\uD83D... [truncated]`. The bundled providers strip lone surrogates before sending (`sanitizeSurrogates` in pi-ai), so the usual effect is a dropped character. But the broken text still reaches the session file, RPC and JSON clients (strict UTF-8 encoders such as Rust's serde_json reject lone surrogates), extensions, and custom providers. This adds `sliceHead(text, n)` and `sliceTail(text, n)` to `core/tools/truncate.ts`. They behave like `slice(0, n)` and `slice(-n)` but return one unit fewer when the cut would split a pair, and are used where truncated text reaches the model: - `truncateLine` (grep match lines), the case reported in #7121 - codemode `truncateOutput` head and tail, and its argument and error previews - compaction `truncateForSummary` (tool results sent to the summarizer) - MCP server summaries in the system prompt and stderr tails in connection errors The cost is two `charCodeAt` checks per truncation, and nothing on the untruncated path. The surrogate check follows the fix in #7122 by @IKEASven69, which @cristinaponcela reviewed in a follow-up branch. That PR was closed before merging. Not changed: TUI-only previews (renderers, visual truncation) and line-based slicing. pi-private's pi-durable codemode `output.ts` has the same head/tail slicing and needs a separate change there. **Testing** - New `test/truncate-surrogates.test.ts`: an emoji at the cut in `truncateLine` is dropped whole, an emoji ending exactly at the limit is kept, `sliceHead`/`sliceTail` never leave a lone surrogate for any length, and they match `slice` on text without surrogates. - `test/tools.test.ts`, compaction, MCP extension, and codemode tests pass. - `npm run check` passes. fixes #7121 This PR is AI-generated.
47 lines
1.9 KiB
TypeScript
47 lines
1.9 KiB
TypeScript
import { describe, expect, it } from "vitest";
|
|
import { sliceHead, sliceTail, truncateLine } from "../src/core/tools/truncate.ts";
|
|
|
|
const EMOJI = "\u{1F600}"; // two UTF-16 code units: \uD83D\uDE00
|
|
|
|
function hasLoneSurrogate(text: string): boolean {
|
|
return /[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/.test(text);
|
|
}
|
|
|
|
// https://github.com/earendil-works/pi/issues/7121
|
|
describe("surrogate-safe slicing", () => {
|
|
it("truncateLine does not cut an emoji in half at the limit", () => {
|
|
const line = `${"a".repeat(499)}${EMOJI}tail`;
|
|
const { text, wasTruncated } = truncateLine(line, 500);
|
|
expect(wasTruncated).toBe(true);
|
|
expect(text).toBe(`${"a".repeat(499)}... [truncated]`);
|
|
expect(hasLoneSurrogate(text)).toBe(false);
|
|
});
|
|
|
|
it("truncateLine keeps an emoji that ends exactly at the limit", () => {
|
|
const line = `${"a".repeat(498)}${EMOJI}tail`;
|
|
expect(truncateLine(line, 500).text).toBe(`${"a".repeat(498)}${EMOJI}... [truncated]`);
|
|
});
|
|
|
|
it("sliceHead and sliceTail never split a pair", () => {
|
|
const text = `${EMOJI}x${EMOJI}`;
|
|
for (let units = 0; units <= text.length + 1; units++) {
|
|
const head = sliceHead(text, units);
|
|
const tail = sliceTail(text, units);
|
|
expect(hasLoneSurrogate(head)).toBe(false);
|
|
expect(hasLoneSurrogate(tail)).toBe(false);
|
|
expect(text.startsWith(head)).toBe(true);
|
|
expect(text.endsWith(tail)).toBe(true);
|
|
expect(head.length).toBeGreaterThanOrEqual(Math.min(units, text.length) - 1);
|
|
expect(tail.length).toBeGreaterThanOrEqual(Math.min(units, text.length) - 1);
|
|
}
|
|
});
|
|
|
|
it("sliceHead and sliceTail match slice on text without surrogates", () => {
|
|
expect(sliceHead("abcdef", 3)).toBe("abc");
|
|
expect(sliceTail("abcdef", 3)).toBe("def");
|
|
expect(sliceHead("abc", 10)).toBe("abc");
|
|
expect(sliceTail("abc", 10)).toBe("abc");
|
|
expect(sliceHead("abc", 0)).toBe("");
|
|
expect(sliceTail("abc", 0)).toBe("");
|
|
});
|
|
});
|