diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 472e4fb5d..0bdab578b 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -27,6 +27,7 @@ - Fixed RPC mode cutting off shutdown cleanup, such as a pending MCP token refresh, when stdin closed and SIGTERM arrived together, which process managers like Paseo and systemd do. Pi lost the refreshed token and the MCP server revoked the sign-in. Once a shutdown has started, a stdin EOF or a single signal no longer interrupts it; a second signal still exits at once. Print mode likewise no longer dies from a signal that arrives during its final cleanup ([#11599](https://github.com/earendil-works/pi/issues/11599)) - Fixed the grep tool reading every matched file into memory when `context` was set, which ran out of memory on large files; context now comes from ripgrep, which also fixes UTF-16 files and lone CRs in matched lines, and the match limit notice no longer appears when exactly `limit` matches exist ([#9276](https://github.com/earendil-works/pi/issues/9276), [#11423](https://github.com/earendil-works/pi/issues/11423)) - Fixed the find tool reporting the result limit when exactly `limit` files match, and `limit: 0` in find and ls returning every result or an empty listing; limits below 1 now mean 1 ([#7121](https://github.com/earendil-works/pi/issues/7121)) +- Fixed grep match lines, codemode output, compaction summaries of tool results, and MCP error and server summaries cutting an emoji or other non-BMP character in half when truncating, which left an invalid lone surrogate in the text ([#7121](https://github.com/earendil-works/pi/issues/7121)) ## [1.5.0] - 2026-11-02 diff --git a/packages/coding-agent/src/core/compaction/utils.ts b/packages/coding-agent/src/core/compaction/utils.ts index 1f05a14b4..b79444cc0 100644 --- a/packages/coding-agent/src/core/compaction/utils.ts +++ b/packages/coding-agent/src/core/compaction/utils.ts @@ -4,6 +4,7 @@ import type { AgentMessage } from "@earendil-works/pi-agent-core"; import { contentText, type Message } from "@earendil-works/pi-ai"; +import { sliceHead } from "../tools/truncate.ts"; // ============================================================================ // File Operation Tracking @@ -100,7 +101,7 @@ const TOOL_RESULT_MAX_CHARS = 2000; function truncateForSummary(text: string, maxChars: number): string { if (text.length <= maxChars) return text; const truncatedChars = text.length - maxChars; - return `${text.slice(0, maxChars)}\n\n[... ${truncatedChars} more characters truncated]`; + return `${sliceHead(text, maxChars)}\n\n[... ${truncatedChars} more characters truncated]`; } /** diff --git a/packages/coding-agent/src/core/tools/truncate.ts b/packages/coding-agent/src/core/tools/truncate.ts index c638ee480..27b415b99 100644 --- a/packages/coding-agent/src/core/tools/truncate.ts +++ b/packages/coding-agent/src/core/tools/truncate.ts @@ -272,5 +272,31 @@ export function truncateLine( if (line.length <= maxChars) { return { text: line, wasTruncated: false }; } - return { text: `${line.slice(0, maxChars)}... [truncated]`, wasTruncated: true }; + return { text: `${sliceHead(line, maxChars)}... [truncated]`, wasTruncated: true }; +} + +/** Returns at most the first `maxUnits` UTF-16 code units of `text`, without splitting a surrogate pair. */ +export function sliceHead(text: string, maxUnits: number): string { + const end = Math.max(0, maxUnits); + // A lone high surrogate before the cut would be invalid UTF-16; keep the whole character out instead. + return isHighSurrogate(text.charCodeAt(end - 1)) && isLowSurrogate(text.charCodeAt(end)) + ? text.slice(0, end - 1) + : text.slice(0, end); +} + +/** Returns at most the last `maxUnits` UTF-16 code units of `text`, without splitting a surrogate pair. */ +export function sliceTail(text: string, maxUnits: number): string { + if (maxUnits <= 0) return ""; + const start = Math.max(0, text.length - maxUnits); + return isLowSurrogate(text.charCodeAt(start)) && isHighSurrogate(text.charCodeAt(start - 1)) + ? text.slice(start + 1) + : text.slice(start); +} + +function isHighSurrogate(code: number): boolean { + return code >= 0xd800 && code <= 0xdbff; +} + +function isLowSurrogate(code: number): boolean { + return code >= 0xdc00 && code <= 0xdfff; } diff --git a/packages/coding-agent/src/extensions/codemode/execute.ts b/packages/coding-agent/src/extensions/codemode/execute.ts index b3635999f..2dc880427 100644 --- a/packages/coding-agent/src/extensions/codemode/execute.ts +++ b/packages/coding-agent/src/extensions/codemode/execute.ts @@ -27,7 +27,7 @@ import { import { getCodemodeWorkerSpecifier, getQuickJSWasmPath } from "../../config.ts"; import type { ExtensionToolContext, ToolNamespace } from "../../core/extensions/types.ts"; import type { SessionEntry } from "../../core/session-manager.ts"; -import { formatSize } from "../../core/tools/truncate.ts"; +import { formatSize, sliceHead, sliceTail } from "../../core/tools/truncate.ts"; import { combineUsage } from "../../core/usage-totals.ts"; import { writeOutputFile } from "../../utils/output-files.ts"; import { Bm25Ranker, createToolSearchDocument, DEFAULT_TOOL_SEARCH_LIMIT } from "../tool-search/tool.ts"; @@ -112,7 +112,7 @@ function releaseCodemodeWorker(): void { const MODEL_TYPES: ReadonlySet = new Set(["chat", "image", "classifier"]); function truncateText(text: string, maxChars: number): string { - return text.length > maxChars ? `${text.slice(0, maxChars - 3)}...` : text; + return text.length > maxChars ? `${sliceHead(text, maxChars - 3)}...` : text; } function previewArgs(args: unknown): string { @@ -457,8 +457,8 @@ async function truncateOutput( const headChars = Math.floor(budget / 2); const tailChars = budget - headChars; const removed = combined.length - headChars - tailChars; - const head = combined.slice(0, headChars); - const tail = tailChars > 0 ? combined.slice(-tailChars) : ""; + const head = sliceHead(combined, headChars); + const tail = sliceTail(combined, tailChars); let text = `Warning: truncated output (original token count: ${Math.ceil(combined.length / CHARS_PER_TOKEN)})\nTotal output lines: ${combined.split("\n").length}\n\n${head}…${Math.ceil(removed / CHARS_PER_TOKEN)} tokens truncated…${tail}`; const spilled = await spillOutput(combined); text += diff --git a/packages/coding-agent/src/extensions/mcp/index.ts b/packages/coding-agent/src/extensions/mcp/index.ts index f2711704a..1a2db568a 100644 --- a/packages/coding-agent/src/extensions/mcp/index.ts +++ b/packages/coding-agent/src/extensions/mcp/index.ts @@ -39,6 +39,7 @@ import type { ToolDefinition, } from "../../core/extensions/types.ts"; import type { ModelRegistry } from "../../core/model-registry.ts"; +import { sliceHead } from "../../core/tools/truncate.ts"; import { openBrowser } from "../../utils/open-browser.ts"; import { CODEMODE_TOOL_NAME } from "../codemode/tool.ts"; import { TOOL_SEARCH_TOOL_NAME } from "../tool-search/tool.ts"; @@ -172,7 +173,7 @@ function serversSectionIntro(reaches: ReadonlySet): string { function truncate(text: string, max: number): string { if (text.length <= max) return text; - return max <= 1 ? "" : `${text.slice(0, max - 1).trimEnd()}…`; + return max <= 1 ? "" : `${sliceHead(text, max - 1).trimEnd()}…`; } /** What the `mcp_servers` section needs of a server. */ diff --git a/packages/coding-agent/src/extensions/mcp/runtime.ts b/packages/coding-agent/src/extensions/mcp/runtime.ts index 39a3d26a5..2e998f86e 100644 --- a/packages/coding-agent/src/extensions/mcp/runtime.ts +++ b/packages/coding-agent/src/extensions/mcp/runtime.ts @@ -36,6 +36,7 @@ import { } from "@earendil-works/pi-mcp/oauth"; import { VERSION } from "../../config.ts"; import { resolveConfigValueOrThrow, resolveHeadersOrThrow } from "../../core/resolve-config-value.ts"; +import { sliceTail } from "../../core/tools/truncate.ts"; import type { McpServerEntry } from "./config.ts"; import type { McpServerLog } from "./log.ts"; import { createMcpAuthProvider, type McpOAuthCredentialStore } from "./oauth.ts"; @@ -399,7 +400,7 @@ export class McpServerConnection implements McpToolCaller, McpResourceServer { } catch (error) { await closeClient(); if (transport instanceof StdioTransport) { - this.stderrTail = transport.stderr.trim().slice(-STDERR_TAIL_CHARS) || undefined; + this.stderrTail = sliceTail(transport.stderr.trim(), STDERR_TAIL_CHARS) || undefined; } throw error; } finally { @@ -423,7 +424,7 @@ export class McpServerConnection implements McpToolCaller, McpResourceServer { if (this.client !== client || this.closed) return; this.client = undefined; this.state = "disconnected"; - const stderr = stdio?.stderr.trim().slice(-STDERR_TAIL_CHARS); + const stderr = stdio ? sliceTail(stdio.stderr.trim(), STDERR_TAIL_CHARS) : undefined; this.error = stderr ? `Connection closed\n${stderr}` : "Connection closed"; this.changed(); } diff --git a/packages/coding-agent/test/truncate-surrogates.test.ts b/packages/coding-agent/test/truncate-surrogates.test.ts new file mode 100644 index 000000000..6806d8111 --- /dev/null +++ b/packages/coding-agent/test/truncate-surrogates.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, it } from "vitest"; +import { sliceHead, sliceTail, truncateLine } from "../src/core/tools/truncate.ts"; + +const EMOJI = "\u{1F600}"; // two UTF-16 code units: \uD83D\uDE00 + +function hasLoneSurrogate(text: string): boolean { + return /[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(? { + it("truncateLine does not cut an emoji in half at the limit", () => { + const line = `${"a".repeat(499)}${EMOJI}tail`; + const { text, wasTruncated } = truncateLine(line, 500); + expect(wasTruncated).toBe(true); + expect(text).toBe(`${"a".repeat(499)}... [truncated]`); + expect(hasLoneSurrogate(text)).toBe(false); + }); + + it("truncateLine keeps an emoji that ends exactly at the limit", () => { + const line = `${"a".repeat(498)}${EMOJI}tail`; + expect(truncateLine(line, 500).text).toBe(`${"a".repeat(498)}${EMOJI}... [truncated]`); + }); + + it("sliceHead and sliceTail never split a pair", () => { + const text = `${EMOJI}x${EMOJI}`; + for (let units = 0; units <= text.length + 1; units++) { + const head = sliceHead(text, units); + const tail = sliceTail(text, units); + expect(hasLoneSurrogate(head)).toBe(false); + expect(hasLoneSurrogate(tail)).toBe(false); + expect(text.startsWith(head)).toBe(true); + expect(text.endsWith(tail)).toBe(true); + expect(head.length).toBeGreaterThanOrEqual(Math.min(units, text.length) - 1); + expect(tail.length).toBeGreaterThanOrEqual(Math.min(units, text.length) - 1); + } + }); + + it("sliceHead and sliceTail match slice on text without surrogates", () => { + expect(sliceHead("abcdef", 3)).toBe("abc"); + expect(sliceTail("abcdef", 3)).toBe("def"); + expect(sliceHead("abc", 10)).toBe("abc"); + expect(sliceTail("abc", 10)).toBe("abc"); + expect(sliceHead("abc", 0)).toBe(""); + expect(sliceTail("abc", 0)).toBe(""); + }); +});