fix(coding-agent): read and edit handle UTF-16 and non-UTF-8 files without corrupting them

One helper, tools/file-encoding.ts, detects UTF-16 by byte order mark and decodes text for read (UTF-8 as before otherwise) and losslessly for edit. Edit writes files back in the encoding they were read with; files that are not valid UTF-8 are decoded as Latin-1 so ASCII edits splice exactly their bytes, and non-ASCII edits, fuzzy matches and changes to the byte after a non-ASCII byte are refused. EditOperations gains an optional writeFileBytes for these files. fixes #10349 fixes #5797
2026-11-05 23:37:25 +00:00 · 2026-11-04 22:30:08 +01:00 · 2026-11-04 22:30:08 +01:00 · d1e8703375
commit d1e8703375
parent 80d4d3c869
6 changed files with 459 additions and 29 deletions
--- a/packages/coding-agent/CHANGELOG.md
+++ b/packages/coding-agent/CHANGELOG.md
@ -37,6 +37,7 @@
 - Fixed OAuth logins such as `openai` and `anthropic` failing with `invalid_grant` after pi was stopped while it refreshed an expired token, for example an RPC client that stops pi right after startup while the background catalog refresh runs. The token endpoint had already replaced the refresh token, but pi exited before saving the new one. On exit, pi now waits for a token refresh in progress to be saved ([#11692](https://github.com/earendil-works/pi/issues/11692))
 - Fixed Ctrl+P not switching to the only model in scope (or the only available model) when the active model is a different one; it showed "Only one model in scope" instead. Cycling forward from a model outside the scope now also selects the first scoped model instead of skipping it ([#11806](https://github.com/earendil-works/pi/issues/11806))
 - Fixed `/reload` and session switches leaving extension overlays on screen when an extension had more than one `ctx.ui.custom({ overlay: true })` open: only the topmost overlay was removed, and the others kept focus and input although the extension that opened them was gone. All extension overlays, including hidden ones, are now removed and disposed; pi's own overlays stay open
+- Fixed the edit tool corrupting files that are not UTF-8: invalid UTF-8 bytes (Windows-1252, GBK and other legacy encodings) were written back as U+FFFD, and UTF-16 files lost their byte order mark. ASCII edits to such files now leave all other bytes unchanged except line endings, which are handled as in UTF-8 files; non-ASCII edits, inexact matches and changes within 3 bytes after a non-ASCII byte are refused with an explanation. read and edit now decode UTF-16 files with a byte order mark, and edit writes them back as UTF-16; a UTF-16 file with an odd number of bytes is refused. Custom `EditOperations` can implement the new optional `writeFileBytes` to support these files ([#10349](https://github.com/earendil-works/pi/issues/10349), [#5797](https://github.com/earendil-works/pi/issues/5797))


 ## [1.5.0] - 2026-11-02
--- a/packages/coding-agent/src/core/tools/edit-diff.ts
+++ b/packages/coding-agent/src/core/tools/edit-diff.ts
@ -5,7 +5,7 @@
 import * as Diff from "diff";
 import { constants } from "fs";
 import { access, readFile } from "fs/promises";
-import { splitBom } from "../../utils/text.ts";
+import { decodeFile, type FileEncoding } from "./file-encoding.ts";
 import { resolveToCwd } from "./path-utils.ts";

 export function detectLineEnding(content: string): "\r\n" | "\n" {
@ -288,6 +288,33 @@ function getNoChangeError(path: string, totalEdits: number): Error {
 	return new Error(`No changes made to ${path}. The replacements produced identical content.`);
 }

+const NOT_UTF8_ALTERNATIVES =
+	"The file was left unchanged. For such changes, use bash (for example iconv, or a script that reads and writes the file in its encoding), or, if converting the file to UTF-8 is acceptable, rewrite it with write.";
+
+function describeEdit(path: string, editIndex: number, totalEdits: number): string {
+	return totalEdits === 1 ? `the edit to ${path}` : `edits[${editIndex}] to ${path}`;
+}
+
+function getNonAsciiEditError(path: string, editIndex: number, totalEdits: number): Error {
+	return new Error(
+		`Cannot apply ${describeEdit(path, editIndex, totalEdits)}: the file is not valid UTF-8 (it may use a legacy encoding such as Windows-1252, GBK or Shift-JIS), so its encoding is unknown. Edits to such files are only applied when oldText and newText are ASCII, which keeps every other byte unchanged. Anchor the edit on ASCII-only text. ${NOT_UTF8_ALTERNATIVES}`,
+	);
+}
+
+function getInexactMatchError(path: string, editIndex: number, totalEdits: number): Error {
+	return new Error(
+		`Cannot apply ${describeEdit(path, editIndex, totalEdits)}: the oldText only matches after normalizing whitespace or Unicode punctuation, and the file is not valid UTF-8, so only exact matches are applied. The oldText must match exactly including all whitespace and newlines.`,
+	);
+}
+
+function getSplitCharacterError(path: string, editIndex: number, totalEdits: number): Error {
+	return new Error(
+		`Cannot apply ${describeEdit(path, editIndex, totalEdits)}: the file is not valid UTF-8, and the edit changes text within 3 bytes after a non-ASCII byte. In multi-byte encodings such as GBK, GB18030, Big5 or Shift-JIS such a byte can belong to the preceding character. ${NOT_UTF8_ALTERNATIVES}`,
+	);
+}
+
+const NON_ASCII = /[\u0080-\uffff]/;
+
 /**
 * Apply one or more exact-text replacements to LF-normalized content.
 *
@ -296,11 +323,17 @@ function getNoChangeError(path: string, totalEdits: number): Error {
 * fuzzy matching, the operation runs in fuzzy-normalized content space and then
 * overlays those line-level changes onto the original content so unchanged line
 * blocks keep their original bytes.
+ *
+ * Content of the `bytes` encoding (not valid UTF-8, one character per byte) has
+ * an unknown encoding, so only ASCII edits that match exactly are accepted, and
+ * none may change the byte right after a non-ASCII byte: those splice exactly
+ * the bytes they replace.
 */
 export function applyEditsToNormalizedContent(
 	normalizedContent: string,
 	edits: Edit[],
 	path: string,
+	encoding: FileEncoding,
 ): AppliedEditsResult {
 	const normalizedEdits = edits.map((edit) => ({
 		oldText: normalizeToLF(edit.oldText),
@ -311,10 +344,17 @@ export function applyEditsToNormalizedContent(
 		if (normalizedEdits[i].oldText.length === 0) {
 			throw getEmptyOldTextError(path, i, normalizedEdits.length);
 		}
+		if (encoding === "bytes" && NON_ASCII.test(normalizedEdits[i].oldText + normalizedEdits[i].newText)) {
+			throw getNonAsciiEditError(path, i, normalizedEdits.length);
+		}
 	}

 	const initialMatches = normalizedEdits.map((edit) => fuzzyFindText(normalizedContent, edit.oldText));
-	const usedFuzzyMatch = initialMatches.some((match) => match.usedFuzzyMatch);
+	const fuzzyEditIndex = initialMatches.findIndex((match) => match.usedFuzzyMatch);
+	const usedFuzzyMatch = fuzzyEditIndex !== -1;
+	if (usedFuzzyMatch && encoding === "bytes") {
+		throw getInexactMatchError(path, fuzzyEditIndex, normalizedEdits.length);
+	}
 	const replacementBaseContent = usedFuzzyMatch ? normalizeForFuzzyMatch(normalizedContent) : normalizedContent;

 	const matchedEdits: MatchedEdit[] = [];
@ -325,7 +365,11 @@ export function applyEditsToNormalizedContent(
 			throw getNotFoundError(path, i, normalizedEdits.length);
 		}

-		const occurrences = countOccurrences(replacementBaseContent, edit.oldText);
+		// Text of unknown encoding only matches exactly, so only exact occurrences are duplicates.
+		const occurrences =
+			encoding === "bytes"
+				? replacementBaseContent.split(edit.oldText).length - 1
+				: countOccurrences(replacementBaseContent, edit.oldText);
 		if (occurrences > 1) {
 			throw getDuplicateError(path, i, normalizedEdits.length, occurrences);
 		}
@ -349,6 +393,25 @@ export function applyEditsToNormalizedContent(
 		}
 	}

+	if (encoding === "bytes") {
+		for (const edit of matchedEdits) {
+			const matched = normalizedContent.slice(edit.matchIndex, edit.matchIndex + edit.matchLength);
+			if (matched === edit.newText) continue;
+			let unchangedPrefix = 0;
+			while (unchangedPrefix < matched.length && matched[unchangedPrefix] === edit.newText[unchangedPrefix]) {
+				unchangedPrefix++;
+			}
+			// In GBK, Big5, Shift-JIS and GB18030 an ASCII byte can be the second or fourth byte of a
+			// character, at most 3 bytes after a non-ASCII byte. The edit is ASCII, so a change starting
+			// further from non-ASCII bytes changes or inserts only whole characters. Line-ending
+			// normalization only shortens CRLF, so 3 characters here cover at least 3 bytes.
+			const changeStart = edit.matchIndex + unchangedPrefix;
+			if (NON_ASCII.test(normalizedContent.slice(Math.max(0, changeStart - 3), changeStart))) {
+				throw getSplitCharacterError(path, edit.editIndex, normalizedEdits.length);
+			}
+		}
+	}
+
 	const baseContent = normalizedContent;
 	const newContent = usedFuzzyMatch
 		? applyReplacementsPreservingUnchangedLines(normalizedContent, replacementBaseContent, matchedEdits)
@ -527,13 +590,10 @@ export async function computeEditsDiff(
 			return { error: `Could not edit file: ${path}. ${errorMessage}.` };
 		}

-		// Read the file
-		const rawContent = await readFile(absolutePath, "utf-8");
-
-		// Strip BOM before matching (LLM won't include invisible BOM in oldText)
-		const { text: content } = splitBom(rawContent);
-		const normalizedContent = normalizeToLF(content);
-		const { baseContent, newContent } = applyEditsToNormalizedContent(normalizedContent, edits, path);
+		// Read the file. The decoded text has no BOM (LLM won't include invisible BOM in oldText).
+		const file = decodeFile(await readFile(absolutePath), path);
+		const normalizedContent = normalizeToLF(file.text);
+		const { baseContent, newContent } = applyEditsToNormalizedContent(normalizedContent, edits, path, file.encoding);

 		// Generate the diff
 		return generateDiffString(baseContent, newContent);
--- a/packages/coding-agent/src/core/tools/edit.ts
+++ b/packages/coding-agent/src/core/tools/edit.ts
@ -1,7 +1,6 @@
 import { constants } from "fs";
 import { access as fsAccess, readFile as fsReadFile, writeFile as fsWriteFile } from "fs/promises";
 import { type Static, Type } from "typebox";
-import { splitBom } from "../../utils/text.ts";
 import type { ExtensionContext, ToolDefinition } from "../extensions/types.ts";
 import {
 	applyEditsToNormalizedContent,
@ -12,6 +11,7 @@ import {
 	normalizeToLF,
 	restoreLineEndings,
 } from "./edit-diff.ts";
+import { type DecodedFile, decodeFile, encodeNonUtf8File } from "./file-encoding.ts";
 import { withFileMutationQueue } from "./file-mutation-queue.ts";
 import { resolveToCwd } from "./path-utils.ts";
 import { type EditRenderState, editRenderers } from "./renderers/edit.ts";
@ -82,6 +82,11 @@ export interface EditOperations {
 	readFile: (absolutePath: string) => Promise<Buffer>;
 	/** Write content to a file */
 	writeFile: (absolutePath: string, content: string) => Promise<void>;
+	/**
+	 * Write raw bytes to a file. Used for files that are not UTF-8 (UTF-16 with BOM, or invalid UTF-8 such as
+	 * legacy encodings), which `writeFile` cannot reproduce. Without it, edits to such files are refused.
+	 */
+	writeFileBytes?: (absolutePath: string, content: Buffer) => Promise<void>;
 	/** Check if file is readable and writable (throw if not) */
 	access: (absolutePath: string) => Promise<void>;
 }
@ -89,6 +94,7 @@ export interface EditOperations {
 const defaultEditOperations: EditOperations = {
 	readFile: (path) => fsReadFile(path),
 	writeFile: (path, content) => fsWriteFile(path, content, "utf-8"),
+	writeFileBytes: (path, content) => fsWriteFile(path, content),
 	access: (path) => fsAccess(path, constants.R_OK | constants.W_OK),
 };

@ -179,20 +185,22 @@ export function createEditToolDefinition(
 				}
 				throwIfAborted();

-				// Read the file.
-				const buffer = await ops.readFile(absolutePath);
-				const rawContent = buffer.toString("utf-8");
+				// Read the file. The decoded text has no BOM: the model will not include an invisible BOM in oldText.
+				const file = decodeFile(await ops.readFile(absolutePath), path);
+				throwIfAborted();
+				const writeText = getTextWriter(ops, absolutePath, path, file);
+
+				const originalEnding = detectLineEnding(file.text);
+				const normalizedContent = normalizeToLF(file.text);
+				const { baseContent, newContent } = applyEditsToNormalizedContent(
+					normalizedContent,
+					edits,
+					path,
+					file.encoding,
+				);
 				throwIfAborted();

-				// Strip BOM before matching. The model will not include an invisible BOM in oldText.
-				const { bom, text: content } = splitBom(rawContent);
-				const originalEnding = detectLineEnding(content);
-				const normalizedContent = normalizeToLF(content);
-				const { baseContent, newContent } = applyEditsToNormalizedContent(normalizedContent, edits, path);
-				throwIfAborted();
-
-				const finalContent = bom + restoreLineEndings(newContent, originalEnding);
-				await ops.writeFile(absolutePath, finalContent);
+				await writeText(restoreLineEndings(newContent, originalEnding));
 				throwIfAborted();

 				const diffResult = generateDiffString(baseContent, newContent);
@ -212,6 +220,31 @@ export function createEditToolDefinition(
 	};
 }

+/**
+ * Writer of edited text in the encoding, and with the BOM, the file was read with.
+ * Throws when the operations cannot write that encoding.
+ */
+function getTextWriter(
+	ops: EditOperations,
+	absolutePath: string,
+	path: string,
+	file: DecodedFile,
+): (text: string) => Promise<void> {
+	if (file.encoding === "utf-8") {
+		return (text) => ops.writeFile(absolutePath, (file.bom ? "\uFEFF" : "") + text);
+	}
+	const { encoding } = file;
+	const { writeFileBytes } = ops;
+	if (!writeFileBytes) {
+		const description = encoding === "bytes" ? "not valid UTF-8" : encoding.toUpperCase();
+		throw new Error(
+			`Could not edit file: ${path}. The file is ${description}, and the configured edit operations can only write UTF-8 text. The file was left unchanged.`,
+		);
+	}
+	// Call as a method: custom operations may be class instances.
+	return (text) => writeFileBytes.call(ops, absolutePath, encodeNonUtf8File(encoding, text));
+}
+
 export function createEditTool(cwd: string, options?: EditToolOptions): TypeBoxAgentTool<typeof editSchema> {
 	return wrapToolDefinition(createEditToolDefinition(cwd, options));
 }
--- a/packages/coding-agent/src/core/tools/file-encoding.ts
+++ b/packages/coding-agent/src/core/tools/file-encoding.ts
@ -0,0 +1,109 @@
+/**
+ * Encodings of text files for the read and edit tools.
+ *
+ * - UTF-16LE and UTF-16BE are detected by their byte order mark (FF FE, FE FF), which is kept out of the text.
+ * - Everything else is UTF-8. `read` shows invalid UTF-8 bytes as U+FFFD, as Buffer.toString("utf-8") does.
+ *   `edit` must write back what it read, so it decodes files that are not valid UTF-8 as Latin-1 instead:
+ *   one character per byte, so ASCII text matches and splices byte-exactly and all other bytes stay as they were.
+ */
+
+import { isUtf8 } from "buffer";
+import { TextDecoder } from "util";
+import { splitBom } from "../../utils/text.ts";
+
+/** How the edit tool maps a file's bytes to text. `bytes` is anything not valid UTF-8 or UTF-16 with BOM. */
+export type FileEncoding = "utf-8" | "utf-16le" | "utf-16be" | "bytes";
+
+/** File text without its BOM, decoded so that encoding it again gives back the same bytes. */
+export type DecodedFile =
+	| { encoding: "utf-8"; bom: boolean; text: string }
+	| { encoding: "utf-16le" | "utf-16be" | "bytes"; text: string };
+
+const UTF16LE_BOM = Buffer.from([0xff, 0xfe]);
+const UTF16BE_BOM = Buffer.from([0xfe, 0xff]);
+
+/** The UTF-16 byte order announced by a leading byte order mark, if any. */
+function getUtf16Encoding(bytes: Uint8Array): "utf-16le" | "utf-16be" | undefined {
+	if (bytes[0] === 0xff && bytes[1] === 0xfe) return "utf-16le";
+	if (bytes[0] === 0xfe && bytes[1] === 0xff) return "utf-16be";
+	return undefined;
+}
+
+/** Decode file contents for editing, losslessly. Throws for a UTF-16 byte order mark with an odd length. */
+export function decodeFile(buffer: Buffer, path: string): DecodedFile {
+	// Buffer's utf16le keeps lone surrogates, so the text round-trips exactly.
+	const utf16 = getUtf16Encoding(buffer);
+	if (utf16 && buffer.length % 2 !== 0) {
+		throw new Error(
+			`Could not edit file: ${path}. It starts with a ${utf16.toUpperCase()} byte order mark but has an odd number of bytes, so it is not valid UTF-16. The file was left unchanged.`,
+		);
+	}
+	if (utf16 === "utf-16le") return { encoding: utf16, text: buffer.toString("utf16le", 2) };
+	if (utf16 === "utf-16be")
+		return { encoding: utf16, text: Buffer.from(buffer.subarray(2)).swap16().toString("utf16le") };
+	if (isUtf8(buffer)) {
+		const { bom, text } = splitBom(buffer.toString("utf-8"));
+		return { encoding: "utf-8", bom: bom !== "", text };
+	}
+	return { encoding: "bytes", text: buffer.toString("latin1") };
+}
+
+/** Encode edited text of a file that is not UTF-8 back to bytes, with the BOM it was read with. */
+export function encodeNonUtf8File(encoding: "utf-16le" | "utf-16be" | "bytes", text: string): Buffer {
+	switch (encoding) {
+		case "utf-16le":
+			return Buffer.concat([UTF16LE_BOM, Buffer.from(text, "utf16le")]);
+		case "utf-16be":
+			return Buffer.concat([UTF16BE_BOM, Buffer.from(text, "utf16le").swap16()]);
+		case "bytes":
+			return Buffer.from(text, "latin1");
+	}
+}
+
+/**
+ * Incremental decoder for the text the read tool shows.
+ *
+ * A file that starts with a UTF-16 byte order mark is decoded as UTF-16 without the mark. Every other file
+ * is decoded as UTF-8 exactly like Buffer.toString("utf-8"), including a UTF-8 byte order mark, which stays
+ * in the text as U+FEFF.
+ */
+export class TextFileDecoder {
+	private decoder: TextDecoder | undefined;
+	private bigEndian = false;
+	// Bytes not decoded yet: the start of the file until the byte order mark can be checked, or the odd
+	// byte of a big endian UTF-16 code unit split across chunks.
+	private pending: Uint8Array = new Uint8Array(0);
+
+	/** Decode the next chunk. Characters split across chunks come out with the chunk that completes them. */
+	write(chunk: Uint8Array): string {
+		return this.decode(chunk, true);
+	}
+
+	/** Decode the bytes still held back; an incomplete character at the end becomes U+FFFD. */
+	end(): string {
+		return this.decode(new Uint8Array(0), false);
+	}
+
+	private decode(chunk: Uint8Array, stream: boolean): string {
+		let bytes = this.pending.length > 0 ? Buffer.concat([this.pending, chunk]) : chunk;
+		this.pending = new Uint8Array(0);
+		if (!this.decoder) {
+			if (bytes.length < 2 && stream) {
+				this.pending = bytes;
+				return "";
+			}
+			const utf16 = getUtf16Encoding(bytes);
+			this.bigEndian = utf16 === "utf-16be";
+			// ignoreBOM keeps a UTF-8 byte order mark in the text, as Buffer.toString("utf-8") does.
+			this.decoder = utf16 ? new TextDecoder("utf-16le") : new TextDecoder("utf-8", { ignoreBOM: true });
+		}
+		if (this.bigEndian) {
+			// Swapping to little endian avoids depending on "utf-16be" support, which runtimes without ICU lack.
+			const evenLength = bytes.length - (bytes.length % 2);
+			if (stream) this.pending = bytes.slice(evenLength);
+			const swapped = Buffer.from(bytes.subarray(0, evenLength)).swap16();
+			bytes = stream ? swapped : Buffer.concat([swapped, bytes.subarray(evenLength)]);
+		}
+		return this.decoder.decode(bytes, { stream });
+	}
+}
--- a/packages/coding-agent/src/core/tools/read.ts
+++ b/packages/coding-agent/src/core/tools/read.ts
@ -2,10 +2,10 @@ import type { Api, ImageContent, Model, ModelImageResizeOptions, TextContent } f
 import { constants } from "fs";
 import { access as fsAccess, open as fsOpen, readFile as fsReadFile } from "fs/promises";
 import { type Static, Type } from "typebox";
-import { TextDecoder } from "util";
 import { processImage } from "../../utils/image-process.ts";
 import { detectSupportedImageMimeTypeFromFile } from "../../utils/mime.ts";
 import type { ExtensionContext, ToolDefinition } from "../extensions/types.ts";
+import { TextFileDecoder } from "./file-encoding.ts";
 import { resolveReadPathAsync } from "./path-utils.ts";
 import { readRenderers } from "./renderers/read.ts";
 import { type TypeBoxAgentTool, wrapToolDefinition } from "./tool-definition-wrapper.ts";
@ -127,7 +127,8 @@ function isStreamableRange(offset: number | undefined, limit: number | undefined

 /** Select lines from a whole file in memory: for operations without createReadStream and unusual ranges. */
 function selectLinesFromBuffer(buffer: Buffer, startLine: number, limit: number | undefined): SelectedLines {
-	const allLines = buffer.toString("utf-8").split("\n");
+	const decoder = new TextFileDecoder();
+	const allLines = (decoder.write(buffer) + decoder.end()).split("\n");
 	const endLine = limit === undefined ? undefined : Math.min(startLine + limit, allLines.length);
 	const truncation = truncateHead(allLines.slice(startLine, endLine).join("\n"));
 	const firstLineBytes = truncation.firstLineExceedsLimit ? Buffer.byteLength(allLines[startLine], "utf-8") : 0;
@ -145,8 +146,8 @@ async function selectLinesFromStream(
 	signal: AbortSignal | undefined,
 ): Promise<SelectedLines> {
 	const endLine = limit === undefined ? Number.POSITIVE_INFINITY : startLine + limit;
-	// Non-fatal UTF-8 that keeps a byte order mark decodes like Buffer.toString("utf-8").
-	const decoder = new TextDecoder("utf-8", { ignoreBOM: true });
+	// UTF-16 by byte order mark, otherwise UTF-8 exactly like Buffer.toString("utf-8").
+	const decoder = new TextFileDecoder();
 	const storedLines: string[] = [];
 	let storedBytes = 0;
 	let storedPastCaps = false;
@ -212,9 +213,9 @@ async function selectLinesFromStream(
 	for await (const chunk of chunks) {
 		// Leaving the loop closes the stream.
 		if (signal?.aborted) throw new Error("Operation aborted");
-		select(decoder.decode(chunk, { stream: true }));
+		select(decoder.write(chunk));
 	}
-	select(decoder.decode());
+	select(decoder.end());
 	if (line >= startLine && line < endLine) finishLine();
 	const truncation: TruncationResult = {
 		...truncateHead(storedLines.join("\n")),
--- a/packages/coding-agent/test/file-encodings.test.ts
+++ b/packages/coding-agent/test/file-encodings.test.ts
@ -0,0 +1,226 @@
+import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "fs";
+import { tmpdir } from "os";
+import { join } from "path";
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import { TextFileDecoder } from "../src/core/tools/file-encoding.ts";
+import { createEditTool, createReadTool } from "../src/index.ts";
+
+function utf16(byteOrder: "le" | "be", text: string): Buffer {
+	const le = Buffer.from(`\uFEFF${text}`, "utf16le");
+	return byteOrder === "le" ? le : Buffer.from(le).swap16();
+}
+
+function textOf(result: { content: Array<{ type: string; text?: string }> }): string {
+	return result.content.map((block) => block.text ?? "").join("\n");
+}
+
+let testDir: string;
+
+beforeEach(() => {
+	testDir = mkdtempSync(join(tmpdir(), "coding-agent-encodings-"));
+});
+
+afterEach(() => {
+	rmSync(testDir, { recursive: true, force: true });
+});
+
+function writeTestFile(name: string, content: Buffer): string {
+	const path = join(testDir, name);
+	writeFileSync(path, content);
+	return path;
+}
+
+describe("edit tool encodings", () => {
+	const editTool = createEditTool(process.cwd());
+	const latin1Content = Buffer.from("caf\xe9\nTIMEOUT = 30\n", "latin1");
+
+	// #10349, #5797
+	it("applies ASCII edits to files that are not valid UTF-8 without changing other bytes", async () => {
+		const path = writeTestFile("latin1.py", latin1Content);
+
+		await editTool.execute("latin1", { path, edits: [{ oldText: "TIMEOUT = 30", newText: "TIMEOUT = 60" }] });
+
+		expect(readFileSync(path).equals(Buffer.from("caf\xe9\nTIMEOUT = 60\n", "latin1"))).toBe(true);
+	});
+
+	// #10349
+	it("refuses non-ASCII edits to files that are not valid UTF-8", async () => {
+		const path = writeTestFile("latin1-non-ascii.py", latin1Content);
+
+		await expect(
+			editTool.execute("latin1-non-ascii", {
+				path,
+				edits: [{ oldText: "TIMEOUT = 30", newText: "TIMEOUT = 30 # caf\u00e9" }],
+			}),
+		).rejects.toThrow(
+			`Cannot apply the edit to ${path}: the file is not valid UTF-8 (it may use a legacy encoding such as Windows-1252, GBK or Shift-JIS), so its encoding is unknown. Edits to such files are only applied when oldText and newText are ASCII, which keeps every other byte unchanged. Anchor the edit on ASCII-only text. The file was left unchanged. For such changes, use bash (for example iconv, or a script that reads and writes the file in its encoding), or, if converting the file to UTF-8 is acceptable, rewrite it with write.`,
+		);
+		expect(readFileSync(path).equals(latin1Content)).toBe(true);
+	});
+
+	// #10349
+	it("refuses fuzzy matches in files that are not valid UTF-8", async () => {
+		const content = Buffer.from("caf\xe9\nTIMEOUT = 30  \n", "latin1");
+		const path = writeTestFile("latin1-fuzzy.py", content);
+
+		await expect(
+			editTool.execute("latin1-fuzzy", { path, edits: [{ oldText: "TIMEOUT = 30\n", newText: "TIMEOUT = 60\n" }] }),
+		).rejects.toThrow(/only matches after normalizing whitespace or Unicode punctuation.*only exact matches/);
+		expect(readFileSync(path).equals(content)).toBe(true);
+	});
+
+	// #10349
+	it("refuses changes within 3 bytes after a non-ASCII byte, which can belong to a multi-byte character", async () => {
+		// U+20000 in GB18030 is 95 32 82 36: its second and fourth bytes read as the digits 2 and 6.
+		const gb18030 = Buffer.from([0x95, 0x32, 0x82, 0x36, 0x0a]);
+		// "表" in Shift-JIS is 95 5C: its second byte reads as a backslash.
+		const shiftJis = Buffer.concat([Buffer.from([0x95, 0x5c]), Buffer.from("abc\n")]);
+		const cases = [
+			{ content: gb18030, edit: { oldText: "2", newText: "2X" } },
+			{ content: shiftJis, edit: { oldText: "\\abc", newText: "Xabc" } },
+			{ content: shiftJis, edit: { oldText: "\\abc", newText: "\\Xbc" } },
+		];
+		for (const [index, { content, edit }] of cases.entries()) {
+			const path = writeTestFile(`split-${index}.txt`, content);
+
+			await expect(editTool.execute("split", { path, edits: [edit] })).rejects.toThrow(
+				/changes text within 3 bytes after a non-ASCII byte/,
+			);
+			expect(readFileSync(path).equals(content)).toBe(true);
+		}
+	});
+
+	it("applies ASCII changes at least 3 bytes after a non-ASCII byte", async () => {
+		const path = writeTestFile("shift-jis-far.txt", Buffer.concat([Buffer.from([0x95, 0x5c]), Buffer.from("abc\n")]));
+
+		await editTool.execute("sjis-far", { path, edits: [{ oldText: "\\abc", newText: "\\abd" }] });
+
+		expect(readFileSync(path).equals(Buffer.concat([Buffer.from([0x95, 0x5c]), Buffer.from("abd\n")]))).toBe(true);
+	});
+
+	// #10349
+	it("counts only exact occurrences as duplicates in files that are not valid UTF-8", async () => {
+		// X\xa0= 1 only matches "X = 1" after fuzzy normalization, which these files do not use.
+		const path = writeTestFile("latin1-nbsp.txt", Buffer.from("X = 1\nX\xa0= 1\n", "latin1"));
+
+		await editTool.execute("latin1-nbsp", { path, edits: [{ oldText: "X = 1", newText: "X = 2" }] });
+
+		expect(readFileSync(path).equals(Buffer.from("X = 2\nX\xa0= 1\n", "latin1"))).toBe(true);
+	});
+
+	it("refuses UTF-16 files with an odd number of bytes", async () => {
+		for (const byteOrder of ["le", "be"] as const) {
+			const content = Buffer.concat([utf16(byteOrder, "X=1\n"), Buffer.from([0x41])]);
+			const path = writeTestFile(`utf16${byteOrder}-odd.txt`, content);
+
+			await expect(
+				editTool.execute("utf16-odd", { path, edits: [{ oldText: "1", newText: "20" }] }),
+				byteOrder,
+			).rejects.toThrow(
+				`Could not edit file: ${path}. It starts with a UTF-16${byteOrder.toUpperCase()} byte order mark but has an odd number of bytes, so it is not valid UTF-16. The file was left unchanged.`,
+			);
+			expect(readFileSync(path).equals(content)).toBe(true);
+		}
+	});
+
+	it("edits UTF-16 files with a byte order mark and writes them back in the same encoding", async () => {
+		for (const byteOrder of ["le", "be"] as const) {
+			const path = writeTestFile(`utf16${byteOrder}.txt`, utf16(byteOrder, "caf\u00e9\r\nX = 1\r\n"));
+
+			await editTool.execute(`utf16${byteOrder}`, {
+				path,
+				edits: [{ oldText: "X = 1", newText: "X = \u00e9\u{1F600}" }],
+			});
+
+			expect(readFileSync(path).equals(utf16(byteOrder, "caf\u00e9\r\nX = \u00e9\u{1F600}\r\n")), byteOrder).toBe(
+				true,
+			);
+		}
+	});
+
+	it("keeps the UTF-8 byte order mark", async () => {
+		const path = writeTestFile("bom.txt", Buffer.from("\uFEFFa\nb\n"));
+
+		await editTool.execute("bom", { path, edits: [{ oldText: "b", newText: "B" }] });
+
+		expect(readFileSync(path).equals(Buffer.from([0xef, 0xbb, 0xbf, ...Buffer.from("a\nB\n")]))).toBe(true);
+	});
+
+	it("refuses files that are not UTF-8 when custom operations cannot write bytes", async () => {
+		let written: string | undefined;
+		const textOnlyTool = createEditTool(testDir, {
+			operations: {
+				access: async () => {},
+				readFile: async () => latin1Content,
+				writeFile: async (_path, content) => {
+					written = content;
+				},
+			},
+		});
+
+		await expect(
+			textOnlyTool.execute("text-only-ops", {
+				path: "latin1.py",
+				edits: [{ oldText: "TIMEOUT = 30", newText: "TIMEOUT = 60" }],
+			}),
+		).rejects.toThrow(
+			"Could not edit file: latin1.py. The file is not valid UTF-8, and the configured edit operations can only write UTF-8 text. The file was left unchanged.",
+		);
+		expect(written).toBeUndefined();
+	});
+});
+
+describe("read tool encodings", () => {
+	const readTool = createReadTool(process.cwd());
+	const text = "h\u00e9llo \u{1F600}\nw\u00f6rld\r\nlast";
+
+	it("decodes UTF-16 files with a byte order mark, without the mark", async () => {
+		for (const byteOrder of ["le", "be"] as const) {
+			const path = writeTestFile(`read-utf16${byteOrder}.txt`, utf16(byteOrder, text));
+
+			expect(textOf(await readTool.execute("read-utf16", { path })), byteOrder).toBe(text);
+		}
+	});
+
+	it("decodes UTF-16 the same when streamed in small chunks and without a stream", async () => {
+		for (const byteOrder of ["le", "be"] as const) {
+			const bytes = utf16(byteOrder, text);
+			const readFile = async () => bytes;
+			const access = async () => {};
+			const chunked = (size: number) =>
+				async function* () {
+					for (let i = 0; i < bytes.length; i += size) yield bytes.subarray(i, i + size);
+				};
+			const tools = [
+				createReadTool(process.cwd(), { operations: { readFile, access } }),
+				createReadTool(process.cwd(), { operations: { readFile, access, createReadStream: chunked(1) } }),
+				createReadTool(process.cwd(), { operations: { readFile, access, createReadStream: chunked(3) } }),
+			];
+			for (const tool of tools) {
+				expect(textOf(await tool.execute("read-utf16-chunks", { path: "x.txt" })), byteOrder).toBe(text);
+			}
+		}
+	});
+
+	it("shows UTF-8 files, a UTF-8 byte order mark and invalid UTF-8 as before", async () => {
+		for (const bytes of [
+			Buffer.from(`\uFEFF${text}`),
+			Buffer.from("caf\xe9\nok", "latin1"),
+			Buffer.from([0xfe, 0x41]),
+		]) {
+			const path = writeTestFile("read-utf8.txt", bytes);
+
+			expect(textOf(await readTool.execute("read-utf8", { path }))).toBe(bytes.toString("utf-8"));
+		}
+	});
+
+	it("decodes UTF-16 across chunk boundaries and replaces a dangling odd byte", () => {
+		for (const byteOrder of ["le", "be"] as const) {
+			const bytes = Buffer.concat([utf16(byteOrder, text), Buffer.from([0x41])]);
+			const decoder = new TextFileDecoder();
+			let decoded = "";
+			for (let i = 0; i < bytes.length; i += 3) decoded += decoder.write(bytes.subarray(i, i + 3));
+			expect(decoded + decoder.end(), byteOrder).toBe(`${text}\uFFFD`);
+		}
+	});
+});