fix(durable,coding-agent): support Unicode word segmentation in tool search (#11037)

This PR uses unicode-aware segmentation and normalization to handle tool
search for multilingual text.

Fixes #10816

---------

Co-authored-by: Petr Houška <houskape@gmail.com>

packages/coding-agent/CHANGELOG.md

@@ -45,6 +45,7 @@
 - Fixed image paste, copy, and opening the browser for sign-in failing silently on WSL when the working directory is deep. Windows programs such as `powershell.exe` and `explorer.exe` cannot start from a directory whose `\\wsl.localhost\...` path exceeds 260 characters; pi now starts these helpers from `/` ([#11592](https://github.com/earendil-works/pi/issues/11592))
 - Fixed the external editor failing silently, for example when the editor command is wrong or a Windows editor cannot start from a deep WSL directory. pi now shows a status message such as `External editor exited with code 1; prompt unchanged` ([#11592](https://github.com/earendil-works/pi/issues/11592))
 - Fixed `tool_search` and codemode `searchTools()` ranking a tool below tools that only mention its name in their descriptions, for example `peekaboo app` ranking `mcp__peekaboo__app` 14th. A tool matching more of the query's words now ranks above one matching fewer, a query word in a tool's name scores more than any number of mentions in its description, a tool whose whole name is in the query gets a large bonus, and long server instructions no longer lower the scores of that server's tools ([#11331](https://github.com/earendil-works/pi/issues/11331))
+- Fixed `tool_search` and codemode `searchTools()` finding nothing for Chinese, Japanese, Thai, and other non-ASCII queries, because the tokenizer kept only ASCII letters and digits. Words in any script now match, text without spaces between words such as `搜索记录` is split into words, and canonically equivalent text such as `café` and `cafe\u0301` matches ([#10816](https://github.com/earendil-works/pi/issues/10816), [#11037](https://github.com/earendil-works/pi/pull/11037) by [@haoqixu](https://github.com/haoqixu))
 
 ## [1.4.0] - 2026-10-21
 

packages/coding-agent/src/extensions/tool-search/tool.ts

@@ -39,6 +39,8 @@ export interface ToolRanker {
 	rank(query: string, documents: readonly ToolSearchDocument[], limit: number): ToolSearchMatch[];
 }
 
+const wordSegmenter = new Intl.Segmenter(undefined, { granularity: "word" });
+
 const STOP_WORDS: ReadonlySet<string> = new Set([
 	"a",
 	"an",
@@ -71,15 +73,22 @@ function stem(term: string): string {
 	return term;
 }
 
-/** Lowercase terms, split at camelCase boundaries and non-alphanumerics, without stop words. */
+/** Unicode word segmentation with identifier boundaries and English stop words/singular forms. */
 export function tokenize(text: string): string[] {
-	return text
+	const normalized = text
+		.normalize("NFC")
 		.replace(/([a-z0-9])([A-Z])/g, "$1 $2")
 		.replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2")
 		.toLowerCase()
-		.split(/[^a-z0-9]+/)
-		.filter((term) => term.length > 0 && !STOP_WORDS.has(term))
-		.map(stem);
+		.replace(/[^\p{L}\p{M}\p{N}]+/gu, " ");
+	// The segmenter costs about 15 times a split. Plain ASCII splits into the same words at spaces, so only other
+	// text, such as Chinese written without spaces, goes through it.
+	const words = /^[a-z0-9 ]*$/.test(normalized)
+		? normalized.split(" ")
+		: Array.from(wordSegmenter.segment(normalized), (segment) => (segment.isWordLike ? segment.segment : ""));
+	return words
+		.filter((word) => word.length > 0 && !STOP_WORDS.has(word))
+		.map((word) => (/^[a-z0-9]+$/.test(word) ? stem(word) : word));
 }
 
 function isObject(value: unknown): value is Record<string, unknown> {

packages/coding-agent/test/tool-search.test.ts

@@ -19,6 +19,40 @@ describe("tokenize", () => {
 	it("splits camelCase and snake_case, drops stop words, and folds plurals", () => {
 		expect(tokenize("listIssues for the GitHub_repo")).toEqual(["list", "issue", "git", "hub", "repo"]);
 		expect(tokenize("searches queries HTTPServer")).toEqual(["search", "query", "http", "server"]);
+		expect(tokenize("repo.issues/search:queries")).toEqual(["repo", "issue", "search", "query"]);
+	});
+
+	it.each([
+		["搜索仓库问题", ["搜索", "仓库", "问题"]],
+		["文書を検索", ["文書", "を", "検索"]],
+		["문서 검색", ["문서", "검색"]],
+		["ค้นหาเอกสาร", ["ค้นหา", "เอกสาร"]],
+		["Поиск документов", ["поиск", "документов"]],
+		["البحث عن الملفات", ["البحث", "عن", "الملفات"]],
+		["फ़ाइल खोजें", ["फ़ाइल", "खोजें"]],
+		["Éléments cafés", ["éléments", "cafés"]],
+	])("segments Unicode words in %s", (text, expected) => {
+		expect(tokenize(text)).toEqual(expected);
+	});
+
+	it("preserves non-ASCII words and splits mixed-language identifiers", () => {
+		expect(tokenize("搜索GitHub_repoIssues résumé cafés")).toEqual([
+			"搜索",
+			"git",
+			"hub",
+			"repo",
+			"issue",
+			"résumé",
+			"cafés",
+		]);
+	});
+
+	it("normalizes canonically equivalent Unicode text", () => {
+		expect(tokenize("CAFÉ cafe\u0301")).toEqual(["café", "café"]);
+	});
+
+	it("ignores whitespace, punctuation, and emoji", () => {
+		expect(tokenize(" \t，。!? 😀 ")).toEqual([]);
 	});
 });
 
@@ -45,6 +79,22 @@ describe("Bm25Ranker", () => {
 		expect(ranker.rank("closed", documents, 8).map((match) => match.name)).toEqual(["mcp__github__list_issues"]);
 	});
 
+	it.each([
+		["搜索仓库问题", "问题"],
+		["文書を検索", "検索"],
+		["문서 검색", "검색"],
+		["ค้นหาเอกสาร", "เอกสาร"],
+		["Поиск документов", "документов"],
+		["البحث عن الملفات", "الملفات"],
+		["फ़ाइल खोजें", "खोजें"],
+		["Rechercher un café", "cafe\u0301"],
+	])("matches Unicode queries against %s", (description, query) => {
+		const localized = createToolSearchDocument(tool("localized", description));
+		expect(new Bm25Ranker().rank(query, [...documents, localized], 8)).toEqual([
+			{ name: "localized", score: expect.any(Number) },
+		]);
+	});
+
 	it("returns nothing for unknown or empty queries", () => {
 		const ranker = new Bm25Ranker();
 		expect(ranker.rank("kubernetes", documents, 8)).toEqual([]);

packages/durable/CHANGELOG.md

@@ -182,6 +182,7 @@
 - Fixed the built-in `read` resolving to `""` for an image in Code Mode scripts, so `image(await tools.read(...))` threw: `read` declares a `structuredOutputSchema`, a text file's text or an image's `ImageContent`, as the coding agent's `read` does ([#11015](https://github.com/earendil-works/pi/issues/11015))
 - Fixed the Code Mode extension loading pi-codemode's WASM QuickJS runtime (`quickjs-wasi`) although it only runs scripts in the sandbox the host passes: it imports `toCodemodeIdentifier` from `@earendil-works/pi-codemode/declarations` instead of `/portable` ([#11254](https://github.com/earendil-works/pi/issues/11254))
 - Fixed `tool_search` and Code Mode's `searchTools()` ranking a tool below tools that only mention its name, as the coding agent did before [#11440](https://github.com/earendil-works/pi/pull/11440) and [#11625](https://github.com/earendil-works/pi/pull/11625): `thread_read` with `limit: 1` returned `queue_read`, and `peekaboo app` missed `mcp__peekaboo__app`. Both now rank like the coding agent: a query that is exactly a tool's name ranks it first, a tool matching more of the query's words ranks above one matching fewer, a query word in the name outweighs any number of mentions in the description, and a tool whose whole name is in the query gets a large bonus. `searchTools()` also searches namespace instructions, as a low-weight field that does not lower a tool's score ([#11331](https://github.com/earendil-works/pi/issues/11331))
+- Fixed `tool_search` and Code Mode's `searchTools()` finding nothing for Chinese, Japanese, Thai, and other non-ASCII queries, because the tokenizer kept only ASCII letters and digits. Words in any script now match, text without spaces between words such as `搜索记录` is split into words, and canonically equivalent text such as `café` and `cafe\u0301` matches ([#10816](https://github.com/earendil-works/pi/issues/10816), [#11037](https://github.com/earendil-works/pi/pull/11037) by [@haoqixu](https://github.com/haoqixu))
 
 ## [1.4.0] - 2026-10-21
 

packages/durable/src/search.ts

@@ -111,6 +111,8 @@ function indexDocument(document: SearchDocument): IndexedDocument {
 	};
 }
 
+const wordSegmenter = new Intl.Segmenter(undefined, { granularity: "word" });
+
 const STOP_WORDS: ReadonlySet<string> = new Set([
 	"a",
 	"an",
@@ -135,15 +137,22 @@ const STOP_WORDS: ReadonlySet<string> = new Set([
 	"with",
 ]);
 
-/** Lowercase terms, split at camelCase boundaries and non-alphanumerics, without stop words. */
+/** Unicode word segmentation with identifier boundaries and English stop words/singular forms. */
 export function tokenize(text: string): string[] {
-	return text
+	const normalized = text
+		.normalize("NFC")
 		.replace(/([a-z0-9])([A-Z])/g, "$1 $2")
 		.replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2")
 		.toLowerCase()
-		.split(/[^a-z0-9]+/)
-		.filter((term) => term.length > 0 && !STOP_WORDS.has(term))
-		.map(stem);
+		.replace(/[^\p{L}\p{M}\p{N}]+/gu, " ");
+	// The segmenter costs about 15 times a split. Plain ASCII splits into the same words at spaces, so only other
+	// text, such as Chinese written without spaces, goes through it.
+	const words = /^[a-z0-9 ]*$/.test(normalized)
+		? normalized.split(" ")
+		: Array.from(wordSegmenter.segment(normalized), (segment) => (segment.isWordLike ? segment.segment : ""));
+	return words
+		.filter((word) => word.length > 0 && !STOP_WORDS.has(word))
+		.map((word) => (/^[a-z0-9]+$/.test(word) ? stem(word) : word));
 }
 
 /** Naive singular form, so `issues` matches `issue` and `searches` matches `search`. */

packages/durable/test/search.test.ts

@@ -9,6 +9,40 @@ describe("tokenize", () => {
 	it("splits camelCase and snake_case, drops stop words, and folds plurals", () => {
 		expect(tokenize("listIssues for the GitHub_repo")).toEqual(["list", "issue", "git", "hub", "repo"]);
 		expect(tokenize("searches queries HTTPServer")).toEqual(["search", "query", "http", "server"]);
+		expect(tokenize("repo.issues/search:queries")).toEqual(["repo", "issue", "search", "query"]);
+	});
+
+	it.each([
+		["搜索仓库问题", ["搜索", "仓库", "问题"]],
+		["文書を検索", ["文書", "を", "検索"]],
+		["문서 검색", ["문서", "검색"]],
+		["ค้นหาเอกสาร", ["ค้นหา", "เอกสาร"]],
+		["Поиск документов", ["поиск", "документов"]],
+		["البحث عن الملفات", ["البحث", "عن", "الملفات"]],
+		["फ़ाइल खोजें", ["फ़ाइल", "खोजें"]],
+		["Éléments cafés", ["éléments", "cafés"]],
+	])("segments Unicode words in %s", (text, expected) => {
+		expect(tokenize(text)).toEqual(expected);
+	});
+
+	it("preserves non-ASCII words and splits mixed-language identifiers", () => {
+		expect(tokenize("搜索GitHub_repoIssues résumé cafés")).toEqual([
+			"搜索",
+			"git",
+			"hub",
+			"repo",
+			"issue",
+			"résumé",
+			"cafés",
+		]);
+	});
+
+	it("normalizes canonically equivalent Unicode text", () => {
+		expect(tokenize("CAFÉ cafe\u0301")).toEqual(["café", "café"]);
+	});
+
+	it("ignores whitespace, punctuation, and emoji", () => {
+		expect(tokenize(" \t，。!? 😀 ")).toEqual([]);
 	});
 });
 
@@ -27,9 +61,23 @@ describe("rankDocuments", () => {
 		expect(rankDocuments("closed", documents, 8)).toEqual(["mcp__github__list_issues"]);
 	});
 
+	it.each([
+		["搜索仓库问题", "问题"],
+		["文書を検索", "検索"],
+		["문서 검색", "검색"],
+		["ค้นหาเอกสาร", "เอกสาร"],
+		["Поиск документов", "документов"],
+		["البحث عن الملفات", "الملفات"],
+		["फ़ाइल खोजें", "खोजें"],
+		["Rechercher un café", "cafe\u0301"],
+	])("matches Unicode queries against %s", (text, query) => {
+		expect(rankDocuments(query, [...documents, doc("localized", text)], 8)).toEqual(["localized"]);
+	});
+
 	it("returns nothing for unknown or empty queries", () => {
 		expect(rankDocuments("kubernetes", documents, 8)).toEqual([]);
 		expect(rankDocuments("the", documents, 8)).toEqual([]);
+		expect(rankDocuments("未知", documents, 8)).toEqual([]);
 	});
 
 	it("searches the namespace text", () => {