diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 4d0dbc921..d10174db2 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -45,6 +45,7 @@ - Fixed image paste, copy, and opening the browser for sign-in failing silently on WSL when the working directory is deep. Windows programs such as `powershell.exe` and `explorer.exe` cannot start from a directory whose `\\wsl.localhost\...` path exceeds 260 characters; pi now starts these helpers from `/` ([#11592](https://github.com/earendil-works/pi/issues/11592)) - Fixed the external editor failing silently, for example when the editor command is wrong or a Windows editor cannot start from a deep WSL directory. pi now shows a status message such as `External editor exited with code 1; prompt unchanged` ([#11592](https://github.com/earendil-works/pi/issues/11592)) - Fixed `tool_search` and codemode `searchTools()` ranking a tool below tools that only mention its name in their descriptions, for example `peekaboo app` ranking `mcp__peekaboo__app` 14th. A tool matching more of the query's words now ranks above one matching fewer, a query word in a tool's name scores more than any number of mentions in its description, a tool whose whole name is in the query gets a large bonus, and long server instructions no longer lower the scores of that server's tools ([#11331](https://github.com/earendil-works/pi/issues/11331)) +- Fixed `tool_search` and codemode `searchTools()` finding nothing for Chinese, Japanese, Thai, and other non-ASCII queries, because the tokenizer kept only ASCII letters and digits. Words in any script now match, text without spaces between words such as `搜索记录` is split into words, and canonically equivalent text such as `café` and `cafe\u0301` matches ([#10816](https://github.com/earendil-works/pi/issues/10816), [#11037](https://github.com/earendil-works/pi/pull/11037) by [@haoqixu](https://github.com/haoqixu)) ## [1.4.0] - 2026-10-21 diff --git a/packages/coding-agent/src/extensions/tool-search/tool.ts b/packages/coding-agent/src/extensions/tool-search/tool.ts index dc743922b..2cf07c769 100644 --- a/packages/coding-agent/src/extensions/tool-search/tool.ts +++ b/packages/coding-agent/src/extensions/tool-search/tool.ts @@ -39,6 +39,8 @@ export interface ToolRanker { rank(query: string, documents: readonly ToolSearchDocument[], limit: number): ToolSearchMatch[]; } +const wordSegmenter = new Intl.Segmenter(undefined, { granularity: "word" }); + const STOP_WORDS: ReadonlySet = new Set([ "a", "an", @@ -71,15 +73,22 @@ function stem(term: string): string { return term; } -/** Lowercase terms, split at camelCase boundaries and non-alphanumerics, without stop words. */ +/** Unicode word segmentation with identifier boundaries and English stop words/singular forms. */ export function tokenize(text: string): string[] { - return text + const normalized = text + .normalize("NFC") .replace(/([a-z0-9])([A-Z])/g, "$1 $2") .replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2") .toLowerCase() - .split(/[^a-z0-9]+/) - .filter((term) => term.length > 0 && !STOP_WORDS.has(term)) - .map(stem); + .replace(/[^\p{L}\p{M}\p{N}]+/gu, " "); + // The segmenter costs about 15 times a split. Plain ASCII splits into the same words at spaces, so only other + // text, such as Chinese written without spaces, goes through it. + const words = /^[a-z0-9 ]*$/.test(normalized) + ? normalized.split(" ") + : Array.from(wordSegmenter.segment(normalized), (segment) => (segment.isWordLike ? segment.segment : "")); + return words + .filter((word) => word.length > 0 && !STOP_WORDS.has(word)) + .map((word) => (/^[a-z0-9]+$/.test(word) ? stem(word) : word)); } function isObject(value: unknown): value is Record { diff --git a/packages/coding-agent/test/tool-search.test.ts b/packages/coding-agent/test/tool-search.test.ts index d60f164f0..68a59cec7 100644 --- a/packages/coding-agent/test/tool-search.test.ts +++ b/packages/coding-agent/test/tool-search.test.ts @@ -19,6 +19,40 @@ describe("tokenize", () => { it("splits camelCase and snake_case, drops stop words, and folds plurals", () => { expect(tokenize("listIssues for the GitHub_repo")).toEqual(["list", "issue", "git", "hub", "repo"]); expect(tokenize("searches queries HTTPServer")).toEqual(["search", "query", "http", "server"]); + expect(tokenize("repo.issues/search:queries")).toEqual(["repo", "issue", "search", "query"]); + }); + + it.each([ + ["搜索仓库问题", ["搜索", "仓库", "问题"]], + ["文書を検索", ["文書", "を", "検索"]], + ["문서 검색", ["문서", "검색"]], + ["ค้นหาเอกสาร", ["ค้นหา", "เอกสาร"]], + ["Поиск документов", ["поиск", "документов"]], + ["البحث عن الملفات", ["البحث", "عن", "الملفات"]], + ["फ़ाइल खोजें", ["फ़ाइल", "खोजें"]], + ["Éléments cafés", ["éléments", "cafés"]], + ])("segments Unicode words in %s", (text, expected) => { + expect(tokenize(text)).toEqual(expected); + }); + + it("preserves non-ASCII words and splits mixed-language identifiers", () => { + expect(tokenize("搜索GitHub_repoIssues résumé cafés")).toEqual([ + "搜索", + "git", + "hub", + "repo", + "issue", + "résumé", + "cafés", + ]); + }); + + it("normalizes canonically equivalent Unicode text", () => { + expect(tokenize("CAFÉ cafe\u0301")).toEqual(["café", "café"]); + }); + + it("ignores whitespace, punctuation, and emoji", () => { + expect(tokenize(" \t,。!? 😀 ")).toEqual([]); }); }); @@ -45,6 +79,22 @@ describe("Bm25Ranker", () => { expect(ranker.rank("closed", documents, 8).map((match) => match.name)).toEqual(["mcp__github__list_issues"]); }); + it.each([ + ["搜索仓库问题", "问题"], + ["文書を検索", "検索"], + ["문서 검색", "검색"], + ["ค้นหาเอกสาร", "เอกสาร"], + ["Поиск документов", "документов"], + ["البحث عن الملفات", "الملفات"], + ["फ़ाइल खोजें", "खोजें"], + ["Rechercher un café", "cafe\u0301"], + ])("matches Unicode queries against %s", (description, query) => { + const localized = createToolSearchDocument(tool("localized", description)); + expect(new Bm25Ranker().rank(query, [...documents, localized], 8)).toEqual([ + { name: "localized", score: expect.any(Number) }, + ]); + }); + it("returns nothing for unknown or empty queries", () => { const ranker = new Bm25Ranker(); expect(ranker.rank("kubernetes", documents, 8)).toEqual([]); diff --git a/packages/durable/CHANGELOG.md b/packages/durable/CHANGELOG.md index 739719b1d..96d53480d 100644 --- a/packages/durable/CHANGELOG.md +++ b/packages/durable/CHANGELOG.md @@ -182,6 +182,7 @@ - Fixed the built-in `read` resolving to `""` for an image in Code Mode scripts, so `image(await tools.read(...))` threw: `read` declares a `structuredOutputSchema`, a text file's text or an image's `ImageContent`, as the coding agent's `read` does ([#11015](https://github.com/earendil-works/pi/issues/11015)) - Fixed the Code Mode extension loading pi-codemode's WASM QuickJS runtime (`quickjs-wasi`) although it only runs scripts in the sandbox the host passes: it imports `toCodemodeIdentifier` from `@earendil-works/pi-codemode/declarations` instead of `/portable` ([#11254](https://github.com/earendil-works/pi/issues/11254)) - Fixed `tool_search` and Code Mode's `searchTools()` ranking a tool below tools that only mention its name, as the coding agent did before [#11440](https://github.com/earendil-works/pi/pull/11440) and [#11625](https://github.com/earendil-works/pi/pull/11625): `thread_read` with `limit: 1` returned `queue_read`, and `peekaboo app` missed `mcp__peekaboo__app`. Both now rank like the coding agent: a query that is exactly a tool's name ranks it first, a tool matching more of the query's words ranks above one matching fewer, a query word in the name outweighs any number of mentions in the description, and a tool whose whole name is in the query gets a large bonus. `searchTools()` also searches namespace instructions, as a low-weight field that does not lower a tool's score ([#11331](https://github.com/earendil-works/pi/issues/11331)) +- Fixed `tool_search` and Code Mode's `searchTools()` finding nothing for Chinese, Japanese, Thai, and other non-ASCII queries, because the tokenizer kept only ASCII letters and digits. Words in any script now match, text without spaces between words such as `搜索记录` is split into words, and canonically equivalent text such as `café` and `cafe\u0301` matches ([#10816](https://github.com/earendil-works/pi/issues/10816), [#11037](https://github.com/earendil-works/pi/pull/11037) by [@haoqixu](https://github.com/haoqixu)) ## [1.4.0] - 2026-10-21 diff --git a/packages/durable/src/search.ts b/packages/durable/src/search.ts index 51534ff23..9dc6ed5ec 100644 --- a/packages/durable/src/search.ts +++ b/packages/durable/src/search.ts @@ -111,6 +111,8 @@ function indexDocument(document: SearchDocument): IndexedDocument { }; } +const wordSegmenter = new Intl.Segmenter(undefined, { granularity: "word" }); + const STOP_WORDS: ReadonlySet = new Set([ "a", "an", @@ -135,15 +137,22 @@ const STOP_WORDS: ReadonlySet = new Set([ "with", ]); -/** Lowercase terms, split at camelCase boundaries and non-alphanumerics, without stop words. */ +/** Unicode word segmentation with identifier boundaries and English stop words/singular forms. */ export function tokenize(text: string): string[] { - return text + const normalized = text + .normalize("NFC") .replace(/([a-z0-9])([A-Z])/g, "$1 $2") .replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2") .toLowerCase() - .split(/[^a-z0-9]+/) - .filter((term) => term.length > 0 && !STOP_WORDS.has(term)) - .map(stem); + .replace(/[^\p{L}\p{M}\p{N}]+/gu, " "); + // The segmenter costs about 15 times a split. Plain ASCII splits into the same words at spaces, so only other + // text, such as Chinese written without spaces, goes through it. + const words = /^[a-z0-9 ]*$/.test(normalized) + ? normalized.split(" ") + : Array.from(wordSegmenter.segment(normalized), (segment) => (segment.isWordLike ? segment.segment : "")); + return words + .filter((word) => word.length > 0 && !STOP_WORDS.has(word)) + .map((word) => (/^[a-z0-9]+$/.test(word) ? stem(word) : word)); } /** Naive singular form, so `issues` matches `issue` and `searches` matches `search`. */ diff --git a/packages/durable/test/search.test.ts b/packages/durable/test/search.test.ts index c9f0ec597..c4510cd46 100644 --- a/packages/durable/test/search.test.ts +++ b/packages/durable/test/search.test.ts @@ -9,6 +9,40 @@ describe("tokenize", () => { it("splits camelCase and snake_case, drops stop words, and folds plurals", () => { expect(tokenize("listIssues for the GitHub_repo")).toEqual(["list", "issue", "git", "hub", "repo"]); expect(tokenize("searches queries HTTPServer")).toEqual(["search", "query", "http", "server"]); + expect(tokenize("repo.issues/search:queries")).toEqual(["repo", "issue", "search", "query"]); + }); + + it.each([ + ["搜索仓库问题", ["搜索", "仓库", "问题"]], + ["文書を検索", ["文書", "を", "検索"]], + ["문서 검색", ["문서", "검색"]], + ["ค้นหาเอกสาร", ["ค้นหา", "เอกสาร"]], + ["Поиск документов", ["поиск", "документов"]], + ["البحث عن الملفات", ["البحث", "عن", "الملفات"]], + ["फ़ाइल खोजें", ["फ़ाइल", "खोजें"]], + ["Éléments cafés", ["éléments", "cafés"]], + ])("segments Unicode words in %s", (text, expected) => { + expect(tokenize(text)).toEqual(expected); + }); + + it("preserves non-ASCII words and splits mixed-language identifiers", () => { + expect(tokenize("搜索GitHub_repoIssues résumé cafés")).toEqual([ + "搜索", + "git", + "hub", + "repo", + "issue", + "résumé", + "cafés", + ]); + }); + + it("normalizes canonically equivalent Unicode text", () => { + expect(tokenize("CAFÉ cafe\u0301")).toEqual(["café", "café"]); + }); + + it("ignores whitespace, punctuation, and emoji", () => { + expect(tokenize(" \t,。!? 😀 ")).toEqual([]); }); }); @@ -27,9 +61,23 @@ describe("rankDocuments", () => { expect(rankDocuments("closed", documents, 8)).toEqual(["mcp__github__list_issues"]); }); + it.each([ + ["搜索仓库问题", "问题"], + ["文書を検索", "検索"], + ["문서 검색", "검색"], + ["ค้นหาเอกสาร", "เอกสาร"], + ["Поиск документов", "документов"], + ["البحث عن الملفات", "الملفات"], + ["फ़ाइल खोजें", "खोजें"], + ["Rechercher un café", "cafe\u0301"], + ])("matches Unicode queries against %s", (text, query) => { + expect(rankDocuments(query, [...documents, doc("localized", text)], 8)).toEqual(["localized"]); + }); + it("returns nothing for unknown or empty queries", () => { expect(rankDocuments("kubernetes", documents, 8)).toEqual([]); expect(rankDocuments("the", documents, 8)).toEqual([]); + expect(rankDocuments("未知", documents, 8)).toEqual([]); }); it("searches the namespace text", () => {