diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 95cebcd08..4d0dbc921 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -44,7 +44,7 @@ - Fixed `pi update` naming pnpm for installs that only live under `PNPM_HOME`, for example a wrapper under `$PNPM_HOME/global-nub/`. pi said "not managed by a global pnpm install"; it now names the package manager only when that manager owns the install ([#7756](https://github.com/earendil-works/pi/issues/7756)) - Fixed image paste, copy, and opening the browser for sign-in failing silently on WSL when the working directory is deep. Windows programs such as `powershell.exe` and `explorer.exe` cannot start from a directory whose `\\wsl.localhost\...` path exceeds 260 characters; pi now starts these helpers from `/` ([#11592](https://github.com/earendil-works/pi/issues/11592)) - Fixed the external editor failing silently, for example when the editor command is wrong or a Windows editor cannot start from a deep WSL directory. pi now shows a status message such as `External editor exited with code 1; prompt unchanged` ([#11592](https://github.com/earendil-works/pi/issues/11592)) -- Fixed `tool_search` and codemode `searchTools()` ranking a tool below tools that only mention its name in their descriptions, for example `peekaboo app` ranking `mcp__peekaboo__app` 14th. A query word in a tool's name now scores more than any number of mentions in its description, a tool whose whole name is in the query gets a large bonus, and long server instructions no longer lower the scores of that server's tools ([#11331](https://github.com/earendil-works/pi/issues/11331)) +- Fixed `tool_search` and codemode `searchTools()` ranking a tool below tools that only mention its name in their descriptions, for example `peekaboo app` ranking `mcp__peekaboo__app` 14th. A tool matching more of the query's words now ranks above one matching fewer, a query word in a tool's name scores more than any number of mentions in its description, a tool whose whole name is in the query gets a large bonus, and long server instructions no longer lower the scores of that server's tools ([#11331](https://github.com/earendil-works/pi/issues/11331)) ## [1.4.0] - 2026-10-21 diff --git a/packages/coding-agent/src/extensions/tool-search/tool.ts b/packages/coding-agent/src/extensions/tool-search/tool.ts index 0d51696eb..dc743922b 100644 --- a/packages/coding-agent/src/extensions/tool-search/tool.ts +++ b/packages/coding-agent/src/extensions/tool-search/tool.ts @@ -177,11 +177,14 @@ const NAMESPACE_WEIGHT = 0.5; /** * Okapi BM25 over the description, with the name and the namespace as separate fields: + * - a tool that matches more of the query's terms, in any field, ranks above one that matches + * fewer, whatever their scores: for `screenshot window`, a tool whose description has both words + * beats `window`, whose name has one; * - a query term in the name scores more than any number of mentions in the description; * - a tool whose whole local name is in the query (`app` in `peekaboo app` for - * `mcp__peekaboo__app`) gets a bonus worth one more match of every query term, so it mostly wins - * over tools that only mention the words; - * - a query that is exactly a tool's name ranks that tool first. + * `mcp__peekaboo__app`) gets a bonus worth one more match of every query term, so among tools + * matching as many terms it mostly wins over tools that only mention the words; + * - a query that is exactly a tool's name ranks that tool first, before coverage. * * The description alone is length-normalized, so a long name or long namespace instructions do * not lower a tool's score. Ties keep document order. @@ -211,25 +214,27 @@ export class Bm25Ranker implements ToolRanker { ); const fullNameBonus = [...idf.values()].reduce((sum, value) => sum + value, 0); const exactQuery = normalizeName(query); - const matches: (ToolSearchMatch & { exact: boolean })[] = []; + const matches: (ToolSearchMatch & { covered: number; exact: boolean })[] = []; for (const document of indexed) { const norm = this.k1 * (1 - this.b + (this.b * document.textLength) / averageLength); let score = 0; + let covered = 0; for (const term of queryTerms) { const count = document.textCounts.get(term) ?? 0; const weight = (document.nameTerms.has(term) ? NAME_WEIGHT : 0) + (document.namespaceTerms.has(term) ? NAMESPACE_WEIGHT : 0) + (count * (this.k1 + 1)) / (count + norm); + if (weight > 0) covered++; score += (idf.get(term) ?? 0) * weight; } if (score === 0) continue; const local = document.localNameTerms; if (local.length > 0 && local.every((term) => queryTerms.has(term))) score += fullNameBonus; - matches.push({ name: document.name, score, exact: isExactNameQuery(exactQuery, document.name) }); + matches.push({ name: document.name, score, covered, exact: isExactNameQuery(exactQuery, document.name) }); } return matches - .sort((a, b) => Number(b.exact) - Number(a.exact) || b.score - a.score) + .sort((a, b) => Number(b.exact) - Number(a.exact) || b.covered - a.covered || b.score - a.score) .slice(0, limit) .map(({ name, score }) => ({ name, score })); } diff --git a/packages/coding-agent/test/tool-search.test.ts b/packages/coding-agent/test/tool-search.test.ts index da5df3433..d60f164f0 100644 --- a/packages/coding-agent/test/tool-search.test.ts +++ b/packages/coding-agent/test/tool-search.test.ts @@ -112,12 +112,28 @@ describe("Bm25Ranker", () => { const first = (query: string) => ranker.rank(query, documents, 1).map((match) => match.name); expect(first("peekaboo app")).toEqual(["mcp__peekaboo__app"]); expect(first("open the peekaboo app tool")).toEqual(["mcp__peekaboo__app"]); - expect(first("describe app")).toEqual(["mcp__peekaboo__app"]); expect(first("read thread_read state")).toEqual(["mcp__dev-app__thread_read"]); // Without the whole name, mentions in descriptions decide. expect(first("describe window")).toEqual(["mcp__peekaboo__see"]); }); + // https://github.com/earendil-works/pi/issues/11331 + it("ranks a tool matching more query terms first, whatever the scores", () => { + const documents = [ + tool("mcp__peekaboo__app", "Launch, quit, focus, or switch applications by name or bundle identifier."), + tool("mcp__peekaboo__see", "Capture the app window and describe the UI elements of the app."), + tool("mcp__peekaboo__window", "Move, resize, or close a window of an app."), + tool("mcp__peekaboo__image", "Take a screenshot of the screen, an app, or a window."), + ].map((entry) => createToolSearchDocument(entry)); + const ranker = new Bm25Ranker(); + const first = (query: string) => ranker.rank(query, documents, 1).map((match) => match.name); + // `window` and `app` are whole names, but each covers one of the two terms. + expect(first("screenshot window")).toEqual(["mcp__peekaboo__image"]); + expect(first("describe app")).toEqual(["mcp__peekaboo__see"]); + // An exact name still ranks first. + expect(first("window")).toEqual(["mcp__peekaboo__window"]); + }); + it("includes the namespace in the search text", () => { const document = createToolSearchDocument(tool("mcp__x__run", "Run it."), { name: "mcp__x",