fix(coding-agent): rank tools first when the search query is their name

A query that equals a tool's full name or the part after a __ separator
(thread_read for mcp__app__thread_read) now ranks that tool first in
tool_search and codemode searchTools(). BM25 length normalization could
otherwise rank a shorter description sharing the name's words above it.

closes #11331
This commit is contained in:
Petr Houška 2026-10-26 14:07:52 +01:00
parent c17be1e982
commit ae823bc0bb
2 changed files with 52 additions and 4 deletions

View file

@ -115,7 +115,28 @@ export function createToolSearchDocument(
return { name: tool.name, text: parts.filter((part) => part.trim()).join(" ") };
}
/** Okapi BM25 with the usual parameters. Ties keep document order. */
/** Lowercase with characters invalid in script identifiers as `_`, so `dev-radius` matches `dev_radius`. */
function normalizeName(name: string): string {
return name
.trim()
.toLowerCase()
.replace(/[^a-z0-9_$]/g, "_");
}
/**
* Whether `query` names the tool: its full name or a suffix after a `__` separator, so `thread_read`
* and `app__thread_read` both name `mcp__app__thread_read`.
*/
function isNameQuery(query: string, name: string): boolean {
const normalized = normalizeName(name);
return normalized === query || normalized.endsWith(`__${query}`);
}
/**
* Okapi BM25 with the usual parameters. A query that names a tool ranks that tool first, because
* length normalization can otherwise rank a short description sharing the name's words above it.
* Ties keep document order.
*/
export class Bm25Ranker implements ToolRanker {
private readonly k1: number;
private readonly b: number;
@ -141,7 +162,8 @@ export class Bm25Ranker implements ToolRanker {
return [term, Math.log(1 + (documents.length - frequency + 0.5) / (frequency + 0.5))] as const;
}),
);
const matches: ToolSearchMatch[] = [];
const nameQuery = normalizeName(query);
const matches: (ToolSearchMatch & { named: boolean })[] = [];
documents.forEach((document, index) => {
let score = 0;
for (const term of queryTerms) {
@ -150,9 +172,12 @@ export class Bm25Ranker implements ToolRanker {
const norm = this.k1 * (1 - this.b + (this.b * lengths[index]) / averageLength);
score += (idf.get(term) ?? 0) * ((count * (this.k1 + 1)) / (count + norm));
}
if (score > 0) matches.push({ name: document.name, score });
if (score > 0) matches.push({ name: document.name, score, named: isNameQuery(nameQuery, document.name) });
});
return matches.sort((a, b) => b.score - a.score).slice(0, limit);
return matches
.sort((a, b) => Number(b.named) - Number(a.named) || b.score - a.score)
.slice(0, limit)
.map(({ name, score }) => ({ name, score }));
}
}

View file

@ -53,6 +53,29 @@ describe("Bm25Ranker", () => {
expect(ranker.rank("tickets", documents, 8)).toEqual([]);
});
// https://github.com/earendil-works/pi/issues/11331
it("ranks a tool first when the query is its name", () => {
const named = [
tool("mcp__app__queue_read", "Read a queued message of this thread."),
tool(
"mcp__dev-app__thread_read",
"Read durable state and a paginated timeline from any thread. The default view returns user messages, assistant messages and plans. Continue with afterPosition until hasMore is false.",
),
].map((entry) => createToolSearchDocument(entry));
const ranker = new Bm25Ranker();
for (const query of [
"thread_read",
"dev_app__thread_read",
"mcp__dev-app__thread_read",
"Thread_Read",
"thread read",
]) {
expect(ranker.rank(query, named, 1).map((match) => match.name)).toEqual(["mcp__dev-app__thread_read"]);
}
// Partial names still rank by BM25.
expect(ranker.rank("read", named, 1).map((match) => match.name)).toEqual(["mcp__app__queue_read"]);
});
it("includes the namespace in the search text", () => {
const document = createToolSearchDocument(tool("mcp__x__run", "Run it."), {
name: "mcp__x",