From 1e39862f677a2a2f5aab73671a43659f338ea461 Mon Sep 17 00:00:00 2001 From: David Brailovsky Date: Fri, 18 Sep 2026 14:54:17 +0200 Subject: [PATCH] fix(coding-agent): detect llama.cpp chat-template thinking Inspect loaded models through /props and configure boolean thinking levels for templates supporting enable_thinking. Fixes #9528 --- packages/coding-agent/CHANGELOG.md | 1 + .../src/extensions/llama/client.ts | 13 +++-- .../src/extensions/llama/provider.ts | 33 +++++++++-- .../coding-agent/test/llama-extension.test.ts | 55 +++++++++++++++++++ 4 files changed, 92 insertions(+), 10 deletions(-) diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 4fca7f40f..b2fddcb33 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -30,6 +30,7 @@ - Fixed fullscreen mode reserving a blank row for custom footers that render zero rows ([#8919](https://github.com/earendil-works/pi/issues/8919)). - Fixed extension tools without parameter schemas to be rejected during registration instead of breaking provider requests ([#9300](https://github.com/earendil-works/pi/issues/9300)). - Fixed `before_agent_start` handlers returning `systemPrompt` (and `forceSystemPrompt`) on models with mid-conversation system messages: the forced prompt is now sent as the provider's leading system prompt instead of being appended as a section patch after the original prompt. +- Fixed loaded llama.cpp models with `enable_thinking` chat templates ignoring Pi's thinking level ([#9528](https://github.com/earendil-works/pi/issues/9528)). ## [0.85.1] - 2026-09-05 diff --git a/packages/coding-agent/src/extensions/llama/client.ts b/packages/coding-agent/src/extensions/llama/client.ts index c071f518c..3233295a8 100644 --- a/packages/coding-agent/src/extensions/llama/client.ts +++ b/packages/coding-agent/src/extensions/llama/client.ts @@ -30,6 +30,7 @@ export interface LlamaModelsResponse { export interface LlamaServerProps { models_autoload?: boolean; + chat_template?: string; } export interface LlamaModelEvent { @@ -193,11 +194,15 @@ export class LlamaClient { return data; } - async props(options: { signal?: AbortSignal } = {}): Promise { - const payload = await this.request("/props", { signal: options.signal }); + async props(options: { model?: string; signal?: AbortSignal } = {}): Promise { + const query = options.model ? `?${new URLSearchParams({ model: options.model, autoload: "false" })}` : ""; + const payload = await this.request(`/props${query}`, { signal: options.signal }); if (typeof payload !== "object" || payload === null) return {}; - const { models_autoload: modelsAutoload } = payload as Record; - return typeof modelsAutoload === "boolean" ? { models_autoload: modelsAutoload } : {}; + const { models_autoload: modelsAutoload, chat_template: chatTemplate } = payload as Record; + return { + ...(typeof modelsAutoload === "boolean" ? { models_autoload: modelsAutoload } : {}), + ...(typeof chatTemplate === "string" ? { chat_template: chatTemplate } : {}), + }; } async load(model: string, signal?: AbortSignal): Promise { diff --git a/packages/coding-agent/src/extensions/llama/provider.ts b/packages/coding-agent/src/extensions/llama/provider.ts index 6a685938d..659ed1758 100644 --- a/packages/coding-agent/src/extensions/llama/provider.ts +++ b/packages/coding-agent/src/extensions/llama/provider.ts @@ -8,7 +8,13 @@ import type { RefreshModelsContext, } from "@earendil-works/pi-ai"; import { stream, streamSimple } from "@earendil-works/pi-ai/compat"; -import { LlamaClient, type LlamaModelInfo, llamaInferenceUrl, normalizeLlamaServerUrl } from "./client.ts"; +import { + LlamaClient, + type LlamaModelInfo, + type LlamaServerProps, + llamaInferenceUrl, + normalizeLlamaServerUrl, +} from "./client.ts"; export const LLAMA_PROVIDER_ID = "llama.cpp"; export const DEFAULT_LLAMA_SERVER_URL = "http://127.0.0.1:8080"; @@ -46,16 +52,20 @@ async function routerAutoloadEnabled( } } -function toPiModel(model: LlamaModelInfo, serverUrl: string): Model<"openai-completions"> { +function toPiModel(model: LlamaModelInfo, serverUrl: string, props?: LlamaServerProps): Model<"openai-completions"> { const reportedContextWindow = model.meta?.n_ctx ?? model.meta?.n_ctx_train; const contextWindow = reportedContextWindow && reportedContextWindow > 0 ? reportedContextWindow : 128000; + const reasoning = props?.chat_template?.includes("enable_thinking") === true; return { id: model.id, name: model.id, api: "openai-completions", provider: LLAMA_PROVIDER_ID, baseUrl: llamaInferenceUrl(serverUrl), - reasoning: false, + reasoning, + ...(reasoning && { + thinkingLevelMap: { off: "off", minimal: null, low: null, medium: "medium", high: null, xhigh: null }, + }), input: model.architecture?.input_modalities?.includes("image") ? ["text", "image"] : ["text"], cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, contextWindow, @@ -67,6 +77,7 @@ function toPiModel(model: LlamaModelInfo, serverUrl: string): Model<"openai-comp supportsUsageInStreaming: true, supportsStrictMode: false, maxTokensField: "max_tokens", + ...(reasoning && { thinkingFormat: "qwen-chat-template" }), }, }; } @@ -162,9 +173,19 @@ export function createLlamaProvider(): LlamaProviderController { if (context.signal.aborted) return; const routerAutoload = await routerAutoloadEnabled(client, catalog, context.signal); if (context.signal.aborted) return; - const refreshed = catalog - .filter((model) => modelIsSelectable(model, routerAutoload)) - .map((model) => toPiModel(model, serverUrl)); + const refreshed = await Promise.all( + catalog + .filter((model) => modelIsSelectable(model, routerAutoload)) + .map(async (model) => { + // Only loaded models expose their template without side effects. Unloaded autoload presets + // would need to be loaded, while querying sleeping models may wake them. Those models remain + // unclassified until they are loaded or woken and a later catalog refresh discovers them. + if (model.status.value !== "loaded") return toPiModel(model, serverUrl); + const props = await client.props({ model: model.id, signal: context.signal }); + return toPiModel(model, serverUrl, props); + }), + ); + if (context.signal.aborted) return; await context.publish({ persist: { models: refreshed, checkedAt: Date.now() }, update: () => { diff --git a/packages/coding-agent/test/llama-extension.test.ts b/packages/coding-agent/test/llama-extension.test.ts index 7c506aafc..5c7db69d2 100644 --- a/packages/coding-agent/test/llama-extension.test.ts +++ b/packages/coding-agent/test/llama-extension.test.ts @@ -91,6 +91,57 @@ describe("llama.cpp extension", () => { ]); }); + // Regression test for #9528. + it("discovers chat-template thinking support for loaded models", async () => { + let propsRequests = 0; + const { url } = await listen((request, response) => { + if (request.url === "/models") { + json(response, { + data: [{ id: "qwen", status: { value: "loaded" }, meta: { n_ctx: 32768 } }], + }); + return; + } + const requestUrl = new URL(request.url ?? "", "http://localhost"); + if (requestUrl.pathname === "/props") { + propsRequests++; + expect(requestUrl.searchParams.get("model")).toBe("qwen"); + expect(requestUrl.searchParams.get("autoload")).toBe("false"); + json(response, { chat_template: "{% if enable_thinking %}think{% endif %}" }); + return; + } + response.writeHead(404).end(); + }); + + const controller = createLlamaProvider(); + await controller.provider.refreshModels?.({ + credential: { type: "api_key", key: "local", env: { LLAMA_BASE_URL: url } }, + stored: undefined, + publish: async (publication) => { + publication.update?.(); + return true; + }, + allowNetwork: true, + signal: new AbortController().signal, + }); + + expect(propsRequests).toBe(1); + expect(controller.provider.getModels()).toEqual([ + expect.objectContaining({ + id: "qwen", + reasoning: true, + thinkingLevelMap: { + off: "off", + minimal: null, + low: null, + medium: "medium", + high: null, + xhigh: null, + }, + compat: expect.objectContaining({ thinkingFormat: "qwen-chat-template" }), + }), + ]); + }); + it("persists and restores selectable models for cache-only startup refreshes", async () => { let cachedEntry: ModelsStoreEntry | undefined; const { url } = await listen((request, response) => { @@ -104,6 +155,10 @@ describe("llama.cpp extension", () => { }); return; } + if (request.url === "/props?model=loaded&autoload=false") { + json(response, {}); + return; + } response.writeHead(404).end(); });