diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 5394de918..a929c54d4 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -25,7 +25,7 @@ - Fixed the subagent example repeatedly prompting before running project-local agents in trusted repositories ([#8261](https://github.com/earendil-works/pi/issues/8261)). - Added `session_compact_failed` extension events so compaction failures and aborts expose their reason, retry state, source, and error message to handlers ([#8175](https://github.com/earendil-works/pi/issues/8175)). - Fixed npm package update checks treating older registry versions as available updates, preventing `pi update` from downgrading already-newer installed packages ([#8226](https://github.com/earendil-works/pi/issues/8226)). -- Fixed built-in llama.cpp models disappearing from `/model` when `/llama` refreshed a configured server under `PI_OFFLINE`, and included idle-slept `sleeping` router models in the selectable catalog ([#8167](https://github.com/earendil-works/pi/issues/8167)). +- Fixed built-in llama.cpp models disappearing from `/model` when `/llama` refreshed a configured server under `PI_OFFLINE`, and included idle-slept `sleeping` router models plus autoloadable unloaded presets in the selectable catalog ([#8167](https://github.com/earendil-works/pi/issues/8167)). - Fixed `pi.registerFlag()` accepting default values that do not match the declared flag type ([#8064](https://github.com/earendil-works/pi/issues/8064)). - Fixed Z.AI Coding Plan defaults referencing the removed GLM-5.1 model ([#8096](https://github.com/earendil-works/pi/issues/8096)). - Fixed repeated ambiguous truncated-response recovery being mislabeled as context overflow ([#8130](https://github.com/earendil-works/pi/issues/8130)). diff --git a/packages/coding-agent/src/extensions/llama/client.ts b/packages/coding-agent/src/extensions/llama/client.ts index 45cd8f3a6..c071f518c 100644 --- a/packages/coding-agent/src/extensions/llama/client.ts +++ b/packages/coding-agent/src/extensions/llama/client.ts @@ -28,6 +28,10 @@ export interface LlamaModelsResponse { object?: string; } +export interface LlamaServerProps { + models_autoload?: boolean; +} + export interface LlamaModelEvent { model: string; event: string; @@ -189,6 +193,13 @@ export class LlamaClient { return data; } + async props(options: { signal?: AbortSignal } = {}): Promise { + const payload = await this.request("/props", { signal: options.signal }); + if (typeof payload !== "object" || payload === null) return {}; + const { models_autoload: modelsAutoload } = payload as Record; + return typeof modelsAutoload === "boolean" ? { models_autoload: modelsAutoload } : {}; + } + async load(model: string, signal?: AbortSignal): Promise { await this.request("/models/load", { method: "POST", body: JSON.stringify({ model }), signal }); } diff --git a/packages/coding-agent/src/extensions/llama/provider.ts b/packages/coding-agent/src/extensions/llama/provider.ts index 83fa72ddc..6a685938d 100644 --- a/packages/coding-agent/src/extensions/llama/provider.ts +++ b/packages/coding-agent/src/extensions/llama/provider.ts @@ -25,9 +25,25 @@ async function resolveServerUrl( return configured ? normalizeLlamaServerUrl(configured) : undefined; } -function modelIsSelectable(model: LlamaModelInfo): boolean { +function modelIsSelectable(model: LlamaModelInfo, routerAutoload: boolean): boolean { + if (model.status.value === "loaded") return true; // llama.cpp reports idle-slept models as "sleeping"; requests wake them automatically. - return model.status.value === "loaded" || model.status.value === "sleeping"; + if (model.status.value === "sleeping") return true; + // Unloaded presets are routable only when llama.cpp router autoload can load them on first use. + return routerAutoload && model.status.value === "unloaded" && !model.status.failed && model.source === "preset"; +} + +async function routerAutoloadEnabled( + client: LlamaClient, + catalog: readonly LlamaModelInfo[], + signal: AbortSignal, +): Promise { + if (!catalog.some((model) => model.status.value === "unloaded" && model.source === "preset")) return false; + try { + return (await client.props({ signal })).models_autoload === true; + } catch { + return false; + } } function toPiModel(model: LlamaModelInfo, serverUrl: string): Model<"openai-completions"> { @@ -57,14 +73,20 @@ function toPiModel(model: LlamaModelInfo, serverUrl: string): Model<"openai-comp export interface LlamaProviderController { provider: Provider<"openai-completions">; - setCatalog(models: readonly LlamaModelInfo[], serverUrl: string): void; + setCatalog(models: readonly LlamaModelInfo[], serverUrl: string, options?: { routerAutoload?: boolean }): void; } export function createLlamaProvider(): LlamaProviderController { let models: readonly Model<"openai-completions">[] = []; - const setCatalog = (catalog: readonly LlamaModelInfo[], serverUrl: string): void => { - models = catalog.filter((model) => modelIsSelectable(model)).map((model) => toPiModel(model, serverUrl)); + const setCatalog = ( + catalog: readonly LlamaModelInfo[], + serverUrl: string, + options: { routerAutoload?: boolean } = {}, + ): void => { + models = catalog + .filter((model) => modelIsSelectable(model, options.routerAutoload === true)) + .map((model) => toPiModel(model, serverUrl)); }; const provider: Provider<"openai-completions"> = { @@ -135,10 +157,13 @@ export function createLlamaProvider(): LlamaProviderController { if (!context.allowNetwork || context.signal.aborted || context.credential?.type !== "api_key") return; const serverUrl = credentialServerUrl(context.credential); if (!serverUrl) return; - const catalog = await new LlamaClient(serverUrl, context.credential.key).list({ signal: context.signal }); + const client = new LlamaClient(serverUrl, context.credential.key); + const catalog = await client.list({ signal: context.signal }); + if (context.signal.aborted) return; + const routerAutoload = await routerAutoloadEnabled(client, catalog, context.signal); if (context.signal.aborted) return; const refreshed = catalog - .filter((model) => modelIsSelectable(model)) + .filter((model) => modelIsSelectable(model, routerAutoload)) .map((model) => toPiModel(model, serverUrl)); await context.publish({ persist: { models: refreshed, checkedAt: Date.now() }, diff --git a/packages/coding-agent/test/llama-extension.test.ts b/packages/coding-agent/test/llama-extension.test.ts index 7d46c76c9..7c506aafc 100644 --- a/packages/coding-agent/test/llama-extension.test.ts +++ b/packages/coding-agent/test/llama-extension.test.ts @@ -138,6 +138,78 @@ describe("llama.cpp extension", () => { ]); }); + it("exposes unloaded presets only when router autoload is enabled", async () => { + let propsRequests = 0; + const { url } = await listen((request, response) => { + expect(request.headers.authorization).toBe("Bearer local"); + if (request.url === "/models") { + json(response, { + data: [ + { id: "preset", status: { value: "unloaded" }, source: "preset", meta: { n_ctx: 65536 } }, + { id: "failed-preset", status: { value: "unloaded", failed: true }, source: "preset" }, + { id: "cache", status: { value: "unloaded" }, source: "cache" }, + { id: "models-dir", status: { value: "unloaded" }, source: "models_dir" }, + ], + }); + return; + } + if (request.url === "/props") { + propsRequests++; + json(response, { role: "router", models_autoload: true }); + return; + } + response.writeHead(404).end(); + }); + + let cachedEntry: ModelsStoreEntry | undefined; + const controller = createLlamaProvider(); + await controller.provider.refreshModels?.({ + credential: { type: "api_key", key: "local", env: { LLAMA_BASE_URL: url } }, + stored: undefined, + publish: async (publication) => { + if (publication.persist !== undefined && publication.persist !== null) { + cachedEntry = structuredClone(publication.persist); + } + publication.update?.(); + return true; + }, + allowNetwork: true, + signal: new AbortController().signal, + }); + + expect(propsRequests).toBe(1); + expect(controller.provider.getModels().map((model) => model.id)).toEqual(["preset"]); + expect(cachedEntry?.models.map((model) => model.id)).toEqual(["preset"]); + }); + + it("hides unloaded presets when router autoload is disabled", async () => { + const { url } = await listen((request, response) => { + if (request.url === "/models") { + json(response, { data: [{ id: "preset", status: { value: "unloaded" }, source: "preset" }] }); + return; + } + if (request.url === "/props") { + json(response, { role: "router", models_autoload: false }); + return; + } + response.writeHead(404).end(); + }); + + const controller = createLlamaProvider(); + await controller.provider.refreshModels?.({ + credential: { type: "api_key", key: "local", env: { LLAMA_BASE_URL: url } }, + stored: undefined, + publish: async (publication) => { + publication.update?.(); + return true; + }, + allowNetwork: true, + signal: new AbortController().signal, + }); + + expect(controller.provider.getModels()).toEqual([]); + }); + it("stays dormant until configured and stores URL plus optional key", async () => { const { provider } = createLlamaProvider(); const auth = provider.auth.apiKey!;