sscoble/pi-mono
1
0
Fork 0
mirror of https://github.com/badlogic/pi-mono.git synced 2026-11-05 23:37:25 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

fix(coding-agent): cache llama.cpp model catalog (#7072)

Browse source
This commit is contained in:
David Brailovsky 2026-07-25 08:27:05 +02:00 • committed by GitHub
parent af3b934fa6
commit 2a2b0a39f0
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 54 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

9
packages/coding-agent/src/extensions/llama/provider.ts
View file

@ -111,11 +111,20 @@ export function createLlamaProvider(): LlamaProviderController {
},
getModels: () => models,
refreshModels: async (context: RefreshModelsContext): Promise<void> => {
const stored = await context.store.read();
if (stored) {
models = stored.models.filter(
(model): model is Model<"openai-completions"> =>
model.provider === LLAMA_PROVIDER_ID && model.api === "openai-completions",
);
}
if (!context.allowNetwork || context.signal?.aborted || context.credential?.type !== "api_key") return;
const serverUrl = credentialServerUrl(context.credential);
if (!serverUrl) return;
const catalog = await new LlamaClient(serverUrl, context.credential.key).list({ signal: context.signal });
setCatalog(catalog, serverUrl);
if (!context.signal?.aborted) await context.store.write({ models, checkedAt: Date.now() });
},
stream: (model, context, options) => stream(model, context, options as ProviderStreamOptions | undefined),
streamSimple: (model, context, options) => streamSimple(model, context, options),

46
packages/coding-agent/test/llama-extension.test.ts
View file

@ -1,7 +1,7 @@
import { once } from "node:events";
import { createServer, type RequestListener, type Server, type ServerResponse } from "node:http";
import type { AddressInfo } from "node:net";
import type { AuthContext, AuthPrompt } from "@earendil-works/pi-ai";
import type { AuthContext, AuthPrompt, ModelsStoreEntry } from "@earendil-works/pi-ai";
import { afterEach, describe, expect, it } from "vitest";
import { createEventBus } from "../src/core/event-bus.ts";
import { createExtensionRuntime, loadExtensionFromFactory } from "../src/core/extensions/loader.ts";
@ -86,6 +86,50 @@ describe("llama.cpp extension", () => {
]);
});
it("persists and restores loaded models for cache-only startup refreshes", async () => {
let cachedEntry: ModelsStoreEntry | undefined;
const store = {
read: async () => cachedEntry,
write: async (entry: ModelsStoreEntry) => {
cachedEntry = structuredClone(entry);
},
delete: async () => {
cachedEntry = undefined;
},
};
const { url } = await listen((request, response) => {
if (request.url === "/models") {
json(response, {
data: [
{ id: "loaded", status: { value: "loaded" }, meta: { n_ctx: 32768 } },
{ id: "unloaded", status: { value: "unloaded" } },
],
});
return;
}
response.writeHead(404).end();
});
const first = createLlamaProvider();
await first.provider.refreshModels?.({
credential: { type: "api_key", key: "local", env: { LLAMA_BASE_URL: url } },
store,
allowNetwork: true,
});
expect(first.provider.getModels().map((model) => model.id)).toEqual(["loaded"]);
expect(cachedEntry?.models.map((model) => model.id)).toEqual(["loaded"]);
const second = createLlamaProvider();
await second.provider.refreshModels?.({
credential: { type: "api_key", key: "local", env: { LLAMA_BASE_URL: url } },
store,
allowNetwork: false,
});
expect(second.provider.getModels()).toEqual([
expect.objectContaining({ id: "loaded", baseUrl: `${url}/v1`, contextWindow: 32768 }),
]);
});
it("stays dormant until configured and stores URL plus optional key", async () => {
const { provider } = createLlamaProvider();
const auth = provider.auth.apiKey!;