From bc3d36474256b271f277d8ffe9ae870fa96fb896 Mon Sep 17 00:00:00 2001 From: David Brailovsky Date: Thu, 23 Jul 2026 13:34:51 +0000 Subject: [PATCH] fix bundled models generation time MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fresh package installs could give bundled model catalog files newer filesystem mtimes than the remote catalog’s Last-Modified timestamp, causing Pi to ignore newer remote models. Use the catalog’s recorded generation time instead of installation-dependent file metadata. --- packages/ai/scripts/generate-models.ts | 4 +++- packages/ai/scripts/model-data.ts | 8 ++++++- packages/ai/src/providers/all.ts | 8 ++++--- .../ai/test/model-data-validation.test.ts | 14 ++++++++++-- .../coding-agent/src/core/model-runtime.ts | 9 ++------ .../src/core/remote-catalog-provider.ts | 22 +++++-------------- .../test/remote-catalog-provider.test.ts | 14 +++++------- 7 files changed, 40 insertions(+), 39 deletions(-) diff --git a/packages/ai/scripts/generate-models.ts b/packages/ai/scripts/generate-models.ts index b63b5039d..782c34959 100644 --- a/packages/ai/scripts/generate-models.ts +++ b/packages/ai/scripts/generate-models.ts @@ -2508,6 +2508,8 @@ async function generateModels() { } } + const generatedAt = new Date().toISOString(); + if (!generatorOptions.jsonOnly) { // Stage and validate all provider values before replacing the current generated data. const providersDir = join(packageRoot, "src/providers"); @@ -2527,7 +2529,7 @@ async function generateModels() { } writeJson( join(stagedDataDir, MODEL_DATA_MANIFEST_FILE), - createModelDataManifest(modelDataStructure, fileContents), + createModelDataManifest(modelDataStructure, fileContents, generatedAt), ); validateModelDataDirectory(modelDataStructure, stagedDataDir); diff --git a/packages/ai/scripts/model-data.ts b/packages/ai/scripts/model-data.ts index e3ca9b2c4..4dd94000e 100644 --- a/packages/ai/scripts/model-data.ts +++ b/packages/ai/scripts/model-data.ts @@ -2,13 +2,14 @@ import { createHash } from "node:crypto"; import { existsSync, readFileSync, readdirSync, statSync } from "node:fs"; import { join } from "node:path"; -export const MODEL_DATA_SCHEMA_VERSION = 2; +export const MODEL_DATA_SCHEMA_VERSION = 3; export const MODEL_DATA_MANIFEST_FILE = ".manifest.json"; export type ModelDataStructure = Record>; export interface ModelDataManifest { schemaVersion: number; + generatedAt: string; structureHash: string; files: Record; } @@ -119,9 +120,11 @@ export function modelDataStructureHash(structure: ModelDataStructure): string { export function createModelDataManifest( structure: ModelDataStructure, fileContents: Readonly>, + generatedAt: string, ): ModelDataManifest { return { schemaVersion: MODEL_DATA_SCHEMA_VERSION, + generatedAt, structureHash: modelDataStructureHash(structure), files: sortedRecord(Object.entries(fileContents).map(([file, content]) => [file, sha256(content)] as const)), }; @@ -203,6 +206,9 @@ export function validateModelDataDirectory(structure: ModelDataStructure, dataDi `model data schema is ${JSON.stringify(manifest?.schemaVersion)}, expected ${MODEL_DATA_SCHEMA_VERSION}`, ); } + if (typeof manifest?.generatedAt !== "string" || Number.isNaN(Date.parse(manifest.generatedAt))) { + errors.push("model data manifest has an invalid generation timestamp"); + } const expectedStructureHash = modelDataStructureHash(structure); if (manifest?.structureHash !== expectedStructureHash) { errors.push("model data generation stamp does not match the generated catalog"); diff --git a/packages/ai/src/providers/all.ts b/packages/ai/src/providers/all.ts index 549880e33..4161a8c65 100644 --- a/packages/ai/src/providers/all.ts +++ b/packages/ai/src/providers/all.ts @@ -9,6 +9,7 @@ import { azureOpenAIResponsesProvider } from "./azure-openai-responses.ts"; import { cerebrasProvider } from "./cerebras.ts"; import { cloudflareAIGatewayProvider } from "./cloudflare-ai-gateway.ts"; import { cloudflareWorkersAIProvider } from "./cloudflare-workers-ai.ts"; +import modelDataManifest from "./data/.manifest.json" with { type: "json" }; import { deepseekProvider } from "./deepseek.ts"; import { fireworksProvider } from "./fireworks.ts"; import { githubCopilotProvider } from "./github-copilot.ts"; @@ -67,9 +68,10 @@ export function getBuiltinProviders(): BuiltinProvider[] { return Object.keys(MODELS) as BuiltinProvider[]; } -/** URL of a generated provider catalog, used to compare its mtime with remote catalogs during development. */ -export function getBuiltinModelDataUrl(provider: BuiltinProvider): URL { - return new URL(`./data/${provider}.json`, import.meta.url); +/** Generation timestamp shared by all built-in provider catalogs. */ +export function getBuiltinModelDataGeneratedAt(): number | undefined { + const generatedAt = Date.parse(modelDataManifest.generatedAt); + return Number.isNaN(generatedAt) ? undefined : generatedAt; } export function getBuiltinModels( diff --git a/packages/ai/test/model-data-validation.test.ts b/packages/ai/test/model-data-validation.test.ts index 29f9572a4..fa84fd1ff 100644 --- a/packages/ai/test/model-data-validation.test.ts +++ b/packages/ai/test/model-data-validation.test.ts @@ -11,6 +11,7 @@ import { validateModelDataDirectory, } from "../scripts/model-data.ts"; +const GENERATED_AT = "2026-07-23T10:00:00.000Z"; const temporaryRoots: string[] = []; afterEach(() => { @@ -70,7 +71,7 @@ function writeFixtureData( const filename = "test-provider.json"; const content = `${JSON.stringify({ [apiGroup]: values })}\n`; writeFileSync(join(dataDir, filename), content); - const manifest = createModelDataManifest(structure, { [filename]: content }); + const manifest = createModelDataManifest(structure, { [filename]: content }, GENERATED_AT); manifest.schemaVersion = manifestSchemaVersion; writeFileSync(join(dataDir, MODEL_DATA_MANIFEST_FILE), `${JSON.stringify(manifest)}\n`); } @@ -120,7 +121,7 @@ describe("generated model data validation", () => { "anthropic-messages": fixture.values, })}\n`; writeFileSync(join(fixture.dataDir, filename), content); - const manifest = createModelDataManifest(fixture.structure, { [filename]: content }); + const manifest = createModelDataManifest(fixture.structure, { [filename]: content }, GENERATED_AT); writeFileSync(join(fixture.dataDir, MODEL_DATA_MANIFEST_FILE), `${JSON.stringify(manifest)}\n`); expect(() => validateModelDataDirectory(fixture.structure, fixture.dataDir)).toThrow("more than one API group"); }); @@ -143,6 +144,15 @@ describe("generated model data validation", () => { expect(() => validateModelDataDirectory(fixture.structure, fixture.dataDir)).toThrow("generation stamp"); }); + it("rejects an invalid generation timestamp", () => { + const fixture = createFixture(); + const manifestPath = join(fixture.dataDir, MODEL_DATA_MANIFEST_FILE); + const manifest = JSON.parse(readFileSync(manifestPath, "utf8")) as Record; + manifest.generatedAt = "invalid"; + writeFileSync(manifestPath, `${JSON.stringify(manifest)}\n`); + expect(() => validateModelDataDirectory(fixture.structure, fixture.dataDir)).toThrow("generation timestamp"); + }); + it("rejects missing provider shards imported by the aggregator", () => { const { packageRoot } = createFixture(); writeFileSync( diff --git a/packages/coding-agent/src/core/model-runtime.ts b/packages/coding-agent/src/core/model-runtime.ts index 2cd85b8d6..6b9639135 100644 --- a/packages/coding-agent/src/core/model-runtime.ts +++ b/packages/coding-agent/src/core/model-runtime.ts @@ -140,18 +140,13 @@ export class ModelRuntime implements Models { (modelsPath ? new FileModelsStore(options.modelsStorePath ?? join(dirname(modelsPath), "models-store.json")) : new InMemoryCodingAgentModelsStore()); + const builtinModelDataGeneratedAt = builtinProviderCatalog.getBuiltinModelDataGeneratedAt(); const providers = builtinProviderCatalog .builtinProviders() .map((provider) => provider.id === "radius" ? provider - : withRemoteCatalog( - provider, - options.catalogBaseUrl, - builtinProviderCatalog.getBuiltinModelDataUrl( - provider.id as builtinProviderCatalog.BuiltinProvider, - ), - ), + : withRemoteCatalog(provider, options.catalogBaseUrl, builtinModelDataGeneratedAt), ); const runtime = new ModelRuntime( credentials, diff --git a/packages/coding-agent/src/core/remote-catalog-provider.ts b/packages/coding-agent/src/core/remote-catalog-provider.ts index b823cf7da..ee5614adf 100644 --- a/packages/coding-agent/src/core/remote-catalog-provider.ts +++ b/packages/coding-agent/src/core/remote-catalog-provider.ts @@ -1,4 +1,3 @@ -import { stat } from "node:fs/promises"; import type { Api, Model, ModelsStoreEntry, Provider } from "@earendil-works/pi-ai"; import { VERSION } from "../config.ts"; import { getPiUserAgent } from "../utils/pi-user-agent.ts"; @@ -32,13 +31,10 @@ function parseCatalog(providerId: string, value: unknown): Model[] { function remoteModels( entry: ModelsStoreEntry | undefined, - localLastModified: number | undefined, + localGeneratedAt: number | undefined, ): readonly Model[] { if (!entry) return []; - if ( - localLastModified !== undefined && - (entry.lastModified === undefined || entry.lastModified <= localLastModified) - ) { + if (localGeneratedAt !== undefined && (entry.lastModified === undefined || entry.lastModified <= localGeneratedAt)) { return []; } return entry.models; @@ -48,7 +44,7 @@ function remoteModels( export function withRemoteCatalog( provider: Provider, catalogBaseUrl: string = DEFAULT_CATALOG_BASE_URL, - localCatalogUrl?: URL, + localGeneratedAt?: number, ): Provider { let dynamicModels: readonly Model[] = []; let inflightRefresh: Promise | undefined; @@ -59,16 +55,8 @@ export function withRemoteCatalog( refreshModels: (context) => { inflightRefresh ??= (async () => { try { - const localLastModified = localCatalogUrl - ? await stat(localCatalogUrl).then( - (value) => value.mtimeMs, - () => undefined, - ) - : undefined; const stored = await context.store.read(); - dynamicModels = remoteModels(stored, localLastModified).filter( - (model) => model.provider === provider.id, - ); + dynamicModels = remoteModels(stored, localGeneratedAt).filter((model) => model.provider === provider.id); if (!context.allowNetwork || context.signal?.aborted) return; if ( !context.force && @@ -105,7 +93,7 @@ export function withRemoteCatalog( checkedAt, lastModified: Number.isNaN(lastModified) ? 0 : lastModified, }; - dynamicModels = remoteModels(entry, localLastModified); + dynamicModels = remoteModels(entry, localGeneratedAt); await context.store.write(entry); } finally { inflightRefresh = undefined; diff --git a/packages/coding-agent/test/remote-catalog-provider.test.ts b/packages/coding-agent/test/remote-catalog-provider.test.ts index 8b43a4f32..510d1460a 100644 --- a/packages/coding-agent/test/remote-catalog-provider.test.ts +++ b/packages/coding-agent/test/remote-catalog-provider.test.ts @@ -1,4 +1,3 @@ -import { statSync } from "node:fs"; import { createProvider, InMemoryModelsStore, @@ -25,7 +24,7 @@ function model(id: string): Model<"openai-completions"> { }; } -function testProvider(localCatalogUrl?: URL) { +function testProvider(localGeneratedAt?: number) { return withRemoteCatalog( createProvider({ id: "test-provider", @@ -41,7 +40,7 @@ function testProvider(localCatalogUrl?: URL) { }, }), "https://pi.dev", - localCatalogUrl, + localGeneratedAt, ); } @@ -80,19 +79,18 @@ describe("remote catalog provider", () => { }); it("prefers the newer of the generated and remote catalogs", async () => { - const localCatalogUrl = new URL(import.meta.url); - const localMtime = statSync(localCatalogUrl).mtimeMs; - const newerHeader = new Date(localMtime + 60_000).toUTCString(); + const localGeneratedAt = Date.parse("2026-07-23T10:00:00.000Z"); + const newerHeader = new Date(localGeneratedAt + 60_000).toUTCString(); const responses = [ new Response(JSON.stringify({ old: model("old") }), { - headers: { "last-modified": new Date(localMtime - 60_000).toUTCString() }, + headers: { "last-modified": new Date(localGeneratedAt - 60_000).toUTCString() }, }), new Response(JSON.stringify({ newer: model("newer") }), { headers: { "last-modified": newerHeader }, }), ]; vi.spyOn(globalThis, "fetch").mockImplementation(async () => responses.shift() as Response); - const provider = testProvider(localCatalogUrl); + const provider = testProvider(localGeneratedAt); const store = new InMemoryModelsStore(); const refresh = { credential: { type: "api_key" } as const, store: scopedStore(store), allowNetwork: true };