feat: check the model catalog against the latest release before publishing (#11007)

pi.dev serves each published catalog to released pi >= MINIMUM_PI_VERSION.
Compile the latest release's pi-ai against its built-in models overlaid by
the new catalog, so a catalog it cannot handle (e.g. kimi-coding moving to
openai-completions) fails in CI instead of breaking released clients.
MINIMUM_PI_VERSION moves to scripts/model-catalog-minimum-version.mjs.

scripts/check-model-catalog-compat.mjs

@@ -0,0 +1,142 @@
+#!/usr/bin/env node
+
+// Checks that released pi versions can use the model catalog we are about to publish.
+//
+// Problem: pi.dev serves each published catalog to every released pi at or above
+// MINIMUM_PI_VERSION. Released clients use it right away, without updating pi.
+// If the catalog needs code they don't have, they break.
+//
+// Example: a catalog moved the kimi-coding models from anthropic-messages to
+// openai-completions. pi 1.2.0's kimi-coding provider only implements
+// anthropic-messages, so 1.2.0 sent requests to the wrong API and they failed.
+//
+// Check: build the model list a released client would end up with (its built-in
+// models, overlaid by the new catalog), and compile the release's pi-ai source
+// against it. pi-ai's types tie each provider to the APIs it implements, so a
+// model with an API its provider lacks is a type error.
+//
+// Only the latest release on npm is checked. If it is older than
+// MINIMUM_PI_VERSION, no released pi receives the catalog, and the check is skipped.
+//
+// Usage: node scripts/check-model-catalog-compat.mjs [--catalog <file>]
+//   --catalog <file>  catalog to check (default: .artifacts/model-catalog/models.all.json,
+//                     written by `npm run generate:model-catalog`)
+
+import { spawnSync } from "node:child_process";
+import { mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { dirname, join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import { parseArgs } from "node:util";
+import { MINIMUM_PI_VERSION } from "./model-catalog-minimum-version.mjs";
+import { compareModelCatalogPiVersions } from "./model-catalog-protocol.ts";
+
+const PI_AI_PACKAGE = "@earendil-works/pi-ai";
+
+/** Read a release's built-in model data (`dist/providers/data`) as model lists by provider. */
+function readBuiltinModels(dataDir) {
+	const builtin = {};
+	for (const file of readdirSync(dataDir)) {
+		if (!file.endsWith(".json") || file === ".manifest.json") continue;
+		const groups = JSON.parse(readFileSync(join(dataDir, file), "utf8"));
+		builtin[file.slice(0, -".json".length)] = Object.values(groups).flatMap((group) => Object.values(group));
+	}
+	return builtin;
+}
+
+/**
+ * Return the models a released client ends up with for each provider it ships:
+ * its built-in models, overlaid by the catalog's models with the same type and
+ * id. Like mergeModels() in remote-catalog-provider.ts, models missing from the
+ * catalog keep their built-in data, and providers the release does not ship are
+ * ignored.
+ */
+function mergeReleaseCatalog(builtin, catalog) {
+	const merged = {};
+	for (const [provider, builtinModels] of Object.entries(builtin)) {
+		const models = new Map(builtinModels.map((model) => [`${model.type}:${model.id}`, model]));
+		for (const model of catalog[provider] ?? []) models.set(`${model.type}:${model.id}`, model);
+		merged[provider] = [...models.values()];
+	}
+	return merged;
+}
+
+function run(command, args, cwd) {
+	const result = spawnSync(command, args, { cwd, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] });
+	if (result.error) throw result.error;
+	if (result.status !== 0) {
+		const output = [result.stdout, result.stderr].filter(Boolean).join("\n").trim();
+		throw new Error(`Command failed: ${[command, ...args].join(" ")}\n${output}`);
+	}
+	return result.stdout;
+}
+
+function main() {
+	const root = join(dirname(fileURLToPath(import.meta.url)), "..");
+	const { values } = parseArgs({ options: { catalog: { type: "string" } } });
+	const catalogPath = resolve(values.catalog ?? join(root, ".artifacts/model-catalog/models.all.json"));
+
+	const release = run("npm", ["view", PI_AI_PACKAGE, "version"], root).trim();
+	if (compareModelCatalogPiVersions(release, MINIMUM_PI_VERSION) < 0) {
+		console.log(
+			`Skipped: the latest release ${release} is older than MINIMUM_PI_VERSION ${MINIMUM_PI_VERSION}, so no released pi receives this catalog.`,
+		);
+		return;
+	}
+
+	const temporaryDir = mkdtempSync(join(tmpdir(), "pi-model-catalog-compat-"));
+	const releaseDir = join(temporaryDir, "release");
+	try {
+		console.log(`Checking the model catalog against pi ${release}...`);
+		// The release's source code.
+		const archive = join(temporaryDir, "release.tar");
+		run("git", ["archive", "--prefix=release/", "--output", archive, `v${release}`], root);
+		run("tar", ["xf", archive, "-C", temporaryDir], root);
+
+		// The built-in models users have: the data shipped in the published package.
+		const [{ filename }] = JSON.parse(
+			run("npm", ["pack", `${PI_AI_PACKAGE}@${release}`, "--pack-destination", temporaryDir, "--json"], root),
+		);
+		run("tar", ["xzf", join(temporaryDir, filename), "-C", temporaryDir, "package/dist/providers/data"], root);
+		const builtin = readBuiltinModels(join(temporaryDir, "package/dist/providers/data"));
+		const catalog = JSON.parse(readFileSync(catalogPath, "utf8"));
+		const mergedPath = join(temporaryDir, "models.all.json");
+		writeFileSync(mergedPath, JSON.stringify(mergeReleaseCatalog(builtin, catalog)));
+
+		console.log("Installing the release's dependencies...");
+		run("npm", ["ci", "--ignore-scripts", "--no-audit", "--no-fund"], releaseDir);
+		// pi-ai's types depend on the declarations of its workspace dependencies.
+		const { dependencies } = JSON.parse(readFileSync(join(releaseDir, "packages/ai/package.json"), "utf8"));
+		for (const name of Object.keys(dependencies).filter((name) => name.startsWith("@earendil-works/"))) {
+			run("npm", ["run", "build", "--workspace", name], releaseDir);
+		}
+
+		console.log("Type-checking the release's pi-ai with the merged models...");
+		run(process.execPath, ["packages/ai/scripts/hydrate-model-catalog.ts", mergedPath], releaseDir);
+		try {
+			run(process.execPath, ["node_modules/typescript/bin/tsc", "-p", "packages/ai/tsconfig.build.json", "--noEmit"], releaseDir);
+		} catch (error) {
+			throw new Error(
+				[
+					error.message,
+					"",
+					`The model catalog does not type-check against pi ${release}'s pi-ai (errors above).`,
+					`pi.dev serves it to released pi >= ${MINIMUM_PI_VERSION}, so those clients would receive models`,
+					"their code cannot handle, e.g. a model whose api its provider does not implement.",
+					"If the change is intended, set MINIMUM_PI_VERSION in scripts/model-catalog-minimum-version.mjs",
+					"to the next release in the same commit.",
+				].join("\n"),
+			);
+		}
+		console.log(`The model catalog is compatible with pi ${release}.`);
+	} finally {
+		rmSync(temporaryDir, { recursive: true, force: true });
+	}
+}
+
+try {
+	main();
+} catch (error) {
+	console.error(error instanceof Error ? error.message : String(error));
+	process.exitCode = 1;
+}

scripts/model-catalog-minimum-version.mjs

@@ -0,0 +1,9 @@
+// The oldest pi version that receives model catalogs published from this checkout.
+// pi.dev keeps one catalog per minimum version; clients older than this keep the
+// last catalog published under the previous minimum and stop receiving updates.
+//
+// Bump this only when generated model metadata requires behavior unavailable in
+// older pi clients, in the same commit as the change that requires it.
+// 1.2.1: kimi-coding models use openai-completions at /coding/v1; older clients only
+// implement anthropic-messages for kimi-coding and ignore a model's api.
+export const MINIMUM_PI_VERSION = "1.2.1";

scripts/publish-model-catalog.mjs

@@ -21,14 +21,10 @@ import {
 	MODEL_CATALOG_SCHEMA_VERSION,
 	parseModelCatalogIndex,
 } from "./model-catalog-protocol.ts";
+import { MINIMUM_PI_VERSION } from "./model-catalog-minimum-version.mjs";
 
 // The storage layout, index format, and version ordering are defined in
 // model-catalog-protocol.ts, which pi.dev shares to serve these artifacts.
-// Bump this only when generated model metadata requires behavior unavailable in older pi clients.
-// Clients older than this keep the last catalog published under the previous minimum.
-// 1.2.1: kimi-coding models use openai-completions at /coding/v1; older clients only
-// implement anthropic-messages for kimi-coding and ignore a model's api.
-const MINIMUM_PI_VERSION = "1.2.1";
 const JSON_CONTENT_TYPE = "application/json; charset=utf-8";
 const IMMUTABLE_CACHE_CONTROL = "public, max-age=31536000, immutable";
 const INDEX_CACHE_CONTROL = "no-store";