diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index a78e43eee..4fe9504d7 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -16,6 +16,7 @@ - Added support for models of every type in `ModelsStoreEntry.models`. Stored and fetched models of unknown types are dropped instead of failing a refresh. - Added classifier models and `Models.classify()` with a provider-neutral JEV-style `choice`/`score`/`bool` contract. The built-in TypeSafe provider exposes models.dev's `jev-latest` through the System One API and translates public `bool` questions to TypeSafe's `noul` wire format. - Added Jev classifier models on OpenRouter (`typesafe/jev-1.13`, `~typesafe/jev-latest`) through its TypeSafe-compatible System One endpoint, and on Cloudflare Workers AI (`typesafe/jev`) through the new `cloudflare-workers-ai-system-one` classifier API. +- Added Jev classifier models on Vercel AI Gateway (`typesafe-ai/jev`, generated from its evaluation model catalog) and OpenCode Zen (`jev-1.13`, `jev-1.13-free`) through their TypeSafe-compatible System One endpoints. - Added a runtime chat-model check to the `Models` stream entry points so non-chat models fail with a clear `ModelsError` instead of a missing-api stream error. - Added array-based `models.all.json` and `providers/{id}.all.json` variants to the generated and published JSON catalog, allowing the same upstream ID once per model type; the existing keyed `models.json` and `providers/{id}.json` stay chat-only for released clients. - Added `onProviderStreamEvent` to observe parsed provider stream events before normalization, including provider-specific fields not retained in assistant messages ([#9784](https://github.com/earendil-works/pi/issues/9784)). diff --git a/packages/ai/README.md b/packages/ai/README.md index e5893e0d1..f6c4be02b 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -897,6 +897,8 @@ Classifier models consume structured JSON state and answer one or more typed que | `typesafe` | `jev-latest` | `TYPESAFE_API_KEY` | | `openrouter` | `typesafe/jev-1.13`, `~typesafe/jev-latest` | `OPENROUTER_API_KEY` or OpenRouter OAuth | | `cloudflare-workers-ai` | `typesafe/jev` | `CLOUDFLARE_API_KEY` and `CLOUDFLARE_ACCOUNT_ID` | +| `vercel-ai-gateway` | `typesafe-ai/jev` | `AI_GATEWAY_API_KEY` | +| `opencode` | `jev-1.13`, `jev-1.13-free` | `OPENCODE_API_KEY` | ```typescript import { builtinModels } from '@earendil-works/pi-ai/providers/all'; diff --git a/packages/ai/scripts/generate-models.ts b/packages/ai/scripts/generate-models.ts index 93ba057b5..6a0bbb85d 100644 --- a/packages/ai/scripts/generate-models.ts +++ b/packages/ai/scripts/generate-models.ts @@ -151,6 +151,7 @@ interface NvidiaNimModelListItem { interface AiGatewayModel { id: string; name?: string; + type?: string; context_window?: number; max_tokens?: number; tags?: string[]; @@ -222,6 +223,9 @@ const TOGETHER_TOGGLE_REASONING_LEVEL_MAP = { const AI_GATEWAY_MODELS_URL = "https://ai-gateway.vercel.sh/v1"; const AI_GATEWAY_BASE_URL = "https://ai-gateway.vercel.sh"; +// TypeSafe-compatible System One endpoint for evaluation models. +// https://vercel.com/docs/ai-gateway/sdks-and-apis/typesafe +const AI_GATEWAY_TYPESAFE_BASE_URL = "https://ai-gateway.vercel.sh/typesafe/v1"; const VERTEX_BASE_URL = "https://{location}-aiplatform.googleapis.com"; const NVIDIA_BASE_URL = "https://integrate.api.nvidia.com/v1"; const NVIDIA_HEADERS = { @@ -1328,13 +1332,17 @@ async function fetchRadiusModels(): Promise[]> { } } -async function fetchAiGatewayModels(): Promise[]> { +async function fetchAiGatewayModels(): Promise<{ + chat: Model[]; + classifiers: ClassifierModel<"typesafe-system-one">[]; +}> { try { console.log("Fetching models from Vercel AI Gateway API..."); const response = await fetch(`${AI_GATEWAY_MODELS_URL}/models`); if (!response.ok) throw new Error(`Vercel AI Gateway API returned ${response.status}`); const data = await response.json(); const models: Model[] = []; + const classifiers: ClassifierModel<"typesafe-system-one">[] = []; const toNumber = (value: string | number | undefined): number => { if (typeof value === "number") { @@ -1346,6 +1354,27 @@ async function fetchAiGatewayModels(): Promise[]> { const items = Array.isArray(data.data) ? (data.data as AiGatewayModel[]) : []; for (const model of items) { + // Evaluation models such as TypeSafe's Jev are served through the + // TypeSafe-compatible System One endpoint. + if (model.type === "evaluation") { + classifiers.push({ + type: "classifier", + id: model.id, + name: model.name || model.id, + api: "typesafe-system-one", + provider: "vercel-ai-gateway", + baseUrl: AI_GATEWAY_TYPESAFE_BASE_URL, + input: ["text"], + cost: { + input: roundCost(toNumber(model.pricing?.input) * 1_000_000), + output: roundCost(toNumber(model.pricing?.output) * 1_000_000), + cacheRead: 0, + cacheWrite: 0, + }, + contextWindow: model.context_window || 4096, + }); + continue; + } const tags = Array.isArray(model.tags) ? model.tags : []; // Only include models that support tools if (!tags.includes("tool-use")) continue; @@ -1380,12 +1409,14 @@ async function fetchAiGatewayModels(): Promise[]> { }); } - console.log(`Fetched ${models.length} tool-capable models from Vercel AI Gateway`); - return models; + console.log( + `Fetched ${models.length} tool-capable and ${classifiers.length} classifier models from Vercel AI Gateway`, + ); + return { chat: models, classifiers }; } catch (error) { console.error("Failed to fetch Vercel AI Gateway models:", error); if (generatorOptions.strict) throw error; - return []; + return { chat: [], classifiers: [] }; } } @@ -2665,6 +2696,34 @@ async function loadModelsDevClassifierModels(): Promise[] = [ + { + type: "classifier", + id: "jev-1.13", + name: "Jev 1.13", + api: "typesafe-system-one", + provider: "opencode", + baseUrl: "https://opencode.ai/zen/v1", + input: ["text"], + cost: { input: 0.042, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 32000, + }, + { + type: "classifier", + id: "jev-1.13-free", + name: "Jev 1.13 Free", + api: "typesafe-system-one", + provider: "opencode", + baseUrl: "https://opencode.ai/zen/v1", + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 32000, + }, +]; + const CLOUDFLARE_WORKERS_AI_CLASSIFIER_MODELS: ClassifierModel<"cloudflare-workers-ai-system-one">[] = [ { type: "classifier", @@ -2688,11 +2747,11 @@ async function generateModels() { const modelsDevModels = await loadModelsDevData(); const modelsDevClassifierModels = await loadModelsDevClassifierModels(); const openRouterCatalog = await fetchOpenRouterModels(); - const aiGatewayModels = await fetchAiGatewayModels(); + const aiGatewayCatalog = await fetchAiGatewayModels(); const radiusModels = await fetchRadiusModels(); // Combine chat models (models.dev has priority where sources overlap). - const allModels = [...modelsDevModels, ...openRouterCatalog.chat, ...aiGatewayModels, ...radiusModels].filter( + const allModels = [...modelsDevModels, ...openRouterCatalog.chat, ...aiGatewayCatalog.chat, ...radiusModels].filter( (model) => !(model.provider === "xai" && XAI_BUILTIN_EXCLUDED_MODEL_IDS.has(model.id)) && !((model.provider === "opencode" || model.provider === "opencode-go") && model.id === "gpt-5.3-codex-spark"), @@ -3357,6 +3416,8 @@ async function generateModels() { const classifierModels: ClassifierModel[] = [ ...modelsDevClassifierModels, ...openRouterCatalog.classifiers, + ...aiGatewayCatalog.classifiers, + ...OPENCODE_CLASSIFIER_MODELS, ...CLOUDFLARE_WORKERS_AI_CLASSIFIER_MODELS, ]; for (const model of classifierModels) { diff --git a/packages/ai/src/providers/opencode.ts b/packages/ai/src/providers/opencode.ts index f6126f68d..7b3aae419 100644 --- a/packages/ai/src/providers/opencode.ts +++ b/packages/ai/src/providers/opencode.ts @@ -2,24 +2,27 @@ import { anthropicMessagesApi } from "../api/anthropic-messages.lazy.ts"; import { googleGenerativeAIApi } from "../api/google-generative-ai.lazy.ts"; import { openAICompletionsApi } from "../api/openai-completions.lazy.ts"; import { openAIResponsesApi } from "../api/openai-responses.lazy.ts"; +import { typesafeSystemOneApi } from "../api/typesafe-system-one.lazy.ts"; import { envApiKeyAuth } from "../auth/helpers.ts"; import { createProvider, type Provider } from "../models.ts"; -import { OPENCODE_MODELS } from "./opencode.models.ts"; +import { OPENCODE_CLASSIFIER_MODELS, OPENCODE_MODELS } from "./opencode.models.ts"; import { withOpenCodeSessionHeader } from "./opencode-headers.ts"; -export function opencodeProvider(): Provider< - "anthropic-messages" | "google-generative-ai" | "openai-completions" | "openai-responses" -> { - return createProvider({ +type OpenCodeApi = "anthropic-messages" | "google-generative-ai" | "openai-completions" | "openai-responses"; + +export function opencodeProvider(): Provider { + return createProvider({ id: "opencode", name: "OpenCode Zen", auth: { apiKey: envApiKeyAuth("OpenCode API key", ["OPENCODE_API_KEY"]) }, - models: Object.values(OPENCODE_MODELS), + models: [...Object.values(OPENCODE_MODELS), ...Object.values(OPENCODE_CLASSIFIER_MODELS)], api: { "anthropic-messages": withOpenCodeSessionHeader(anthropicMessagesApi()), "google-generative-ai": withOpenCodeSessionHeader(googleGenerativeAIApi()), "openai-completions": withOpenCodeSessionHeader(openAICompletionsApi()), "openai-responses": withOpenCodeSessionHeader(openAIResponsesApi()), }, + // OpenCode Zen serves TypeSafe's System One protocol at /zen/v1/systemone. + classifiers: { "typesafe-system-one": typesafeSystemOneApi() }, }); } diff --git a/packages/ai/src/providers/vercel-ai-gateway.ts b/packages/ai/src/providers/vercel-ai-gateway.ts index 3aca03283..ec2155793 100644 --- a/packages/ai/src/providers/vercel-ai-gateway.ts +++ b/packages/ai/src/providers/vercel-ai-gateway.ts @@ -1,15 +1,18 @@ import { anthropicMessagesApi } from "../api/anthropic-messages.lazy.ts"; +import { typesafeSystemOneApi } from "../api/typesafe-system-one.lazy.ts"; import { envApiKeyAuth } from "../auth/helpers.ts"; import { createProvider, type Provider } from "../models.ts"; -import { VERCEL_AI_GATEWAY_MODELS } from "./vercel-ai-gateway.models.ts"; +import { VERCEL_AI_GATEWAY_CLASSIFIER_MODELS, VERCEL_AI_GATEWAY_MODELS } from "./vercel-ai-gateway.models.ts"; export function vercelAIGatewayProvider(): Provider<"anthropic-messages"> { - return createProvider({ + return createProvider<"anthropic-messages">({ id: "vercel-ai-gateway", name: "Vercel AI Gateway", baseUrl: "https://ai-gateway.vercel.sh", auth: { apiKey: envApiKeyAuth("Vercel AI Gateway API key", ["AI_GATEWAY_API_KEY"]) }, - models: Object.values(VERCEL_AI_GATEWAY_MODELS), + models: [...Object.values(VERCEL_AI_GATEWAY_MODELS), ...Object.values(VERCEL_AI_GATEWAY_CLASSIFIER_MODELS)], api: anthropicMessagesApi(), + // AI Gateway serves TypeSafe's System One protocol at /typesafe/v1/systemone. + classifiers: { "typesafe-system-one": typesafeSystemOneApi() }, }); } diff --git a/packages/ai/test/classifier-models.test.ts b/packages/ai/test/classifier-models.test.ts index f1e7d2c2f..9ef488e35 100644 --- a/packages/ai/test/classifier-models.test.ts +++ b/packages/ai/test/classifier-models.test.ts @@ -127,6 +127,41 @@ describe("Models with classifier models", () => { expect(models.getModelOfType("classifier", "typesafe", "jev-latest")).toEqual(jev); }); + it.each([ + ["vercel-ai-gateway", "typesafe-ai/jev", "https://ai-gateway.vercel.sh/typesafe/v1/systemone"], + ["opencode", "jev-1.13", "https://opencode.ai/zen/v1/systemone"], + ["opencode", "jev-1.13-free", "https://opencode.ai/zen/v1/systemone"], + ])("routes %s Jev (%s) to its TypeSafe-compatible endpoint", async (provider, id, url) => { + const models = builtinModels(); + const jev = models.getModelOfType("classifier", provider, id); + if (!jev) throw new Error(`missing ${provider} Jev model`); + expect(jev).toMatchObject({ api: "typesafe-system-one", contextWindow: 32000 }); + expect(models.getModel(provider, id)).toBeUndefined(); + + const requests: Array<{ url: string; body: Record; authorization: string | null }> = []; + const result = await models.classify(jev, context, { + apiKey: "secret", + fetch: async (input, init) => { + requests.push({ + url: String(input), + body: JSON.parse(String(init?.body)) as Record, + authorization: new Headers(init?.headers).get("authorization"), + }); + return Response.json({ model: id, answers: { approved: { type: "noul", noul: 0.8 } } }); + }, + }); + + expect(requests).toEqual([ + { + url, + body: expect.objectContaining({ model: id, state: context.state }), + authorization: "Bearer secret", + }, + ]); + expect(result.stopReason).toBe("stop"); + expect(result.answers.approved).toEqual({ type: "bool", probability: 0.8 }); + }); + it("routes OpenRouter classifier models through the System One API", () => { const models = builtinModels(); for (const model of getBuiltinClassifierModels("openrouter")) { diff --git a/packages/coding-agent/docs/models.md b/packages/coding-agent/docs/models.md index 01ad0ee16..bae53947d 100644 --- a/packages/coding-agent/docs/models.md +++ b/packages/coding-agent/docs/models.md @@ -102,13 +102,15 @@ Compatibility settings should describe verified differences in the endpoint's re ## Use classifier models -Classifier models do not chat. They answer typed questions about JSON state: pick one of several choices, answer yes or no, or give a score, each with probabilities. Pi includes TypeSafe's Jev model from three providers: +Classifier models do not chat. They answer typed questions about JSON state: pick one of several choices, answer yes or no, or give a score, each with probabilities. Pi includes TypeSafe's Jev model from these providers: | Provider | Model IDs | Authentication | |---|---|---| | `typesafe` | `jev-latest` | `TYPESAFE_API_KEY` | | `openrouter` | `typesafe/jev-1.13`, `~typesafe/jev-latest` | `OPENROUTER_API_KEY` or `/login` | | `cloudflare-workers-ai` | `typesafe/jev` | `CLOUDFLARE_API_KEY` and `CLOUDFLARE_ACCOUNT_ID` | +| `vercel-ai-gateway` | `typesafe-ai/jev` | `AI_GATEWAY_API_KEY` | +| `opencode` | `jev-1.13`, `jev-1.13-free` | `OPENCODE_API_KEY` | Chat models on a [llama.cpp router](llama-cpp.md#classification) are also listed as classifier models.