diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index c0f5e5ed9..1715cde27 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -15,6 +15,7 @@ - Added `getModelsOfType()`, `getModelOfType()`, `getAvailableOfType()`, `getAllModels()`, and `getAllAvailable()` on `Models`; optional `Provider.getAllModels()` and `Provider.filterAllModels()`; corresponding generated-catalog accessors; and the `AnyModel` and `ModelTypeMap` types. `hasApi()`, `calculateCost()`, and `modelsAreEqual()` accept `AnyModel`. - Added support for models of every type in `ModelsStoreEntry.models`. Stored and fetched models of unknown types are dropped instead of failing a refresh. - Added classifier models and `Models.classify()` with a provider-neutral JEV-style `choice`/`score`/`bool` contract. The built-in TypeSafe provider exposes models.dev's `jev-latest` through the System One API and translates public `bool` questions to TypeSafe's `noul` wire format. +- Added Jev classifier models on OpenRouter (`typesafe/jev-1.13`, `~typesafe/jev-latest`) through its TypeSafe-compatible System One endpoint, and on Cloudflare Workers AI (`typesafe/jev`) through the new `cloudflare-workers-ai-system-one` classifier API. - Added a runtime chat-model check to the `Models` stream entry points so non-chat models fail with a clear `ModelsError` instead of a missing-api stream error. - Added array-based `models.all.json` and `providers/{id}.all.json` variants to the generated and published JSON catalog, allowing the same upstream ID once per model type; the existing keyed `models.json` and `providers/{id}.json` stay chat-only for released clients. - Added `onProviderStreamEvent` to observe parsed provider stream events before normalization, including provider-specific fields not retained in assistant messages ([#9784](https://github.com/earendil-works/pi/issues/9784)). diff --git a/packages/ai/README.md b/packages/ai/README.md index 11979c1c2..a4a2a99a3 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -890,7 +890,13 @@ console.log(model.output); // ['image'] or ['image', 'text'] ## Classification -Classifier models consume structured JSON state and answer one or more typed questions. They do not use chat or image-generation APIs. The built-in TypeSafe provider exposes models.dev's `jev-latest` model and reads `TYPESAFE_API_KEY`. +Classifier models consume structured JSON state and answer one or more typed questions. They do not use chat or image-generation APIs. TypeSafe's Jev model is available from these built-in providers: + +| Provider | Model IDs | Auth | +| --- | --- | --- | +| `typesafe` | `jev-latest` | `TYPESAFE_API_KEY` | +| `openrouter` | `typesafe/jev-1.13`, `~typesafe/jev-latest` | `OPENROUTER_API_KEY` or OpenRouter OAuth | +| `cloudflare-workers-ai` | `typesafe/jev` | `CLOUDFLARE_API_KEY` and `CLOUDFLARE_ACCOUNT_ID` | ```typescript import { builtinModels } from '@earendil-works/pi-ai/providers/all'; diff --git a/packages/ai/scripts/generate-models.ts b/packages/ai/scripts/generate-models.ts index c787395b2..b4068cfdb 100644 --- a/packages/ai/scripts/generate-models.ts +++ b/packages/ai/scripts/generate-models.ts @@ -10,6 +10,7 @@ import { CLOUDFLARE_AI_GATEWAY_COMPAT_BASE_URL, CLOUDFLARE_AI_GATEWAY_OPENAI_BASE_URL, CLOUDFLARE_WORKERS_AI_BASE_URL, + CLOUDFLARE_WORKERS_AI_REST_BASE_URL, } from "../src/api/cloudflare.ts"; import type { AnthropicMessagesCompat, @@ -1284,12 +1285,15 @@ async function fetchOpenRouterList(query: string): Promise { try { console.log("Fetching models from OpenRouter API..."); - const [listed, imageListed] = await Promise.all([ + const [listed, imageListed, decisionListed] = await Promise.all([ fetchOpenRouterList(""), fetchOpenRouterList("?output_modalities=image"), + fetchOpenRouterList("?output_modalities=decisions"), ]); - const catalog = buildOpenRouterCatalog(listed, imageListed); - console.log(`Fetched ${catalog.chat.length} tool-capable and ${catalog.images.length} image models from OpenRouter`); + const catalog = buildOpenRouterCatalog(listed, imageListed, decisionListed); + console.log( + `Fetched ${catalog.chat.length} tool-capable, ${catalog.images.length} image, and ${catalog.classifiers.length} classifier models from OpenRouter`, + ); if (generatorOptions.strict && catalog.images.length === 0) { throw new Error("OpenRouter API returned no usable image models"); } @@ -1297,7 +1301,7 @@ async function fetchOpenRouterModels(): Promise { } catch (error) { console.error("Failed to fetch OpenRouter models:", error); if (generatorOptions.strict) throw error; - return { chat: [], images: [] }; + return { chat: [], images: [], classifiers: [] }; } } @@ -2646,6 +2650,23 @@ async function loadModelsDevClassifierModels(): Promise[] = [ + { + type: "classifier", + id: "typesafe/jev", + name: "Jev", + api: "cloudflare-workers-ai-system-one", + provider: "cloudflare-workers-ai", + baseUrl: CLOUDFLARE_WORKERS_AI_REST_BASE_URL, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 32000, + }, +]; + async function generateModels() { // Fetch models from all upstream catalogs. // models.dev: Anthropic, Google, OpenAI, Groq, Cerebras, and others @@ -2653,7 +2674,7 @@ async function generateModels() { // AI Gateway: OpenAI-compatible catalog with tool-capable models // Radius: its unauthenticated public catalog; authenticated clients overlay it at runtime const modelsDevModels = await loadModelsDevData(); - const classifierModels = await loadModelsDevClassifierModels(); + const modelsDevClassifierModels = await loadModelsDevClassifierModels(); const openRouterCatalog = await fetchOpenRouterModels(); const aiGatewayModels = await fetchAiGatewayModels(); const radiusModels = await fetchRadiusModels(); @@ -3293,6 +3314,11 @@ async function generateModels() { providers[model.provider] ??= { chat: {}, image: {}, classifier: {} }; providers[model.provider].image[model.id] ??= model; } + const classifierModels: ClassifierModel[] = [ + ...modelsDevClassifierModels, + ...openRouterCatalog.classifiers, + ...CLOUDFLARE_WORKERS_AI_CLASSIFIER_MODELS, + ]; for (const model of classifierModels) { providers[model.provider] ??= { chat: {}, image: {}, classifier: {} }; providers[model.provider].classifier[model.id] ??= model; diff --git a/packages/ai/scripts/openrouter-catalog.ts b/packages/ai/scripts/openrouter-catalog.ts index a6e7f2c18..1faf0febe 100644 --- a/packages/ai/scripts/openrouter-catalog.ts +++ b/packages/ai/scripts/openrouter-catalog.ts @@ -1,4 +1,4 @@ -import type { ImageModel, Model, ModelCost } from "../src/types.ts"; +import type { ClassifierModel, ImageModel, Model, ModelCost } from "../src/types.ts"; import { getOpenRouterThinkingLevelMap, type OpenRouterReasoningMetadata } from "./openrouter-reasoning-options.ts"; export interface OpenRouterModelListItem { @@ -23,6 +23,7 @@ export interface OpenRouterModelListItem { export interface OpenRouterCatalog { chat: Model<"anthropic-messages" | "openai-completions">[]; images: ImageModel<"openrouter-images">[]; + classifiers: ClassifierModel<"typesafe-system-one">[]; } function roundCost(value: number): number { @@ -47,13 +48,15 @@ function cost(model: OpenRouterModelListItem): ModelCost { /** * Build the OpenRouter catalog from the default listing and the - * `output_modalities=image` listing. The default listing omits image-only - * models, so image models come from the second one. An upstream model may - * appear in both results; it then gets separate chat and image entries. + * `output_modalities=image` and `output_modalities=decisions` listings. The + * default listing omits image-only and decision models, so those come from + * the other listings. An upstream model may appear in several results; it then + * gets separate entries per operation. */ export function buildOpenRouterCatalog( listed: readonly OpenRouterModelListItem[], imageListed: readonly OpenRouterModelListItem[], + decisionListed: readonly OpenRouterModelListItem[], ): OpenRouterCatalog { const chat: OpenRouterCatalog["chat"] = []; @@ -103,5 +106,25 @@ export function buildOpenRouterCatalog( }); } - return { chat, images }; + // Decision models such as TypeSafe's Jev are served through OpenRouter's + // TypeSafe-compatible System One endpoint. + const classifiers: OpenRouterCatalog["classifiers"] = []; + for (const model of decisionListed) { + if (classifiers.some((entry) => entry.id === model.id)) continue; + if (!model.architecture?.output_modalities?.includes("decisions")) continue; + const input = modalities(model.architecture.input_modalities); + classifiers.push({ + type: "classifier", + id: model.id, + name: model.name, + api: "typesafe-system-one", + provider: "openrouter", + baseUrl: "https://openrouter.ai/api/v1", + input: input.length > 0 ? input : ["text"], + cost: cost(model), + contextWindow: model.top_provider?.context_length || model.context_length || 4096, + }); + } + + return { chat, images, classifiers }; } diff --git a/packages/ai/src/api/cloudflare-workers-ai-system-one.lazy.ts b/packages/ai/src/api/cloudflare-workers-ai-system-one.lazy.ts new file mode 100644 index 000000000..87d13d915 --- /dev/null +++ b/packages/ai/src/api/cloudflare-workers-ai-system-one.lazy.ts @@ -0,0 +1,6 @@ +import type { ProviderClassifier } from "../types.ts"; + +export const cloudflareWorkersAISystemOneApi = (): ProviderClassifier => ({ + classify: async (model, context, options) => + (await import("./cloudflare-workers-ai-system-one.ts")).classify(model, context, options), +}); diff --git a/packages/ai/src/api/cloudflare-workers-ai-system-one.ts b/packages/ai/src/api/cloudflare-workers-ai-system-one.ts new file mode 100644 index 000000000..9f4c50653 --- /dev/null +++ b/packages/ai/src/api/cloudflare-workers-ai-system-one.ts @@ -0,0 +1,43 @@ +import type { ClassifierFunction, ClassifierOptions } from "../types.ts"; +import { classifySystemOne, isRecord, type SystemOneTransport } from "./system-one-shared.ts"; + +const LABEL = "Cloudflare Workers AI"; + +function cloudflareErrorMessage(errors: unknown): string { + if (Array.isArray(errors)) { + const messages = errors + .map((error) => (isRecord(error) && typeof error.message === "string" ? error.message : undefined)) + .filter((message): message is string => message !== undefined); + if (messages.length > 0) return `${LABEL} error: ${messages.join("; ")}`; + } + return `${LABEL} request failed`; +} + +/** + * System One models on the Workers AI REST endpoint: + * `POST /accounts/{account}/ai/run` with `{ model, input }`. The REST API + * wraps the model output in Cloudflare's API envelope and a run record: + * `{ success, result: { state: "Completed", result: { answers, usage } } }`. + * https://developers.cloudflare.com/ai/models/typesafe/jev/ + */ +const transport: SystemOneTransport = { + api: "cloudflare-workers-ai-system-one", + label: LABEL, + url: (model) => new URL("run", `${model.baseUrl.replace(/\/+$/u, "")}/`), + payload: (model, request) => ({ model: model.id, input: request }), + answers: (body) => { + if (!isRecord(body)) throw new Error(`${LABEL} returned an unexpected response`); + if (body.success === false) throw new Error(cloudflareErrorMessage(body.errors)); + const run = body.result; + if (!isRecord(run)) throw new Error(`${LABEL} returned an unexpected response`); + if (run.state !== "Completed") { + throw new Error(`${LABEL} run did not complete (state: ${String(run.state)})`); + } + if (!isRecord(run.result)) throw new Error(`${LABEL} returned an unexpected response`); + return run.result.answers; + }, +}; + +/** Cloudflare Workers AI System One classification with public `bool` values mapped to wire-level `noul`. */ +export const classify: ClassifierFunction = (model, context, options) => + classifySystemOne(transport, model, context, options); diff --git a/packages/ai/src/api/cloudflare.ts b/packages/ai/src/api/cloudflare.ts index f8b1138ce..238c9eaab 100644 --- a/packages/ai/src/api/cloudflare.ts +++ b/packages/ai/src/api/cloudflare.ts @@ -2,6 +2,10 @@ export const CLOUDFLARE_WORKERS_AI_BASE_URL = "https://api.cloudflare.com/client/v4/accounts/{CLOUDFLARE_ACCOUNT_ID}/ai/v1"; +/** Workers AI REST endpoint root; model runs are posted to `run`. */ +export const CLOUDFLARE_WORKERS_AI_REST_BASE_URL = + "https://api.cloudflare.com/client/v4/accounts/{CLOUDFLARE_ACCOUNT_ID}/ai"; + /** AI Gateway Unified API. https://developers.cloudflare.com/ai-gateway/usage/unified-api/ */ export const CLOUDFLARE_AI_GATEWAY_COMPAT_BASE_URL = "https://gateway.ai.cloudflare.com/v1/{CLOUDFLARE_ACCOUNT_ID}/{CLOUDFLARE_GATEWAY_ID}/compat"; diff --git a/packages/ai/src/api/system-one-shared.ts b/packages/ai/src/api/system-one-shared.ts new file mode 100644 index 000000000..1752d28b7 --- /dev/null +++ b/packages/ai/src/api/system-one-shared.ts @@ -0,0 +1,207 @@ +import type { + ClassifierAnswer, + ClassifierApi, + ClassifierContext, + ClassifierModel, + ClassifierOptions, + ClassifierResult, + ProviderHeaders, +} from "../types.ts"; +import { formatProviderError, normalizeProviderError } from "../utils/error-body.ts"; +import { headersToRecord, providerHeadersToRecord } from "../utils/headers.ts"; +import { retryProviderRequest } from "../utils/provider-retry.ts"; + +/** TypeSafe System One request body without the transport-specific envelope. */ +export interface SystemOneWireRequest { + state: ClassifierContext["state"]; + questions: Record; +} + +/** Differences between services that serve System One models. */ +export interface SystemOneTransport { + /** Classifier API implemented by this transport. */ + api: ClassifierApi; + /** Service name used in error messages. */ + label: string; + /** Absolute request URL. */ + url(model: ClassifierModel): URL; + /** Wraps the System One request in the service's request envelope. */ + payload(model: ClassifierModel, request: SystemOneWireRequest): unknown; + /** Extracts the System One `answers` object from the service's response envelope. */ + answers(body: unknown): unknown; +} + +interface SystemOneHttpError extends Error { + status: number | undefined; + headers: Headers | undefined; + body: string; +} + +function httpError(label: string, response: Response, body: string): SystemOneHttpError { + const error = new Error(`${label} returned ${response.status}`) as SystemOneHttpError; + error.status = response.status; + error.headers = response.headers; + error.body = body; + return error; +} + +function timeoutError(timeoutMs: number): SystemOneHttpError { + const error = new Error(`Request timed out after ${timeoutMs}ms`) as SystemOneHttpError; + error.name = "TimeoutError"; + error.status = undefined; + error.headers = undefined; + error.body = ""; + return error; +} + +export function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function requiredNumber(label: string, value: unknown, field: string): number { + if (typeof value !== "number" || !Number.isFinite(value)) { + throw new Error(`${label} returned an invalid ${field}`); + } + return value; +} + +function probabilities(label: string, value: unknown, id: string): Record { + if (!isRecord(value)) throw new Error(`${label} returned invalid probabilities for ${id}`); + return Object.fromEntries( + Object.entries(value).map(([key, probability]) => [ + key, + requiredNumber(label, probability, `probability for ${id}.${key}`), + ]), + ); +} + +function parseAnswers(label: string, value: unknown, context: ClassifierContext): Record { + if (!isRecord(value)) throw new Error(`${label} returned an unexpected response`); + const answers: Array<[string, ClassifierAnswer]> = []; + for (const [id, question] of Object.entries(context.questions)) { + const answer = value[id]; + if (!isRecord(answer)) throw new Error(`${label} did not return an answer for ${id}`); + if (question.type === "choice") { + if (answer.type !== "choice" || typeof answer.choice !== "string") { + throw new Error(`${label} did not return a choice answer for ${id}`); + } + answers.push([ + id, + { + type: "choice", + choice: answer.choice, + probabilities: probabilities(label, answer.probabilities, id), + confidence: requiredNumber(label, answer.confidence, `confidence for ${id}`), + }, + ]); + } else if (question.type === "score") { + if (answer.type !== "score") throw new Error(`${label} did not return a score answer for ${id}`); + answers.push([ + id, + { + type: "score", + score: requiredNumber(label, answer.score, `score for ${id}`), + confidence: requiredNumber(label, answer.confidence, `confidence for ${id}`), + }, + ]); + } else { + if (answer.type !== "noul") throw new Error(`${label} did not return a bool answer for ${id}`); + answers.push([ + id, + { + type: "bool", + probability: requiredNumber(label, answer.noul, `probability for ${id}`), + }, + ]); + } + } + return Object.fromEntries(answers); +} + +/** Maps public `bool` questions to TypeSafe's wire-level `noul` type. */ +function wireRequest(context: ClassifierContext): SystemOneWireRequest { + return { + state: context.state, + questions: Object.fromEntries( + Object.entries(context.questions).map(([id, question]) => [ + id, + question.type === "bool" ? { ...question, type: "noul" } : question, + ]), + ), + }; +} + +function requestHeaders( + model: ClassifierModel, + apiKey: string, + optionsHeaders?: ProviderHeaders, +): Record { + return ( + providerHeadersToRecord( + { authorization: `Bearer ${apiKey}`, "content-type": "application/json" }, + model.headers, + optionsHeaders, + ) ?? {} + ); +} + +/** Runs one System One classification over the given transport. */ +export async function classifySystemOne( + transport: SystemOneTransport, + model: ClassifierModel, + context: ClassifierContext, + options: ClassifierOptions | undefined, +): Promise { + const output: ClassifierResult = { + api: model.api, + provider: model.provider, + model: model.id, + answers: {}, + stopReason: "stop", + timestamp: Date.now(), + }; + + try { + if (model.api !== transport.api) throw new Error(`Unsupported classifier API: ${model.api}`); + if (!options?.apiKey) throw new Error(`No API key for provider: ${model.provider}`); + const apiKey = options.apiKey; + let payload = transport.payload(model, wireRequest(context)); + const transformed = await options.onPayload?.(payload, model); + if (transformed !== undefined) payload = transformed; + const requestFetch = options.fetch ?? globalThis.fetch; + const { response, body } = await retryProviderRequest( + async () => { + const timeoutSignal = options.timeoutMs !== undefined ? AbortSignal.timeout(options.timeoutMs) : undefined; + const signal = + options.signal && timeoutSignal + ? AbortSignal.any([options.signal, timeoutSignal]) + : (options.signal ?? timeoutSignal); + try { + const next = await requestFetch(transport.url(model), { + method: "POST", + headers: requestHeaders(model, apiKey, options.headers), + body: JSON.stringify(payload), + signal, + }); + if (!next.ok) throw httpError(transport.label, next, await next.text()); + return { response: next, body: (await next.json()) as unknown }; + } catch (error) { + if (timeoutSignal?.aborted && !options.signal?.aborted) throw timeoutError(options.timeoutMs!); + throw error; + } + }, + { + maxRetries: options.maxRetries ?? 2, + maxRetryDelayMs: options.maxRetryDelayMs, + signal: options.signal, + }, + ); + await options.onResponse?.({ status: response.status, headers: headersToRecord(response.headers) }, model); + output.answers = parseAnswers(transport.label, transport.answers(body), context); + return output; + } catch (error) { + output.stopReason = options?.signal?.aborted ? "aborted" : "error"; + output.errorMessage = formatProviderError(normalizeProviderError(error), `${transport.label} error`); + return output; + } +} diff --git a/packages/ai/src/api/typesafe-system-one.ts b/packages/ai/src/api/typesafe-system-one.ts index 256139b41..c49761d8a 100644 --- a/packages/ai/src/api/typesafe-system-one.ts +++ b/packages/ai/src/api/typesafe-system-one.ts @@ -1,187 +1,21 @@ -import type { - ClassifierAnswer, - ClassifierApi, - ClassifierContext, - ClassifierFunction, - ClassifierModel, - ClassifierOptions, - ClassifierResult, - ProviderHeaders, -} from "../types.ts"; -import { formatProviderError, normalizeProviderError } from "../utils/error-body.ts"; -import { headersToRecord, providerHeadersToRecord } from "../utils/headers.ts"; -import { retryProviderRequest } from "../utils/provider-retry.ts"; +import type { ClassifierFunction, ClassifierOptions } from "../types.ts"; +import { classifySystemOne, isRecord, type SystemOneTransport } from "./system-one-shared.ts"; -interface TypeSafeHttpError extends Error { - status: number | undefined; - headers: Headers | undefined; - body: string; -} - -function httpError(response: Response, body: string): TypeSafeHttpError { - const error = new Error(`TypeSafe API returned ${response.status}`) as TypeSafeHttpError; - error.status = response.status; - error.headers = response.headers; - error.body = body; - return error; -} - -function timeoutError(timeoutMs: number): TypeSafeHttpError { - const error = new Error(`Request timed out after ${timeoutMs}ms`) as TypeSafeHttpError; - error.name = "TimeoutError"; - error.status = undefined; - error.headers = undefined; - error.body = ""; - return error; -} - -function isRecord(value: unknown): value is Record { - return typeof value === "object" && value !== null && !Array.isArray(value); -} - -function requiredNumber(value: unknown, label: string): number { - if (typeof value !== "number" || !Number.isFinite(value)) { - throw new Error(`TypeSafe API returned an invalid ${label}`); - } - return value; -} - -function probabilities(value: unknown, label: string): Record { - if (!isRecord(value)) throw new Error(`TypeSafe API returned invalid probabilities for ${label}`); - return Object.fromEntries( - Object.entries(value).map(([key, probability]) => [ - key, - requiredNumber(probability, `probability for ${label}.${key}`), - ]), - ); -} - -function parseAnswers(value: unknown, context: ClassifierContext): Record { - if (!isRecord(value)) throw new Error("TypeSafe API returned an unexpected response"); - const answers: Array<[string, ClassifierAnswer]> = []; - for (const [id, question] of Object.entries(context.questions)) { - const answer = value[id]; - if (!isRecord(answer)) throw new Error(`TypeSafe API did not return an answer for ${id}`); - if (question.type === "choice") { - if (answer.type !== "choice" || typeof answer.choice !== "string") { - throw new Error(`TypeSafe API did not return a choice answer for ${id}`); - } - answers.push([ - id, - { - type: "choice", - choice: answer.choice, - probabilities: probabilities(answer.probabilities, id), - confidence: requiredNumber(answer.confidence, `confidence for ${id}`), - }, - ]); - } else if (question.type === "score") { - if (answer.type !== "score") throw new Error(`TypeSafe API did not return a score answer for ${id}`); - answers.push([ - id, - { - type: "score", - score: requiredNumber(answer.score, `score for ${id}`), - confidence: requiredNumber(answer.confidence, `confidence for ${id}`), - }, - ]); - } else { - if (answer.type !== "noul") throw new Error(`TypeSafe API did not return a bool answer for ${id}`); - answers.push([ - id, - { - type: "bool", - probability: requiredNumber(answer.noul, `probability for ${id}`), - }, - ]); - } - } - return Object.fromEntries(answers); -} - -function wirePayload(model: ClassifierModel, context: ClassifierContext): Record { - return { - model: model.id, - state: context.state, - questions: Object.fromEntries( - Object.entries(context.questions).map(([id, question]) => [ - id, - question.type === "bool" ? { ...question, type: "noul" } : question, - ]), - ), - }; -} - -function requestHeaders( - model: ClassifierModel, - apiKey: string, - optionsHeaders?: ProviderHeaders, -): Record { - return ( - providerHeadersToRecord( - { authorization: `Bearer ${apiKey}`, "content-type": "application/json" }, - model.headers, - optionsHeaders, - ) ?? {} - ); -} +/** + * TypeSafe's native System One protocol. OpenRouter serves the same protocol, + * so both providers use this API with different base URLs. + */ +const transport: SystemOneTransport = { + api: "typesafe-system-one", + label: "System One API", + url: (model) => new URL("systemone", `${model.baseUrl.replace(/\/+$/u, "")}/`), + payload: (model, request) => ({ model: model.id, ...request }), + answers: (body) => { + if (!isRecord(body)) throw new Error("System One API returned an unexpected response"); + return body.answers; + }, +}; /** TypeSafe System One classification with public `bool` values mapped to wire-level `noul`. */ -export const classify: ClassifierFunction = async ( - model, - context, - options, -): Promise => { - const output: ClassifierResult = { - api: model.api, - provider: model.provider, - model: model.id, - answers: {}, - stopReason: "stop", - timestamp: Date.now(), - }; - - try { - if (model.api !== "typesafe-system-one") throw new Error(`Unsupported classifier API: ${model.api}`); - if (!options?.apiKey) throw new Error(`No API key for provider: ${model.provider}`); - let payload: unknown = wirePayload(model, context); - const transformed = await options.onPayload?.(payload, model); - if (transformed !== undefined) payload = transformed; - const requestFetch = options.fetch ?? globalThis.fetch; - const { response, body } = await retryProviderRequest( - async () => { - const timeoutSignal = options.timeoutMs !== undefined ? AbortSignal.timeout(options.timeoutMs) : undefined; - const signal = - options.signal && timeoutSignal - ? AbortSignal.any([options.signal, timeoutSignal]) - : (options.signal ?? timeoutSignal); - try { - const next = await requestFetch(new URL("systemone", `${model.baseUrl.replace(/\/+$/u, "")}/`), { - method: "POST", - headers: requestHeaders(model, options.apiKey!, options.headers), - body: JSON.stringify(payload), - signal, - }); - if (!next.ok) throw httpError(next, await next.text()); - return { response: next, body: (await next.json()) as unknown }; - } catch (error) { - if (timeoutSignal?.aborted && !options.signal?.aborted) throw timeoutError(options.timeoutMs!); - throw error; - } - }, - { - maxRetries: options.maxRetries ?? 2, - maxRetryDelayMs: options.maxRetryDelayMs, - signal: options.signal, - }, - ); - await options.onResponse?.({ status: response.status, headers: headersToRecord(response.headers) }, model); - if (!isRecord(body)) throw new Error("TypeSafe API returned an unexpected response"); - output.answers = parseAnswers(body.answers, context); - return output; - } catch (error) { - output.stopReason = options?.signal?.aborted ? "aborted" : "error"; - output.errorMessage = formatProviderError(normalizeProviderError(error), "TypeSafe API error"); - return output; - } -}; +export const classify: ClassifierFunction = (model, context, options) => + classifySystemOne(transport, model, context, options); diff --git a/packages/ai/src/providers/cloudflare-stream.ts b/packages/ai/src/providers/cloudflare-stream.ts index 7284f6562..c8acc48af 100644 --- a/packages/ai/src/providers/cloudflare-stream.ts +++ b/packages/ai/src/providers/cloudflare-stream.ts @@ -1,12 +1,12 @@ -import type { Api, Model, ProviderEnv, ProviderStreams } from "../types.ts"; +import type { ProviderClassifier, ProviderEnv, ProviderStreams } from "../types.ts"; const CLOUDFLARE_ACCOUNT_ID = "CLOUDFLARE_ACCOUNT_ID"; const CLOUDFLARE_GATEWAY_ID = "CLOUDFLARE_GATEWAY_ID"; -export function resolveCloudflareModel( - model: Model, +export function resolveCloudflareModel( + model: TModel, env: ProviderEnv | undefined, -): Model { +): TModel { if (!env) return model; const baseUrl = model.baseUrl .replaceAll(`{${CLOUDFLARE_ACCOUNT_ID}}`, env[CLOUDFLARE_ACCOUNT_ID] ?? `{${CLOUDFLARE_ACCOUNT_ID}}`) @@ -26,3 +26,11 @@ export function cloudflareStreams(streams: ProviderStreams): ProviderStreams { streams.streamSimple(resolveCloudflareModel(model, options?.env), context, options), }; } + +/** Classifier counterpart of {@link cloudflareStreams}. */ +export function cloudflareClassifier(classifier: ProviderClassifier): ProviderClassifier { + return { + classify: (model, context, options) => + classifier.classify(resolveCloudflareModel(model, options?.env), context, options), + }; +} diff --git a/packages/ai/src/providers/cloudflare-workers-ai.ts b/packages/ai/src/providers/cloudflare-workers-ai.ts index 89216b5ef..4798fa8a3 100644 --- a/packages/ai/src/providers/cloudflare-workers-ai.ts +++ b/packages/ai/src/providers/cloudflare-workers-ai.ts @@ -1,15 +1,25 @@ +import { cloudflareWorkersAISystemOneApi } from "../api/cloudflare-workers-ai-system-one.lazy.ts"; import { openAICompletionsApi } from "../api/openai-completions.lazy.ts"; import { createProvider, type Provider } from "../models.ts"; import { cloudflareWorkersAIAuth } from "./cloudflare-auth.ts"; -import { cloudflareStreams } from "./cloudflare-stream.ts"; -import { CLOUDFLARE_WORKERS_AI_MODELS } from "./cloudflare-workers-ai.models.ts"; +import { cloudflareClassifier, cloudflareStreams } from "./cloudflare-stream.ts"; +import { + CLOUDFLARE_WORKERS_AI_CLASSIFIER_MODELS, + CLOUDFLARE_WORKERS_AI_MODELS, +} from "./cloudflare-workers-ai.models.ts"; export function cloudflareWorkersAIProvider(): Provider<"openai-completions"> { - return createProvider({ + return createProvider<"openai-completions">({ id: "cloudflare-workers-ai", name: "Cloudflare Workers AI", auth: { apiKey: cloudflareWorkersAIAuth() }, - models: Object.values(CLOUDFLARE_WORKERS_AI_MODELS), + models: [ + ...Object.values(CLOUDFLARE_WORKERS_AI_MODELS), + ...Object.values(CLOUDFLARE_WORKERS_AI_CLASSIFIER_MODELS), + ], api: cloudflareStreams(openAICompletionsApi()), + classifiers: { + "cloudflare-workers-ai-system-one": cloudflareClassifier(cloudflareWorkersAISystemOneApi()), + }, }); } diff --git a/packages/ai/src/providers/openrouter.ts b/packages/ai/src/providers/openrouter.ts index b5e6698c0..86efb8d9d 100644 --- a/packages/ai/src/providers/openrouter.ts +++ b/packages/ai/src/providers/openrouter.ts @@ -1,10 +1,11 @@ import { anthropicMessagesApi } from "../api/anthropic-messages.lazy.ts"; import { openAICompletionsApi } from "../api/openai-completions.lazy.ts"; import { openrouterImagesApi } from "../api/openrouter-images.lazy.ts"; +import { typesafeSystemOneApi } from "../api/typesafe-system-one.lazy.ts"; import { envApiKeyAuth, lazyOAuth } from "../auth/helpers.ts"; import { loadOpenRouterOAuth } from "../auth/oauth/load.ts"; import { createProvider, type Provider } from "../models.ts"; -import { OPENROUTER_IMAGE_MODELS, OPENROUTER_MODELS } from "./openrouter.models.ts"; +import { OPENROUTER_CLASSIFIER_MODELS, OPENROUTER_IMAGE_MODELS, OPENROUTER_MODELS } from "./openrouter.models.ts"; export function openrouterProvider(): Provider<"anthropic-messages" | "openai-completions"> { return createProvider<"anthropic-messages" | "openai-completions">({ @@ -19,11 +20,17 @@ export function openrouterProvider(): Provider<"anthropic-messages" | "openai-co load: loadOpenRouterOAuth, }), }, - models: [...Object.values(OPENROUTER_MODELS), ...Object.values(OPENROUTER_IMAGE_MODELS)], + models: [ + ...Object.values(OPENROUTER_MODELS), + ...Object.values(OPENROUTER_IMAGE_MODELS), + ...Object.values(OPENROUTER_CLASSIFIER_MODELS), + ], api: { "anthropic-messages": anthropicMessagesApi(), "openai-completions": openAICompletionsApi(), }, images: { "openrouter-images": openrouterImagesApi() }, + // OpenRouter serves TypeSafe's System One protocol at /api/v1/systemone. + classifiers: { "typesafe-system-one": typesafeSystemOneApi() }, }); } diff --git a/packages/ai/src/types.ts b/packages/ai/src/types.ts index 00483dd8b..14e9bdbd5 100644 --- a/packages/ai/src/types.ts +++ b/packages/ai/src/types.ts @@ -32,7 +32,7 @@ export type KnownImageApi = "openrouter-images"; export type ImageApi = KnownImageApi | (string & {}); -export type KnownClassifierApi = "typesafe-system-one"; +export type KnownClassifierApi = "typesafe-system-one" | "cloudflare-workers-ai-system-one"; export type ClassifierApi = KnownClassifierApi | (string & {}); diff --git a/packages/ai/test/classifier-models.test.ts b/packages/ai/test/classifier-models.test.ts index 44f076855..f1e7d2c2f 100644 --- a/packages/ai/test/classifier-models.test.ts +++ b/packages/ai/test/classifier-models.test.ts @@ -126,4 +126,13 @@ describe("Models with classifier models", () => { expect(models.getModel("typesafe", "jev-latest")).toBeUndefined(); expect(models.getModelOfType("classifier", "typesafe", "jev-latest")).toEqual(jev); }); + + it("routes OpenRouter classifier models through the System One API", () => { + const models = builtinModels(); + for (const model of getBuiltinClassifierModels("openrouter")) { + expect(model).toMatchObject({ api: "typesafe-system-one", baseUrl: "https://openrouter.ai/api/v1" }); + expect(models.getModel("openrouter", model.id)).toBeUndefined(); + expect(models.getModelOfType("classifier", "openrouter", model.id)).toEqual(model); + } + }); }); diff --git a/packages/ai/test/cloudflare-workers-ai-system-one.test.ts b/packages/ai/test/cloudflare-workers-ai-system-one.test.ts new file mode 100644 index 000000000..0a04041cd --- /dev/null +++ b/packages/ai/test/cloudflare-workers-ai-system-one.test.ts @@ -0,0 +1,111 @@ +import { describe, expect, it, vi } from "vitest"; +import { createModels } from "../src/models.ts"; +import { getBuiltinClassifierModel } from "../src/providers/all.ts"; +import { cloudflareWorkersAIProvider } from "../src/providers/cloudflare-workers-ai.ts"; +import type { ClassifierContext } from "../src/types.ts"; + +const context: ClassifierContext = { + state: { message: "Help! My payouts have been failing for 3 days." }, + questions: { + is_urgent: { + type: "bool", + instructions: "Does this convey urgency?", + criteria: { true: "Explicitly time-sensitive", false: "No urgency expressed" }, + }, + department: { + type: "choice", + instructions: "Which team should handle this?", + criteria: { billing: "Payments", technical: "Bugs" }, + }, + }, +}; + +// Model output from https://developers.cloudflare.com/ai/models/typesafe/jev/ +const jevOutput = { + model: "jev-1.13.0", + answers: { + is_urgent: { type: "noul", noul: 0.95 }, + department: { + type: "choice", + choice: "billing", + confidence: 0.8, + probabilities: { billing: 0.87, technical: 0.13 }, + }, + }, + usage: { input_tokens: 426, output_tokens: 73 }, +}; + +// REST envelope observed from the live /ai/run endpoint. +function restResponse(state: string, result: unknown = jevOutput) { + return { + result: { state, result, gatewayMetadata: { keySource: "Unified" } }, + success: true, + errors: [], + messages: [], + }; +} + +function setup() { + const models = createModels(); + models.setProvider(cloudflareWorkersAIProvider()); + const jev = models.getModelOfType("classifier", "cloudflare-workers-ai", "typesafe/jev"); + if (!jev) throw new Error("missing Cloudflare Jev model"); + return { models, jev }; +} + +const auth = { apiKey: "cf-key", env: { CLOUDFLARE_ACCOUNT_ID: "account-id" } }; + +describe("Cloudflare Workers AI System One", () => { + it("exposes Jev only through classifier catalog accessors", () => { + const { models, jev } = setup(); + expect(jev).toEqual(getBuiltinClassifierModel("cloudflare-workers-ai", "typesafe/jev")); + expect(jev).toMatchObject({ type: "classifier", api: "cloudflare-workers-ai-system-one" }); + expect(models.getModel("cloudflare-workers-ai", "typesafe/jev")).toBeUndefined(); + }); + + it("runs Jev through the account-scoped /ai/run endpoint", async () => { + const { models, jev } = setup(); + const fetch = vi.fn(async (_input: string | URL | Request, init?: RequestInit) => { + const payload = JSON.parse(String(init?.body)) as { + model: string; + input: { state: unknown; questions: Record }; + }; + expect(payload.model).toBe("typesafe/jev"); + expect(payload.input.state).toEqual(context.state); + expect(payload.input.questions.is_urgent?.type).toBe("noul"); + expect(payload.input.questions.department?.type).toBe("choice"); + expect(new Headers(init?.headers).get("authorization")).toBe("Bearer cf-key"); + return Response.json(restResponse("Completed")); + }); + + const result = await models.classify(jev, context, { ...auth, fetch }); + + expect(String(fetch.mock.calls[0]?.[0])).toBe("https://api.cloudflare.com/client/v4/accounts/account-id/ai/run"); + expect(result.stopReason).toBe("stop"); + expect(result.answers.is_urgent).toEqual({ type: "bool", probability: 0.95 }); + expect(result.answers.department).toMatchObject({ type: "choice", choice: "billing", confidence: 0.8 }); + }); + + it("reports runs that did not complete", async () => { + const { models, jev } = setup(); + const result = await models.classify(jev, context, { + ...auth, + fetch: async () => Response.json(restResponse("Queued", null)), + }); + + expect(result.stopReason).toBe("error"); + expect(result.errorMessage).toContain("run did not complete (state: Queued)"); + }); + + it("reports Cloudflare envelope errors", async () => { + const { models, jev } = setup(); + const result = await models.classify(jev, context, { + ...auth, + fetch: async () => + Response.json({ success: false, errors: [{ code: 5007, message: "No such model" }], result: null }), + }); + + expect(result.stopReason).toBe("error"); + expect(result.errorMessage).toContain("Cloudflare Workers AI error: No such model"); + }); +}); diff --git a/packages/ai/test/image-model-data.test.ts b/packages/ai/test/image-model-data.test.ts index 84ac8015b..9b40486bb 100644 --- a/packages/ai/test/image-model-data.test.ts +++ b/packages/ai/test/image-model-data.test.ts @@ -33,9 +33,19 @@ const chatOnly: OpenRouterModelListItem = { architecture: { modality: "text->text", output_modalities: ["text"] }, }; +const decisionModel: OpenRouterModelListItem = { + id: "typesafe/jev-1.13", + name: "TypeSafe: Jev 1.13", + supported_parameters: [], + architecture: { modality: "text->decisions", input_modalities: ["text"], output_modalities: ["decisions"] }, + pricing: { prompt: "0.000000042", completion: "0" }, + context_length: 32000, + top_provider: { context_length: 32000, max_completion_tokens: 28800 }, +}; + describe("OpenRouter catalog parsing", () => { it("emits image-only models from the image listing as image models", () => { - const catalog = buildOpenRouterCatalog([], [imageOnly]); + const catalog = buildOpenRouterCatalog([], [imageOnly], []); expect(catalog.chat).toEqual([]); expect(catalog.images).toEqual([ expect.objectContaining({ @@ -50,7 +60,7 @@ describe("OpenRouter catalog parsing", () => { }); it("emits separate chat and image entries for an id that supports both operations", () => { - const catalog = buildOpenRouterCatalog([chatWithImages, chatOnly], [chatWithImages, imageOnly]); + const catalog = buildOpenRouterCatalog([chatWithImages, chatOnly], [chatWithImages, imageOnly], []); expect(catalog.chat.map((model) => model.id)).toEqual(["example/multimodal", "example/chat"]); expect(catalog.chat.map((model) => model.type)).toEqual(["chat", "chat"]); expect(catalog.images.map((model) => model.id)).toEqual(["example/multimodal", "example/image-model"]); @@ -62,8 +72,28 @@ describe("OpenRouter catalog parsing", () => { const catalog = buildOpenRouterCatalog( [{ ...chatOnly, supported_parameters: [] }], [{ ...imageOnly, architecture: { output_modalities: ["text"] } }], + [{ ...decisionModel, architecture: { output_modalities: ["text"] } }], ); expect(catalog.chat).toEqual([]); expect(catalog.images).toEqual([]); + expect(catalog.classifiers).toEqual([]); + }); + + it("emits decision models as System One classifier models", () => { + const catalog = buildOpenRouterCatalog([], [], [decisionModel, decisionModel]); + expect(catalog.chat).toEqual([]); + expect(catalog.classifiers).toEqual([ + { + type: "classifier", + id: "typesafe/jev-1.13", + name: "TypeSafe: Jev 1.13", + api: "typesafe-system-one", + provider: "openrouter", + baseUrl: "https://openrouter.ai/api/v1", + input: ["text"], + cost: { input: 0.042, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 32000, + }, + ]); }); }); diff --git a/packages/ai/test/typesafe-system-one.test.ts b/packages/ai/test/typesafe-system-one.test.ts index 2e5f26d51..b00491d17 100644 --- a/packages/ai/test/typesafe-system-one.test.ts +++ b/packages/ai/test/typesafe-system-one.test.ts @@ -71,6 +71,37 @@ describe("TypeSafe System One", () => { expect(result.answers.satisfaction).toEqual({ type: "score", score: 2, confidence: 0.7 }); }); + it("posts OpenRouter System One requests to its TypeSafe-compatible endpoint", async () => { + const fetch = vi.fn(async (_input: string | URL | Request, init?: RequestInit) => { + expect(JSON.parse(String(init?.body))).toMatchObject({ model: "typesafe/jev-1.13", state: context.state }); + return Response.json({ id: "gen-dec-1", provider: "TypeSafe", answers: wireAnswers, usage: { cost: 0.1 } }); + }); + const openRouterModel = { + ...model, + id: "typesafe/jev-1.13", + provider: "openrouter", + baseUrl: "https://openrouter.ai/api/v1", + }; + + const result = await classify(openRouterModel, context, { apiKey: "secret", fetch }); + + expect(String(fetch.mock.calls[0]?.[0])).toBe("https://openrouter.ai/api/v1/systemone"); + expect(result.stopReason).toBe("stop"); + expect(result.answers.approved).toEqual({ type: "bool", probability: 0.95 }); + }); + + it("rejects models for other classifier APIs", async () => { + const fetch = vi.fn(async () => Response.json({ answers: wireAnswers })); + const result = await classify({ ...model, api: "cloudflare-workers-ai-system-one" }, context, { + apiKey: "secret", + fetch, + }); + + expect(fetch).not.toHaveBeenCalled(); + expect(result.stopReason).toBe("error"); + expect(result.errorMessage).toContain("Unsupported classifier API: cloudflare-workers-ai-system-one"); + }); + it("merges headers case-insensitively and supports null suppression", async () => { const requests: Array> = []; const fetch = vi.fn(async (_input: string | URL | Request, init?: RequestInit) => {