From ce8972a0e99cab79f2620a594a1c17baef436c67 Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Wed, 7 Oct 2026 08:27:56 +0200 Subject: [PATCH] feat(ai,coding-agent): add OpenAI Decisions classifier and classifier images Adds the openai-decisions classifier API with gpt-6-luna on the openai provider. bool questions map to predicates with the criteria in the instructions, refusals fail the result, and gateway timeouts (504) are not retried. The model needs an API key and is hidden from availability while openai uses Sign in with ChatGPT. Adds optional images to ClassifierContext. Models without image input and APIs that cannot send images return an error result. Codemode's models.classify() accepts images. Moves the shared classifier HTTP code out of system-one-shared.ts into classifier-shared.ts. --- packages/ai/CHANGELOG.md | 2 + packages/ai/README.md | 15 +- packages/ai/scripts/generate-models.ts | 22 ++ packages/ai/src/api/classifier-shared.ts | 128 ++++++++ .../api/cloudflare-workers-ai-system-one.ts | 3 +- packages/ai/src/api/llama-cpp-classify.ts | 1 + packages/ai/src/api/openai-decisions.lazy.ts | 6 + packages/ai/src/api/openai-decisions.ts | 197 ++++++++++++ packages/ai/src/api/system-one-shared.ts | 120 +------- packages/ai/src/api/typesafe-system-one.ts | 3 +- packages/ai/src/models.ts | 2 + packages/ai/src/providers/openai.ts | 15 +- packages/ai/src/types.ts | 11 +- packages/ai/src/utils/model-operations.ts | 11 + packages/ai/src/utils/provider-retry.ts | 3 + packages/ai/test/classifier-models.test.ts | 85 ++++- packages/ai/test/openai-decisions.test.ts | 291 ++++++++++++++++++ packages/ai/test/provider-retry.test.ts | 8 + packages/ai/test/typesafe-system-one.test.ts | 13 + packages/coding-agent/CHANGELOG.md | 2 + packages/coding-agent/docs/codemode.md | 22 +- packages/coding-agent/docs/models.md | 5 +- .../coding-agent/src/core/model-runtime.ts | 2 + .../src/extensions/codemode/execute.ts | 16 +- .../test/suite/agent-session-codemode.test.ts | 14 + 25 files changed, 872 insertions(+), 125 deletions(-) create mode 100644 packages/ai/src/api/classifier-shared.ts create mode 100644 packages/ai/src/api/openai-decisions.lazy.ts create mode 100644 packages/ai/src/api/openai-decisions.ts create mode 100644 packages/ai/test/openai-decisions.test.ts diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index 7c1ed3af6..d04a17432 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -10,6 +10,8 @@ - Added `durationMs` to `AssistantMessage`: `AssistantMessageEventStream` measures each response with a monotonic clock from the start of the request to its final message, for every API implementation, including direct calls. Deferred results fetched later stay untimed ([#10549](https://github.com/earendil-works/pi/issues/10549)) - Added an optional `durationMs` to `ToolResultMessage` for the execution time of the tool ([#10549](https://github.com/earendil-works/pi/issues/10549)) +- Added the `openai-decisions` classifier API for OpenAI's Decisions API, with `gpt-6-luna` as a classifier model of the `openai` provider. It needs an API key, so it is not listed as available while `openai` uses Sign in with ChatGPT +- Added optional `images` to `ClassifierContext`. Models whose `input` includes `"image"` judge them with the state; `classify()` returns an error result for other models and for APIs that cannot send images ### Changed diff --git a/packages/ai/README.md b/packages/ai/README.md index 0ad23aa31..cdb65b6aa 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -60,7 +60,7 @@ Unified LLM API with provider collections, automatic auth resolution, token and ## Supported Providers -- **OpenAI** +- **OpenAI** (including the Decisions classifier API) - **Ant Ling** - **Azure OpenAI (Responses)** - **OpenAI Codex (legacy)** (ChatGPT Plus/Pro subscription, requires OAuth, see below) @@ -890,7 +890,7 @@ console.log(model.output); // ['image'] or ['image', 'text'] ## Classification -Classifier models consume structured JSON state and answer one or more typed questions. They do not use chat or image-generation APIs. TypeSafe's Jev model is available from these built-in providers: +Classifier models consume structured JSON state, and optionally images, and answer one or more typed questions. They do not use chat or image-generation APIs. TypeSafe's Jev model, Cloudflare's Clef models, and OpenAI's GPT-6 Luna are available from these built-in providers: | Provider | Model IDs | Auth | | --- | --- | --- | @@ -899,6 +899,7 @@ Classifier models consume structured JSON state and answer one or more typed que | `cloudflare-workers-ai` | `typesafe/jev`, `@cf/cloudflare/clef`, `@cf/cloudflare/clef-flash` | `CLOUDFLARE_API_KEY` and `CLOUDFLARE_ACCOUNT_ID` | | `vercel-ai-gateway` | `typesafe-ai/jev` | `AI_GATEWAY_API_KEY` | | `opencode` | `jev-1.13`, `jev-1.13-free` | `OPENCODE_API_KEY` | +| `openai` | `gpt-6-luna` | `OPENAI_API_KEY` | ```typescript import { builtinModels } from '@earendil-works/pi-ai/providers/all'; @@ -936,7 +937,15 @@ The public contract uses `bool` questions and `{ type: "bool", probability }` an When the service reports token counts, `result.usage` carries them with their cost at the model's catalog price, the same `Usage` shape as chat messages. All System One services report token counts; a request that was answered with malformed answers keeps its usage. Local classifiers such as `llama-cpp-classify` report no usage. -`ClassifierOptions.temperature` divides the answer logits by the given value before they are normalized; values above 1 soften the distribution. APIs that cannot apply it, such as System One, ignore it. +`ClassifierOptions.temperature` divides the answer logits by the given value before they are normalized; values above 1 soften the distribution. APIs that cannot apply it, such as System One and OpenAI Decisions, ignore it. + +`ClassifierContext.images` holds image blocks (`{ type: "image", data, mimeType }`) judged together with the state. Only models whose `input` includes `"image"` accept them; `classify()` returns an error result for other models. + +### OpenAI Decisions + +The `openai-decisions` API calls OpenAI's [Decisions API](https://developers.openai.com/api/docs/guides/decisions) (`POST /v1/decisions`). GPT-6 Luna is its only model. The state is sent as JSON text; with images, the input is one user message with the state followed by the images as data URLs (at most 128). `choice` and `score` questions map to the Decisions types of the same name. `bool` questions become `predicate` questions; predicates have no criteria field, so the meanings of true and false are appended to the instructions. If OpenAI refuses to answer a question, the whole result is an error. Usage reports input tokens only, priced at the catalog input rate with OpenAI's long-context multiplier. + +The endpoint needs an OpenAI API key; Sign in with ChatGPT credentials are rejected, so the `openai` provider hides `gpt-6-luna` from `getAvailableOfType('classifier')` while it uses OAuth. The API rejects inputs above 922K tokens, but requests that run longer than about five seconds, currently above roughly 600K input tokens, fail with a gateway timeout (504). ### Chat models on llama.cpp diff --git a/packages/ai/scripts/generate-models.ts b/packages/ai/scripts/generate-models.ts index 8deeb1014..1d4a0ab1e 100644 --- a/packages/ai/scripts/generate-models.ts +++ b/packages/ai/scripts/generate-models.ts @@ -2785,6 +2785,27 @@ const CLOUDFLARE_WORKERS_AI_CLASSIFIER_MODELS: ClassifierModel<"cloudflare-worke }, ]; +// OpenAI Decisions API (public beta): gpt-6-luna is its only model. It bills input tokens only, +// with the same long-context multiplier as chat requests. Only API keys work: Sign in with ChatGPT +// tokens are rejected on /v1/decisions, and the Codex backend has no Decisions route. +// The endpoint rejects inputs above 922K tokens (the model's documented maximum input), but +// requests running longer than about five seconds, currently above roughly 600K input tokens, +// fail with a gateway timeout. +// https://developers.openai.com/api/docs/guides/decisions +const OPENAI_CLASSIFIER_MODELS: ClassifierModel<"openai-decisions">[] = [ + { + type: "classifier", + id: "gpt-6-luna", + name: "GPT-6 Luna", + api: "openai-decisions", + provider: "openai", + baseUrl: "https://api.openai.com/v1", + input: ["text", "image"], + cost: withOpenAiLongContextPricing({ input: 0.1, output: 0, cacheRead: 0, cacheWrite: 0 }), + contextWindow: 922000, + }, +]; + async function generateModels() { // Fetch models from all upstream catalogs. // models.dev: Anthropic, Google, OpenAI, Groq, Cerebras, and others @@ -3512,6 +3533,7 @@ async function generateModels() { ...aiGatewayCatalog.classifiers, ...OPENCODE_CLASSIFIER_MODELS, ...CLOUDFLARE_WORKERS_AI_CLASSIFIER_MODELS, + ...OPENAI_CLASSIFIER_MODELS, ]; for (const model of classifierModels) { providers[model.provider] ??= { chat: {}, image: {}, classifier: {} }; diff --git a/packages/ai/src/api/classifier-shared.ts b/packages/ai/src/api/classifier-shared.ts new file mode 100644 index 000000000..9788fc072 --- /dev/null +++ b/packages/ai/src/api/classifier-shared.ts @@ -0,0 +1,128 @@ +import { calculateCost } from "../models.ts"; +import type { ClassifierApi, ClassifierModel, ClassifierOptions, ProviderHeaders, Usage } from "../types.ts"; +import { headersToRecord, providerHeadersToRecord } from "../utils/headers.ts"; +import { retryProviderRequest } from "../utils/provider-retry.ts"; + +/** An HTTP failure in the shape `retryProviderRequest` and `normalizeProviderError` understand. */ +export interface ClassifierHttpError extends Error { + status: number | undefined; + headers: Headers | undefined; + body: string; +} + +function httpError(label: string, response: Response, body: string): ClassifierHttpError { + const error = new Error(`${label} returned ${response.status}`) as ClassifierHttpError; + error.status = response.status; + error.headers = response.headers; + error.body = body; + return error; +} + +function timeoutError(timeoutMs: number): ClassifierHttpError { + const error = new Error(`Request timed out after ${timeoutMs}ms`) as ClassifierHttpError; + error.name = "TimeoutError"; + error.status = undefined; + error.headers = undefined; + error.body = ""; + return error; +} + +export function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +export function requiredNumber(label: string, value: unknown, field: string): number { + if (typeof value !== "number" || !Number.isFinite(value)) { + throw new Error(`${label} returned an invalid ${field}`); + } + return value; +} + +function requestHeaders( + model: ClassifierModel, + apiKey: string, + optionsHeaders?: ProviderHeaders, +): Record { + return ( + providerHeadersToRecord( + { authorization: `Bearer ${apiKey}`, "content-type": "application/json" }, + model.headers, + optionsHeaders, + ) ?? {} + ); +} + +/** + * Posts one JSON classifier request with bearer auth, `onPayload`/`onResponse` hooks, a fresh + * timeout per attempt, and provider retries. Returns the parsed response body; throws on failure. + * `noRetryStatuses` lists HTTP statuses that fail at once although they are normally retried. + */ +export async function postClassifierRequest( + label: string, + url: URL, + model: ClassifierModel, + body: unknown, + options: ClassifierOptions | undefined, + noRetryStatuses?: readonly number[], +): Promise { + if (!options?.apiKey) throw new Error(`No API key for provider: ${model.provider}`); + const apiKey = options.apiKey; + let payload = body; + const transformed = await options.onPayload?.(payload, model); + if (transformed !== undefined) payload = transformed; + const requestFetch = options.fetch ?? globalThis.fetch; + const { response, json } = await retryProviderRequest( + async () => { + const timeoutSignal = options.timeoutMs !== undefined ? AbortSignal.timeout(options.timeoutMs) : undefined; + const signal = + options.signal && timeoutSignal + ? AbortSignal.any([options.signal, timeoutSignal]) + : (options.signal ?? timeoutSignal); + try { + const next = await requestFetch(url, { + method: "POST", + headers: requestHeaders(model, apiKey, options.headers), + body: JSON.stringify(payload), + signal, + }); + if (!next.ok) throw httpError(label, next, await next.text()); + return { response: next, json: (await next.json()) as unknown }; + } catch (error) { + if (timeoutSignal?.aborted && !options.signal?.aborted) throw timeoutError(options.timeoutMs!); + throw error; + } + }, + { + maxRetries: options.maxRetries ?? 2, + maxRetryDelayMs: options.maxRetryDelayMs, + signal: options.signal, + noRetryStatuses, + }, + ); + await options.onResponse?.({ status: response.status, headers: headersToRecord(response.headers) }, model); + return json; +} + +function tokenCount(value: unknown): number { + return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : 0; +} + +/** + * Usage from a `{ input_tokens, output_tokens }` object, priced from the model catalog like chat + * usage. A missing or malformed usage object leaves the result without usage instead of failing it. + */ +export function parseClassifierUsage(value: unknown, model: ClassifierModel): Usage | undefined { + if (!isRecord(value) || (value.input_tokens === undefined && value.output_tokens === undefined)) return undefined; + const input = tokenCount(value.input_tokens); + const output = tokenCount(value.output_tokens); + const usage: Usage = { + input, + output, + cacheRead: 0, + cacheWrite: 0, + totalTokens: input + output, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }; + calculateCost(model, usage); + return usage; +} diff --git a/packages/ai/src/api/cloudflare-workers-ai-system-one.ts b/packages/ai/src/api/cloudflare-workers-ai-system-one.ts index 2aaae8c18..1f08d3f5e 100644 --- a/packages/ai/src/api/cloudflare-workers-ai-system-one.ts +++ b/packages/ai/src/api/cloudflare-workers-ai-system-one.ts @@ -1,5 +1,6 @@ import type { ClassifierFunction, ClassifierOptions } from "../types.ts"; -import { classifySystemOne, isRecord, type SystemOneTransport } from "./system-one-shared.ts"; +import { isRecord } from "./classifier-shared.ts"; +import { classifySystemOne, type SystemOneTransport } from "./system-one-shared.ts"; const LABEL = "Cloudflare Workers AI"; diff --git a/packages/ai/src/api/llama-cpp-classify.ts b/packages/ai/src/api/llama-cpp-classify.ts index fcedc374b..d69bb3329 100644 --- a/packages/ai/src/api/llama-cpp-classify.ts +++ b/packages/ai/src/api/llama-cpp-classify.ts @@ -434,6 +434,7 @@ export const classify: ClassifierFunction = async (model, con try { if (model.api !== "llama-cpp-classify") throw new Error(`Unsupported classifier API: ${model.api}`); + if (context.images?.length) throw new Error(`${LABEL} classification does not support image input`); const temperature = options?.temperature ?? 1; if (!(temperature > 0) || !Number.isFinite(temperature)) { throw new Error(`Temperature must be a positive number, got ${temperature}`); diff --git a/packages/ai/src/api/openai-decisions.lazy.ts b/packages/ai/src/api/openai-decisions.lazy.ts new file mode 100644 index 000000000..c2031e6ae --- /dev/null +++ b/packages/ai/src/api/openai-decisions.lazy.ts @@ -0,0 +1,6 @@ +import type { ProviderClassifier } from "../types.ts"; + +export const openAIDecisionsApi = (): ProviderClassifier => ({ + classify: async (model, context, options) => + (await import("./openai-decisions.ts")).classify(model, context, options), +}); diff --git a/packages/ai/src/api/openai-decisions.ts b/packages/ai/src/api/openai-decisions.ts new file mode 100644 index 000000000..5912781d6 --- /dev/null +++ b/packages/ai/src/api/openai-decisions.ts @@ -0,0 +1,197 @@ +import type { + ClassifierAnswer, + ClassifierContext, + ClassifierFunction, + ClassifierOptions, + ClassifierQuestion, + ClassifierResult, +} from "../types.ts"; +import { formatProviderError, normalizeProviderError } from "../utils/error-body.ts"; +import { + type ClassifierHttpError, + isRecord, + parseClassifierUsage, + postClassifierRequest, + requiredNumber, +} from "./classifier-shared.ts"; + +/** + * OpenAI's Decisions API: `POST /v1/decisions` with `{ model, input, questions }`. + * https://developers.openai.com/api/docs/guides/decisions + * + * The state is sent as JSON text. With images, the input becomes one user message with the state + * as `input_text` followed by `input_image` data URLs. Questions map to Decisions types: + * `choice` to `choice`, `score` to `score`, and `bool` to `predicate`. Predicates have no + * criteria field, so the meanings of true and false are appended to the instructions. + * + * Only OpenAI API keys work: Sign in with ChatGPT tokens are rejected on this route. + */ + +const LABEL = "OpenAI Decisions"; + +/** The endpoint accepts at most this many image parts per request. */ +const MAX_IMAGES = 128; + +interface DecisionQuestion { + type: "predicate" | "choice" | "score"; + name: string; + instructions: string; + choices?: Array<{ value: string; description?: string }>; + levels?: Array<{ label: string }>; +} + +function predicateInstructions(question: Extract): string { + const meanings = [ + question.criteria.true ? `True means: ${question.criteria.true}` : "", + question.criteria.false ? `False means: ${question.criteria.false}` : "", + ].filter(Boolean); + return meanings.length > 0 ? `${question.instructions}\n\n${meanings.join("\n")}` : question.instructions; +} + +function wireQuestion(name: string, question: ClassifierQuestion): DecisionQuestion { + if (question.type === "choice") { + return { + type: "choice", + name, + instructions: question.instructions, + choices: Object.entries(question.criteria).map(([value, description]) => + description ? { value, description } : { value }, + ), + }; + } + if (question.type === "score") { + return { + type: "score", + name, + instructions: question.instructions, + levels: question.criteria.map((label) => ({ label })), + }; + } + return { type: "predicate", name, instructions: predicateInstructions(question) }; +} + +function wireInput(context: ClassifierContext): unknown { + const state = JSON.stringify(context.state); + const images = context.images ?? []; + if (images.length === 0) return state; + if (images.length > MAX_IMAGES) { + throw new Error(`${LABEL} accepts at most ${MAX_IMAGES} images, got ${images.length}`); + } + return [ + { + role: "user", + content: [ + { type: "input_text", text: state }, + ...images.map((image) => ({ + type: "input_image", + image_url: `data:${image.mimeType};base64,${image.data}`, + })), + ], + }, + ]; +} + +function choiceProbabilities(value: unknown, id: string): Record { + if (!Array.isArray(value)) throw new Error(`${LABEL} returned invalid probabilities for ${id}`); + return Object.fromEntries( + value.map((entry) => { + if (!isRecord(entry) || typeof entry.value !== "string") { + throw new Error(`${LABEL} returned invalid probabilities for ${id}`); + } + return [entry.value, requiredNumber(LABEL, entry.probability, `probability for ${id}.${entry.value}`)]; + }), + ); +} + +function parseAnswer(id: string, question: ClassifierQuestion, answer: Record): ClassifierAnswer { + if (answer.type === "refusal") throw new Error(`${LABEL} refused to answer ${id}`); + if (question.type === "choice") { + if (answer.type !== "choice" || typeof answer.choice !== "string") { + throw new Error(`${LABEL} did not return a choice answer for ${id}`); + } + return { + type: "choice", + choice: answer.choice, + probabilities: choiceProbabilities(answer.probabilities, id), + confidence: requiredNumber(LABEL, answer.confidence, `confidence for ${id}`), + }; + } + if (question.type === "score") { + if (answer.type !== "score") throw new Error(`${LABEL} did not return a score answer for ${id}`); + return { + type: "score", + score: requiredNumber(LABEL, answer.score, `score for ${id}`), + confidence: requiredNumber(LABEL, answer.confidence, `confidence for ${id}`), + }; + } + if (answer.type !== "predicate") throw new Error(`${LABEL} did not return a predicate answer for ${id}`); + return { type: "bool", probability: requiredNumber(LABEL, answer.probability, `probability for ${id}`) }; +} + +function parseAnswers(value: unknown, context: ClassifierContext): Record { + if (!Array.isArray(value)) throw new Error(`${LABEL} returned an unexpected response`); + const byName = new Map>(); + for (const answer of value) { + if (isRecord(answer) && typeof answer.name === "string") byName.set(answer.name, answer); + } + return Object.fromEntries( + Object.entries(context.questions).map(([id, question]) => { + const answer = byName.get(id); + if (!answer) throw new Error(`${LABEL} did not return an answer for ${id}`); + return [id, parseAnswer(id, question, answer)]; + }), + ); +} + +/** + * Cloudflare in front of api.openai.com answers 504 with an HTML page when a request runs longer + * than about five seconds. Large inputs, currently above roughly 600K tokens, hit this limit, and + * retrying the same input runs into it again, so 504 is not retried. + */ +const NO_RETRY_STATUSES = [504]; + +function errorMessage(error: unknown): string { + if ((error as Partial).status === 504) { + return `${LABEL} error (504): the request timed out at the gateway. Very large inputs (above roughly 600K tokens) currently exceed its time limit.`; + } + return formatProviderError(normalizeProviderError(error), `${LABEL} error`); +} + +/** Classification through OpenAI's Decisions API. */ +export const classify: ClassifierFunction = async (model, context, options) => { + const output: ClassifierResult = { + api: model.api, + provider: model.provider, + model: model.id, + answers: {}, + stopReason: "stop", + timestamp: Date.now(), + }; + + try { + if (model.api !== "openai-decisions") throw new Error(`Unsupported classifier API: ${model.api}`); + const body = await postClassifierRequest( + LABEL, + new URL("decisions", `${model.baseUrl.replace(/\/+$/u, "")}/`), + model, + { + model: model.id, + input: wireInput(context), + questions: Object.entries(context.questions).map(([id, question]) => wireQuestion(id, question)), + }, + options, + NO_RETRY_STATUSES, + ); + if (!isRecord(body)) throw new Error(`${LABEL} returned an unexpected response`); + // Set before parsing answers: a request with malformed or refused answers was still billed. + const usage = parseClassifierUsage(body.usage, model); + if (usage) output.usage = usage; + output.answers = parseAnswers(body.answers, context); + return output; + } catch (error) { + output.answers = {}; + output.stopReason = options?.signal?.aborted ? "aborted" : "error"; + output.errorMessage = errorMessage(error); + return output; + } +}; diff --git a/packages/ai/src/api/system-one-shared.ts b/packages/ai/src/api/system-one-shared.ts index e10295887..6f15f044f 100644 --- a/packages/ai/src/api/system-one-shared.ts +++ b/packages/ai/src/api/system-one-shared.ts @@ -1,4 +1,3 @@ -import { calculateCost } from "../models.ts"; import type { ClassifierAnswer, ClassifierApi, @@ -6,12 +5,9 @@ import type { ClassifierModel, ClassifierOptions, ClassifierResult, - ProviderHeaders, - Usage, } from "../types.ts"; import { formatProviderError, normalizeProviderError } from "../utils/error-body.ts"; -import { headersToRecord, providerHeadersToRecord } from "../utils/headers.ts"; -import { retryProviderRequest } from "../utils/provider-retry.ts"; +import { isRecord, parseClassifierUsage, postClassifierRequest, requiredNumber } from "./classifier-shared.ts"; /** TypeSafe System One request body without the transport-specific envelope. */ export interface SystemOneWireRequest { @@ -33,40 +29,6 @@ export interface SystemOneTransport { output(body: unknown): Record; } -interface SystemOneHttpError extends Error { - status: number | undefined; - headers: Headers | undefined; - body: string; -} - -function httpError(label: string, response: Response, body: string): SystemOneHttpError { - const error = new Error(`${label} returned ${response.status}`) as SystemOneHttpError; - error.status = response.status; - error.headers = response.headers; - error.body = body; - return error; -} - -function timeoutError(timeoutMs: number): SystemOneHttpError { - const error = new Error(`Request timed out after ${timeoutMs}ms`) as SystemOneHttpError; - error.name = "TimeoutError"; - error.status = undefined; - error.headers = undefined; - error.body = ""; - return error; -} - -export function isRecord(value: unknown): value is Record { - return typeof value === "object" && value !== null && !Array.isArray(value); -} - -function requiredNumber(label: string, value: unknown, field: string): number { - if (typeof value !== "number" || !Number.isFinite(value)) { - throw new Error(`${label} returned an invalid ${field}`); - } - return value; -} - function probabilities(label: string, value: unknown, id: string): Record { if (!isRecord(value)) throw new Error(`${label} returned invalid probabilities for ${id}`); return Object.fromEntries( @@ -120,30 +82,6 @@ function parseAnswers(label: string, value: unknown, context: ClassifierContext) return Object.fromEntries(answers); } -function tokenCount(value: unknown): number { - return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : 0; -} - -/** - * Usage from System One's `{ input_tokens, output_tokens }`, priced from the model catalog like chat - * usage. A missing or malformed usage object leaves the result without usage instead of failing it. - */ -function parseUsage(value: unknown, model: ClassifierModel): Usage | undefined { - if (!isRecord(value) || (value.input_tokens === undefined && value.output_tokens === undefined)) return undefined; - const input = tokenCount(value.input_tokens); - const output = tokenCount(value.output_tokens); - const usage: Usage = { - input, - output, - cacheRead: 0, - cacheWrite: 0, - totalTokens: input + output, - cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, - }; - calculateCost(model, usage); - return usage; -} - /** Maps public `bool` questions to TypeSafe's wire-level `noul` type. */ function wireRequest(context: ClassifierContext): SystemOneWireRequest { return { @@ -157,20 +95,6 @@ function wireRequest(context: ClassifierContext): SystemOneWireRequest { }; } -function requestHeaders( - model: ClassifierModel, - apiKey: string, - optionsHeaders?: ProviderHeaders, -): Record { - return ( - providerHeadersToRecord( - { authorization: `Bearer ${apiKey}`, "content-type": "application/json" }, - model.headers, - optionsHeaders, - ) ?? {} - ); -} - /** Runs one System One classification over the given transport. */ export async function classifySystemOne( transport: SystemOneTransport, @@ -189,43 +113,17 @@ export async function classifySystemOne( try { if (model.api !== transport.api) throw new Error(`Unsupported classifier API: ${model.api}`); - if (!options?.apiKey) throw new Error(`No API key for provider: ${model.provider}`); - const apiKey = options.apiKey; - let payload = transport.payload(model, wireRequest(context)); - const transformed = await options.onPayload?.(payload, model); - if (transformed !== undefined) payload = transformed; - const requestFetch = options.fetch ?? globalThis.fetch; - const { response, body } = await retryProviderRequest( - async () => { - const timeoutSignal = options.timeoutMs !== undefined ? AbortSignal.timeout(options.timeoutMs) : undefined; - const signal = - options.signal && timeoutSignal - ? AbortSignal.any([options.signal, timeoutSignal]) - : (options.signal ?? timeoutSignal); - try { - const next = await requestFetch(transport.url(model), { - method: "POST", - headers: requestHeaders(model, apiKey, options.headers), - body: JSON.stringify(payload), - signal, - }); - if (!next.ok) throw httpError(transport.label, next, await next.text()); - return { response: next, body: (await next.json()) as unknown }; - } catch (error) { - if (timeoutSignal?.aborted && !options.signal?.aborted) throw timeoutError(options.timeoutMs!); - throw error; - } - }, - { - maxRetries: options.maxRetries ?? 2, - maxRetryDelayMs: options.maxRetryDelayMs, - signal: options.signal, - }, + if (context.images?.length) throw new Error(`${transport.label} does not support image input`); + const body = await postClassifierRequest( + transport.label, + transport.url(model), + model, + transport.payload(model, wireRequest(context)), + options, ); - await options.onResponse?.({ status: response.status, headers: headersToRecord(response.headers) }, model); const result = transport.output(body); // Set before parsing answers: a request with malformed answers was still billed. - const usage = parseUsage(result.usage, model); + const usage = parseClassifierUsage(result.usage, model); if (usage) output.usage = usage; output.answers = parseAnswers(transport.label, result.answers, context); return output; diff --git a/packages/ai/src/api/typesafe-system-one.ts b/packages/ai/src/api/typesafe-system-one.ts index 30dc25b16..542fc476b 100644 --- a/packages/ai/src/api/typesafe-system-one.ts +++ b/packages/ai/src/api/typesafe-system-one.ts @@ -1,5 +1,6 @@ import type { ClassifierFunction, ClassifierOptions } from "../types.ts"; -import { classifySystemOne, isRecord, type SystemOneTransport } from "./system-one-shared.ts"; +import { isRecord } from "./classifier-shared.ts"; +import { classifySystemOne, type SystemOneTransport } from "./system-one-shared.ts"; /** * TypeSafe's native System One protocol. OpenRouter serves the same protocol, diff --git a/packages/ai/src/models.ts b/packages/ai/src/models.ts index 98aa27ce0..fbcc52231 100644 --- a/packages/ai/src/models.ts +++ b/packages/ai/src/models.ts @@ -57,6 +57,7 @@ import type { import { operationSignal, raceWithAbortSignal } from "./utils/abort.ts"; import { assertChatModel, + assertClassifierInputSupported, assertClassifierModel, assertImageModel, classifierErrorResult, @@ -975,6 +976,7 @@ class ModelsImpl implements MutableModels { ): Promise { try { assertClassifierModel(model); + assertClassifierInputSupported(model, context); const provider = this.requireProvider(model); if (!provider.classify) { throw new ModelsError("provider", `Provider ${model.provider} does not support classification`); diff --git a/packages/ai/src/providers/openai.ts b/packages/ai/src/providers/openai.ts index f34b62034..0b2ef380b 100644 --- a/packages/ai/src/providers/openai.ts +++ b/packages/ai/src/providers/openai.ts @@ -1,11 +1,12 @@ +import { openAIDecisionsApi } from "../api/openai-decisions.lazy.ts"; import { openAIResponsesApi } from "../api/openai-responses.lazy.ts"; import { envApiKeyAuth, lazyOAuth } from "../auth/helpers.ts"; import { loadOpenAIChatGPTOAuth } from "../auth/oauth/load.ts"; -import { createProvider, type Provider } from "../models.ts"; -import { OPENAI_MODELS } from "./openai.models.ts"; +import { createProvider, isModelType, type Provider } from "../models.ts"; +import { OPENAI_CLASSIFIER_MODELS, OPENAI_MODELS } from "./openai.models.ts"; export function openaiProvider(): Provider<"openai-responses"> { - return createProvider({ + return createProvider<"openai-responses">({ id: "openai", name: "OpenAI", baseUrl: "https://api.openai.com/v1", @@ -18,7 +19,13 @@ export function openaiProvider(): Provider<"openai-responses"> { load: loadOpenAIChatGPTOAuth, }), }, - models: Object.values(OPENAI_MODELS), + models: [...Object.values(OPENAI_MODELS), ...Object.values(OPENAI_CLASSIFIER_MODELS)], + // Sign in with ChatGPT tokens only reach the Responses API; the Decisions API rejects them. + filterAllModels: (models, credential) => + credential?.type === "oauth" ? models.filter((model) => !isModelType(model, "classifier")) : models, api: openAIResponsesApi(), + classifiers: { + "openai-decisions": openAIDecisionsApi(), + }, }); } diff --git a/packages/ai/src/types.ts b/packages/ai/src/types.ts index 99a8fc4ef..4b5f2038a 100644 --- a/packages/ai/src/types.ts +++ b/packages/ai/src/types.ts @@ -32,7 +32,11 @@ export type KnownImageApi = "openrouter-images"; export type ImageApi = KnownImageApi | (string & {}); -export type KnownClassifierApi = "typesafe-system-one" | "cloudflare-workers-ai-system-one" | "llama-cpp-classify"; +export type KnownClassifierApi = + | "typesafe-system-one" + | "cloudflare-workers-ai-system-one" + | "llama-cpp-classify" + | "openai-decisions"; export type ClassifierApi = KnownClassifierApi | (string & {}); @@ -664,6 +668,11 @@ export type ClassifierQuestion = ClassifierChoiceQuestion | ClassifierScoreQuest export interface ClassifierContext { state: JsonObject; + /** + * Images judged together with `state`. Only models whose `input` includes `"image"` accept them; + * other models return an error result. + */ + images?: ImageContent[]; questions: Record; } diff --git a/packages/ai/src/utils/model-operations.ts b/packages/ai/src/utils/model-operations.ts index bdf510f9c..b23e06104 100644 --- a/packages/ai/src/utils/model-operations.ts +++ b/packages/ai/src/utils/model-operations.ts @@ -3,6 +3,7 @@ import type { Api, AssistantImages, ClassifierApi, + ClassifierContext, ClassifierModel, ClassifierResult, ImageApi, @@ -41,6 +42,16 @@ export function assertClassifierModel(model: AnyModel): asserts model is Classif } } +/** Rejects classifier images for models whose catalog entry does not accept image input. */ +export function assertClassifierInputSupported( + model: ClassifierModel, + context: ClassifierContext, +): void { + if (context.images?.length && !model.input.includes("image")) { + throw new ModelsError("provider", `Model ${model.provider}/${model.id} does not accept image input`); + } +} + export function imageErrorResult(model: ImageModel, error: unknown, aborted = false): AssistantImages { return { api: model.api, diff --git a/packages/ai/src/utils/provider-retry.ts b/packages/ai/src/utils/provider-retry.ts index 0cd097d23..9d7ef81f2 100644 --- a/packages/ai/src/utils/provider-retry.ts +++ b/packages/ai/src/utils/provider-retry.ts @@ -4,6 +4,8 @@ interface ProviderRetryOptions { maxRetries?: number; maxRetryDelayMs?: number; signal?: AbortSignal; + /** HTTP statuses that fail at once although the default policy would retry them. */ + noRetryStatuses?: readonly number[]; } interface ProviderError extends Error { @@ -116,6 +118,7 @@ export async function retryProviderRequest( } catch (error) { if (options.signal?.aborted) throw createAbortError(); if (retriesRemaining <= 0 || !isProviderError(error) || !isRetryableProviderError(error)) throw error; + if (error.status !== undefined && options.noRetryStatuses?.includes(error.status)) throw error; const retryIndex = maxRetries - retriesRemaining; retriesRemaining--; diff --git a/packages/ai/test/classifier-models.test.ts b/packages/ai/test/classifier-models.test.ts index 9ef488e35..e57b4a67b 100644 --- a/packages/ai/test/classifier-models.test.ts +++ b/packages/ai/test/classifier-models.test.ts @@ -1,4 +1,5 @@ -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; +import { InMemoryCredentialStore } from "../src/auth/credential-store.ts"; import { createModels, createProvider, getModelType } from "../src/models.ts"; import { builtinModels, @@ -162,6 +163,88 @@ describe("Models with classifier models", () => { expect(result.answers.approved).toEqual({ type: "bool", probability: 0.8 }); }); + it("rejects images for classifier models without image input before calling the provider", async () => { + const classifier = classifierModel("test", "text-only"); + const classify = vi.fn( + async (): Promise => ({ + api: classifier.api, + provider: classifier.provider, + model: classifier.id, + answers: {}, + stopReason: "stop", + timestamp: Date.now(), + }), + ); + const models = createModels(); + models.setProvider( + createProvider({ + id: "test", + auth: { apiKey: { name: "Test", resolve: async () => ({ auth: {} }) } }, + models: [classifier], + classifiers: { "test-classifier": { classify } }, + }), + ); + + const result = await models.classify(classifier, { + ...context, + images: [{ type: "image", data: "aW1hZ2U=", mimeType: "image/png" }], + }); + const withoutImages = await models.classify(classifier, { ...context, images: [] }); + + expect(result.stopReason).toBe("error"); + expect(result.errorMessage).toBe("Model test/text-only does not accept image input"); + expect(withoutImages.stopReason).toBe("stop"); + expect(classify).toHaveBeenCalledOnce(); + }); + + it("routes OpenAI GPT-6 Luna through the Decisions API with images", async () => { + const models = builtinModels(); + const luna = models.getModelOfType("classifier", "openai", "gpt-6-luna"); + if (!luna) throw new Error("missing OpenAI Decisions model"); + expect(luna).toMatchObject({ api: "openai-decisions", input: ["text", "image"], contextWindow: 922000 }); + // The chat entry with the same id stays separate. + expect(models.getModel("openai", "gpt-6-luna")?.api).toBe("openai-responses"); + + const urls: string[] = []; + const result = await models.classify( + luna, + { ...context, images: [{ type: "image", data: "aW1hZ2U=", mimeType: "image/png" }] }, + { + apiKey: "secret", + fetch: async (input) => { + urls.push(String(input)); + return Response.json({ answers: [{ type: "predicate", name: "approved", probability: 0.8 }] }); + }, + }, + ); + + expect(urls).toEqual(["https://api.openai.com/v1/decisions"]); + expect(result.stopReason).toBe("stop"); + expect(result.answers.approved).toEqual({ type: "bool", probability: 0.8 }); + }); + + it("lists OpenAI Decisions models only for API key credentials", async () => { + const apiKeyStore = new InMemoryCredentialStore(); + await apiKeyStore.modify("openai", async () => ({ type: "api_key", key: "secret" })); + const oauthStore = new InMemoryCredentialStore(); + await oauthStore.modify("openai", async () => ({ + type: "oauth", + access: "access", + refresh: "refresh", + expires: Date.now() + 3_600_000, + })); + + const withApiKey = builtinModels({ credentials: apiKeyStore }); + const withOAuth = builtinModels({ credentials: oauthStore }); + + expect((await withApiKey.getAvailableOfType("classifier", "openai")).map((model) => model.id)).toEqual([ + "gpt-6-luna", + ]); + expect(await withOAuth.getAvailableOfType("classifier", "openai")).toEqual([]); + // Chat models stay available with ChatGPT OAuth. + expect((await withOAuth.getAvailable("openai")).some((model) => model.id === "gpt-6-luna")).toBe(true); + }); + it("routes OpenRouter classifier models through the System One API", () => { const models = builtinModels(); for (const model of getBuiltinClassifierModels("openrouter")) { diff --git a/packages/ai/test/openai-decisions.test.ts b/packages/ai/test/openai-decisions.test.ts new file mode 100644 index 000000000..191d2f9b8 --- /dev/null +++ b/packages/ai/test/openai-decisions.test.ts @@ -0,0 +1,291 @@ +import { describe, expect, it, vi } from "vitest"; +import { classify } from "../src/api/openai-decisions.ts"; +import type { ClassifierContext, ClassifierModel, ImageContent } from "../src/types.ts"; + +const model: ClassifierModel<"openai-decisions"> = { + type: "classifier", + id: "gpt-6-luna", + name: "GPT-6 Luna", + api: "openai-decisions", + provider: "openai", + baseUrl: "https://api.openai.com/v1", + input: ["text", "image"], + cost: { + input: 0.1, + output: 0, + cacheRead: 0, + cacheWrite: 0, + tiers: [{ inputTokensAbove: 272000, input: 0.2, output: 0, cacheRead: 0, cacheWrite: 0 }], + }, + contextWindow: 922000, +}; + +const context: ClassifierContext = { + state: { text: "The deployment succeeded, thank you." }, + questions: { + category: { + type: "choice", + instructions: "Classify the message", + criteria: { success: "Successful", failure: "" }, + }, + satisfaction: { + type: "score", + instructions: "Score satisfaction", + criteria: ["low", "neutral", "high"], + }, + approved: { + type: "bool", + instructions: "Does the user approve?", + criteria: { true: "Approval", false: "No approval" }, + }, + }, +}; + +/** Response shape from the API reference and live `gpt-6-luna` requests. */ +const wireAnswers = [ + { + type: "choice", + name: "category", + choice: "success", + probabilities: [ + { value: "success", probability: 0.9 }, + { value: "failure", probability: 0.1 }, + ], + confidence: 0.8, + }, + { + type: "score", + name: "satisfaction", + score: 1.8, + probabilities: [ + { value: 0, label: "low", probability: 0.05 }, + { value: 1, label: "neutral", probability: 0.1 }, + { value: 2, label: "high", probability: 0.85 }, + ], + confidence: 0.7, + }, + { type: "predicate", name: "approved", probability: 0.95 }, +]; + +const wireUsage = { + input_tokens: 164, + input_tokens_details: { cached_tokens: 0, cache_write_tokens: 0 }, + output_tokens: 0, + output_tokens_details: { reasoning_tokens: 0 }, + total_tokens: 164, +}; + +const image: ImageContent = { type: "image", data: "aW1hZ2U=", mimeType: "image/png" }; + +describe("OpenAI Decisions", () => { + it("maps questions to Decisions types and answers back by name", async () => { + const fetch = vi.fn(async (_input: string | URL | Request, _init?: RequestInit) => + // Answers out of question order: they are matched by name. + Response.json({ model: "gpt-6-luna", answers: [...wireAnswers].reverse(), usage: wireUsage }), + ); + + const result = await classify(model, context, { apiKey: "secret", fetch, temperature: 1.5 }); + + expect(fetch).toHaveBeenCalledOnce(); + const [url, init] = fetch.mock.calls[0]!; + expect(String(url)).toBe("https://api.openai.com/v1/decisions"); + expect(new Headers(init?.headers).get("authorization")).toBe("Bearer secret"); + expect(JSON.parse(String(init?.body))).toEqual({ + model: "gpt-6-luna", + input: JSON.stringify(context.state), + questions: [ + { + type: "choice", + name: "category", + instructions: "Classify the message", + // Empty descriptions are omitted. + choices: [{ value: "success", description: "Successful" }, { value: "failure" }], + }, + { + type: "score", + name: "satisfaction", + instructions: "Score satisfaction", + levels: [{ label: "low" }, { label: "neutral" }, { label: "high" }], + }, + { + type: "predicate", + name: "approved", + instructions: "Does the user approve?\n\nTrue means: Approval\nFalse means: No approval", + }, + ], + }); + expect(result.stopReason).toBe("stop"); + expect(result.answers).toEqual({ + category: { + type: "choice", + choice: "success", + probabilities: { success: 0.9, failure: 0.1 }, + confidence: 0.8, + }, + satisfaction: { type: "score", score: 1.8, confidence: 0.7 }, + approved: { type: "bool", probability: 0.95 }, + }); + expect(result.usage).toMatchObject({ input: 164, output: 0, cacheRead: 0, totalTokens: 164 }); + expect(result.usage?.cost.total).toBeCloseTo(0.0000164, 12); + }); + + it("prices long-context requests at the long-context input rate", async () => { + const result = await classify(model, context, { + apiKey: "secret", + fetch: async () => Response.json({ answers: wireAnswers, usage: { input_tokens: 300000, output_tokens: 0 } }), + }); + + expect(result.usage?.cost.total).toBeCloseTo(0.06, 12); + }); + + it("sends images after the state in one user message", async () => { + const fetch = vi.fn(async (_input: string | URL | Request, _init?: RequestInit) => + Response.json({ answers: wireAnswers }), + ); + + const result = await classify( + model, + { ...context, images: [image, { ...image, mimeType: "image/jpeg" }] }, + { + apiKey: "secret", + fetch, + }, + ); + + expect(result.stopReason).toBe("stop"); + expect(JSON.parse(String(fetch.mock.calls[0]?.[1]?.body)).input).toEqual([ + { + role: "user", + content: [ + { type: "input_text", text: JSON.stringify(context.state) }, + { type: "input_image", image_url: "data:image/png;base64,aW1hZ2U=" }, + { type: "input_image", image_url: "data:image/jpeg;base64,aW1hZ2U=" }, + ], + }, + ]); + }); + + it("rejects more than 128 images before sending", async () => { + const fetch = vi.fn(async () => Response.json({ answers: wireAnswers })); + + const result = await classify( + model, + { ...context, images: Array.from({ length: 129 }, () => image) }, + { + apiKey: "secret", + fetch, + }, + ); + + expect(fetch).not.toHaveBeenCalled(); + expect(result.stopReason).toBe("error"); + expect(result.errorMessage).toContain("at most 128 images, got 129"); + }); + + it("fails the result when a question is refused and keeps the billed usage", async () => { + const result = await classify(model, context, { + apiKey: "secret", + fetch: async () => + Response.json({ + answers: [wireAnswers[0], wireAnswers[1], { type: "refusal", name: "approved" }], + usage: wireUsage, + }), + }); + + expect(result.stopReason).toBe("error"); + expect(result.answers).toEqual({}); + expect(result.errorMessage).toBe("OpenAI Decisions refused to answer approved"); + expect(result.usage).toMatchObject({ input: 164 }); + }); + + it("returns missing and mistyped answers as classifier errors", async () => { + const missing = await classify(model, context, { + apiKey: "secret", + fetch: async () => Response.json({ answers: wireAnswers.slice(0, 2) }), + }); + const mistyped = await classify(model, context, { + apiKey: "secret", + fetch: async () => + Response.json({ answers: [wireAnswers[0], wireAnswers[1], { type: "score", name: "approved" }] }), + }); + + expect(missing.stopReason).toBe("error"); + expect(missing.errorMessage).toContain("did not return an answer for approved"); + expect(mistyped.stopReason).toBe("error"); + expect(mistyped.errorMessage).toContain("did not return a predicate answer for approved"); + }); + + it("preserves prototype-sensitive question IDs in answers", async () => { + const prototypeContext: ClassifierContext = { + state: {}, + questions: JSON.parse( + '{"__proto__":{"type":"bool","instructions":"Is this true?","criteria":{"true":"Yes","false":"No"}}}', + ), + }; + const result = await classify(model, prototypeContext, { + apiKey: "secret", + fetch: async () => Response.json({ answers: [{ type: "predicate", name: "__proto__", probability: 0.75 }] }), + }); + + expect(result.stopReason).toBe("stop"); + expect(Object.hasOwn(result.answers, "__proto__")).toBe(true); + expect(result.answers.__proto__).toEqual({ type: "bool", probability: 0.75 }); + }); + + it("does not retry gateway timeouts and explains them instead of returning the HTML page", async () => { + const fetch = vi.fn( + async () => + new Response("Gateway time-out", { + status: 504, + headers: { "retry-after-ms": "0" }, + }), + ); + // Default retries: the same input would time out again. + const result = await classify(model, context, { apiKey: "secret", fetch }); + + expect(fetch).toHaveBeenCalledOnce(); + expect(result.stopReason).toBe("error"); + expect(result.errorMessage).toContain("OpenAI Decisions error (504): the request timed out at the gateway"); + expect(result.errorMessage).not.toContain(""); + }); + + it("still retries other server errors", async () => { + let attempt = 0; + const result = await classify(model, context, { + apiKey: "secret", + fetch: async () => + ++attempt === 1 + ? new Response("busy", { status: 503, headers: { "retry-after-ms": "0" } }) + : Response.json({ answers: wireAnswers }), + }); + + expect(attempt).toBe(2); + expect(result.stopReason).toBe("stop"); + }); + + it("includes the API error body for other HTTP failures", async () => { + const result = await classify(model, context, { + apiKey: "secret", + maxRetries: 0, + fetch: async () => + Response.json( + { error: { message: "Decision input exceeds the token limit.", type: "invalid_request_error" } }, + { status: 400 }, + ), + }); + + expect(result.stopReason).toBe("error"); + expect(result.errorMessage).toContain("OpenAI Decisions error (400)"); + expect(result.errorMessage).toContain("Decision input exceeds the token limit."); + }); + + it("rejects models for other classifier APIs and missing API keys", async () => { + const fetch = vi.fn(async () => Response.json({ answers: wireAnswers })); + const otherApi = await classify({ ...model, api: "typesafe-system-one" }, context, { apiKey: "secret", fetch }); + const noKey = await classify(model, context, { fetch }); + + expect(fetch).not.toHaveBeenCalled(); + expect(otherApi.errorMessage).toContain("Unsupported classifier API: typesafe-system-one"); + expect(noKey.errorMessage).toContain("No API key for provider: openai"); + }); +}); diff --git a/packages/ai/test/provider-retry.test.ts b/packages/ai/test/provider-retry.test.ts index 0c5313ec7..b8e5c212a 100644 --- a/packages/ai/test/provider-retry.test.ts +++ b/packages/ai/test/provider-retry.test.ts @@ -37,6 +37,14 @@ describe("provider request retries", () => { expect(request).toHaveBeenCalledTimes(1); }); + it("does not retry statuses listed in noRetryStatuses", async () => { + const error = providerError(504, { "retry-after-ms": "0" }); + const request = vi.fn<() => Promise>().mockRejectedValue(error); + + await expect(retryProviderRequest(request, { maxRetries: 2, noRetryStatuses: [504] })).rejects.toBe(error); + expect(request).toHaveBeenCalledTimes(1); + }); + it("rejects a provider-requested retry delay above the limit", async () => { const request = vi.fn<() => Promise>().mockRejectedValue(providerError(429, { "retry-after": "277403" })); diff --git a/packages/ai/test/typesafe-system-one.test.ts b/packages/ai/test/typesafe-system-one.test.ts index 821115bed..97e56f0aa 100644 --- a/packages/ai/test/typesafe-system-one.test.ts +++ b/packages/ai/test/typesafe-system-one.test.ts @@ -120,6 +120,19 @@ describe("TypeSafe System One", () => { expect(result.errorMessage).toContain("Unsupported classifier API: cloudflare-workers-ai-system-one"); }); + it("rejects image input before sending", async () => { + const fetch = vi.fn(async () => Response.json({ answers: wireAnswers })); + const result = await classify( + { ...model, input: ["text", "image"] }, + { ...context, images: [{ type: "image", data: "aW1hZ2U=", mimeType: "image/png" }] }, + { apiKey: "secret", fetch }, + ); + + expect(fetch).not.toHaveBeenCalled(); + expect(result.stopReason).toBe("error"); + expect(result.errorMessage).toBe("System One API does not support image input"); + }); + it("merges headers case-insensitively and supports null suppression", async () => { const requests: Array> = []; const fetch = vi.fn(async (_input: string | URL | Request, init?: RequestInit) => { diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index f233300ff..ed8ee0b7c 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -7,6 +7,8 @@ - Added `+name` and `-name` entries to `--tools`, which change the default tool selection instead of replacing it, for example `pi -t +codemode` - Added `durationMs` to the tool render context and to `tool_execution_end` extension events: the recorded execution time of a final tool result ([#10549](https://github.com/earendil-works/pi/issues/10549)) - Added `outputPad` to the tool render context ([#10557](https://github.com/earendil-works/pi/pull/10557) by [@rwachtler](https://github.com/rwachtler)) +- Added OpenAI's GPT-6 Luna as a classifier model through the Decisions API, available with `OPENAI_API_KEY` (see [Use classifier models](docs/models.md#use-classifier-models)) +- Added `images` to codemode's `models.classify()` context, so classifiers that accept images, such as GPT-6 Luna, can judge them ### Changed diff --git a/packages/coding-agent/docs/codemode.md b/packages/coding-agent/docs/codemode.md index 1a873e36f..0d8dd795a 100644 --- a/packages/coding-agent/docs/codemode.md +++ b/packages/coding-agent/docs/codemode.md @@ -59,7 +59,7 @@ The store is for small state such as IDs, cursors, or summaries. One value may h ## Models -`models` reaches the model catalog and runs non-LLM models with the session's credentials: classifiers, which answer typed questions about JSON state, and image models, which generate images. Chat models are listed but cannot be run from scripts. Which classifier and image models exist is described in [Use classifier models](models.md#use-classifier-models) and [Use image models](models.md#use-image-models). +`models` reaches the model catalog and runs non-LLM models with the session's credentials: classifiers, which answer typed questions about JSON state and, for some models, images, and image models, which generate images. Chat models are listed but cannot be run from scripts. Which classifier and image models exist is described in [Use classifier models](models.md#use-classifier-models) and [Use image models](models.md#use-image-models). ```ts type ModelType = "chat" | "image" | "classifier"; @@ -100,6 +100,8 @@ Model IDs differ between providers, for example `typesafe/jev-latest` and `openr interface ClassifierContext { /** The data to classify. */ state: Record; + /** Images judged together with `state`. Only models whose `input` includes "image" accept them. */ + images?: { type: "image"; data: string; mimeType: string }[]; /** Questions by ID. One call answers all of them. */ questions: Record; } @@ -163,6 +165,24 @@ return results.map((result, i) => ); ``` +Classifiers whose `input` includes `"image"` also judge images. `tools.read()` returns an image file as an image block that `images` accepts. Other classifiers return an error result when `images` is not empty. + +```js +const luna = await models.getModelOfType("classifier", "openai", "gpt-6-luna"); +const photo = await tools.read({ path: "screenshot.png" }); +const result = await models.classify(luna, { + state: { task: "Settings page redesign" }, + images: [photo], + questions: { + broken: { + type: "bool", + instructions: "Does the screenshot show a broken layout?", + criteria: { true: "Overlapping, cut-off, or misaligned elements", false: "Clean layout" }, + }, + }, +}); +``` + ### Generate images ```ts diff --git a/packages/coding-agent/docs/models.md b/packages/coding-agent/docs/models.md index 838eec835..be39ce8a7 100644 --- a/packages/coding-agent/docs/models.md +++ b/packages/coding-agent/docs/models.md @@ -128,7 +128,7 @@ Compatibility settings should describe verified differences in the endpoint's re ## Use classifier models -Classifier models do not chat. They answer typed questions about JSON state: pick one of several choices, answer yes or no, or give a score, each with probabilities. Pi includes TypeSafe's Jev model from these providers, and Cloudflare's Clef and Clef Flash models from Workers AI: +Classifier models do not chat. They answer typed questions about JSON state: pick one of several choices, answer yes or no, or give a score, each with probabilities. Pi includes TypeSafe's Jev model from these providers, Cloudflare's Clef and Clef Flash models from Workers AI, and OpenAI's GPT-6 Luna through the [Decisions API](https://developers.openai.com/api/docs/guides/decisions): | Provider | Model IDs | Authentication | |---|---|---| @@ -137,9 +137,12 @@ Classifier models do not chat. They answer typed questions about JSON state: pic | `cloudflare-workers-ai` | `typesafe/jev`, `@cf/cloudflare/clef`, `@cf/cloudflare/clef-flash` | `CLOUDFLARE_API_KEY` and `CLOUDFLARE_ACCOUNT_ID` | | `vercel-ai-gateway` | `typesafe-ai/jev` | `AI_GATEWAY_API_KEY` | | `opencode` | `jev-1.13`, `jev-1.13-free` | `OPENCODE_API_KEY` | +| `openai` | `gpt-6-luna` | `OPENAI_API_KEY` | Chat models on a [llama.cpp router](llama-cpp.md#classification) are also listed as classifier models. +OpenAI's Decisions API needs an API key. Sign in with ChatGPT credentials do not work with it, so `gpt-6-luna` is not listed as available while `openai` is logged in through `/login`, even when `OPENAI_API_KEY` is set; log out of `openai` to use the key. GPT-6 Luna also judges images passed in `images` (see [Codemode](codemode.md#classify)); other classifier models return an error for them. The API rejects inputs above 922K tokens, but requests that run longer than about five seconds, currently above roughly 600K input tokens, fail with a gateway timeout. + Classifier models do not appear in `/model`. The model reaches them through the [`codemode`](cli.md#enable-codemode) tool, which is off unless an MCP server turned it on. Enable it with `"defaultTools": ["+codemode"]` in [settings](settings.md#tools). Scripts then list classifier models with `models.getAvailableOfType("classifier")` and call `models.classify(model, { state, questions })`: ```js diff --git a/packages/coding-agent/src/core/model-runtime.ts b/packages/coding-agent/src/core/model-runtime.ts index 7edd44048..81acdbb74 100644 --- a/packages/coding-agent/src/core/model-runtime.ts +++ b/packages/coding-agent/src/core/model-runtime.ts @@ -59,6 +59,7 @@ import { import * as builtinProviderCatalog from "@earendil-works/pi-ai/providers/all"; import { assertChatModel, + assertClassifierInputSupported, assertClassifierModel, assertImageModel, classifierErrorResult, @@ -804,6 +805,7 @@ export class ModelRuntime implements Models { ): Promise { try { assertClassifierModel(model); + assertClassifierInputSupported(model, context); const prepared = await this.prepareRequest(model, options); if (!prepared.provider.classify) { throw new ModelsError("provider", `Provider ${model.provider} does not support classification`); diff --git a/packages/coding-agent/src/extensions/codemode/execute.ts b/packages/coding-agent/src/extensions/codemode/execute.ts index 306af4552..3aafd6104 100644 --- a/packages/coding-agent/src/extensions/codemode/execute.ts +++ b/packages/coding-agent/src/extensions/codemode/execute.ts @@ -116,7 +116,7 @@ function describeValue(value: unknown): string { } const CLASSIFIER_CONTEXT_SHAPE = - '{ state: { ... }, questions: { : { type: "choice", instructions, criteria: {