diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index ae39a9dd3..ca0381b6f 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -12,6 +12,7 @@ - Fixed resumed sessions dropping tools activated during the session. Resume now restores the session's active tools, with `--tools +name`/`-name` entries applied to them, unless `--tools` with tool names, `--no-tools`, or `--no-builtin-tools` selects tools explicitly ([#10785](https://github.com/earendil-works/pi/issues/10785)) - Fixed stopping a codemode script or reaching its deadline waiting for a non-interruptible built-in to return on Bun ([#10748](https://github.com/earendil-works/pi/issues/10748)) - Fixed repeated codemode timeouts or aborts accumulating surviving workers; admission now limits live workers to two across sessions, counting cancelled workers until actual exit ([#10748](https://github.com/earendil-works/pi/issues/10748)) +- Fixed `--model` choosing a provider by catalog order when several providers offer the match; Pi now prefers the provider with credentials and otherwise reports the ambiguity. `--models` and `enabledModels` add a model for each provider that offers it and warn about the ambiguity ([#10160](https://github.com/earendil-works/pi/issues/10160)) ## [1.2.0] - 2026-10-12 diff --git a/packages/coding-agent/docs/cli.md b/packages/coding-agent/docs/cli.md index 5a107fd63..6719bd355 100644 --- a/packages/coding-agent/docs/cli.md +++ b/packages/coding-agent/docs/cli.md @@ -63,13 +63,13 @@ See [Choose a Model](models.md) for model selection and [Providers](providers.md - `--provider `
Restricts `--model` lookup to one provider. It requires `--model`. - `--model `
- Selects by exact ID or fuzzy ID/name match. It accepts `provider/id` and an optional `:` suffix. + Selects by exact ID or fuzzy ID/name match. It accepts `provider/id` and an optional `:` suffix. If several providers match, Pi prefers the one with credentials, otherwise it asks for `--provider`. - `--api-key `
- Uses a non-persistent API-key override. It requires a model selected through `--model` or `--models`. + Uses a non-persistent API-key override. It requires a model selected through `--model` or `--models`. It does not choose between providers that offer the same model. - `--thinking `
Sets `off`, `minimal`, `low`, `medium`, `high`, `xhigh`, or `max`. It overrides a `--model` suffix and is clamped to the model's capabilities. - `--models `
- Sets a comma-separated scope for startup and cycling. It accepts exact IDs, fuzzy matches, case-insensitive globs, and optional `:` suffixes. + Sets a comma-separated scope for startup and cycling. It accepts exact IDs, fuzzy matches, case-insensitive globs, and optional `:` suffixes. A model offered by several providers is added for each, with a warning. - `--list-models [search]`
Lists available models, optionally filtered by a fuzzy search, then exits. diff --git a/packages/coding-agent/src/cli/credential-print.ts b/packages/coding-agent/src/cli/credential-print.ts index 304876659..f5250b978 100644 --- a/packages/coding-agent/src/cli/credential-print.ts +++ b/packages/coding-agent/src/cli/credential-print.ts @@ -1,5 +1,5 @@ import type { Api, CredentialInfo, Model } from "@earendil-works/pi-ai"; -import { resolveCliModel } from "../core/model-resolver.ts"; +import { resolveCliModel, resolveModelReference } from "../core/model-resolver.ts"; import type { ModelRuntime } from "../core/model-runtime.ts"; import type { Args } from "./args.ts"; import { AuthCommandError, type AuthCommandKind, getAuthCredential, validateAuthCommandArgs } from "./auth-command.ts"; @@ -41,12 +41,15 @@ export async function resolveCredentialForPrint( providers.push({ id: provider.id }); } } else { + const models = modelRuntime.getModels(); for (const provider of modelRuntime.getProviders()) { if (!credentialTypes.has(provider.id)) continue; - const resolved = resolveCliModel({ cliProvider: provider.id, cliModel: cliModel!, modelRuntime }); - if (resolved.model && !resolved.error && !resolved.warning?.includes("Using custom model id")) { - providers.push({ id: provider.id, model: resolved.model }); - } + const resolved = resolveModelReference( + { text: cliModel!, provider: provider.id, thinkingSuffix: "strict", fuzzy: true }, + models, + () => true, + ); + if (resolved.status === "resolved") providers.push({ id: provider.id, model: resolved.model }); } if (providers.length === 0) { throw new AuthCommandError(`Model "${cliModel}" not found. Use --list-models to see available models.`); diff --git a/packages/coding-agent/src/core/model-resolver.ts b/packages/coding-agent/src/core/model-resolver.ts index a59d78e36..0e9312074 100644 --- a/packages/coding-agent/src/core/model-resolver.ts +++ b/packages/coding-agent/src/core/model-resolver.ts @@ -67,116 +67,234 @@ export interface ScopedModel { thinkingLevel?: ThinkingLevel; } -/** - * Helper to check if a model ID looks like an alias (no date suffix) - * Dates are typically in format: -20241022 or -20250929 +/* + * Model reference resolution. Candidates are collected before a provider is chosen, so catalog order + * never decides the provider and an ambiguous match is reported instead of falling through to a + * weaker one. */ -function isAlias(id: string): boolean { - // Check if ID ends with -latest - if (id.endsWith("-latest")) return true; - // Check if ID ends with a date pattern (-YYYYMMDD) - const datePattern = /-\d{8}$/; - return !datePattern.test(id); +export interface ModelReferenceRequest { + /** Bare model id or `provider/id`, optionally followed by `:`. */ + text: string; + /** Only consider models from this provider. */ + provider?: string; + /** + * A trailing `:` is only considered when the full text matches no model, since ids such as + * OpenRouter's `qwen/qwen3-coder:exacto` contain colons. + * - `none`: the suffix stays part of the id + * - `strict`: a valid thinking level is parsed; any other suffix stays part of the id + * - `lenient`: like `strict`, but an invalid suffix is dropped with a warning + */ + thinkingSuffix: "none" | "strict" | "lenient"; + /** Match id and name substrings when no exact match exists. */ + fuzzy: boolean; +} + +export type ModelReferenceResult = + | { status: "resolved"; model: Model; thinkingLevel?: ThinkingLevel; warning?: string } + | { + status: "ambiguous"; + candidates: Model[]; + usableCount: number; + thinkingLevel?: ThinkingLevel; + warning?: string; + } + | { + status: "not-found"; + /** Set when a provider is explicit or named by a `provider/` prefix, for building a custom model id. */ + target?: ModelReferenceTarget; + }; + +export interface ModelReferenceTarget { + provider: string; + providerModels: [Model, ...Model[]]; + /** Text after the provider prefix as typed, including any `:`. */ + modelText: string; + /** `modelText` without a valid thinking suffix (unless `thinkingSuffix` is `none`). */ + modelId: string; + thinkingLevel?: ThinkingLevel; +} + +type ResolvedOrAmbiguous = Exclude; + +function splitThinkingSuffix(text: string): { base: string; thinkingLevel: ThinkingLevel } | undefined { + const colonIndex = text.lastIndexOf(":"); + const suffix = text.substring(colonIndex + 1); + if (colonIndex === -1 || !isValidThinkingLevel(suffix)) return undefined; + return { base: text.substring(0, colonIndex), thinkingLevel: suffix }; +} + +function splitProviderPrefix( + text: string, + models: readonly Model[], +): { provider: string; modelText: string } | undefined { + const slashIndex = text.indexOf("/"); + if (slashIndex === -1) return undefined; + const prefix = text.substring(0, slashIndex).trim().toLowerCase(); + const modelText = text.substring(slashIndex + 1).trim(); + if (!prefix || !modelText) return undefined; + const provider = models.find((model) => model.provider.toLowerCase() === prefix)?.provider; + return provider ? { provider, modelText } : undefined; } /** - * Find an exact model reference match. - * Supports either a bare model id or a canonical provider/modelId reference. - * When matching by bare id, ambiguous matches across providers are rejected. + * `usable` only breaks ties between providers, so a unique match resolves even without credentials + * (`--api-key` first-time setup). */ +export function resolveModelReference( + request: ModelReferenceRequest, + models: readonly Model[], + usable: (provider: string) => boolean, +): ModelReferenceResult { + const text = request.text.trim(); + if (!text) return { status: "not-found" }; + const constraint = request.provider?.toLowerCase(); + const pool = constraint ? models.filter((model) => model.provider.toLowerCase() === constraint) : models; + const result = resolveText(text, request, pool, usable); + if (result.status !== "not-found") return result; + + const split = splitProviderPrefix(text, pool); + const anchor = constraint ? pool[0] : pool.find((model) => model.provider === split?.provider); + if (!anchor) return result; + const modelText = split?.modelText ?? text; + const suffix = request.thinkingSuffix === "none" ? undefined : splitThinkingSuffix(modelText); + return { + status: "not-found", + target: { + provider: anchor.provider, + providerModels: [anchor, ...pool.filter((model) => model.provider === anchor.provider && model !== anchor)], + modelText, + modelId: suffix?.base ?? modelText, + thinkingLevel: suffix?.thinkingLevel, + }, + }; +} + +/** Exact `provider/id` or bare id lookup without thinking suffixes or fuzzy matching. */ export function findExactModelReferenceMatch( modelReference: string, - availableModels: Model[], + models: readonly Model[], ): Model | undefined { - const trimmedReference = modelReference.trim(); - if (!trimmedReference) { - return undefined; - } - - const normalizedReference = trimmedReference.toLowerCase(); - - const canonicalMatches = availableModels.filter( - (model) => `${model.provider}/${model.id}`.toLowerCase() === normalizedReference, + const result = resolveModelReference( + { text: modelReference, thinkingSuffix: "none", fuzzy: false }, + models, + () => true, ); - if (canonicalMatches.length === 1) { - return canonicalMatches[0]; - } - if (canonicalMatches.length > 1) { - return undefined; + return result.status === "resolved" ? result.model : undefined; +} + +function resolveText( + text: string, + request: ModelReferenceRequest, + pool: readonly Model[], + usable: (provider: string) => boolean, +): ModelReferenceResult { + const reference = selectReference(text, pool, request.fuzzy, usable); + if (reference) return reference; + + if (request.fuzzy) { + const fuzzy = selectFuzzy(fuzzyMatches(text, pool), usable); + if (fuzzy) return fuzzy; } - const slashIndex = trimmedReference.indexOf("/"); - if (slashIndex !== -1) { - const provider = trimmedReference.substring(0, slashIndex).trim(); - const modelId = trimmedReference.substring(slashIndex + 1).trim(); - if (provider && modelId) { - const providerMatches = availableModels.filter( - (model) => - model.provider.toLowerCase() === provider.toLowerCase() && - model.id.toLowerCase() === modelId.toLowerCase(), - ); - if (providerMatches.length === 1) { - return providerMatches[0]; - } - if (providerMatches.length > 1) { - return undefined; - } - } + if (request.thinkingSuffix === "none") return { status: "not-found" }; + const thinking = splitThinkingSuffix(text); + if (thinking) { + const inner = resolveText(thinking.base, request, pool, usable); + if (inner.status === "not-found") return inner; + // Only use this thinking level if no warning from inner recursion + return { ...inner, thinkingLevel: inner.warning ? undefined : thinking.thinkingLevel }; } + // Keep an invalid suffix as part of the id and fail, rather than resolve a different model + if (request.thinkingSuffix === "strict") return { status: "not-found" }; + const colonIndex = text.lastIndexOf(":"); + if (colonIndex === -1) return { status: "not-found" }; - const idMatches = availableModels.filter((model) => model.id.toLowerCase() === normalizedReference); - return idMatches.length === 1 ? idMatches[0] : undefined; + const suffix = text.substring(colonIndex + 1); + const inner = resolveText(text.substring(0, colonIndex), request, pool, usable); + if (inner.status === "not-found") return inner; + return { + ...inner, + thinkingLevel: undefined, + warning: `Invalid thinking level "${suffix}" in pattern "${text}". Using default instead.`, + }; } /** - * Try to match a pattern to a model from the available models list. - * Returns the matched model or undefined if no match found. + * `provider/rest` is only a hint: the whole text may also be a model id on another provider, such as + * gateway id `zai/glm-5` or commandcode id `xiaomi/mimo-v2.5-pro`. The named provider wins when usable, + * since that is what the syntax usually means; otherwise a sole usable raw match wins, since the named + * provider could not serve the request. An unusable named match is still preferred over unusable raw + * matches, so `--api-key` can set up the named provider. */ -function tryMatchModel(modelPattern: string, availableModels: Model[]): Model | undefined { - const exactMatch = findExactModelReferenceMatch(modelPattern, availableModels); - if (exactMatch) { - return exactMatch; +function selectReference( + text: string, + pool: readonly Model[], + fuzzy: boolean, + usable: (provider: string) => boolean, +): ResolvedOrAmbiguous | undefined { + const lowerText = text.toLowerCase(); + const raw = pool.filter((model) => model.id.toLowerCase() === lowerText); + const split = splitProviderPrefix(text, pool); + let named: Model | undefined; + if (split) { + const providerModels = pool.filter((model) => model.provider === split.provider); + const lowerModelText = split.modelText.toLowerCase(); + // The provider's own literal id (OpenRouter's `openrouter/free`) beats its fuzzy matches + named = + providerModels.find((model) => model.id.toLowerCase() === lowerModelText) ?? + raw.find((model) => model.provider === split.provider) ?? + (fuzzy ? bestFuzzyMatch(fuzzyMatches(split.modelText, providerModels)) : undefined); } - // No exact match - fall back to partial matching - const matches = availableModels.filter( - (m) => - m.id.toLowerCase().includes(modelPattern.toLowerCase()) || - m.name?.toLowerCase().includes(modelPattern.toLowerCase()), + const others = raw.filter((model) => model !== named); + const candidates = named ? [named, ...others] : others; + if (candidates.length === 0) return undefined; + if (new Set(candidates.map((model) => model.provider)).size === 1) + return { status: "resolved", model: candidates[0] }; + if (named && usable(named.provider)) return { status: "resolved", model: named }; + const usableOthers = others.filter((model) => usable(model.provider)); + if (usableOthers.length === 1) return { status: "resolved", model: usableOthers[0] }; + if (usableOthers.length === 0 && named) return { status: "resolved", model: named }; + return { status: "ambiguous", candidates, usableCount: usableOthers.length }; +} + +function fuzzyMatches(text: string, models: readonly Model[]): Model[] { + const lowerText = text.toLowerCase(); + return models.filter( + (model) => model.id.toLowerCase().includes(lowerText) || model.name?.toLowerCase().includes(lowerText), ); - - if (matches.length === 0) { - return undefined; - } - - // Separate into aliases and dated versions - const aliases = matches.filter((m) => isAlias(m.id)); - const datedVersions = matches.filter((m) => !isAlias(m.id)); - - if (aliases.length > 0) { - // Prefer alias - if multiple aliases, pick the one that sorts highest - aliases.sort((a, b) => b.id.localeCompare(a.id)); - return aliases[0]; - } else { - // No alias found, pick latest dated version - datedVersions.sort((a, b) => b.id.localeCompare(a.id)); - return datedVersions[0]; - } } -export interface ParsedModelResult { - model: Model | undefined; - /** Thinking level if explicitly specified in pattern, undefined otherwise */ - thinkingLevel?: ThinkingLevel; - warning: string | undefined; +function selectFuzzy(matches: Model[], usable: (provider: string) => boolean): ResolvedOrAmbiguous | undefined { + if (matches.length === 0) return undefined; + const multipleProviders = new Set(matches.map((model) => model.provider)).size > 1; + const usableMatches = multipleProviders ? matches.filter((model) => usable(model.provider)) : []; + const pool = usableMatches.length > 0 ? usableMatches : matches; + const best = bestFuzzyMatch(pool); + if (!best) return undefined; + // The same id on several providers would otherwise be decided by catalog order + const tied = pool.filter((model) => model.id === best.id); + if (new Set(tied.map((model) => model.provider)).size === 1) return { status: "resolved", model: best }; + return { + status: "ambiguous", + candidates: tied, + usableCount: usableMatches.length > 0 ? tied.length : 0, + }; } -function buildFallbackModel(provider: string, modelId: string, availableModels: Model[]): Model | undefined { - const providerModels = availableModels.filter((m) => m.provider === provider); - if (providerModels.length === 0) return undefined; +/** Prefer aliases (`claude-sonnet-4-5`) over dated versions (`claude-sonnet-4-5-20250929`), then the latest. */ +function bestFuzzyMatch(matches: readonly Model[]): Model | undefined { + const aliases = matches.filter((model) => isAlias(model.id)); + return [...(aliases.length > 0 ? aliases : matches)].sort((a, b) => b.id.localeCompare(a.id))[0]; +} - const defaultId = defaultModelPerProvider[provider as KnownProvider]; +function isAlias(id: string): boolean { + return id.endsWith("-latest") || !/-\d{8}$/.test(id); +} + +function buildFallbackModel(providerModels: [Model, ...Model[]], modelId: string): Model { + const defaultId = defaultModelPerProvider[providerModels[0].provider as KnownProvider]; const baseModel = defaultId ? (providerModels.find((m) => m.id === defaultId) ?? providerModels[0]) : providerModels[0]; @@ -188,88 +306,9 @@ function buildFallbackModel(provider: string, modelId: string, availableModels: }; } -/** - * Parse a pattern to extract model and thinking level. - * Handles models with colons in their IDs (e.g., OpenRouter's :exacto suffix). - * - * Algorithm: - * 1. Try to match full pattern as a model - * 2. If found, return it with "off" thinking level - * 3. If not found and has colons, split on last colon: - * - If suffix is valid thinking level, use it and recurse on prefix - * - If suffix is invalid, warn and recurse on prefix with "off" - * - * @internal Exported for testing - */ -export function parseModelPattern( - pattern: string, - availableModels: Model[], - options?: { allowInvalidThinkingLevelFallback?: boolean }, -): ParsedModelResult { - // Try exact match first - const exactMatch = tryMatchModel(pattern, availableModels); - if (exactMatch) { - return { model: exactMatch, thinkingLevel: undefined, warning: undefined }; - } - - // No match - try splitting on last colon if present - const lastColonIndex = pattern.lastIndexOf(":"); - if (lastColonIndex === -1) { - // No colons, pattern simply doesn't match any model - return { model: undefined, thinkingLevel: undefined, warning: undefined }; - } - - const prefix = pattern.substring(0, lastColonIndex); - const suffix = pattern.substring(lastColonIndex + 1); - - if (isValidThinkingLevel(suffix)) { - // Valid thinking level - recurse on prefix and use this level - const result = parseModelPattern(prefix, availableModels, options); - if (result.model) { - // Only use this thinking level if no warning from inner recursion - return { - model: result.model, - thinkingLevel: result.warning ? undefined : suffix, - warning: result.warning, - }; - } - return result; - } else { - // Invalid suffix - const allowFallback = options?.allowInvalidThinkingLevelFallback ?? true; - if (!allowFallback) { - // In strict mode (CLI --model parsing), treat it as part of the model id and fail. - // This avoids accidentally resolving to a different model. - return { model: undefined, thinkingLevel: undefined, warning: undefined }; - } - - // Scope mode: recurse on prefix and warn - const result = parseModelPattern(prefix, availableModels, options); - if (result.model) { - return { - model: result.model, - thinkingLevel: undefined, - warning: `Invalid thinking level "${suffix}" in pattern "${pattern}". Using default instead.`, - }; - } - return result; - } -} - -/** - * Resolve model patterns to actual Model objects with optional thinking levels - * Format: "pattern:level" where :level is optional - * For each pattern, finds all matching models and picks the best version: - * 1. Prefer alias (e.g., claude-sonnet-4-5) over dated versions (claude-sonnet-4-5-20250929) - * 2. If no alias, pick the latest dated version - * - * Supports models with colons in their IDs (e.g., OpenRouter's model:exacto). - * The algorithm tries to match the full pattern first, then progressively - * strips colon-suffixes to find a match. - */ export interface ModelScopeDiagnostic { type: "warning"; - code: "no-match" | "invalid-thinking-level"; + code: "no-match" | "invalid-thinking-level" | "ambiguous"; message: string; pattern: string; } @@ -279,83 +318,83 @@ export interface ResolveModelScopeResult { diagnostics: ModelScopeDiagnostic[]; } +/** + * Resolve scope patterns (`--models`, `enabledModels`) to models with optional thinking levels. + * + * Every model in `models` is usable, so credentials cannot choose between providers offering the same + * id. All of them are added, with a warning so the user can pick one with `provider/id`. + */ export function resolveModelScopeFromModels( patterns: string[], models: readonly Model[], ): ResolveModelScopeResult { - const availableModels = [...models]; const scopedModels: ScopedModel[] = []; const diagnostics: ModelScopeDiagnostic[] = []; + const add = (model: Model, thinkingLevel: ThinkingLevel | undefined) => { + if (!scopedModels.some((scoped) => modelsAreEqual(scoped.model, model))) { + scopedModels.push({ model, thinkingLevel }); + } + }; + const noMatch = (pattern: string) => + diagnostics.push({ type: "warning", code: "no-match", message: `No models match pattern "${pattern}"`, pattern }); + const ambiguous = (pattern: string, candidates: readonly Model[]) => { + const providers = [...new Set(candidates.map((m) => m.provider))].sort((a, b) => a.localeCompare(b)); + diagnostics.push({ + type: "warning", + code: "ambiguous", + message: `Pattern "${pattern}" matches several providers: ${providers.join(", ")}. Use provider/id to choose one.`, + pattern, + }); + }; for (const pattern of patterns) { // Check if pattern contains glob characters if (pattern.includes("*") || pattern.includes("?") || pattern.includes("[")) { // Extract optional thinking level suffix (e.g., "provider/*:high") - const colonIdx = pattern.lastIndexOf(":"); - let globPattern = pattern; - let thinkingLevel: ThinkingLevel | undefined; + const thinking = splitThinkingSuffix(pattern); + const globPattern = thinking?.base ?? pattern; + const thinkingLevel = thinking?.thinkingLevel; - if (colonIdx !== -1) { - const suffix = pattern.substring(colonIdx + 1); - if (isValidThinkingLevel(suffix)) { - thinkingLevel = suffix; - globPattern = pattern.substring(0, colonIdx); - } - } - - const exactMatch = findExactModelReferenceMatch(globPattern, availableModels); - if (exactMatch) { - if (!scopedModels.find((sm) => modelsAreEqual(sm.model, exactMatch))) { - scopedModels.push({ model: exactMatch, thinkingLevel }); - } - continue; - } - - // Match against "provider/modelId" format OR just model ID - // This allows "*sonnet*" to match without requiring "anthropic/*sonnet*" - const matchingModels = availableModels.filter((m) => { - const fullId = `${m.provider}/${m.id}`; - return minimatch(fullId, globPattern, { nocase: true }) || minimatch(m.id, globPattern, { nocase: true }); - }); - - if (matchingModels.length === 0) { - diagnostics.push({ - type: "warning", - code: "no-match", - message: `No models match pattern "${pattern}"`, - pattern, - }); - continue; - } - - for (const model of matchingModels) { - if (!scopedModels.find((sm) => modelsAreEqual(sm.model, model))) { - scopedModels.push({ model, thinkingLevel }); - } + // Ids such as `bracketed-model[1m]` are not globs + const exact = resolveModelReference( + { text: globPattern, thinkingSuffix: "none", fuzzy: false }, + models, + () => true, + ); + let matches: readonly Model[] = []; + if (exact.status === "resolved") matches = [exact.model]; + else if (exact.status === "ambiguous") { + matches = exact.candidates; + ambiguous(pattern, matches); + } else { + // Match against "provider/modelId" format OR just model ID + // This allows "*sonnet*" to match without requiring "anthropic/*sonnet*" + matches = models.filter( + (m) => + minimatch(`${m.provider}/${m.id}`, globPattern, { nocase: true }) || + minimatch(m.id, globPattern, { nocase: true }), + ); } + if (matches.length === 0) noMatch(pattern); + for (const model of matches) add(model, thinkingLevel); continue; } - const { model, thinkingLevel, warning } = parseModelPattern(pattern, availableModels); - - if (warning) { - diagnostics.push({ type: "warning", code: "invalid-thinking-level", message: warning, pattern }); - } - - if (!model) { - diagnostics.push({ - type: "warning", - code: "no-match", - message: `No models match pattern "${pattern}"`, - pattern, - }); + const result = resolveModelReference( + { text: pattern, thinkingSuffix: "lenient", fuzzy: true }, + models, + () => true, + ); + if (result.status === "not-found") { + noMatch(pattern); continue; } - - // Avoid duplicates - if (!scopedModels.find((sm) => modelsAreEqual(sm.model, model))) { - scopedModels.push({ model, thinkingLevel }); + if (result.warning) { + diagnostics.push({ type: "warning", code: "invalid-thinking-level", message: result.warning, pattern }); } + if (result.status === "ambiguous") ambiguous(pattern, result.candidates); + const matches = result.status === "resolved" ? [result.model] : result.candidates; + for (const model of matches) add(model, result.thinkingLevel); } return { scopedModels, diagnostics }; @@ -407,9 +446,11 @@ export function resolveCliModel(options: { cliProvider?: string; cliModel?: string; cliThinking?: ThinkingLevel; + /** `--api-key` was supplied for the selected provider. */ + inlineApiKey?: boolean; modelRuntime: ModelRuntime; }): ResolveCliModelResult { - const { cliProvider, cliModel, cliThinking, modelRuntime } = options; + const { cliProvider, cliModel, cliThinking, inlineApiKey, modelRuntime } = options; if (!cliModel) { return { model: undefined, warning: undefined, error: undefined }; @@ -417,8 +458,8 @@ export function resolveCliModel(options: { // Important: use *all* models here, not just models with pre-configured auth. // This allows "--api-key" to be used for first-time setup. - const availableModels = [...modelRuntime.getModels()]; - if (availableModels.length === 0) { + const models = [...modelRuntime.getModels()]; + if (models.length === 0) { return { model: undefined, warning: undefined, @@ -426,13 +467,9 @@ export function resolveCliModel(options: { }; } - // Build canonical provider lookup (case-insensitive) - const providerMap = new Map(); - for (const m of availableModels) { - providerMap.set(m.provider.toLowerCase(), m.provider); - } - - let provider = cliProvider ? providerMap.get(cliProvider.toLowerCase()) : undefined; + const provider = cliProvider + ? models.find((m) => m.provider.toLowerCase() === cliProvider.toLowerCase())?.provider + : undefined; if (cliProvider && !provider) { return { model: undefined, @@ -441,167 +478,53 @@ export function resolveCliModel(options: { }; } - // If no explicit --provider, try to interpret "provider/model" format first. - // When the prefix before the first slash matches a known provider, prefer that - // interpretation over matching models whose IDs literally contain slashes - // (e.g. "zai/glm-5" should resolve to provider=zai, model=glm-5, not to a - // vercel-ai-gateway model with id "zai/glm-5"). - let pattern = cliModel; - let inferredProvider = false; - - if (!provider) { - const slashIndex = cliModel.indexOf("/"); - if (slashIndex !== -1) { - const maybeProvider = cliModel.substring(0, slashIndex); - const canonical = providerMap.get(maybeProvider.toLowerCase()); - if (canonical) { - provider = canonical; - pattern = cliModel.substring(slashIndex + 1); - inferredProvider = true; - } - } + const result = resolveModelReference( + { text: cliModel, provider, thinkingSuffix: "strict", fuzzy: true }, + models, + // An inline key goes to whichever provider is selected, so stored credentials must not choose it + inlineApiKey ? () => false : (providerId) => modelRuntime.hasConfiguredAuth(providerId), + ); + if (result.status === "resolved") { + return { model: result.model, thinkingLevel: result.thinkingLevel, warning: result.warning, error: undefined }; + } + if (result.status === "ambiguous") { + const matches = result.candidates + .map((m) => `${m.provider}/${m.id}`) + .sort((a, b) => a.localeCompare(b)) + .join(", "); + const authHint = inlineApiKey + ? "--api-key does not choose a provider." + : result.usableCount === 0 + ? "No matching provider is authenticated." + : "More than one matching provider is authenticated."; + return { + model: undefined, + warning: undefined, + error: `Model "${cliModel}" is ambiguous across providers: ${matches}. ${authHint} Use --provider or provider/model.`, + }; } - // If no provider was inferred from the slash, try exact matches without provider inference. - // This handles models whose IDs naturally contain slashes (e.g. OpenRouter-style IDs). - // Bare exact IDs can exist in multiple providers, so do not choose by catalog order. - // Prefer the sole authenticated provider when there is one; otherwise require an - // explicit provider to avoid silently selecting an unusable provider. - if (!provider) { - const lower = cliModel.toLowerCase(); - const exactMatches = availableModels.filter( - (m) => m.id.toLowerCase() === lower || `${m.provider}/${m.id}`.toLowerCase() === lower, - ); - if (exactMatches.length === 1) { - return { model: exactMatches[0], warning: undefined, thinkingLevel: undefined, error: undefined }; - } - if (exactMatches.length > 1) { - const authenticatedExactMatches = exactMatches.filter((m) => modelRuntime.hasConfiguredAuth(m.provider)); - if (authenticatedExactMatches.length === 1) { - return { - model: authenticatedExactMatches[0], - warning: undefined, - thinkingLevel: undefined, - error: undefined, - }; - } - - const matches = exactMatches - .map((m) => `${m.provider}/${m.id}`) - .sort((a, b) => a.localeCompare(b)) - .join(", "); - const authHint = - authenticatedExactMatches.length === 0 - ? "No matching provider is authenticated." - : "More than one matching provider is authenticated."; - return { - model: undefined, - warning: undefined, - thinkingLevel: undefined, - error: `Model "${cliModel}" is ambiguous across providers: ${matches}. ${authHint} Use --provider or provider/model.`, - }; - } + const { target } = result; + if (!target) { + return { + model: undefined, + warning: undefined, + error: `Model "${cliModel}" not found. Use --list-models to see available models.`, + }; } - if (cliProvider && provider) { - // If both were provided, tolerate --model / by stripping the provider prefix - const prefix = `${provider}/`; - if (cliModel.toLowerCase().startsWith(prefix.toLowerCase())) { - pattern = cliModel.substring(prefix.length); - } - } - - const candidates = provider ? availableModels.filter((m) => m.provider === provider) : availableModels; - const { model, thinkingLevel, warning } = parseModelPattern(pattern, candidates, { - allowInvalidThinkingLevelFallback: false, - }); - - if (model) { - // If provider inference matched an unauthenticated provider/model pair, prefer - // one exact raw model-id match that is authenticated. This keeps - // "provider/model" syntax preferred when usable, but handles models whose - // literal id starts with a known provider name (for example - // commandcode model id "xiaomi/mimo-v2.5-pro"). - if (inferredProvider) { - const rawExactMatches = availableModels.filter( - (m) => m.id.toLowerCase() === cliModel.toLowerCase() && !modelsAreEqual(m, model), - ); - if (rawExactMatches.length > 0 && !modelRuntime.hasConfiguredAuth(model.provider)) { - const authenticatedRawMatches = rawExactMatches.filter((m) => modelRuntime.hasConfiguredAuth(m.provider)); - if (authenticatedRawMatches.length === 1) { - return { - model: authenticatedRawMatches[0], - thinkingLevel: undefined, - warning: undefined, - error: undefined, - }; - } - } - } - return { model, thinkingLevel, warning, error: undefined }; - } - - // If we inferred a provider from the slash but found no match within that provider, - // fall back to matching the full input as a raw model id across all models. - // This handles OpenRouter-style IDs like "openai/gpt-4o:extended" where "openai" - // looks like a provider but the full string is actually a model id on openrouter. - if (inferredProvider) { - const lower = cliModel.toLowerCase(); - const exact = availableModels.find( - (m) => m.id.toLowerCase() === lower || `${m.provider}/${m.id}`.toLowerCase() === lower, - ); - if (exact) { - return { model: exact, warning: undefined, thinkingLevel: undefined, error: undefined }; - } - // Also try parseModelPattern on the full input against all models - const fallback = parseModelPattern(cliModel, availableModels, { - allowInvalidThinkingLevelFallback: false, - }); - if (fallback.model) { - return { - model: fallback.model, - thinkingLevel: fallback.thinkingLevel, - warning: fallback.warning, - error: undefined, - }; - } - } - - if (provider) { - // Parse thinking level suffix from the pattern before building the fallback model, - // but only when --thinking is not explicitly provided. - // e.g. "zai-org/GLM-5.1-FP8:high" → modelId="zai-org/GLM-5.1-FP8", fallbackThinking="high" - let fallbackPattern = pattern; - let fallbackThinking: ThinkingLevel | undefined; - if (!cliThinking) { - const lastColon = pattern.lastIndexOf(":"); - if (lastColon !== -1) { - const suffix = pattern.substring(lastColon + 1); - if (isValidThinkingLevel(suffix)) { - fallbackPattern = pattern.substring(0, lastColon); - fallbackThinking = suffix; - } - } - } - - const fallbackModel = buildFallbackModel(provider, fallbackPattern, availableModels); - if (fallbackModel) { - const requestedThinking = cliThinking ?? fallbackThinking; - const model = - requestedThinking && requestedThinking !== "off" ? { ...fallbackModel, reasoning: true } : fallbackModel; - const fallbackWarning = warning - ? `${warning} Model "${fallbackPattern}" not found for provider "${provider}". Using custom model id.` - : `Model "${fallbackPattern}" not found for provider "${provider}". Using custom model id.`; - return { model, thinkingLevel: fallbackThinking, warning: fallbackWarning, error: undefined }; - } - } - - const display = provider ? `${provider}/${pattern}` : cliModel; + // Use the thinking level suffix only when --thinking is not explicitly provided; otherwise it + // stays part of the custom model id. + // e.g. "zai-org/GLM-5.1-FP8:high" → modelId="zai-org/GLM-5.1-FP8", fallbackThinking="high" + const modelId = cliThinking ? target.modelText : target.modelId; + const fallbackThinking = cliThinking ? undefined : target.thinkingLevel; + const fallbackModel = buildFallbackModel(target.providerModels, modelId); + const requestedThinking = cliThinking ?? fallbackThinking; return { - model: undefined, - thinkingLevel: undefined, - warning, - error: `Model "${display}" not found. Use --list-models to see available models.`, + model: requestedThinking && requestedThinking !== "off" ? { ...fallbackModel, reasoning: true } : fallbackModel, + thinkingLevel: fallbackThinking, + warning: `Model "${modelId}" not found for provider "${target.provider}". Using custom model id.`, + error: undefined, }; } @@ -613,15 +536,11 @@ export interface InitialModelResult { /** * Find the initial model to use based on priority: - * 1. CLI args (provider + model) - * 2. First model from scoped models (if not continuing/resuming) - * 3. Restored from session (if continuing/resuming) - * 4. Saved default from settings - * 5. First available model with valid API key + * 1. First model from scoped models (if not continuing/resuming) + * 2. Saved default from settings + * 3. First available model with valid API key */ export async function findInitialModel(options: { - cliProvider?: string; - cliModel?: string; scopedModels: ScopedModel[]; isContinuing: boolean; defaultProvider?: string; @@ -631,8 +550,6 @@ export async function findInitialModel(options: { modelRuntime: ModelRuntime; }): Promise { const { - cliProvider, - cliModel, scopedModels, isContinuing, defaultProvider, @@ -645,23 +562,7 @@ export async function findInitialModel(options: { let model: Model | undefined; let thinkingLevel: ThinkingLevel = DEFAULT_THINKING_LEVEL; - // 1. CLI args take priority - if (cliProvider && cliModel) { - const resolved = resolveCliModel({ - cliProvider, - cliModel, - modelRuntime, - }); - if (resolved.error) { - console.error(chalk.red(resolved.error)); - process.exit(1); - } - if (resolved.model) { - return { model: resolved.model, thinkingLevel: DEFAULT_THINKING_LEVEL, fallbackMessage: undefined }; - } - } - - // 2. Use first model from scoped models (skip if continuing/resuming) + // 1. Use first model from scoped models (skip if continuing/resuming) if (scopedModels.length > 0 && !isContinuing) { const scopedModel = scopedModels[0]; const perModel = modelThinkingLevels?.[`${scopedModel.model.provider}/${scopedModel.model.id}`]; @@ -672,7 +573,7 @@ export async function findInitialModel(options: { }; } - // 3. Try saved default from settings if auth is configured. + // 2. Try saved default from settings if auth is configured. if (defaultProvider && defaultModelId) { const found = modelRuntime.getModel(defaultProvider, defaultModelId); if (found && modelRuntime.hasConfiguredAuth(found.provider)) { @@ -687,7 +588,7 @@ export async function findInitialModel(options: { } } - // 4. Try first available model with valid API key + // 3. Try first available model with valid API key const availableModels = [...modelRuntime.getAvailableSnapshot()]; if (availableModels.length > 0) { @@ -704,7 +605,7 @@ export async function findInitialModel(options: { return { model: availableModels[0], thinkingLevel: DEFAULT_THINKING_LEVEL, fallbackMessage: undefined }; } - // 5. No model found + // 4. No model found return { model: undefined, thinkingLevel: DEFAULT_THINKING_LEVEL, fallbackMessage: undefined }; } diff --git a/packages/coding-agent/src/main.ts b/packages/coding-agent/src/main.ts index 0710ed9e6..51df447c5 100644 --- a/packages/coding-agent/src/main.ts +++ b/packages/coding-agent/src/main.ts @@ -477,6 +477,7 @@ function buildSessionOptions( cliProvider: parsed.provider, cliModel: parsed.model, cliThinking: parsed.thinking, + inlineApiKey: Boolean(parsed.apiKey), modelRuntime, }); if (resolved.warning) { diff --git a/packages/coding-agent/test/credential-print.test.ts b/packages/coding-agent/test/credential-print.test.ts index a1ca56d9c..ebf306f4b 100644 --- a/packages/coding-agent/test/credential-print.test.ts +++ b/packages/coding-agent/test/credential-print.test.ts @@ -24,6 +24,20 @@ describe("credential print commands", () => { await expect(resolveCredentialForPrint(args, runtime, "api_key")).resolves.toBe("test-api-key"); }); + test("resolves --model without --provider only among catalog models of configured providers", async () => { + const runtime = await createRuntime( + AuthStorage.inMemory({ + openai: { type: "api_key", key: "openai-test-key" }, + anthropic: { type: "api_key", key: "anthropic-test-key" }, + }), + ); + + // anthropic has no gpt-5.5; an uncatalogued id on a configured provider must not count as a match. + await expect(resolveCredentialForPrint(parseArgs(["--model", "gpt-5.5"]), runtime, "api_key")).resolves.toBe( + "openai-test-key", + ); + }); + test("prints bearer tokens resolved from an Authorization header", async () => { const runtime = await createRuntime( AuthStorage.inMemory({ diff --git a/packages/coding-agent/test/model-resolver.test.ts b/packages/coding-agent/test/model-resolver.test.ts index 866a6990f..f8226a2be 100644 --- a/packages/coding-agent/test/model-resolver.test.ts +++ b/packages/coding-agent/test/model-resolver.test.ts @@ -2,7 +2,7 @@ import { mkdirSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { Agent } from "@earendil-works/pi-agent-core"; -import type { Model } from "@earendil-works/pi-ai"; +import type { Api, Model } from "@earendil-works/pi-ai"; import { getModel, streamSimple } from "@earendil-works/pi-ai/compat"; import { getBuiltinModels, getBuiltinProviders } from "@earendil-works/pi-ai/providers/all"; import { afterEach, describe, expect, test, vi } from "vitest"; @@ -11,8 +11,9 @@ import { AuthStorage } from "../src/core/auth-storage.ts"; import { defaultModelPerProvider, findInitialModel, - parseModelPattern, + type ModelReferenceRequest, resolveCliModel, + resolveModelReference, resolveModelScope, resolveModelScopeWithDiagnostics, } from "../src/core/model-resolver.ts"; @@ -79,144 +80,258 @@ const mockOpenRouterModels: Model<"anthropic-messages">[] = [ const allModels = [...mockModels, ...mockOpenRouterModels]; -describe("parseModelPattern", () => { - describe("simple patterns without colons", () => { - test("exact match returns model with undefined thinking level", () => { - const result = parseModelPattern("claude-sonnet-4-5", allModels); - expect(result.model?.id).toBe("claude-sonnet-4-5"); - expect(result.thinkingLevel).toBeUndefined(); - expect(result.warning).toBeUndefined(); - }); +function referenceModel(provider: string, id: string): Model { + return { + id, + name: id, + api: "openai-completions", + provider, + baseUrl: "https://example.invalid", + reasoning: true, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 128000, + maxTokens: 8192, + }; +} - test("partial match returns best model with undefined thinking level", () => { - const result = parseModelPattern("sonnet", allModels); - expect(result.model?.id).toBe("claude-sonnet-4-5"); - expect(result.thinkingLevel).toBeUndefined(); - expect(result.warning).toBeUndefined(); - }); +const referenceCatalog = [ + referenceModel("anthropic", "claude-sonnet-4-5"), + referenceModel("anthropic", "claude-sonnet-4-5-20250929"), + referenceModel("anthropic", "claude-opus-4-8"), + referenceModel("anthropic", "exact"), + // Fuzzy-matches "shared"; an ambiguous exact "shared" must not fall through to it. + referenceModel("anthropic", "shared-pro"), + referenceModel("openai", "gpt-4o"), + referenceModel("openrouter", "qwen/qwen3-coder:exacto"), + referenceModel("openrouter", "openai/gpt-4o:extended"), + referenceModel("gw-a", "shared"), + referenceModel("gw-a", "anthropic/future"), + referenceModel("gw-a", "anthropic/exact"), + referenceModel("gw-a", "anthropic/claude-opus-4.8"), + referenceModel("gw-a", "anthropic/claude-opus"), + referenceModel("gw-b", "shared"), + referenceModel("gw-b", "anthropic/future"), + referenceModel("gw-b", "claude-opus-4-8"), + referenceModel("gw-b", "gw-b/free"), + referenceModel("gw-b", "zz:free"), +]; - test("no match returns undefined model and thinking level", () => { - const result = parseModelPattern("nonexistent", allModels); - expect(result.model).toBeUndefined(); - expect(result.thinkingLevel).toBeUndefined(); - expect(result.warning).toBeUndefined(); - }); - }); +type ReferenceExpected = + | { model: string; thinkingLevel?: string; warning?: string } + | { ambiguous: string[]; usableCount: number } + | "not-found"; - describe("patterns with valid thinking levels", () => { - test("sonnet:high returns sonnet with high thinking level", () => { - const result = parseModelPattern("sonnet:high", allModels); - expect(result.model?.id).toBe("claude-sonnet-4-5"); - expect(result.thinkingLevel).toBe("high"); - expect(result.warning).toBeUndefined(); - }); +interface ReferenceCase { + name: string; + text: string; + expected: ReferenceExpected; + usable?: string[]; + provider?: string; + thinkingSuffix?: ModelReferenceRequest["thinkingSuffix"]; + fuzzy?: boolean; +} - test("gpt-4o:medium returns gpt-4o with medium thinking level", () => { - const result = parseModelPattern("gpt-4o:medium", allModels); - expect(result.model?.id).toBe("gpt-4o"); - expect(result.thinkingLevel).toBe("medium"); - expect(result.warning).toBeUndefined(); - }); +const referenceCases: ReferenceCase[] = [ + // Reference syntax + { name: "exact id", text: "claude-sonnet-4-5", expected: { model: "anthropic/claude-sonnet-4-5" } }, + { name: "exact id is case-insensitive", text: "GPT-4O", expected: { model: "openai/gpt-4o" } }, + { name: "fuzzy prefers aliases over dated ids", text: "sonnet", expected: { model: "anthropic/claude-sonnet-4-5" } }, + { + name: "thinking suffix", + text: "sonnet:high", + expected: { model: "anthropic/claude-sonnet-4-5", thinkingLevel: "high" }, + }, + { + name: "colon in a literal id is not a thinking suffix", + text: "qwen/qwen3-coder:exacto", + expected: { model: "openrouter/qwen/qwen3-coder:exacto" }, + }, + { + name: "provider prefix, colon id, and thinking suffix", + text: "openrouter/qwen/qwen3-coder:exacto:high", + expected: { model: "openrouter/qwen/qwen3-coder:exacto", thinkingLevel: "high" }, + }, + { + name: "raw id wins when the provider prefix names no matching model", + text: "openai/gpt-4o:extended", + expected: { model: "openrouter/openai/gpt-4o:extended" }, + }, + { + name: "fuzzy searches the prefixed provider first", + text: "openrouter/qwen", + expected: { model: "openrouter/qwen/qwen3-coder:exacto" }, + }, + { name: "strict keeps an invalid suffix in the id", text: "gpt-4o:random", expected: "not-found" }, + { + name: "lenient drops an invalid suffix with a warning", + text: "gpt-4o:random", + thinkingSuffix: "lenient", + expected: { model: "openai/gpt-4o", warning: 'Invalid thinking level "random"' }, + }, + { + name: "lenient invalid outer suffix discards the inner thinking level", + text: "qwen/qwen3-coder:exacto:high:random", + thinkingSuffix: "lenient", + expected: { model: "openrouter/qwen/qwen3-coder:exacto", warning: 'Invalid thinking level "random"' }, + }, + { name: "none keeps the suffix in the id", text: "gpt-4o:high", thinkingSuffix: "none", expected: "not-found" }, + { name: "exact-only skips fuzzy matches", text: "sonnet", fuzzy: false, expected: "not-found" }, + { name: "empty text", text: " ", expected: "not-found" }, - test("all valid thinking levels work", () => { - for (const level of ["off", "minimal", "low", "medium", "high", "xhigh", "max"]) { - const result = parseModelPattern(`sonnet:${level}`, allModels); - expect(result.model?.id).toBe("claude-sonnet-4-5"); - expect(result.thinkingLevel).toBe(level); - expect(result.warning).toBeUndefined(); + // Exact matches across providers + // #7327 + { name: "sole usable provider wins", text: "shared", usable: ["gw-b"], expected: { model: "gw-b/shared" } }, + { + name: "no usable provider is ambiguous instead of falling through to fuzzy", + text: "shared", + expected: { ambiguous: ["gw-a/shared", "gw-b/shared"], usableCount: 0 }, + }, + { + name: "several usable providers are ambiguous", + text: "shared", + usable: ["gw-a", "gw-b"], + expected: { ambiguous: ["gw-a/shared", "gw-b/shared"], usableCount: 2 }, + }, + { + name: "thinking suffix does not change provider selection", + text: "shared:high", + usable: ["gw-b"], + expected: { model: "gw-b/shared", thinkingLevel: "high" }, + }, + { + name: "thinking suffix does not bypass ambiguity", + text: "shared:high", + usable: ["gw-a", "gw-b"], + expected: { ambiguous: ["gw-a/shared", "gw-b/shared"], usableCount: 2 }, + }, + { + // #10160 + name: "provider prefix without a direct model selects the usable raw id", + text: "anthropic/future", + usable: ["gw-b"], + expected: { model: "gw-b/anthropic/future" }, + }, + { + // #5823 + name: "usable qualified match beats a usable raw id", + text: "anthropic/exact", + usable: ["anthropic", "gw-a"], + expected: { model: "anthropic/exact" }, + }, + { + // #5643 + name: "usable raw id beats an unusable qualified match", + text: "anthropic/exact", + usable: ["gw-a"], + expected: { model: "gw-a/anthropic/exact" }, + }, + { + name: "unusable qualified match is kept when nothing is usable", + text: "anthropic/exact", + expected: { model: "anthropic/exact" }, + }, + + { + // Like `mistral/codestral` -> mistral/codestral-latest despite gateway id `mistral/codestral` + name: "named provider's usable fuzzy match beats a raw id elsewhere", + text: "anthropic/claude-opus", + usable: ["anthropic"], + expected: { model: "anthropic/claude-opus-4-8" }, + }, + { + name: "usable raw id beats the named provider's unusable fuzzy match", + text: "anthropic/claude-opus", + usable: ["gw-a"], + expected: { model: "gw-a/anthropic/claude-opus" }, + }, + { + name: "named provider's fuzzy match is kept when nothing is usable", + text: "anthropic/claude-opus", + expected: { model: "anthropic/claude-opus-4-8" }, + }, + { + name: "named provider's literal id beats its fuzzy matches", + text: "gw-b/free", + expected: { model: "gw-b/gw-b/free" }, + }, + + // Fuzzy matches across providers + { + name: "fuzzy prefers usable providers", + text: "opus", + usable: ["gw-a"], + expected: { model: "gw-a/anthropic/claude-opus-4.8" }, + }, + { + name: "fuzzy id tie across usable providers is ambiguous", + text: "opus", + usable: ["anthropic", "gw-a", "gw-b"], + expected: { ambiguous: ["anthropic/claude-opus-4-8", "gw-b/claude-opus-4-8"], usableCount: 2 }, + }, + { + name: "fuzzy id tie resolves to the sole usable provider", + text: "opus", + usable: ["anthropic"], + expected: { model: "anthropic/claude-opus-4-8" }, + }, + + // Explicit provider constraint + { + name: "explicit provider ignores usable providers elsewhere", + text: "shared", + provider: "gw-a", + usable: ["gw-b"], + expected: { model: "gw-a/shared" }, + }, + { + name: "explicit provider accepts its own prefix", + text: "GW-B/shared", + provider: "gw-b", + expected: { model: "gw-b/shared" }, + }, + { + name: "explicit provider allows fuzzy matches", + text: "4o", + provider: "openai", + expected: { model: "openai/gpt-4o" }, + }, + { name: "explicit provider never leaves the provider", text: "gpt-4o", provider: "gw-a", expected: "not-found" }, +]; + +function modelRef(m: Model): string { + return `${m.provider}/${m.id}`; +} + +describe("resolveModelReference", () => { + test.each(referenceCases)("$name", (testCase) => { + const usable = new Set(testCase.usable); + const request: ModelReferenceRequest = { + text: testCase.text, + provider: testCase.provider, + thinkingSuffix: testCase.thinkingSuffix ?? "strict", + fuzzy: testCase.fuzzy ?? true, + }; + // Catalog order must never decide the result. + for (const models of [referenceCatalog, [...referenceCatalog].reverse()]) { + const result = resolveModelReference(request, models, (provider) => usable.has(provider)); + const expected = testCase.expected; + if (expected === "not-found") { + expect(result.status).toBe("not-found"); + } else if ("ambiguous" in expected) { + expect(result.status).toBe("ambiguous"); + if (result.status !== "ambiguous") continue; + expect(result.candidates.map(modelRef).sort()).toEqual(expected.ambiguous); + expect(result.usableCount).toBe(expected.usableCount); + } else { + expect(result.status).toBe("resolved"); + if (result.status !== "resolved") continue; + expect(modelRef(result.model)).toBe(expected.model); + expect(result.thinkingLevel).toBe(expected.thinkingLevel); + if (expected.warning) expect(result.warning).toContain(expected.warning); + else expect(result.warning).toBeUndefined(); } - }); - }); - - describe("patterns with invalid thinking levels", () => { - test("sonnet:random returns sonnet with undefined thinking level and warning", () => { - const result = parseModelPattern("sonnet:random", allModels); - expect(result.model?.id).toBe("claude-sonnet-4-5"); - expect(result.thinkingLevel).toBeUndefined(); - expect(result.warning).toContain("Invalid thinking level"); - expect(result.warning).toContain("random"); - }); - - test("gpt-4o:invalid returns gpt-4o with undefined thinking level and warning", () => { - const result = parseModelPattern("gpt-4o:invalid", allModels); - expect(result.model?.id).toBe("gpt-4o"); - expect(result.thinkingLevel).toBeUndefined(); - expect(result.warning).toContain("Invalid thinking level"); - }); - }); - - describe("OpenRouter models with colons in IDs", () => { - test("qwen3-coder:exacto matches the model with undefined thinking level", () => { - const result = parseModelPattern("qwen/qwen3-coder:exacto", allModels); - expect(result.model?.id).toBe("qwen/qwen3-coder:exacto"); - expect(result.thinkingLevel).toBeUndefined(); - expect(result.warning).toBeUndefined(); - }); - - test("openrouter/qwen/qwen3-coder:exacto matches with provider prefix", () => { - const result = parseModelPattern("openrouter/qwen/qwen3-coder:exacto", allModels); - expect(result.model?.id).toBe("qwen/qwen3-coder:exacto"); - expect(result.model?.provider).toBe("openrouter"); - expect(result.thinkingLevel).toBeUndefined(); - expect(result.warning).toBeUndefined(); - }); - - test("qwen3-coder:exacto:high matches model with high thinking level", () => { - const result = parseModelPattern("qwen/qwen3-coder:exacto:high", allModels); - expect(result.model?.id).toBe("qwen/qwen3-coder:exacto"); - expect(result.thinkingLevel).toBe("high"); - expect(result.warning).toBeUndefined(); - }); - - test("openrouter/qwen/qwen3-coder:exacto:high matches with provider and thinking level", () => { - const result = parseModelPattern("openrouter/qwen/qwen3-coder:exacto:high", allModels); - expect(result.model?.id).toBe("qwen/qwen3-coder:exacto"); - expect(result.model?.provider).toBe("openrouter"); - expect(result.thinkingLevel).toBe("high"); - expect(result.warning).toBeUndefined(); - }); - - test("gpt-4o:extended matches the extended model with undefined thinking level", () => { - const result = parseModelPattern("openai/gpt-4o:extended", allModels); - expect(result.model?.id).toBe("openai/gpt-4o:extended"); - expect(result.thinkingLevel).toBeUndefined(); - expect(result.warning).toBeUndefined(); - }); - }); - - describe("invalid thinking levels with OpenRouter models", () => { - test("qwen3-coder:exacto:random returns model with undefined thinking level and warning", () => { - const result = parseModelPattern("qwen/qwen3-coder:exacto:random", allModels); - expect(result.model?.id).toBe("qwen/qwen3-coder:exacto"); - expect(result.thinkingLevel).toBeUndefined(); - expect(result.warning).toContain("Invalid thinking level"); - expect(result.warning).toContain("random"); - }); - - test("qwen3-coder:exacto:high:random returns model with undefined thinking level and warning", () => { - const result = parseModelPattern("qwen/qwen3-coder:exacto:high:random", allModels); - expect(result.model?.id).toBe("qwen/qwen3-coder:exacto"); - expect(result.thinkingLevel).toBeUndefined(); - expect(result.warning).toContain("Invalid thinking level"); - expect(result.warning).toContain("random"); - }); - }); - - describe("edge cases", () => { - test("empty pattern matches via partial matching", () => { - // Empty string is included in all model IDs, so partial matching finds a match - const result = parseModelPattern("", allModels); - expect(result.model).not.toBeNull(); - expect(result.thinkingLevel).toBeUndefined(); - }); - - test("pattern ending with colon treats empty suffix as invalid", () => { - const result = parseModelPattern("sonnet:", allModels); - // Empty string after colon is not a valid thinking level - // So it tries to match "sonnet:" which won't match, then tries "sonnet" - expect(result.model?.id).toBe("claude-sonnet-4-5"); - expect(result.warning).toContain("Invalid thinking level"); - }); + } }); }); @@ -270,6 +385,41 @@ describe("resolveModelScopeWithDiagnostics", () => { } }); + test("adds every provider for a pattern matching the same id on several providers, with a warning", async () => { + const registry = { + getAvailable: () => [ + ...allModels, + { ...mockModels[1], provider: "azure" }, + { ...mockModels[1], provider: "openai-codex", id: "gpt-4o-mini" }, + ], + } as unknown as Parameters[1]; + + for (const [pattern, thinkingLevel] of [ + ["gpt-4o:high", "high"], + ["GPT-4o", undefined], + ] as const) { + const result = await resolveModelScopeWithDiagnostics([pattern], registry); + + expect( + result.scopedModels.map((scoped) => [`${scoped.model.provider}/${scoped.model.id}`, scoped.thinkingLevel]), + ).toEqual( + expect.arrayContaining([ + ["azure/gpt-4o", thinkingLevel], + ["openai/gpt-4o", thinkingLevel], + ]), + ); + expect(result.scopedModels).toHaveLength(2); + expect(result.diagnostics).toEqual([ + { + type: "warning", + code: "ambiguous", + message: `Pattern "${pattern}" matches several providers: azure, openai. Use provider/id to choose one.`, + pattern, + }, + ]); + } + }); + test("resolves bracketed model ids as exact references before glob matching", async () => { const bracketedModel: Model<"anthropic-messages"> = { id: "bracketed-model[1m]", @@ -319,67 +469,6 @@ describe("resolveModelScopeWithDiagnostics", () => { }); describe("resolveCliModel", () => { - test("resolves --model provider/id without --provider", () => { - const registry = { - getModels: () => allModels, - } as unknown as Parameters[0]["modelRuntime"]; - - const result = resolveCliModel({ - cliModel: "openai/gpt-4o", - modelRuntime: registry, - }); - - expect(result.error).toBeUndefined(); - expect(result.model?.provider).toBe("openai"); - expect(result.model?.id).toBe("gpt-4o"); - }); - - test("resolves fuzzy patterns within an explicit provider", () => { - const registry = { - getModels: () => allModels, - } as unknown as Parameters[0]["modelRuntime"]; - - const result = resolveCliModel({ - cliProvider: "openai", - cliModel: "4o", - modelRuntime: registry, - }); - - expect(result.error).toBeUndefined(); - expect(result.model?.provider).toBe("openai"); - expect(result.model?.id).toBe("gpt-4o"); - }); - - test("supports --model : (without explicit --thinking)", () => { - const registry = { - getModels: () => allModels, - } as unknown as Parameters[0]["modelRuntime"]; - - const result = resolveCliModel({ - cliModel: "sonnet:high", - modelRuntime: registry, - }); - - expect(result.error).toBeUndefined(); - expect(result.model?.id).toBe("claude-sonnet-4-5"); - expect(result.thinkingLevel).toBe("high"); - }); - - test("prefers exact model id match over provider inference (OpenRouter-style ids)", () => { - const registry = { - getModels: () => allModels, - } as unknown as Parameters[0]["modelRuntime"]; - - const result = resolveCliModel({ - cliModel: "openai/gpt-4o:extended", - modelRuntime: registry, - }); - - expect(result.error).toBeUndefined(); - expect(result.model?.provider).toBe("openrouter"); - expect(result.model?.id).toBe("openai/gpt-4o:extended"); - }); - test("does not strip invalid :suffix as thinking level in --model (treat as raw id)", () => { const registry = { getModels: () => allModels, @@ -427,159 +516,49 @@ describe("resolveCliModel", () => { expect(result.error).toContain("No models available"); }); - test("prefers the sole authenticated provider for an ambiguous bare exact model id", () => { - const azureModel: Model<"anthropic-messages"> = { - ...mockModels[1], - id: "gpt-5.6-sol", - name: "GPT 5.6 Sol", - provider: "azure", - }; - const codexModel: Model<"anthropic-messages"> = { - ...mockModels[1], - id: "gpt-5.6-sol", - name: "GPT 5.6 Sol", - provider: "openai-codex", - }; - const registry = { - getModels: () => [azureModel, codexModel], - hasConfiguredAuth: (provider: string) => provider === "openai-codex", - } as unknown as Parameters[0]["modelRuntime"]; + describe("duplicate raw ids behind a known provider prefix (#10160)", () => { + const direct = { ...mockModels[0], id: "claude-sonnet-4-5", provider: "anthropic" }; + const openrouter = { ...mockModels[0], id: "anthropic/claude-opus-5.5", provider: "openrouter" }; + const vercel = { ...mockModels[0], id: "anthropic/claude-opus-5.5", provider: "vercel-ai-gateway" }; + const registry = (configured: string[]) => + ({ + getModels: () => [direct, openrouter, vercel], + hasConfiguredAuth: (provider: string) => configured.includes(provider), + }) as unknown as Parameters[0]["modelRuntime"]; - const result = resolveCliModel({ - cliModel: "gpt-5.6-sol", - modelRuntime: registry, + test("selects the authenticated gateway, with or without a thinking suffix", () => { + for (const [cliModel, thinkingLevel] of [ + ["anthropic/claude-opus-5.5", undefined], + ["anthropic/claude-opus-5.5:high", "high"], + ] as const) { + const result = resolveCliModel({ cliModel, modelRuntime: registry(["vercel-ai-gateway"]) }); + + expect(result.error).toBeUndefined(); + expect(result.model?.provider).toBe("vercel-ai-gateway"); + expect(result.thinkingLevel).toBe(thinkingLevel); + } }); - expect(result.error).toBeUndefined(); - expect(result.model?.provider).toBe("openai-codex"); - expect(result.model?.id).toBe("gpt-5.6-sol"); - }); + test("reports ambiguity instead of choosing by catalog order", () => { + const result = resolveCliModel({ cliModel: "anthropic/claude-opus-5.5", modelRuntime: registry([]) }); - test("requires an explicit provider for an ambiguous bare exact model id without a unique authenticated provider", () => { - const azureModel: Model<"anthropic-messages"> = { - ...mockModels[1], - id: "gpt-5.6-sol", - name: "GPT 5.6 Sol", - provider: "azure", - }; - const codexModel: Model<"anthropic-messages"> = { - ...mockModels[1], - id: "gpt-5.6-sol", - name: "GPT 5.6 Sol", - provider: "openai-codex", - }; - const registry = { - getModels: () => [azureModel, codexModel], - hasConfiguredAuth: () => false, - } as unknown as Parameters[0]["modelRuntime"]; - - const result = resolveCliModel({ - cliModel: "gpt-5.6-sol", - modelRuntime: registry, + expect(result.model).toBeUndefined(); + expect(result.error).toContain( + "openrouter/anthropic/claude-opus-5.5, vercel-ai-gateway/anthropic/claude-opus-5.5", + ); + expect(result.error).toContain("No matching provider is authenticated. Use --provider or provider/model."); }); - expect(result.model).toBeUndefined(); - expect(result.error).toContain('Model "gpt-5.6-sol" is ambiguous across providers'); - expect(result.error).toContain("azure/gpt-5.6-sol"); - expect(result.error).toContain("openai-codex/gpt-5.6-sol"); - expect(result.error).toContain("Use --provider or provider/model"); - }); + test("does not let stored credentials choose the provider for --api-key", () => { + const result = resolveCliModel({ + cliModel: "anthropic/claude-opus-5.5", + inlineApiKey: true, + modelRuntime: registry(["vercel-ai-gateway"]), + }); - test("prefers provider/model split over gateway model with matching id", () => { - // When a user writes "zai/glm-5", and both a zai provider model (id: "glm-5") - // and a gateway model (id: "zai/glm-5") exist, prefer the zai provider model. - const zaiModel: Model<"anthropic-messages"> = { - id: "glm-5", - name: "GLM-5", - api: "anthropic-messages", - provider: "zai", - baseUrl: "https://open.bigmodel.cn/api/paas/v4", - reasoning: true, - input: ["text"], - cost: { input: 1, output: 2, cacheRead: 0.1, cacheWrite: 1 }, - contextWindow: 128000, - maxTokens: 8192, - }; - const gatewayModel: Model<"anthropic-messages"> = { - id: "zai/glm-5", - name: "GLM-5", - api: "anthropic-messages", - provider: "vercel-ai-gateway", - baseUrl: "https://ai-gateway.vercel.sh", - reasoning: true, - input: ["text"], - cost: { input: 1, output: 2, cacheRead: 0.1, cacheWrite: 1 }, - contextWindow: 128000, - maxTokens: 8192, - }; - const registry = { - getModels: () => [...allModels, zaiModel, gatewayModel], - hasConfiguredAuth: () => true, - } as unknown as Parameters[0]["modelRuntime"]; - - const result = resolveCliModel({ - cliModel: "zai/glm-5", - modelRuntime: registry, + expect(result.model).toBeUndefined(); + expect(result.error).toContain("--api-key does not choose a provider."); }); - - expect(result.error).toBeUndefined(); - expect(result.model?.provider).toBe("zai"); - expect(result.model?.id).toBe("glm-5"); - }); - - test("prefers an authenticated exact raw model id over an unauthenticated inferred provider", () => { - const commandcodeModel: Model<"anthropic-messages"> = { - id: "xiaomi/mimo-v2.5-pro", - name: "Xiaomi MiMo via Commandcode", - api: "anthropic-messages", - provider: "commandcode", - baseUrl: "https://example.invalid", - reasoning: false, - input: ["text"], - cost: { input: 1, output: 2, cacheRead: 0.1, cacheWrite: 1 }, - contextWindow: 128000, - maxTokens: 8192, - }; - const xiaomiModel: Model<"anthropic-messages"> = { - id: "mimo-v2.5-pro", - name: "Xiaomi MiMo", - api: "anthropic-messages", - provider: "xiaomi", - baseUrl: "https://api.xiaomimimo.com", - reasoning: false, - input: ["text"], - cost: { input: 1, output: 2, cacheRead: 0.1, cacheWrite: 1 }, - contextWindow: 128000, - maxTokens: 8192, - }; - const registry = { - getModels: () => [...allModels, commandcodeModel, xiaomiModel], - hasConfiguredAuth: (provider: string) => provider === "commandcode", - } as unknown as Parameters[0]["modelRuntime"]; - - const result = resolveCliModel({ - cliModel: "xiaomi/mimo-v2.5-pro", - modelRuntime: registry, - }); - - expect(result.error).toBeUndefined(); - expect(result.model?.provider).toBe("commandcode"); - expect(result.model?.id).toBe("xiaomi/mimo-v2.5-pro"); - }); - - test("resolves provider-prefixed fuzzy patterns (openrouter/qwen -> openrouter model)", () => { - const registry = { - getModels: () => allModels, - } as unknown as Parameters[0]["modelRuntime"]; - - const result = resolveCliModel({ - cliModel: "openrouter/qwen", - modelRuntime: registry, - }); - - expect(result.error).toBeUndefined(); - expect(result.model?.provider).toBe("openrouter"); - expect(result.model?.id).toBe("qwen/qwen3-coder:exacto"); }); describe("custom model fallback with :thinking suffix (#5552)", () => { @@ -747,23 +726,6 @@ describe("default model selection", () => { expect(defaultModelPerProvider["qwen-token-plan-individual"]).toBe("qwen3.8-max"); }); - test("findInitialModel accepts explicit provider custom model ids", async () => { - const registry = { - getModels: () => allModels, - } as unknown as Parameters[0]["modelRuntime"]; - - const result = await findInitialModel({ - cliProvider: "openrouter", - cliModel: "openrouter/openai/ghost-model", - scopedModels: [], - isContinuing: false, - modelRuntime: registry, - }); - - expect(result.model?.provider).toBe("openrouter"); - expect(result.model?.id).toBe("openai/ghost-model"); - }); - test("findInitialModel selects ai-gateway default when available", async () => { const aiGatewayModel: Model<"anthropic-messages"> = { id: "anthropic/claude-opus-4-6",