diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index e63db04d3..cbc0f46cc 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -80,6 +80,7 @@ - Added Sign in with ChatGPT to the `openai` provider: an OAuth login that uses a ChatGPT subscription with the OpenAI API. `Models.login()` accepts `LoginOptions` with `getDeviceId()`, which supplies a stable installation ID to login flows that need one. Subscription usage-limit errors are not retried and link to the ChatGPT usage page; temporary usage errors are retried. - Added the `llama-cpp-classify` classifier API, which answers classifier questions from llama-server's next-token probabilities for single-token answer labels. - Added optional `AssistantMessage.thinkingLevel`, which records the thinking level the agent loop requested for a response. +- Added per-thinking-level model sampling parameter overrides for OpenAI-compatible requests. ### Changed diff --git a/packages/ai/src/api/azure-openai-responses.ts b/packages/ai/src/api/azure-openai-responses.ts index 9089fd0e8..614965fbd 100644 --- a/packages/ai/src/api/azure-openai-responses.ts +++ b/packages/ai/src/api/azure-openai-responses.ts @@ -20,7 +20,7 @@ import { getDeclaredTools, resolveTranscript, resolveTranscriptTools } from "../ import { createGrammarToolInputProperties } from "./constrained-sampling.ts"; import { clampOpenAIPromptCacheKey } from "./openai-prompt-cache.ts"; import { convertResponsesMessages, convertResponsesTools, processResponsesStream } from "./openai-responses-shared.ts"; -import { buildBaseOptions } from "./simple-options.ts"; +import { buildBaseOptions, resolveSamplingParams } from "./simple-options.ts"; const DEFAULT_AZURE_API_VERSION = "v1"; const AZURE_TOOL_CALL_PROVIDERS = new Set(["openai", "openai-codex", "opencode", "azure-openai-responses"]); @@ -327,11 +327,12 @@ function buildParams( params.tool_choice = options.toolChoice; } + const reasoningEffort = options?.reasoningEffort ?? (options?.reasoningSummary ? "medium" : undefined); if (model.reasoning) { - if (options?.reasoningEffort || options?.reasoningSummary) { + if (reasoningEffort) { const effort = options?.reasoningEffort ? (model.thinkingLevelMap?.[options.reasoningEffort] ?? options.reasoningEffort) - : "medium"; + : reasoningEffort; params.reasoning = { effort: effort as NonNullable["effort"], summary: options?.reasoningSummary || "auto", @@ -344,8 +345,11 @@ function buildParams( } } - // Last so custom keys override the named request fields. Per-request keys override model defaults. - Object.assign(params, model.samplingParams, options?.samplingParams); + // Last so model and request sampling parameters override named request fields. + const samplingParams = resolveSamplingParams(model, reasoningEffort ?? "off", options?.samplingParams); + if (samplingParams) { + Object.assign(params, samplingParams); + } return params; } diff --git a/packages/ai/src/api/openai-completions.ts b/packages/ai/src/api/openai-completions.ts index e4ebb7ce6..96cb23ab3 100644 --- a/packages/ai/src/api/openai-completions.ts +++ b/packages/ai/src/api/openai-completions.ts @@ -62,7 +62,12 @@ import { } from "./constrained-sampling.ts"; import { buildCopilotDynamicHeaders, hasCopilotVisionInput } from "./github-copilot-headers.ts"; import { clampOpenAIPromptCacheKey } from "./openai-prompt-cache.ts"; -import { buildBaseOptions, clampThinkingBudgetToAnswerRoom, thinkingBudgetForLevel } from "./simple-options.ts"; +import { + buildBaseOptions, + clampThinkingBudgetToAnswerRoom, + resolveSamplingParams, + thinkingBudgetForLevel, +} from "./simple-options.ts"; import { transformMessages } from "./transform-messages.ts"; /** @@ -995,8 +1000,11 @@ function buildParams( } } - // Last so custom keys override the named request fields. Per-request keys override model defaults. - Object.assign(params, model.samplingParams, options?.samplingParams); + // Last so model and request sampling parameters override named request fields. + const samplingParams = resolveSamplingParams(model, options?.reasoningEffort ?? "off", options?.samplingParams); + if (samplingParams) { + Object.assign(params, samplingParams); + } return params; } diff --git a/packages/ai/src/api/openai-responses.ts b/packages/ai/src/api/openai-responses.ts index addc3a5f7..07e5624e9 100644 --- a/packages/ai/src/api/openai-responses.ts +++ b/packages/ai/src/api/openai-responses.ts @@ -26,7 +26,7 @@ import { createGrammarToolInputProperties } from "./constrained-sampling.ts"; import { buildCopilotDynamicHeaders, hasCopilotVisionInput } from "./github-copilot-headers.ts"; import { clampOpenAIPromptCacheKey } from "./openai-prompt-cache.ts"; import { convertResponsesMessages, convertResponsesTools, processResponsesStream } from "./openai-responses-shared.ts"; -import { buildBaseOptions } from "./simple-options.ts"; +import { buildBaseOptions, resolveSamplingParams } from "./simple-options.ts"; const OPENAI_TOOL_CALL_PROVIDERS = new Set(["openai", "openai-codex", "opencode"]); // OpenAI Responses rejects max_output_tokens below 16: https://github.com/earendil-works/pi/issues/6265 @@ -360,11 +360,12 @@ function buildParams( params.tool_choice = options.toolChoice; } + const reasoningEffort = options?.reasoningEffort ?? (options?.reasoningSummary ? "medium" : undefined); if (model.reasoning) { - if (options?.reasoningEffort || options?.reasoningSummary) { + if (reasoningEffort) { const effort = options?.reasoningEffort ? (model.thinkingLevelMap?.[options.reasoningEffort] ?? options.reasoningEffort) - : "medium"; + : reasoningEffort; params.reasoning = { effort: effort as NonNullable["effort"], summary: options?.reasoningSummary || "auto", @@ -378,8 +379,11 @@ function buildParams( if (model.provider === "xai") params.include = ["reasoning.encrypted_content"]; } - // Last so custom keys override the named request fields. Per-request keys override model defaults. - Object.assign(params, model.samplingParams, options?.samplingParams); + // Last so model and request sampling parameters override named request fields. + const samplingParams = resolveSamplingParams(model, reasoningEffort ?? "off", options?.samplingParams); + if (samplingParams) { + Object.assign(params, samplingParams); + } return params; } diff --git a/packages/ai/src/api/simple-options.ts b/packages/ai/src/api/simple-options.ts index f2b888814..7d287aa87 100644 --- a/packages/ai/src/api/simple-options.ts +++ b/packages/ai/src/api/simple-options.ts @@ -1,6 +1,9 @@ +import { clampThinkingLevel } from "../models.ts"; import type { Api, Model, + ModelThinkingLevel, + SamplingParams, SimpleStreamOptions, StreamOptions, ThinkingBudgets, @@ -18,15 +21,28 @@ export function clampMaxTokensToContext(model: Model, context: TranscriptCo return Math.min(maxTokens, Math.max(MIN_MAX_TOKENS, available)); } +export function resolveSamplingParams( + model: Model, + thinkingLevel: ModelThinkingLevel, + requestParams?: SamplingParams, +): SamplingParams | undefined { + const effectiveThinkingLevel = clampThinkingLevel(model, thinkingLevel); + const thinkingLevelParams = model.samplingParamsByThinkingLevel?.[effectiveThinkingLevel]; + return model.samplingParams || thinkingLevelParams || requestParams + ? { ...model.samplingParams, ...thinkingLevelParams, ...requestParams } + : undefined; +} + export function buildBaseOptions( model: Model, context: TranscriptContext, options?: SimpleStreamOptions, apiKey?: string, ): StreamOptions { + const samplingParams = resolveSamplingParams(model, options?.reasoning ?? "off", options?.samplingParams); return { temperature: options?.temperature, - samplingParams: options?.samplingParams, + samplingParams, maxTokens: clampMaxTokensToContext(model, context, options?.maxTokens ?? model.maxTokens), signal: options?.signal, telemetryContext: options?.telemetryContext, diff --git a/packages/ai/src/types.ts b/packages/ai/src/types.ts index 0358843d1..374f2274a 100644 --- a/packages/ai/src/types.ts +++ b/packages/ai/src/types.ts @@ -85,6 +85,8 @@ export type ToolChoice = "auto" | "none"; export type ThinkingLevel = "minimal" | "low" | "medium" | "high" | "xhigh" | "max"; export type ModelThinkingLevel = "off" | ThinkingLevel; export type ThinkingLevelMap = Partial>; +export type SamplingParams = Record; +export type SamplingParamsByThinkingLevel = Partial>; export type ChatTemplateKwargValue = | string | number @@ -204,7 +206,7 @@ export interface StreamOptions extends ProviderRequestOptions> { * `repetition_penalty`. Merged over `Model.samplingParams` per key. Only applied by * OpenAI-compatible adapters (completions, responses, Azure responses); other APIs ignore it. */ - samplingParams?: Record; + samplingParams?: SamplingParams; maxTokens?: number; /** * Preferred transport for providers that support multiple transports. @@ -1126,7 +1128,9 @@ export interface Model extends BaseModel { contextWindow: number; maxTokens: number; /** Default sampling parameters for this model. See {@link StreamOptions.samplingParams}; per-request keys override these. */ - samplingParams?: Record; + samplingParams?: SamplingParams; + /** Sampling parameter overrides selected by the effective pi thinking level. */ + samplingParamsByThinkingLevel?: SamplingParamsByThinkingLevel; /** Compatibility overrides for OpenAI-compatible APIs. If not set, auto-detected from baseUrl. */ compat?: TApi extends "openai-completions" ? OpenAICompletionsCompat diff --git a/packages/ai/test/sampling-options.test.ts b/packages/ai/test/sampling-options.test.ts index a3c80489d..a880ee1be 100644 --- a/packages/ai/test/sampling-options.test.ts +++ b/packages/ai/test/sampling-options.test.ts @@ -1,12 +1,18 @@ import { describe, expect, it } from "vitest"; import { stream, streamSimple } from "../src/compat.ts"; -import type { Api, Context, Model, StreamOptions } from "../src/types.ts"; +import type { Api, Context, Model, SimpleStreamOptions, StreamOptions, ThinkingLevel } from "../src/types.ts"; + +type DirectSamplingOptions = StreamOptions & { + reasoningEffort?: ThinkingLevel; + reasoningSummary?: "auto" | "detailed" | "concise" | null; +}; interface SamplingPayload { temperature?: number; top_p?: number; top_k?: number; min_p?: number; + reasoning?: { effort?: string }; } class PayloadCaptured extends Error { @@ -22,7 +28,11 @@ function makeContext(): Context { }; } -function makeModel(api: Api, samplingParams?: Record): Model { +function makeModel( + api: Api, + samplingParams?: Record, + overrides: Partial> = {}, +): Model { return { id: "custom-model", name: "Custom Model", @@ -35,6 +45,7 @@ function makeModel(api: Api, samplingParams?: Record): Model void) { }; } -async function capturePayload(model: Model, options?: StreamOptions): Promise { +async function capturePayload(model: Model, options?: DirectSamplingOptions): Promise { let capturedPayload: SamplingPayload | undefined; await stream(model, makeContext(), { @@ -65,6 +76,23 @@ async function capturePayload(model: Model, options?: StreamOptions): Promi return capturedPayload; } +async function captureSimplePayload(model: Model, options?: SimpleStreamOptions): Promise { + let capturedPayload: SamplingPayload | undefined; + + await streamSimple(model, makeContext(), { + ...options, + ...capturingOptions((payload) => { + capturedPayload = payload; + }), + }).result(); + + if (!capturedPayload) { + throw new Error("Expected payload to be captured before request failure"); + } + + return capturedPayload; +} + describe("sampling params", () => { it("merges request sampling params into the request body", async () => { const payload = await capturePayload(makeModel("openai-completions"), { @@ -97,18 +125,95 @@ describe("sampling params", () => { ); it("passes request sampling params through streamSimple", async () => { - let payload: SamplingPayload | undefined; - - await streamSimple(makeModel("openai-completions"), makeContext(), { + const payload = await captureSimplePayload(makeModel("openai-completions"), { samplingParams: { top_p: 0.5 }, - ...capturingOptions((captured) => { - payload = captured; - }), - }).result(); + }); - expect(payload?.top_p).toBe(0.5); + expect(payload.top_p).toBe(0.5); }); + it("applies sampling params for the effective thinking level over model defaults", async () => { + const payload = await captureSimplePayload( + makeModel( + "openai-completions", + { temperature: 1, top_p: 0.95 }, + { + reasoning: true, + thinkingLevelMap: { low: null, medium: null }, + samplingParamsByThinkingLevel: { high: { temperature: 0.8, top_k: 64 } }, + }, + ), + { reasoning: "low" }, + ); + + expect(payload.temperature).toBe(0.8); + expect(payload.top_p).toBe(0.95); + expect(payload.top_k).toBe(64); + }); + + it("applies off sampling params when reasoning is disabled", async () => { + const payload = await captureSimplePayload( + makeModel("openai-completions", undefined, { + samplingParamsByThinkingLevel: { off: { temperature: 0.7 } }, + }), + ); + + expect(payload.temperature).toBe(0.7); + }); + + it("merges stream-option keys over thinking-level keys", async () => { + const payload = await captureSimplePayload( + makeModel("openai-completions", undefined, { + reasoning: true, + samplingParamsByThinkingLevel: { low: { temperature: 0.6, top_p: 0.95 } }, + }), + { reasoning: "low", samplingParams: { top_p: 0.5 } }, + ); + + expect(payload.temperature).toBe(0.6); + expect(payload.top_p).toBe(0.5); + }); + + it.each(["openai-completions", "openai-responses", "azure-openai-responses"] as const)( + "applies thinking-level params between model and request params for %s", + async (api) => { + const payload = await capturePayload( + makeModel( + api, + { temperature: 1, top_p: 0.95 }, + { + reasoning: true, + samplingParamsByThinkingLevel: { low: { temperature: 0.6, top_k: 64 } }, + }, + ), + { reasoningEffort: "low", samplingParams: { top_p: 0.5 } }, + ); + + expect(payload.temperature).toBe(0.6); + expect(payload.top_p).toBe(0.5); + expect(payload.top_k).toBe(64); + }, + ); + + it.each(["openai-responses", "azure-openai-responses"] as const)( + "uses medium sampling params for summary-only %s requests", + async (api) => { + const payload = await capturePayload( + makeModel(api, undefined, { + reasoning: true, + samplingParamsByThinkingLevel: { + off: { temperature: 0.7 }, + medium: { temperature: 0.8 }, + }, + }), + { reasoningSummary: "auto" }, + ); + + expect(payload.reasoning?.effort).toBe("medium"); + expect(payload.temperature).toBe(0.8); + }, + ); + it("overrides named request fields", async () => { const payload = await capturePayload(makeModel("openai-completions"), { temperature: 0, diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 511dc6161..76dcb49d3 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -182,6 +182,7 @@ - Added a Built-in section in `pi config` to disable the built-in `mcp`, `llama.cpp`, `codemode`, and `tool-search` extensions globally or per project, stored as `-builtin:` in the `extensions` setting. SDK inline extensions opt in with `builtin: true`. - Added `+name` and `-name` entries to the `defaultTools` setting to add or remove tools without repeating the defaults, for example `"defaultTools": ["+codemode"]`. Project entries of this form apply on top of the user setting. Documented how to enable `codemode` without MCP and how to use classifier models such as Jev from codemode scripts. - Added the token usage and cost of codemode `models.classify()` calls to the codemode tool result, so they count toward the session cost; the codemode result shows each call's cost. +- Added `samplingParamsByThinkingLevel` to `models.json` for per-thinking-level sampling parameter overrides. ### Changed diff --git a/packages/coding-agent/docs/models.md b/packages/coding-agent/docs/models.md index af3fb7e8a..838eec835 100644 --- a/packages/coding-agent/docs/models.md +++ b/packages/coding-agent/docs/models.md @@ -98,6 +98,32 @@ Use `promptCache` to declare the provider's best-effort cache lifetime in second Choose the conservative end of any published range. A model without a lifetime for the active tier is not eligible for cache warming. A `modelOverrides` entry can set `inputLimits` or `promptCache` for a built-in or extension model, including a model accessed through a validated proxy. See [`cacheWarming`](settings.md#model-and-thinking). +### Configure sampling by thinking level + +OpenAI-compatible APIs support free-form `samplingParams` model defaults and `samplingParamsByThinkingLevel` overrides. The latter uses Pi thinking-level keys (`off`, `minimal`, `low`, `medium`, `high`, `xhigh`, and `max`), not provider values from `thinkingLevelMap`: + +```json +{ + "id": "qwen-thinking-model", + "reasoning": true, + "samplingParams": { + "temperature": 1.0, + "top_p": 0.95 + }, + "samplingParamsByThinkingLevel": { + "off": { + "temperature": 0.7, + "top_p": 0.8 + }, + "high": { + "top_k": 20 + } + } +} +``` + +Pi first clamps unsupported thinking levels, then merges model `samplingParams`, the effective level's override, and request-level `samplingParams` in that order. Later values win per key. Missing levels inherit the model defaults. `modelOverrides` merges per-level entries per key with the base model. These fields apply only to `openai-completions`, `openai-responses`, and `azure-openai-responses`; other APIs ignore them. + Compatibility settings should describe verified differences in the endpoint's request or response behavior. Do not enable them based only on an endpoint advertising OpenAI or Anthropic compatibility. ## Use classifier models diff --git a/packages/coding-agent/src/core/model-config.ts b/packages/coding-agent/src/core/model-config.ts index e0df3473b..8e06116a9 100644 --- a/packages/coding-agent/src/core/model-config.ts +++ b/packages/coding-agent/src/core/model-config.ts @@ -63,6 +63,17 @@ const ThinkingLevelMapSchema = Type.Object({ max: Type.Optional(ThinkingLevelMapValueSchema), }); +const SamplingParamsSchema = Type.Record(Type.String(), Type.Unknown()); +const SamplingParamsByThinkingLevelSchema = Type.Object({ + off: Type.Optional(SamplingParamsSchema), + minimal: Type.Optional(SamplingParamsSchema), + low: Type.Optional(SamplingParamsSchema), + medium: Type.Optional(SamplingParamsSchema), + high: Type.Optional(SamplingParamsSchema), + xhigh: Type.Optional(SamplingParamsSchema), + max: Type.Optional(SamplingParamsSchema), +}); + const ChatTemplateKwargScalarSchema = Type.Union([Type.String(), Type.Number(), Type.Boolean(), Type.Null()]); const ChatTemplateKwargVariableSchema = Type.Object({ $var: Type.Union([Type.Literal("thinking.enabled"), Type.Literal("thinking.effort")]), @@ -198,7 +209,8 @@ const ModelDefinitionSchema = Type.Object({ promptCache: Type.Optional(ModelPromptCacheSchema), contextWindow: Type.Optional(Type.Number()), maxTokens: Type.Optional(Type.Number()), - samplingParams: Type.Optional(Type.Record(Type.String(), Type.Unknown())), + samplingParams: Type.Optional(SamplingParamsSchema), + samplingParamsByThinkingLevel: Type.Optional(SamplingParamsByThinkingLevelSchema), headers: Type.Optional(Type.Record(Type.String(), Type.String())), compat: Type.Optional(ProviderCompatSchema), }); @@ -221,7 +233,8 @@ const ModelOverrideSchema = Type.Object({ promptCache: Type.Optional(ModelPromptCacheSchema), contextWindow: Type.Optional(Type.Number()), maxTokens: Type.Optional(Type.Number()), - samplingParams: Type.Optional(Type.Record(Type.String(), Type.Unknown())), + samplingParams: Type.Optional(SamplingParamsSchema), + samplingParamsByThinkingLevel: Type.Optional(SamplingParamsByThinkingLevelSchema), headers: Type.Optional(Type.Record(Type.String(), Type.String())), compat: Type.Optional(ProviderCompatSchema), }); diff --git a/packages/coding-agent/src/core/provider-composer.ts b/packages/coding-agent/src/core/provider-composer.ts index caa502352..94bd2ba3e 100644 --- a/packages/coding-agent/src/core/provider-composer.ts +++ b/packages/coding-agent/src/core/provider-composer.ts @@ -68,7 +68,8 @@ export interface ProviderChatModelConfig extends ProviderModelConfigBase { promptCache?: Model["promptCache"]; contextWindow: number; maxTokens: number; - samplingParams?: Record; + samplingParams?: Model["samplingParams"]; + samplingParamsByThinkingLevel?: Model["samplingParamsByThinkingLevel"]; compat?: Model["compat"]; } @@ -160,6 +161,19 @@ function mergeInputLimits( }; } +function mergeSamplingParamsByThinkingLevel( + base: Model["samplingParamsByThinkingLevel"], + override: Model["samplingParamsByThinkingLevel"], +): Model["samplingParamsByThinkingLevel"] { + if (!override) return base; + const merged = { ...base }; + for (const level of ["off", "minimal", "low", "medium", "high", "xhigh", "max"] as const) { + const params = override[level]; + if (params) merged[level] = { ...base?.[level], ...params }; + } + return merged; +} + function applyModelOverride(model: Model, override: ModelsJsonModelOverride): Model { return { ...model, @@ -185,6 +199,10 @@ function applyModelOverride(model: Model, override: ModelsJsonModelOverride samplingParams: override.samplingParams ? { ...model.samplingParams, ...override.samplingParams } : model.samplingParams, + samplingParamsByThinkingLevel: mergeSamplingParamsByThinkingLevel( + model.samplingParamsByThinkingLevel, + override.samplingParamsByThinkingLevel, + ), compat: mergeCompat(model.compat, override.compat), }; } @@ -224,6 +242,7 @@ function modelFromJson( contextWindow: definition.contextWindow ?? 128000, maxTokens: definition.maxTokens ?? 16384, samplingParams: definition.samplingParams, + samplingParamsByThinkingLevel: definition.samplingParamsByThinkingLevel, headers: undefined, compat: mergeCompat(providerConfig.compat, definition.compat), }; diff --git a/packages/coding-agent/test/model-registry.test.ts b/packages/coding-agent/test/model-registry.test.ts index ccf925f09..c495e3565 100644 --- a/packages/coding-agent/test/model-registry.test.ts +++ b/packages/coding-agent/test/model-registry.test.ts @@ -754,7 +754,7 @@ describe("ModelRegistry", () => { expect(compat?.allowedFallbackModels).toEqual([]); }); - test("custom model and model override carry sampling params", async () => { + test("custom models and model overrides carry sampling params", async () => { writeRawModelsJson({ openrouter: { baseUrl: "https://my-proxy.example.com/v1", @@ -763,11 +763,22 @@ describe("ModelRegistry", () => { { id: "custom/sampling-model", samplingParams: { temperature: 1, top_p: 0.95, top_k: 0 }, + samplingParamsByThinkingLevel: { + low: { temperature: 0.6, top_p: 0.95 }, + high: { temperature: 0.8 }, + }, }, ], modelOverrides: { + "custom/sampling-model": { + samplingParamsByThinkingLevel: { + low: { temperature: 0.5, top_k: 20 }, + max: { temperature: 1 }, + }, + }, "anthropic/claude-sonnet-4": { samplingParams: { top_p: 0.9 }, + samplingParamsByThinkingLevel: { high: { temperature: 0.8 } }, }, }, }, @@ -778,13 +789,20 @@ describe("ModelRegistry", () => { const custom = models.find((m) => m.id === "custom/sampling-model"); expect(custom?.samplingParams).toEqual({ temperature: 1, top_p: 0.95, top_k: 0 }); + expect(custom?.samplingParamsByThinkingLevel).toEqual({ + low: { temperature: 0.5, top_p: 0.95, top_k: 20 }, + high: { temperature: 0.8 }, + max: { temperature: 1 }, + }); const sonnet = models.find((m) => m.id === "anthropic/claude-sonnet-4"); expect(sonnet?.samplingParams).toEqual({ top_p: 0.9 }); + expect(sonnet?.samplingParamsByThinkingLevel).toEqual({ high: { temperature: 0.8 } }); // Models without sampling config keep it unset. const opus = models.find((m) => m.id === "anthropic/claude-opus-4.1"); expect(opus?.samplingParams).toBeUndefined(); + expect(opus?.samplingParamsByThinkingLevel).toBeUndefined(); }); test("custom model and model override carry prompt cache lifetimes", async () => {