feat(ai): add per-thinking-level sampling parameters (#9776)

This commit is contained in:
Duncan Ogilvie 2026-10-04 01:29:01 +02:00 • committed by GitHub
parent 70eceaade6
commit 76dfb88f63
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
12 changed files with 250 additions and 31 deletions

View file

@ -80,6 +80,7 @@
- Added Sign in with ChatGPT to the `openai` provider: an OAuth login that uses a ChatGPT subscription with the OpenAI API. `Models.login()` accepts `LoginOptions` with `getDeviceId()`, which supplies a stable installation ID to login flows that need one. Subscription usage-limit errors are not retried and link to the ChatGPT usage page; temporary usage errors are retried.
- Added the `llama-cpp-classify` classifier API, which answers classifier questions from llama-server's next-token probabilities for single-token answer labels.
- Added optional `AssistantMessage.thinkingLevel`, which records the thinking level the agent loop requested for a response.
- Added per-thinking-level model sampling parameter overrides for OpenAI-compatible requests.
### Changed

View file

@ -20,7 +20,7 @@ import { getDeclaredTools, resolveTranscript, resolveTranscriptTools } from "../
import { createGrammarToolInputProperties } from "./constrained-sampling.ts";
import { clampOpenAIPromptCacheKey } from "./openai-prompt-cache.ts";
import { convertResponsesMessages, convertResponsesTools, processResponsesStream } from "./openai-responses-shared.ts";
import { buildBaseOptions } from "./simple-options.ts";
import { buildBaseOptions, resolveSamplingParams } from "./simple-options.ts";
const DEFAULT_AZURE_API_VERSION = "v1";
const AZURE_TOOL_CALL_PROVIDERS = new Set(["openai", "openai-codex", "opencode", "azure-openai-responses"]);
@ -327,11 +327,12 @@ function buildParams(
params.tool_choice = options.toolChoice;
}
const reasoningEffort = options?.reasoningEffort ?? (options?.reasoningSummary ? "medium" : undefined);
if (model.reasoning) {
if (options?.reasoningEffort || options?.reasoningSummary) {
if (reasoningEffort) {
const effort = options?.reasoningEffort
? (model.thinkingLevelMap?.[options.reasoningEffort] ?? options.reasoningEffort)
: "medium";
: reasoningEffort;
params.reasoning = {
effort: effort as NonNullable<typeof params.reasoning>["effort"],
summary: options?.reasoningSummary || "auto",
@ -344,8 +345,11 @@ function buildParams(
}
}
// Last so custom keys override the named request fields. Per-request keys override model defaults.
Object.assign(params, model.samplingParams, options?.samplingParams);
// Last so model and request sampling parameters override named request fields.
const samplingParams = resolveSamplingParams(model, reasoningEffort ?? "off", options?.samplingParams);
if (samplingParams) {
Object.assign(params, samplingParams);
}
return params;
}

View file

@ -62,7 +62,12 @@ import {
} from "./constrained-sampling.ts";
import { buildCopilotDynamicHeaders, hasCopilotVisionInput } from "./github-copilot-headers.ts";
import { clampOpenAIPromptCacheKey } from "./openai-prompt-cache.ts";
import { buildBaseOptions, clampThinkingBudgetToAnswerRoom, thinkingBudgetForLevel } from "./simple-options.ts";
import {
buildBaseOptions,
clampThinkingBudgetToAnswerRoom,
resolveSamplingParams,
thinkingBudgetForLevel,
} from "./simple-options.ts";
import { transformMessages } from "./transform-messages.ts";
/**
@ -995,8 +1000,11 @@ function buildParams(
}
}
// Last so custom keys override the named request fields. Per-request keys override model defaults.
Object.assign(params, model.samplingParams, options?.samplingParams);
// Last so model and request sampling parameters override named request fields.
const samplingParams = resolveSamplingParams(model, options?.reasoningEffort ?? "off", options?.samplingParams);
if (samplingParams) {
Object.assign(params, samplingParams);
}
return params;
}

View file

@ -26,7 +26,7 @@ import { createGrammarToolInputProperties } from "./constrained-sampling.ts";
import { buildCopilotDynamicHeaders, hasCopilotVisionInput } from "./github-copilot-headers.ts";
import { clampOpenAIPromptCacheKey } from "./openai-prompt-cache.ts";
import { convertResponsesMessages, convertResponsesTools, processResponsesStream } from "./openai-responses-shared.ts";
import { buildBaseOptions } from "./simple-options.ts";
import { buildBaseOptions, resolveSamplingParams } from "./simple-options.ts";
const OPENAI_TOOL_CALL_PROVIDERS = new Set(["openai", "openai-codex", "opencode"]);
// OpenAI Responses rejects max_output_tokens below 16: https://github.com/earendil-works/pi/issues/6265
@ -360,11 +360,12 @@ function buildParams(
params.tool_choice = options.toolChoice;
}
const reasoningEffort = options?.reasoningEffort ?? (options?.reasoningSummary ? "medium" : undefined);
if (model.reasoning) {
if (options?.reasoningEffort || options?.reasoningSummary) {
if (reasoningEffort) {
const effort = options?.reasoningEffort
? (model.thinkingLevelMap?.[options.reasoningEffort] ?? options.reasoningEffort)
: "medium";
: reasoningEffort;
params.reasoning = {
effort: effort as NonNullable<typeof params.reasoning>["effort"],
summary: options?.reasoningSummary || "auto",
@ -378,8 +379,11 @@ function buildParams(
if (model.provider === "xai") params.include = ["reasoning.encrypted_content"];
}
// Last so custom keys override the named request fields. Per-request keys override model defaults.
Object.assign(params, model.samplingParams, options?.samplingParams);
// Last so model and request sampling parameters override named request fields.
const samplingParams = resolveSamplingParams(model, reasoningEffort ?? "off", options?.samplingParams);
if (samplingParams) {
Object.assign(params, samplingParams);
}
return params;
}

View file

@ -1,6 +1,9 @@
import { clampThinkingLevel } from "../models.ts";
import type {
Api,
Model,
ModelThinkingLevel,
SamplingParams,
SimpleStreamOptions,
StreamOptions,
ThinkingBudgets,
@ -18,15 +21,28 @@ export function clampMaxTokensToContext(model: Model<Api>, context: TranscriptCo
return Math.min(maxTokens, Math.max(MIN_MAX_TOKENS, available));
}
export function resolveSamplingParams(
model: Model<Api>,
thinkingLevel: ModelThinkingLevel,
requestParams?: SamplingParams,
): SamplingParams | undefined {
const effectiveThinkingLevel = clampThinkingLevel(model, thinkingLevel);
const thinkingLevelParams = model.samplingParamsByThinkingLevel?.[effectiveThinkingLevel];
return model.samplingParams || thinkingLevelParams || requestParams
? { ...model.samplingParams, ...thinkingLevelParams, ...requestParams }
: undefined;
}
export function buildBaseOptions(
model: Model<Api>,
context: TranscriptContext,
options?: SimpleStreamOptions,
apiKey?: string,
): StreamOptions {
const samplingParams = resolveSamplingParams(model, options?.reasoning ?? "off", options?.samplingParams);
return {
temperature: options?.temperature,
samplingParams: options?.samplingParams,
samplingParams,
maxTokens: clampMaxTokensToContext(model, context, options?.maxTokens ?? model.maxTokens),
signal: options?.signal,
telemetryContext: options?.telemetryContext,

View file

@ -85,6 +85,8 @@ export type ToolChoice = "auto" | "none";
export type ThinkingLevel = "minimal" | "low" | "medium" | "high" | "xhigh" | "max";
export type ModelThinkingLevel = "off" | ThinkingLevel;
export type ThinkingLevelMap = Partial<Record<ModelThinkingLevel, string | null>>;
export type SamplingParams = Record<string, unknown>;
export type SamplingParamsByThinkingLevel = Partial<Record<ModelThinkingLevel, SamplingParams>>;
export type ChatTemplateKwargValue =
| string
| number
@ -204,7 +206,7 @@ export interface StreamOptions extends ProviderRequestOptions<Model<Api>> {
* `repetition_penalty`. Merged over `Model.samplingParams` per key. Only applied by
* OpenAI-compatible adapters (completions, responses, Azure responses); other APIs ignore it.
*/
samplingParams?: Record<string, unknown>;
samplingParams?: SamplingParams;
maxTokens?: number;
/**
* Preferred transport for providers that support multiple transports.
@ -1126,7 +1128,9 @@ export interface Model<TApi extends Api> extends BaseModel<TApi> {
contextWindow: number;
maxTokens: number;
/** Default sampling parameters for this model. See {@link StreamOptions.samplingParams}; per-request keys override these. */
samplingParams?: Record<string, unknown>;
samplingParams?: SamplingParams;
/** Sampling parameter overrides selected by the effective pi thinking level. */
samplingParamsByThinkingLevel?: SamplingParamsByThinkingLevel;
/** Compatibility overrides for OpenAI-compatible APIs. If not set, auto-detected from baseUrl. */
compat?: TApi extends "openai-completions"
? OpenAICompletionsCompat

View file

@ -1,12 +1,18 @@
import { describe, expect, it } from "vitest";
import { stream, streamSimple } from "../src/compat.ts";
import type { Api, Context, Model, StreamOptions } from "../src/types.ts";
import type { Api, Context, Model, SimpleStreamOptions, StreamOptions, ThinkingLevel } from "../src/types.ts";
type DirectSamplingOptions = StreamOptions & {
reasoningEffort?: ThinkingLevel;
reasoningSummary?: "auto" | "detailed" | "concise" | null;
};
interface SamplingPayload {
temperature?: number;
top_p?: number;
top_k?: number;
min_p?: number;
reasoning?: { effort?: string };
}
class PayloadCaptured extends Error {
@ -22,7 +28,11 @@ function makeContext(): Context {
};
}
function makeModel(api: Api, samplingParams?: Record<string, unknown>): Model<Api> {
function makeModel(
api: Api,
samplingParams?: Record<string, unknown>,
overrides: Partial<Model<Api>> = {},
): Model<Api> {
return {
id: "custom-model",
name: "Custom Model",
@ -35,6 +45,7 @@ function makeModel(api: Api, samplingParams?: Record<string, unknown>): Model<Ap
contextWindow: 128000,
maxTokens: 16384,
samplingParams,
...overrides,
};
}
@ -48,7 +59,7 @@ function capturingOptions(onCapture: (payload: SamplingPayload) => void) {
};
}
async function capturePayload(model: Model<Api>, options?: StreamOptions): Promise<SamplingPayload> {
async function capturePayload(model: Model<Api>, options?: DirectSamplingOptions): Promise<SamplingPayload> {
let capturedPayload: SamplingPayload | undefined;
await stream(model, makeContext(), {
@ -65,6 +76,23 @@ async function capturePayload(model: Model<Api>, options?: StreamOptions): Promi
return capturedPayload;
}
async function captureSimplePayload(model: Model<Api>, options?: SimpleStreamOptions): Promise<SamplingPayload> {
let capturedPayload: SamplingPayload | undefined;
await streamSimple(model, makeContext(), {
...options,
...capturingOptions((payload) => {
capturedPayload = payload;
}),
}).result();
if (!capturedPayload) {
throw new Error("Expected payload to be captured before request failure");
}
return capturedPayload;
}
describe("sampling params", () => {
it("merges request sampling params into the request body", async () => {
const payload = await capturePayload(makeModel("openai-completions"), {
@ -97,18 +125,95 @@ describe("sampling params", () => {
);
it("passes request sampling params through streamSimple", async () => {
let payload: SamplingPayload | undefined;
await streamSimple(makeModel("openai-completions"), makeContext(), {
const payload = await captureSimplePayload(makeModel("openai-completions"), {
samplingParams: { top_p: 0.5 },
...capturingOptions((captured) => {
payload = captured;
}),
}).result();
});
expect(payload?.top_p).toBe(0.5);
expect(payload.top_p).toBe(0.5);
});
it("applies sampling params for the effective thinking level over model defaults", async () => {
const payload = await captureSimplePayload(
makeModel(
"openai-completions",
{ temperature: 1, top_p: 0.95 },
{
reasoning: true,
thinkingLevelMap: { low: null, medium: null },
samplingParamsByThinkingLevel: { high: { temperature: 0.8, top_k: 64 } },
},
),
{ reasoning: "low" },
);
expect(payload.temperature).toBe(0.8);
expect(payload.top_p).toBe(0.95);
expect(payload.top_k).toBe(64);
});
it("applies off sampling params when reasoning is disabled", async () => {
const payload = await captureSimplePayload(
makeModel("openai-completions", undefined, {
samplingParamsByThinkingLevel: { off: { temperature: 0.7 } },
}),
);
expect(payload.temperature).toBe(0.7);
});
it("merges stream-option keys over thinking-level keys", async () => {
const payload = await captureSimplePayload(
makeModel("openai-completions", undefined, {
reasoning: true,
samplingParamsByThinkingLevel: { low: { temperature: 0.6, top_p: 0.95 } },
}),
{ reasoning: "low", samplingParams: { top_p: 0.5 } },
);
expect(payload.temperature).toBe(0.6);
expect(payload.top_p).toBe(0.5);
});
it.each(["openai-completions", "openai-responses", "azure-openai-responses"] as const)(
"applies thinking-level params between model and request params for %s",
async (api) => {
const payload = await capturePayload(
makeModel(
api,
{ temperature: 1, top_p: 0.95 },
{
reasoning: true,
samplingParamsByThinkingLevel: { low: { temperature: 0.6, top_k: 64 } },
},
),
{ reasoningEffort: "low", samplingParams: { top_p: 0.5 } },
);
expect(payload.temperature).toBe(0.6);
expect(payload.top_p).toBe(0.5);
expect(payload.top_k).toBe(64);
},
);
it.each(["openai-responses", "azure-openai-responses"] as const)(
"uses medium sampling params for summary-only %s requests",
async (api) => {
const payload = await capturePayload(
makeModel(api, undefined, {
reasoning: true,
samplingParamsByThinkingLevel: {
off: { temperature: 0.7 },
medium: { temperature: 0.8 },
},
}),
{ reasoningSummary: "auto" },
);
expect(payload.reasoning?.effort).toBe("medium");
expect(payload.temperature).toBe(0.8);
},
);
it("overrides named request fields", async () => {
const payload = await capturePayload(makeModel("openai-completions"), {
temperature: 0,

View file

@ -182,6 +182,7 @@
- Added a Built-in section in `pi config` to disable the built-in `mcp`, `llama.cpp`, `codemode`, and `tool-search` extensions globally or per project, stored as `-builtin:<name>` in the `extensions` setting. SDK inline extensions opt in with `builtin: true`.
- Added `+name` and `-name` entries to the `defaultTools` setting to add or remove tools without repeating the defaults, for example `"defaultTools": ["+codemode"]`. Project entries of this form apply on top of the user setting. Documented how to enable `codemode` without MCP and how to use classifier models such as Jev from codemode scripts.
- Added the token usage and cost of codemode `models.classify()` calls to the codemode tool result, so they count toward the session cost; the codemode result shows each call's cost.
- Added `samplingParamsByThinkingLevel` to `models.json` for per-thinking-level sampling parameter overrides.
### Changed

View file

@ -98,6 +98,32 @@ Use `promptCache` to declare the provider's best-effort cache lifetime in second
Choose the conservative end of any published range. A model without a lifetime for the active tier is not eligible for cache warming. A `modelOverrides` entry can set `inputLimits` or `promptCache` for a built-in or extension model, including a model accessed through a validated proxy. See [`cacheWarming`](settings.md#model-and-thinking).
### Configure sampling by thinking level
OpenAI-compatible APIs support free-form `samplingParams` model defaults and `samplingParamsByThinkingLevel` overrides. The latter uses Pi thinking-level keys (`off`, `minimal`, `low`, `medium`, `high`, `xhigh`, and `max`), not provider values from `thinkingLevelMap`:
```json
{
"id": "qwen-thinking-model",
"reasoning": true,
"samplingParams": {
"temperature": 1.0,
"top_p": 0.95
},
"samplingParamsByThinkingLevel": {
"off": {
"temperature": 0.7,
"top_p": 0.8
},
"high": {
"top_k": 20
}
}
}
```
Pi first clamps unsupported thinking levels, then merges model `samplingParams`, the effective level's override, and request-level `samplingParams` in that order. Later values win per key. Missing levels inherit the model defaults. `modelOverrides` merges per-level entries per key with the base model. These fields apply only to `openai-completions`, `openai-responses`, and `azure-openai-responses`; other APIs ignore them.
Compatibility settings should describe verified differences in the endpoint's request or response behavior. Do not enable them based only on an endpoint advertising OpenAI or Anthropic compatibility.
## Use classifier models

View file

@ -63,6 +63,17 @@ const ThinkingLevelMapSchema = Type.Object({
max: Type.Optional(ThinkingLevelMapValueSchema),
});
const SamplingParamsSchema = Type.Record(Type.String(), Type.Unknown());
const SamplingParamsByThinkingLevelSchema = Type.Object({
off: Type.Optional(SamplingParamsSchema),
minimal: Type.Optional(SamplingParamsSchema),
low: Type.Optional(SamplingParamsSchema),
medium: Type.Optional(SamplingParamsSchema),
high: Type.Optional(SamplingParamsSchema),
xhigh: Type.Optional(SamplingParamsSchema),
max: Type.Optional(SamplingParamsSchema),
});
const ChatTemplateKwargScalarSchema = Type.Union([Type.String(), Type.Number(), Type.Boolean(), Type.Null()]);
const ChatTemplateKwargVariableSchema = Type.Object({
$var: Type.Union([Type.Literal("thinking.enabled"), Type.Literal("thinking.effort")]),
@ -198,7 +209,8 @@ const ModelDefinitionSchema = Type.Object({
promptCache: Type.Optional(ModelPromptCacheSchema),
contextWindow: Type.Optional(Type.Number()),
maxTokens: Type.Optional(Type.Number()),
samplingParams: Type.Optional(Type.Record(Type.String(), Type.Unknown())),
samplingParams: Type.Optional(SamplingParamsSchema),
samplingParamsByThinkingLevel: Type.Optional(SamplingParamsByThinkingLevelSchema),
headers: Type.Optional(Type.Record(Type.String(), Type.String())),
compat: Type.Optional(ProviderCompatSchema),
});
@ -221,7 +233,8 @@ const ModelOverrideSchema = Type.Object({
promptCache: Type.Optional(ModelPromptCacheSchema),
contextWindow: Type.Optional(Type.Number()),
maxTokens: Type.Optional(Type.Number()),
samplingParams: Type.Optional(Type.Record(Type.String(), Type.Unknown())),
samplingParams: Type.Optional(SamplingParamsSchema),
samplingParamsByThinkingLevel: Type.Optional(SamplingParamsByThinkingLevelSchema),
headers: Type.Optional(Type.Record(Type.String(), Type.String())),
compat: Type.Optional(ProviderCompatSchema),
});

View file

@ -68,7 +68,8 @@ export interface ProviderChatModelConfig extends ProviderModelConfigBase {
promptCache?: Model<Api>["promptCache"];
contextWindow: number;
maxTokens: number;
samplingParams?: Record<string, unknown>;
samplingParams?: Model<Api>["samplingParams"];
samplingParamsByThinkingLevel?: Model<Api>["samplingParamsByThinkingLevel"];
compat?: Model<Api>["compat"];
}
@ -160,6 +161,19 @@ function mergeInputLimits(
};
}
function mergeSamplingParamsByThinkingLevel(
base: Model<Api>["samplingParamsByThinkingLevel"],
override: Model<Api>["samplingParamsByThinkingLevel"],
): Model<Api>["samplingParamsByThinkingLevel"] {
if (!override) return base;
const merged = { ...base };
for (const level of ["off", "minimal", "low", "medium", "high", "xhigh", "max"] as const) {
const params = override[level];
if (params) merged[level] = { ...base?.[level], ...params };
}
return merged;
}
function applyModelOverride(model: Model<Api>, override: ModelsJsonModelOverride): Model<Api> {
return {
...model,
@ -185,6 +199,10 @@ function applyModelOverride(model: Model<Api>, override: ModelsJsonModelOverride
samplingParams: override.samplingParams
? { ...model.samplingParams, ...override.samplingParams }
: model.samplingParams,
samplingParamsByThinkingLevel: mergeSamplingParamsByThinkingLevel(
model.samplingParamsByThinkingLevel,
override.samplingParamsByThinkingLevel,
),
compat: mergeCompat(model.compat, override.compat),
};
}
@ -224,6 +242,7 @@ function modelFromJson(
contextWindow: definition.contextWindow ?? 128000,
maxTokens: definition.maxTokens ?? 16384,
samplingParams: definition.samplingParams,
samplingParamsByThinkingLevel: definition.samplingParamsByThinkingLevel,
headers: undefined,
compat: mergeCompat(providerConfig.compat, definition.compat),
};

View file

@ -754,7 +754,7 @@ describe("ModelRegistry", () => {
expect(compat?.allowedFallbackModels).toEqual([]);
});
test("custom model and model override carry sampling params", async () => {
test("custom models and model overrides carry sampling params", async () => {
writeRawModelsJson({
openrouter: {
baseUrl: "https://my-proxy.example.com/v1",
@ -763,11 +763,22 @@ describe("ModelRegistry", () => {
{
id: "custom/sampling-model",
samplingParams: { temperature: 1, top_p: 0.95, top_k: 0 },
samplingParamsByThinkingLevel: {
low: { temperature: 0.6, top_p: 0.95 },
high: { temperature: 0.8 },
},
},
],
modelOverrides: {
"custom/sampling-model": {
samplingParamsByThinkingLevel: {
low: { temperature: 0.5, top_k: 20 },
max: { temperature: 1 },
},
},
"anthropic/claude-sonnet-4": {
samplingParams: { top_p: 0.9 },
samplingParamsByThinkingLevel: { high: { temperature: 0.8 } },
},
},
},
@ -778,13 +789,20 @@ describe("ModelRegistry", () => {
const custom = models.find((m) => m.id === "custom/sampling-model");
expect(custom?.samplingParams).toEqual({ temperature: 1, top_p: 0.95, top_k: 0 });
expect(custom?.samplingParamsByThinkingLevel).toEqual({
low: { temperature: 0.5, top_p: 0.95, top_k: 20 },
high: { temperature: 0.8 },
max: { temperature: 1 },
});
const sonnet = models.find((m) => m.id === "anthropic/claude-sonnet-4");
expect(sonnet?.samplingParams).toEqual({ top_p: 0.9 });
expect(sonnet?.samplingParamsByThinkingLevel).toEqual({ high: { temperature: 0.8 } });
// Models without sampling config keep it unset.
const opus = models.find((m) => m.id === "anthropic/claude-opus-4.1");
expect(opus?.samplingParams).toBeUndefined();
expect(opus?.samplingParamsByThinkingLevel).toBeUndefined();
});
test("custom model and model override carry prompt cache lifetimes", async () => {