mirror of
https://github.com/badlogic/pi-mono.git
synced 2026-11-05 23:37:25 +00:00
feat(ai): add per-thinking-level sampling parameters (#9776)
This commit is contained in:
parent
70eceaade6
commit
76dfb88f63
12 changed files with 250 additions and 31 deletions
|
|
@ -80,6 +80,7 @@
|
|||
- Added Sign in with ChatGPT to the `openai` provider: an OAuth login that uses a ChatGPT subscription with the OpenAI API. `Models.login()` accepts `LoginOptions` with `getDeviceId()`, which supplies a stable installation ID to login flows that need one. Subscription usage-limit errors are not retried and link to the ChatGPT usage page; temporary usage errors are retried.
|
||||
- Added the `llama-cpp-classify` classifier API, which answers classifier questions from llama-server's next-token probabilities for single-token answer labels.
|
||||
- Added optional `AssistantMessage.thinkingLevel`, which records the thinking level the agent loop requested for a response.
|
||||
- Added per-thinking-level model sampling parameter overrides for OpenAI-compatible requests.
|
||||
|
||||
### Changed
|
||||
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ import { getDeclaredTools, resolveTranscript, resolveTranscriptTools } from "../
|
|||
import { createGrammarToolInputProperties } from "./constrained-sampling.ts";
|
||||
import { clampOpenAIPromptCacheKey } from "./openai-prompt-cache.ts";
|
||||
import { convertResponsesMessages, convertResponsesTools, processResponsesStream } from "./openai-responses-shared.ts";
|
||||
import { buildBaseOptions } from "./simple-options.ts";
|
||||
import { buildBaseOptions, resolveSamplingParams } from "./simple-options.ts";
|
||||
|
||||
const DEFAULT_AZURE_API_VERSION = "v1";
|
||||
const AZURE_TOOL_CALL_PROVIDERS = new Set(["openai", "openai-codex", "opencode", "azure-openai-responses"]);
|
||||
|
|
@ -327,11 +327,12 @@ function buildParams(
|
|||
params.tool_choice = options.toolChoice;
|
||||
}
|
||||
|
||||
const reasoningEffort = options?.reasoningEffort ?? (options?.reasoningSummary ? "medium" : undefined);
|
||||
if (model.reasoning) {
|
||||
if (options?.reasoningEffort || options?.reasoningSummary) {
|
||||
if (reasoningEffort) {
|
||||
const effort = options?.reasoningEffort
|
||||
? (model.thinkingLevelMap?.[options.reasoningEffort] ?? options.reasoningEffort)
|
||||
: "medium";
|
||||
: reasoningEffort;
|
||||
params.reasoning = {
|
||||
effort: effort as NonNullable<typeof params.reasoning>["effort"],
|
||||
summary: options?.reasoningSummary || "auto",
|
||||
|
|
@ -344,8 +345,11 @@ function buildParams(
|
|||
}
|
||||
}
|
||||
|
||||
// Last so custom keys override the named request fields. Per-request keys override model defaults.
|
||||
Object.assign(params, model.samplingParams, options?.samplingParams);
|
||||
// Last so model and request sampling parameters override named request fields.
|
||||
const samplingParams = resolveSamplingParams(model, reasoningEffort ?? "off", options?.samplingParams);
|
||||
if (samplingParams) {
|
||||
Object.assign(params, samplingParams);
|
||||
}
|
||||
|
||||
return params;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -62,7 +62,12 @@ import {
|
|||
} from "./constrained-sampling.ts";
|
||||
import { buildCopilotDynamicHeaders, hasCopilotVisionInput } from "./github-copilot-headers.ts";
|
||||
import { clampOpenAIPromptCacheKey } from "./openai-prompt-cache.ts";
|
||||
import { buildBaseOptions, clampThinkingBudgetToAnswerRoom, thinkingBudgetForLevel } from "./simple-options.ts";
|
||||
import {
|
||||
buildBaseOptions,
|
||||
clampThinkingBudgetToAnswerRoom,
|
||||
resolveSamplingParams,
|
||||
thinkingBudgetForLevel,
|
||||
} from "./simple-options.ts";
|
||||
import { transformMessages } from "./transform-messages.ts";
|
||||
|
||||
/**
|
||||
|
|
@ -995,8 +1000,11 @@ function buildParams(
|
|||
}
|
||||
}
|
||||
|
||||
// Last so custom keys override the named request fields. Per-request keys override model defaults.
|
||||
Object.assign(params, model.samplingParams, options?.samplingParams);
|
||||
// Last so model and request sampling parameters override named request fields.
|
||||
const samplingParams = resolveSamplingParams(model, options?.reasoningEffort ?? "off", options?.samplingParams);
|
||||
if (samplingParams) {
|
||||
Object.assign(params, samplingParams);
|
||||
}
|
||||
|
||||
return params;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ import { createGrammarToolInputProperties } from "./constrained-sampling.ts";
|
|||
import { buildCopilotDynamicHeaders, hasCopilotVisionInput } from "./github-copilot-headers.ts";
|
||||
import { clampOpenAIPromptCacheKey } from "./openai-prompt-cache.ts";
|
||||
import { convertResponsesMessages, convertResponsesTools, processResponsesStream } from "./openai-responses-shared.ts";
|
||||
import { buildBaseOptions } from "./simple-options.ts";
|
||||
import { buildBaseOptions, resolveSamplingParams } from "./simple-options.ts";
|
||||
|
||||
const OPENAI_TOOL_CALL_PROVIDERS = new Set(["openai", "openai-codex", "opencode"]);
|
||||
// OpenAI Responses rejects max_output_tokens below 16: https://github.com/earendil-works/pi/issues/6265
|
||||
|
|
@ -360,11 +360,12 @@ function buildParams(
|
|||
params.tool_choice = options.toolChoice;
|
||||
}
|
||||
|
||||
const reasoningEffort = options?.reasoningEffort ?? (options?.reasoningSummary ? "medium" : undefined);
|
||||
if (model.reasoning) {
|
||||
if (options?.reasoningEffort || options?.reasoningSummary) {
|
||||
if (reasoningEffort) {
|
||||
const effort = options?.reasoningEffort
|
||||
? (model.thinkingLevelMap?.[options.reasoningEffort] ?? options.reasoningEffort)
|
||||
: "medium";
|
||||
: reasoningEffort;
|
||||
params.reasoning = {
|
||||
effort: effort as NonNullable<typeof params.reasoning>["effort"],
|
||||
summary: options?.reasoningSummary || "auto",
|
||||
|
|
@ -378,8 +379,11 @@ function buildParams(
|
|||
if (model.provider === "xai") params.include = ["reasoning.encrypted_content"];
|
||||
}
|
||||
|
||||
// Last so custom keys override the named request fields. Per-request keys override model defaults.
|
||||
Object.assign(params, model.samplingParams, options?.samplingParams);
|
||||
// Last so model and request sampling parameters override named request fields.
|
||||
const samplingParams = resolveSamplingParams(model, reasoningEffort ?? "off", options?.samplingParams);
|
||||
if (samplingParams) {
|
||||
Object.assign(params, samplingParams);
|
||||
}
|
||||
|
||||
return params;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,9 @@
|
|||
import { clampThinkingLevel } from "../models.ts";
|
||||
import type {
|
||||
Api,
|
||||
Model,
|
||||
ModelThinkingLevel,
|
||||
SamplingParams,
|
||||
SimpleStreamOptions,
|
||||
StreamOptions,
|
||||
ThinkingBudgets,
|
||||
|
|
@ -18,15 +21,28 @@ export function clampMaxTokensToContext(model: Model<Api>, context: TranscriptCo
|
|||
return Math.min(maxTokens, Math.max(MIN_MAX_TOKENS, available));
|
||||
}
|
||||
|
||||
export function resolveSamplingParams(
|
||||
model: Model<Api>,
|
||||
thinkingLevel: ModelThinkingLevel,
|
||||
requestParams?: SamplingParams,
|
||||
): SamplingParams | undefined {
|
||||
const effectiveThinkingLevel = clampThinkingLevel(model, thinkingLevel);
|
||||
const thinkingLevelParams = model.samplingParamsByThinkingLevel?.[effectiveThinkingLevel];
|
||||
return model.samplingParams || thinkingLevelParams || requestParams
|
||||
? { ...model.samplingParams, ...thinkingLevelParams, ...requestParams }
|
||||
: undefined;
|
||||
}
|
||||
|
||||
export function buildBaseOptions(
|
||||
model: Model<Api>,
|
||||
context: TranscriptContext,
|
||||
options?: SimpleStreamOptions,
|
||||
apiKey?: string,
|
||||
): StreamOptions {
|
||||
const samplingParams = resolveSamplingParams(model, options?.reasoning ?? "off", options?.samplingParams);
|
||||
return {
|
||||
temperature: options?.temperature,
|
||||
samplingParams: options?.samplingParams,
|
||||
samplingParams,
|
||||
maxTokens: clampMaxTokensToContext(model, context, options?.maxTokens ?? model.maxTokens),
|
||||
signal: options?.signal,
|
||||
telemetryContext: options?.telemetryContext,
|
||||
|
|
|
|||
|
|
@ -85,6 +85,8 @@ export type ToolChoice = "auto" | "none";
|
|||
export type ThinkingLevel = "minimal" | "low" | "medium" | "high" | "xhigh" | "max";
|
||||
export type ModelThinkingLevel = "off" | ThinkingLevel;
|
||||
export type ThinkingLevelMap = Partial<Record<ModelThinkingLevel, string | null>>;
|
||||
export type SamplingParams = Record<string, unknown>;
|
||||
export type SamplingParamsByThinkingLevel = Partial<Record<ModelThinkingLevel, SamplingParams>>;
|
||||
export type ChatTemplateKwargValue =
|
||||
| string
|
||||
| number
|
||||
|
|
@ -204,7 +206,7 @@ export interface StreamOptions extends ProviderRequestOptions<Model<Api>> {
|
|||
* `repetition_penalty`. Merged over `Model.samplingParams` per key. Only applied by
|
||||
* OpenAI-compatible adapters (completions, responses, Azure responses); other APIs ignore it.
|
||||
*/
|
||||
samplingParams?: Record<string, unknown>;
|
||||
samplingParams?: SamplingParams;
|
||||
maxTokens?: number;
|
||||
/**
|
||||
* Preferred transport for providers that support multiple transports.
|
||||
|
|
@ -1126,7 +1128,9 @@ export interface Model<TApi extends Api> extends BaseModel<TApi> {
|
|||
contextWindow: number;
|
||||
maxTokens: number;
|
||||
/** Default sampling parameters for this model. See {@link StreamOptions.samplingParams}; per-request keys override these. */
|
||||
samplingParams?: Record<string, unknown>;
|
||||
samplingParams?: SamplingParams;
|
||||
/** Sampling parameter overrides selected by the effective pi thinking level. */
|
||||
samplingParamsByThinkingLevel?: SamplingParamsByThinkingLevel;
|
||||
/** Compatibility overrides for OpenAI-compatible APIs. If not set, auto-detected from baseUrl. */
|
||||
compat?: TApi extends "openai-completions"
|
||||
? OpenAICompletionsCompat
|
||||
|
|
|
|||
|
|
@ -1,12 +1,18 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import { stream, streamSimple } from "../src/compat.ts";
|
||||
import type { Api, Context, Model, StreamOptions } from "../src/types.ts";
|
||||
import type { Api, Context, Model, SimpleStreamOptions, StreamOptions, ThinkingLevel } from "../src/types.ts";
|
||||
|
||||
type DirectSamplingOptions = StreamOptions & {
|
||||
reasoningEffort?: ThinkingLevel;
|
||||
reasoningSummary?: "auto" | "detailed" | "concise" | null;
|
||||
};
|
||||
|
||||
interface SamplingPayload {
|
||||
temperature?: number;
|
||||
top_p?: number;
|
||||
top_k?: number;
|
||||
min_p?: number;
|
||||
reasoning?: { effort?: string };
|
||||
}
|
||||
|
||||
class PayloadCaptured extends Error {
|
||||
|
|
@ -22,7 +28,11 @@ function makeContext(): Context {
|
|||
};
|
||||
}
|
||||
|
||||
function makeModel(api: Api, samplingParams?: Record<string, unknown>): Model<Api> {
|
||||
function makeModel(
|
||||
api: Api,
|
||||
samplingParams?: Record<string, unknown>,
|
||||
overrides: Partial<Model<Api>> = {},
|
||||
): Model<Api> {
|
||||
return {
|
||||
id: "custom-model",
|
||||
name: "Custom Model",
|
||||
|
|
@ -35,6 +45,7 @@ function makeModel(api: Api, samplingParams?: Record<string, unknown>): Model<Ap
|
|||
contextWindow: 128000,
|
||||
maxTokens: 16384,
|
||||
samplingParams,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -48,7 +59,7 @@ function capturingOptions(onCapture: (payload: SamplingPayload) => void) {
|
|||
};
|
||||
}
|
||||
|
||||
async function capturePayload(model: Model<Api>, options?: StreamOptions): Promise<SamplingPayload> {
|
||||
async function capturePayload(model: Model<Api>, options?: DirectSamplingOptions): Promise<SamplingPayload> {
|
||||
let capturedPayload: SamplingPayload | undefined;
|
||||
|
||||
await stream(model, makeContext(), {
|
||||
|
|
@ -65,6 +76,23 @@ async function capturePayload(model: Model<Api>, options?: StreamOptions): Promi
|
|||
return capturedPayload;
|
||||
}
|
||||
|
||||
async function captureSimplePayload(model: Model<Api>, options?: SimpleStreamOptions): Promise<SamplingPayload> {
|
||||
let capturedPayload: SamplingPayload | undefined;
|
||||
|
||||
await streamSimple(model, makeContext(), {
|
||||
...options,
|
||||
...capturingOptions((payload) => {
|
||||
capturedPayload = payload;
|
||||
}),
|
||||
}).result();
|
||||
|
||||
if (!capturedPayload) {
|
||||
throw new Error("Expected payload to be captured before request failure");
|
||||
}
|
||||
|
||||
return capturedPayload;
|
||||
}
|
||||
|
||||
describe("sampling params", () => {
|
||||
it("merges request sampling params into the request body", async () => {
|
||||
const payload = await capturePayload(makeModel("openai-completions"), {
|
||||
|
|
@ -97,18 +125,95 @@ describe("sampling params", () => {
|
|||
);
|
||||
|
||||
it("passes request sampling params through streamSimple", async () => {
|
||||
let payload: SamplingPayload | undefined;
|
||||
|
||||
await streamSimple(makeModel("openai-completions"), makeContext(), {
|
||||
const payload = await captureSimplePayload(makeModel("openai-completions"), {
|
||||
samplingParams: { top_p: 0.5 },
|
||||
...capturingOptions((captured) => {
|
||||
payload = captured;
|
||||
}),
|
||||
}).result();
|
||||
});
|
||||
|
||||
expect(payload?.top_p).toBe(0.5);
|
||||
expect(payload.top_p).toBe(0.5);
|
||||
});
|
||||
|
||||
it("applies sampling params for the effective thinking level over model defaults", async () => {
|
||||
const payload = await captureSimplePayload(
|
||||
makeModel(
|
||||
"openai-completions",
|
||||
{ temperature: 1, top_p: 0.95 },
|
||||
{
|
||||
reasoning: true,
|
||||
thinkingLevelMap: { low: null, medium: null },
|
||||
samplingParamsByThinkingLevel: { high: { temperature: 0.8, top_k: 64 } },
|
||||
},
|
||||
),
|
||||
{ reasoning: "low" },
|
||||
);
|
||||
|
||||
expect(payload.temperature).toBe(0.8);
|
||||
expect(payload.top_p).toBe(0.95);
|
||||
expect(payload.top_k).toBe(64);
|
||||
});
|
||||
|
||||
it("applies off sampling params when reasoning is disabled", async () => {
|
||||
const payload = await captureSimplePayload(
|
||||
makeModel("openai-completions", undefined, {
|
||||
samplingParamsByThinkingLevel: { off: { temperature: 0.7 } },
|
||||
}),
|
||||
);
|
||||
|
||||
expect(payload.temperature).toBe(0.7);
|
||||
});
|
||||
|
||||
it("merges stream-option keys over thinking-level keys", async () => {
|
||||
const payload = await captureSimplePayload(
|
||||
makeModel("openai-completions", undefined, {
|
||||
reasoning: true,
|
||||
samplingParamsByThinkingLevel: { low: { temperature: 0.6, top_p: 0.95 } },
|
||||
}),
|
||||
{ reasoning: "low", samplingParams: { top_p: 0.5 } },
|
||||
);
|
||||
|
||||
expect(payload.temperature).toBe(0.6);
|
||||
expect(payload.top_p).toBe(0.5);
|
||||
});
|
||||
|
||||
it.each(["openai-completions", "openai-responses", "azure-openai-responses"] as const)(
|
||||
"applies thinking-level params between model and request params for %s",
|
||||
async (api) => {
|
||||
const payload = await capturePayload(
|
||||
makeModel(
|
||||
api,
|
||||
{ temperature: 1, top_p: 0.95 },
|
||||
{
|
||||
reasoning: true,
|
||||
samplingParamsByThinkingLevel: { low: { temperature: 0.6, top_k: 64 } },
|
||||
},
|
||||
),
|
||||
{ reasoningEffort: "low", samplingParams: { top_p: 0.5 } },
|
||||
);
|
||||
|
||||
expect(payload.temperature).toBe(0.6);
|
||||
expect(payload.top_p).toBe(0.5);
|
||||
expect(payload.top_k).toBe(64);
|
||||
},
|
||||
);
|
||||
|
||||
it.each(["openai-responses", "azure-openai-responses"] as const)(
|
||||
"uses medium sampling params for summary-only %s requests",
|
||||
async (api) => {
|
||||
const payload = await capturePayload(
|
||||
makeModel(api, undefined, {
|
||||
reasoning: true,
|
||||
samplingParamsByThinkingLevel: {
|
||||
off: { temperature: 0.7 },
|
||||
medium: { temperature: 0.8 },
|
||||
},
|
||||
}),
|
||||
{ reasoningSummary: "auto" },
|
||||
);
|
||||
|
||||
expect(payload.reasoning?.effort).toBe("medium");
|
||||
expect(payload.temperature).toBe(0.8);
|
||||
},
|
||||
);
|
||||
|
||||
it("overrides named request fields", async () => {
|
||||
const payload = await capturePayload(makeModel("openai-completions"), {
|
||||
temperature: 0,
|
||||
|
|
|
|||
|
|
@ -182,6 +182,7 @@
|
|||
- Added a Built-in section in `pi config` to disable the built-in `mcp`, `llama.cpp`, `codemode`, and `tool-search` extensions globally or per project, stored as `-builtin:<name>` in the `extensions` setting. SDK inline extensions opt in with `builtin: true`.
|
||||
- Added `+name` and `-name` entries to the `defaultTools` setting to add or remove tools without repeating the defaults, for example `"defaultTools": ["+codemode"]`. Project entries of this form apply on top of the user setting. Documented how to enable `codemode` without MCP and how to use classifier models such as Jev from codemode scripts.
|
||||
- Added the token usage and cost of codemode `models.classify()` calls to the codemode tool result, so they count toward the session cost; the codemode result shows each call's cost.
|
||||
- Added `samplingParamsByThinkingLevel` to `models.json` for per-thinking-level sampling parameter overrides.
|
||||
|
||||
### Changed
|
||||
|
||||
|
|
|
|||
|
|
@ -98,6 +98,32 @@ Use `promptCache` to declare the provider's best-effort cache lifetime in second
|
|||
|
||||
Choose the conservative end of any published range. A model without a lifetime for the active tier is not eligible for cache warming. A `modelOverrides` entry can set `inputLimits` or `promptCache` for a built-in or extension model, including a model accessed through a validated proxy. See [`cacheWarming`](settings.md#model-and-thinking).
|
||||
|
||||
### Configure sampling by thinking level
|
||||
|
||||
OpenAI-compatible APIs support free-form `samplingParams` model defaults and `samplingParamsByThinkingLevel` overrides. The latter uses Pi thinking-level keys (`off`, `minimal`, `low`, `medium`, `high`, `xhigh`, and `max`), not provider values from `thinkingLevelMap`:
|
||||
|
||||
```json
|
||||
{
|
||||
"id": "qwen-thinking-model",
|
||||
"reasoning": true,
|
||||
"samplingParams": {
|
||||
"temperature": 1.0,
|
||||
"top_p": 0.95
|
||||
},
|
||||
"samplingParamsByThinkingLevel": {
|
||||
"off": {
|
||||
"temperature": 0.7,
|
||||
"top_p": 0.8
|
||||
},
|
||||
"high": {
|
||||
"top_k": 20
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Pi first clamps unsupported thinking levels, then merges model `samplingParams`, the effective level's override, and request-level `samplingParams` in that order. Later values win per key. Missing levels inherit the model defaults. `modelOverrides` merges per-level entries per key with the base model. These fields apply only to `openai-completions`, `openai-responses`, and `azure-openai-responses`; other APIs ignore them.
|
||||
|
||||
Compatibility settings should describe verified differences in the endpoint's request or response behavior. Do not enable them based only on an endpoint advertising OpenAI or Anthropic compatibility.
|
||||
|
||||
## Use classifier models
|
||||
|
|
|
|||
|
|
@ -63,6 +63,17 @@ const ThinkingLevelMapSchema = Type.Object({
|
|||
max: Type.Optional(ThinkingLevelMapValueSchema),
|
||||
});
|
||||
|
||||
const SamplingParamsSchema = Type.Record(Type.String(), Type.Unknown());
|
||||
const SamplingParamsByThinkingLevelSchema = Type.Object({
|
||||
off: Type.Optional(SamplingParamsSchema),
|
||||
minimal: Type.Optional(SamplingParamsSchema),
|
||||
low: Type.Optional(SamplingParamsSchema),
|
||||
medium: Type.Optional(SamplingParamsSchema),
|
||||
high: Type.Optional(SamplingParamsSchema),
|
||||
xhigh: Type.Optional(SamplingParamsSchema),
|
||||
max: Type.Optional(SamplingParamsSchema),
|
||||
});
|
||||
|
||||
const ChatTemplateKwargScalarSchema = Type.Union([Type.String(), Type.Number(), Type.Boolean(), Type.Null()]);
|
||||
const ChatTemplateKwargVariableSchema = Type.Object({
|
||||
$var: Type.Union([Type.Literal("thinking.enabled"), Type.Literal("thinking.effort")]),
|
||||
|
|
@ -198,7 +209,8 @@ const ModelDefinitionSchema = Type.Object({
|
|||
promptCache: Type.Optional(ModelPromptCacheSchema),
|
||||
contextWindow: Type.Optional(Type.Number()),
|
||||
maxTokens: Type.Optional(Type.Number()),
|
||||
samplingParams: Type.Optional(Type.Record(Type.String(), Type.Unknown())),
|
||||
samplingParams: Type.Optional(SamplingParamsSchema),
|
||||
samplingParamsByThinkingLevel: Type.Optional(SamplingParamsByThinkingLevelSchema),
|
||||
headers: Type.Optional(Type.Record(Type.String(), Type.String())),
|
||||
compat: Type.Optional(ProviderCompatSchema),
|
||||
});
|
||||
|
|
@ -221,7 +233,8 @@ const ModelOverrideSchema = Type.Object({
|
|||
promptCache: Type.Optional(ModelPromptCacheSchema),
|
||||
contextWindow: Type.Optional(Type.Number()),
|
||||
maxTokens: Type.Optional(Type.Number()),
|
||||
samplingParams: Type.Optional(Type.Record(Type.String(), Type.Unknown())),
|
||||
samplingParams: Type.Optional(SamplingParamsSchema),
|
||||
samplingParamsByThinkingLevel: Type.Optional(SamplingParamsByThinkingLevelSchema),
|
||||
headers: Type.Optional(Type.Record(Type.String(), Type.String())),
|
||||
compat: Type.Optional(ProviderCompatSchema),
|
||||
});
|
||||
|
|
|
|||
|
|
@ -68,7 +68,8 @@ export interface ProviderChatModelConfig extends ProviderModelConfigBase {
|
|||
promptCache?: Model<Api>["promptCache"];
|
||||
contextWindow: number;
|
||||
maxTokens: number;
|
||||
samplingParams?: Record<string, unknown>;
|
||||
samplingParams?: Model<Api>["samplingParams"];
|
||||
samplingParamsByThinkingLevel?: Model<Api>["samplingParamsByThinkingLevel"];
|
||||
compat?: Model<Api>["compat"];
|
||||
}
|
||||
|
||||
|
|
@ -160,6 +161,19 @@ function mergeInputLimits(
|
|||
};
|
||||
}
|
||||
|
||||
function mergeSamplingParamsByThinkingLevel(
|
||||
base: Model<Api>["samplingParamsByThinkingLevel"],
|
||||
override: Model<Api>["samplingParamsByThinkingLevel"],
|
||||
): Model<Api>["samplingParamsByThinkingLevel"] {
|
||||
if (!override) return base;
|
||||
const merged = { ...base };
|
||||
for (const level of ["off", "minimal", "low", "medium", "high", "xhigh", "max"] as const) {
|
||||
const params = override[level];
|
||||
if (params) merged[level] = { ...base?.[level], ...params };
|
||||
}
|
||||
return merged;
|
||||
}
|
||||
|
||||
function applyModelOverride(model: Model<Api>, override: ModelsJsonModelOverride): Model<Api> {
|
||||
return {
|
||||
...model,
|
||||
|
|
@ -185,6 +199,10 @@ function applyModelOverride(model: Model<Api>, override: ModelsJsonModelOverride
|
|||
samplingParams: override.samplingParams
|
||||
? { ...model.samplingParams, ...override.samplingParams }
|
||||
: model.samplingParams,
|
||||
samplingParamsByThinkingLevel: mergeSamplingParamsByThinkingLevel(
|
||||
model.samplingParamsByThinkingLevel,
|
||||
override.samplingParamsByThinkingLevel,
|
||||
),
|
||||
compat: mergeCompat(model.compat, override.compat),
|
||||
};
|
||||
}
|
||||
|
|
@ -224,6 +242,7 @@ function modelFromJson(
|
|||
contextWindow: definition.contextWindow ?? 128000,
|
||||
maxTokens: definition.maxTokens ?? 16384,
|
||||
samplingParams: definition.samplingParams,
|
||||
samplingParamsByThinkingLevel: definition.samplingParamsByThinkingLevel,
|
||||
headers: undefined,
|
||||
compat: mergeCompat(providerConfig.compat, definition.compat),
|
||||
};
|
||||
|
|
|
|||
|
|
@ -754,7 +754,7 @@ describe("ModelRegistry", () => {
|
|||
expect(compat?.allowedFallbackModels).toEqual([]);
|
||||
});
|
||||
|
||||
test("custom model and model override carry sampling params", async () => {
|
||||
test("custom models and model overrides carry sampling params", async () => {
|
||||
writeRawModelsJson({
|
||||
openrouter: {
|
||||
baseUrl: "https://my-proxy.example.com/v1",
|
||||
|
|
@ -763,11 +763,22 @@ describe("ModelRegistry", () => {
|
|||
{
|
||||
id: "custom/sampling-model",
|
||||
samplingParams: { temperature: 1, top_p: 0.95, top_k: 0 },
|
||||
samplingParamsByThinkingLevel: {
|
||||
low: { temperature: 0.6, top_p: 0.95 },
|
||||
high: { temperature: 0.8 },
|
||||
},
|
||||
},
|
||||
],
|
||||
modelOverrides: {
|
||||
"custom/sampling-model": {
|
||||
samplingParamsByThinkingLevel: {
|
||||
low: { temperature: 0.5, top_k: 20 },
|
||||
max: { temperature: 1 },
|
||||
},
|
||||
},
|
||||
"anthropic/claude-sonnet-4": {
|
||||
samplingParams: { top_p: 0.9 },
|
||||
samplingParamsByThinkingLevel: { high: { temperature: 0.8 } },
|
||||
},
|
||||
},
|
||||
},
|
||||
|
|
@ -778,13 +789,20 @@ describe("ModelRegistry", () => {
|
|||
|
||||
const custom = models.find((m) => m.id === "custom/sampling-model");
|
||||
expect(custom?.samplingParams).toEqual({ temperature: 1, top_p: 0.95, top_k: 0 });
|
||||
expect(custom?.samplingParamsByThinkingLevel).toEqual({
|
||||
low: { temperature: 0.5, top_p: 0.95, top_k: 20 },
|
||||
high: { temperature: 0.8 },
|
||||
max: { temperature: 1 },
|
||||
});
|
||||
|
||||
const sonnet = models.find((m) => m.id === "anthropic/claude-sonnet-4");
|
||||
expect(sonnet?.samplingParams).toEqual({ top_p: 0.9 });
|
||||
expect(sonnet?.samplingParamsByThinkingLevel).toEqual({ high: { temperature: 0.8 } });
|
||||
|
||||
// Models without sampling config keep it unset.
|
||||
const opus = models.find((m) => m.id === "anthropic/claude-opus-4.1");
|
||||
expect(opus?.samplingParams).toBeUndefined();
|
||||
expect(opus?.samplingParamsByThinkingLevel).toBeUndefined();
|
||||
});
|
||||
|
||||
test("custom model and model override carry prompt cache lifetimes", async () => {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue