mirror of
https://github.com/badlogic/pi-mono.git
synced 2026-11-05 23:37:25 +00:00
fix: cap agent retry backoff
Add a configurable 60-second default cap for assistant and summarization retry delays across coding-agent and the agent harness. Fixes #8826
This commit is contained in:
David Brailovsky
parent
be26e32704
commit
c37b0e03b5
24 changed files with 151 additions and 47 deletions
|
|
@ -132,7 +132,7 @@ Every tool call has a result and nothing ran twice; without a committed checkpoi
|
|||
|
||||
Path convention: `src/...` is relative to `packages/agent/`; bare harness paths such as `session/types.ts` or `agent-harness.ts` are relative to `packages/agent/src/harness/`; `docs/...` is relative to `packages/agent/`; paths beginning `packages/` are repository-root relative. Source type provenance: `AgentMessage`, `AgentTool`, `AgentToolResult`, `QueueMode`, `ThinkingLevel` — `packages/agent/src/types.ts`. `Skill`, `PromptTemplate`, `AgentHarnessResources` (`Resources` below), the `AgentHarnessTool*` family, `AgentHarnessStreamOptions`/`Patch` — `packages/agent/src/harness/types.ts`. `Model`, `Models`, `Tool`, `Usage`, `RetryPolicy`, `StopReason`, `AssistantMessage`, `ImageContent`, provider messages, stream options, deferred handles — `packages/ai`; `AiContext` aliases pi-ai's provider request `Context` to distinguish it from the harness invocation `Context`. `AssistantMessageFrame`, `AssistantMessageFrameEncoder`, `reduceAssistantMessageFrames` — `packages/ai` `src/utils/assistant-message-frame.ts`; the harness defines no second frame codec or reducer. `CompactionSettings`, `CompactionPreparation`, `CompactResult`, `BranchPreparation`, `BranchSummaryResult` — `packages/agent/src/harness/compaction/`; existing preparation and split-turn algorithms remain the implementation unless this document changes them. `TelemetryContext` and schema helpers — `packages/telemetry`; agent-owned schemas — `src/harness/telemetry.ts`. `Context`, `ContextKey`, `BACKGROUND_CONTEXT`, derivation helpers — `src/harness/context.ts`. Harness/lane public declarations — `src/harness/agent-harness.ts`; Session/storage declarations — `src/harness/session/types.ts` and `session/values.ts`.
|
||||
|
||||
Public `QueueMode` is `"all" | "one-at-a-time"`. Public `RetryPolicy` is `{ enabled, maxRetries, baseDelayMs }`; operation state stores the normalized `{ maxAttempts, baseDelayMs }`. `maxRetries` and `baseDelayMs` must be finite non-negative safe integers and `maxRetries + 1` must remain safe; disabled retry normalizes to one attempt; delay and `notBefore` arithmetic saturate at `Number.MAX_SAFE_INTEGER`. Public `CompactionSettings` is `{ enabled, reserveTokens, keepRecentTokens }`; both token counts must be finite non-negative safe integers. Constructors and setters reject invalid settings before publication. `AgentHarnessStreamOptions` and its patch include `deferred?: boolean | { window?: "15m" | "1h" | "24h" }`; structural requests always force it to false. `SettledAssistantMessage` is `AssistantMessage & { stopReason: Exclude<StopReason, "pending"> }`. Provider dispatch resolves the durable `{ provider, modelId }` identity through `Models` at request time (which also applies auth); a missing or swapped registry entry fails the request in-band, like an unknown tool.
|
||||
Public `QueueMode` is `"all" | "one-at-a-time"`. Public `RetryPolicy` is `{ enabled, maxRetries, baseDelayMs, maxAgentDelayMs? }`; operation state stores the normalized `{ maxAttempts, baseDelayMs, maxAgentDelayMs }`. `maxRetries`, `baseDelayMs`, and optional `maxAgentDelayMs` must be finite non-negative safe integers and `maxRetries + 1` must remain safe; disabled retry normalizes to one attempt; omitted `maxAgentDelayMs` defaults to 60 seconds; delay and `notBefore` arithmetic saturate at `Number.MAX_SAFE_INTEGER`. Public `CompactionSettings` is `{ enabled, reserveTokens, keepRecentTokens }`; both token counts must be finite non-negative safe integers. Constructors and setters reject invalid settings before publication. `AgentHarnessStreamOptions` and its patch include `deferred?: boolean | { window?: "15m" | "1h" | "24h" }`; structural requests always force it to false. `SettledAssistantMessage` is `AssistantMessage & { stopReason: Exclude<StopReason, "pending"> }`. Provider dispatch resolves the durable `{ provider, modelId }` identity through `Models` at request time (which also applies auth); a missing or swapped registry entry fails the request in-band, like an unknown tool.
|
||||
|
||||
## 0.8 Validation boundary
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,12 @@
|
|||
import type { RetryPolicy } from "@earendil-works/pi-ai";
|
||||
import { DEFAULT_MAX_AGENT_RETRY_DELAY_MS, type RetryPolicy } from "@earendil-works/pi-ai";
|
||||
import type { CompactionSettings } from "./compaction/compaction.ts";
|
||||
|
||||
export const DEFAULT_RETRY_POLICY: RetryPolicy = { enabled: true, maxRetries: 3, baseDelayMs: 1_000 };
|
||||
export const DEFAULT_RETRY_POLICY: RetryPolicy = {
|
||||
enabled: true,
|
||||
maxRetries: 3,
|
||||
baseDelayMs: 1_000,
|
||||
maxAgentDelayMs: DEFAULT_MAX_AGENT_RETRY_DELAY_MS,
|
||||
};
|
||||
|
||||
export function validateToolNames(tools: readonly { name: string }[]): void {
|
||||
const names = new Set<string>();
|
||||
|
|
@ -17,7 +22,9 @@ export function validateRetryPolicy(policy: RetryPolicy): void {
|
|||
policy.maxRetries < 0 ||
|
||||
policy.maxRetries === Number.MAX_SAFE_INTEGER ||
|
||||
!Number.isSafeInteger(policy.baseDelayMs) ||
|
||||
policy.baseDelayMs < 0
|
||||
policy.baseDelayMs < 0 ||
|
||||
(policy.maxAgentDelayMs !== undefined &&
|
||||
(!Number.isSafeInteger(policy.maxAgentDelayMs) || policy.maxAgentDelayMs < 0))
|
||||
) {
|
||||
throw new RangeError("Retry policy values must be finite non-negative safe integers");
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import { DEFAULT_MAX_AGENT_RETRY_DELAY_MS } from "@earendil-works/pi-ai";
|
||||
import type { HarnessEvent, LaneQueuedItem } from "../../agent-harness.ts";
|
||||
import { insertEntry } from "../../session/commit.ts";
|
||||
import { SessionInvariantError } from "../../session/session.ts";
|
||||
|
|
@ -44,9 +45,11 @@ export function normalizedRetryPolicy<TContext extends object | undefined>(
|
|||
lane: Lane<TContext>,
|
||||
): NormalizedRetryPolicy {
|
||||
const retry = lane.readConfig().retryPolicy;
|
||||
return retry.enabled
|
||||
? { maxAttempts: retry.maxRetries + 1, baseDelayMs: retry.baseDelayMs }
|
||||
: { maxAttempts: 1, baseDelayMs: retry.baseDelayMs };
|
||||
return {
|
||||
maxAttempts: retry.enabled ? retry.maxRetries + 1 : 1,
|
||||
baseDelayMs: retry.baseDelayMs,
|
||||
maxAgentDelayMs: retry.maxAgentDelayMs ?? DEFAULT_MAX_AGENT_RETRY_DELAY_MS,
|
||||
};
|
||||
}
|
||||
|
||||
export function assistantReadyAtBoundary<TContext extends object | undefined>(
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import {
|
|||
isContextOverflow,
|
||||
isRecoverableLength,
|
||||
isRetryableAssistantError,
|
||||
retryDelayMs,
|
||||
} from "@earendil-works/pi-ai";
|
||||
import type { HarnessEvent } from "../../agent-harness.ts";
|
||||
import type { Context } from "../../context.ts";
|
||||
|
|
@ -28,7 +29,7 @@ import { branchTip, deleteList, operationPreparation, pendingAssistantFrames, se
|
|||
import type { Lane } from "../lane.ts";
|
||||
import { openFrameProgress } from "../progress.ts";
|
||||
import type { Drive, ProcedureResult } from "../types.ts";
|
||||
import { retryDelay, retryNotBefore } from "./retry.ts";
|
||||
import { retryNotBefore } from "./retry.ts";
|
||||
import { prepareOverflowCompaction } from "./structural.ts";
|
||||
import { operationCleanupWrites, operationResultRecord } from "./terminal.ts";
|
||||
|
||||
|
|
@ -283,7 +284,7 @@ export async function publishResponse<TContext extends object | undefined>(
|
|||
at: "assistant.retry_wait",
|
||||
generationContext: current.generationContext,
|
||||
nextAttempt: current.attempt + 1,
|
||||
notBefore: retryNotBefore(current.generationContext.retryPolicy.baseDelayMs, current.attempt),
|
||||
notBefore: retryNotBefore(current.generationContext.retryPolicy, current.attempt),
|
||||
errorMessage: response.errorMessage ?? "Assistant request failed",
|
||||
};
|
||||
} else {
|
||||
|
|
@ -395,7 +396,7 @@ export async function publishResponse<TContext extends object | undefined>(
|
|||
step: turnId,
|
||||
attempt: settled.nextAttempt,
|
||||
maxAttempts: settled.generationContext.retryPolicy.maxAttempts,
|
||||
delayMs: retryDelay(current.generationContext.retryPolicy.baseDelayMs, current.attempt),
|
||||
delayMs: retryDelayMs(current.generationContext.retryPolicy, current.attempt),
|
||||
notBefore: settled.notBefore,
|
||||
errorMessage: settled.errorMessage,
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,10 +1,11 @@
|
|||
export function retryDelay(baseDelayMs: number, attempt: number): number {
|
||||
const delay = baseDelayMs * 2 ** Math.max(0, attempt - 1);
|
||||
return Number.isSafeInteger(delay) ? delay : Number.MAX_SAFE_INTEGER;
|
||||
}
|
||||
import { type RetryPolicy, retryDelayMs } from "@earendil-works/pi-ai";
|
||||
|
||||
export function retryNotBefore(baseDelayMs: number, attempt: number, now = Date.now()): number {
|
||||
const sum = now + retryDelay(baseDelayMs, attempt);
|
||||
export function retryNotBefore(
|
||||
policy: Pick<RetryPolicy, "baseDelayMs" | "maxAgentDelayMs">,
|
||||
attempt: number,
|
||||
now = Date.now(),
|
||||
): number {
|
||||
const sum = now + retryDelayMs(policy, attempt);
|
||||
return Number.isSafeInteger(sum) ? sum : Number.MAX_SAFE_INTEGER;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import {
|
|||
type AssistantMessage,
|
||||
isRetryableAssistantError,
|
||||
type Model,
|
||||
retryDelayMs,
|
||||
type SimpleStreamOptions,
|
||||
} from "@earendil-works/pi-ai";
|
||||
import type { HarnessEvent } from "../../agent-harness.ts";
|
||||
|
|
@ -53,7 +54,7 @@ import {
|
|||
normalizedRetryPolicy,
|
||||
planBoundaryInbox,
|
||||
} from "./boundary.ts";
|
||||
import { retryDelay, retryNotBefore, waitUntil } from "./retry.ts";
|
||||
import { retryNotBefore, waitUntil } from "./retry.ts";
|
||||
import { operationCleanupWrites, operationResultRecord } from "./terminal.ts";
|
||||
|
||||
class StructuralCancelled extends Error {
|
||||
|
|
@ -688,7 +689,7 @@ function retryWaitFromEffect(effect: SummaryEffectPendingOperation, errorMessage
|
|||
task: effect.task,
|
||||
summaryContext: effect.summaryContext,
|
||||
nextAttempt: effect.attempt + 1,
|
||||
notBefore: retryNotBefore(effect.summaryContext.retryPolicy.baseDelayMs, effect.attempt),
|
||||
notBefore: retryNotBefore(effect.summaryContext.retryPolicy, effect.attempt),
|
||||
errorMessage,
|
||||
};
|
||||
}
|
||||
|
|
@ -992,7 +993,7 @@ async function publishAttemptResult<TContext extends object | undefined>(
|
|||
step: effect.task.taskId,
|
||||
attempt: retryWait.nextAttempt,
|
||||
maxAttempts: effect.summaryContext.retryPolicy.maxAttempts,
|
||||
delayMs: retryDelay(effect.summaryContext.retryPolicy.baseDelayMs, effect.attempt),
|
||||
delayMs: retryDelayMs(effect.summaryContext.retryPolicy, effect.attempt),
|
||||
notBefore: retryWait.notBefore,
|
||||
errorMessage: result.error.message,
|
||||
},
|
||||
|
|
@ -1101,7 +1102,7 @@ export async function recoverStructuralGeneration<TContext extends object | unde
|
|||
step: effect.task.taskId,
|
||||
attempt: retryWait.nextAttempt,
|
||||
maxAttempts: effect.summaryContext.retryPolicy.maxAttempts,
|
||||
delayMs: retryDelay(effect.summaryContext.retryPolicy.baseDelayMs, effect.attempt),
|
||||
delayMs: retryDelayMs(effect.summaryContext.retryPolicy, effect.attempt),
|
||||
notBefore: retryWait.notBefore,
|
||||
errorMessage: error.message,
|
||||
recovery: true,
|
||||
|
|
|
|||
|
|
@ -136,6 +136,7 @@ export interface InboxItem {
|
|||
export interface NormalizedRetryPolicy {
|
||||
maxAttempts: number;
|
||||
baseDelayMs: number;
|
||||
maxAgentDelayMs: number;
|
||||
}
|
||||
|
||||
export interface GenerationContext {
|
||||
|
|
|
|||
|
|
@ -217,7 +217,7 @@ describe("runtime generation checkpoint", () => {
|
|||
expect(run.generationContext).toMatchObject({
|
||||
configuration: fixture.lane.state.configuration,
|
||||
streamOptions: {},
|
||||
retryPolicy: { maxAttempts: 4, baseDelayMs: 1 },
|
||||
retryPolicy: { maxAttempts: 4, baseDelayMs: 1, maxAgentDelayMs: 60_000 },
|
||||
overflowRecoveryUsed: false,
|
||||
});
|
||||
await expectProjectionRestores(fixture);
|
||||
|
|
@ -632,7 +632,10 @@ describe("runtime assistant generation", () => {
|
|||
const pending: AssistantEffectPendingOperation = {
|
||||
...operationScopeOf(ready),
|
||||
at: "assistant.effect_pending",
|
||||
generationContext: { ...ready.generationContext, retryPolicy: { maxAttempts: 2, baseDelayMs: 1 } },
|
||||
generationContext: {
|
||||
...ready.generationContext,
|
||||
retryPolicy: { maxAttempts: 2, baseDelayMs: 1, maxAgentDelayMs: 30_000 },
|
||||
},
|
||||
attempt: 1,
|
||||
responseEntryId,
|
||||
usageId,
|
||||
|
|
@ -681,7 +684,10 @@ describe("runtime assistant generation", () => {
|
|||
const pending: AssistantEffectPendingOperation = {
|
||||
...operationScopeOf(ready),
|
||||
at: "assistant.effect_pending",
|
||||
generationContext: { ...ready.generationContext, retryPolicy: { maxAttempts: 1, baseDelayMs: 1 } },
|
||||
generationContext: {
|
||||
...ready.generationContext,
|
||||
retryPolicy: { maxAttempts: 1, baseDelayMs: 1, maxAgentDelayMs: 30_000 },
|
||||
},
|
||||
attempt: 1,
|
||||
responseEntryId,
|
||||
usageId,
|
||||
|
|
|
|||
|
|
@ -92,7 +92,7 @@ function summaryContext(configuration: LaneConfiguration): SummaryContext {
|
|||
resultEntryId: "summary-entry",
|
||||
configuration,
|
||||
streamOptions: {},
|
||||
retryPolicy: { maxAttempts: 2, baseDelayMs: 10 },
|
||||
retryPolicy: { maxAttempts: 2, baseDelayMs: 10, maxAgentDelayMs: 30_000 },
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -242,7 +242,7 @@ function cases(fixture: Fixture): InstalledOperation[] {
|
|||
triggerEntryId: "tip",
|
||||
configuration: fixture.configuration,
|
||||
streamOptions: {},
|
||||
retryPolicy: { maxAttempts: 2, baseDelayMs: 10 },
|
||||
retryPolicy: { maxAttempts: 2, baseDelayMs: 10, maxAgentDelayMs: 30_000 },
|
||||
overflowRecoveryUsed: false,
|
||||
};
|
||||
const runTask: SummaryTask = {
|
||||
|
|
@ -606,7 +606,7 @@ describe("runtime cancellation reconciliation", () => {
|
|||
triggerEntryId: "tip",
|
||||
configuration: fixture.configuration,
|
||||
streamOptions: {},
|
||||
retryPolicy: { maxAttempts: 2, baseDelayMs: 10 },
|
||||
retryPolicy: { maxAttempts: 2, baseDelayMs: 10, maxAgentDelayMs: 30_000 },
|
||||
overflowRecoveryUsed: false,
|
||||
};
|
||||
await installOperation(fixture, {
|
||||
|
|
|
|||
9
packages/agent/test/harness/runtime/drive-retry.test.ts
Normal file
9
packages/agent/test/harness/runtime/drive-retry.test.ts
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import { retryNotBefore } from "../../../src/harness/runtime/drive/retry.ts";
|
||||
|
||||
describe("runtime retry delay", () => {
|
||||
it("uses capped delay when computing retry readiness", () => {
|
||||
// Regression for #8826.
|
||||
expect(retryNotBefore({ baseDelayMs: 2000, maxAgentDelayMs: 30000 }, 5, 100)).toBe(30100);
|
||||
});
|
||||
});
|
||||
|
|
@ -101,7 +101,7 @@ function summaryReady(
|
|||
scope: OperationScope,
|
||||
task: SummaryTask,
|
||||
configuration: LaneConfiguration,
|
||||
retryPolicy = { maxAttempts: 2, baseDelayMs: 10 },
|
||||
retryPolicy = { maxAttempts: 2, baseDelayMs: 10, maxAgentDelayMs: 30_000 },
|
||||
): SummaryReadyOperation {
|
||||
return {
|
||||
...scope,
|
||||
|
|
@ -700,7 +700,7 @@ describe("runtime structural drive", () => {
|
|||
triggerEntryId: "tip",
|
||||
configuration: fixture.configuration,
|
||||
streamOptions: {},
|
||||
retryPolicy: { maxAttempts: 2, baseDelayMs: 10 },
|
||||
retryPolicy: { maxAttempts: 2, baseDelayMs: 10, maxAgentDelayMs: 30_000 },
|
||||
overflowRecoveryUsed: false,
|
||||
},
|
||||
nextAttempt: 1,
|
||||
|
|
@ -1055,6 +1055,7 @@ describe("runtime structural drive", () => {
|
|||
const ready = summaryReady(runScope(), standaloneCompactionTask(), fixture.configuration, {
|
||||
maxAttempts: 2,
|
||||
baseDelayMs: 10,
|
||||
maxAgentDelayMs: 30_000,
|
||||
});
|
||||
await installOperation(
|
||||
fixture,
|
||||
|
|
@ -1105,6 +1106,7 @@ describe("runtime structural drive", () => {
|
|||
const ready = summaryReady(runScope(), standaloneCompactionTask(), fixture.configuration, {
|
||||
maxAttempts: 2,
|
||||
baseDelayMs: 10,
|
||||
maxAgentDelayMs: 30_000,
|
||||
});
|
||||
await installOperation(
|
||||
fixture,
|
||||
|
|
@ -1143,6 +1145,7 @@ describe("runtime structural drive", () => {
|
|||
const ready = summaryReady(runScope(), standaloneCompactionTask(), fixture.configuration, {
|
||||
maxAttempts: 1,
|
||||
baseDelayMs: 10,
|
||||
maxAgentDelayMs: 30_000,
|
||||
});
|
||||
await installOperation(
|
||||
fixture,
|
||||
|
|
@ -1356,7 +1359,7 @@ describe("runtime structural drive", () => {
|
|||
resultEntryId: "summary-entry",
|
||||
configuration: fixture.configuration,
|
||||
streamOptions: {},
|
||||
retryPolicy: { maxAttempts: 2, baseDelayMs: 10 },
|
||||
retryPolicy: { maxAttempts: 2, baseDelayMs: 10, maxAgentDelayMs: 30_000 },
|
||||
},
|
||||
attempt: 1,
|
||||
request: { index: 1, usageId: "abandoned-usage" },
|
||||
|
|
@ -1400,7 +1403,7 @@ describe("runtime structural drive", () => {
|
|||
resultEntryId: "summary-entry",
|
||||
configuration: fixture.configuration,
|
||||
streamOptions: {},
|
||||
retryPolicy: { maxAttempts: 1, baseDelayMs: 10 },
|
||||
retryPolicy: { maxAttempts: 1, baseDelayMs: 10, maxAgentDelayMs: 30_000 },
|
||||
},
|
||||
attempt: 1,
|
||||
request: { index: 0, usageId: "abandoned-usage" },
|
||||
|
|
|
|||
|
|
@ -106,7 +106,7 @@ describe("runtime terminal cleanup mechanics", () => {
|
|||
triggerEntryId: "trigger",
|
||||
configuration,
|
||||
streamOptions: {},
|
||||
retryPolicy: { maxAttempts: 2, baseDelayMs: 1 },
|
||||
retryPolicy: { maxAttempts: 2, baseDelayMs: 1, maxAgentDelayMs: 30_000 },
|
||||
overflowRecoveryUsed: false,
|
||||
},
|
||||
attempt: 1,
|
||||
|
|
|
|||
|
|
@ -85,7 +85,7 @@ function assistantEffectPending(responseEntryId: string): AssistantEffectPending
|
|||
triggerEntryId: "trigger",
|
||||
configuration,
|
||||
streamOptions: {},
|
||||
retryPolicy: { maxAttempts: 2, baseDelayMs: 1 },
|
||||
retryPolicy: { maxAttempts: 2, baseDelayMs: 1, maxAgentDelayMs: 30_000 },
|
||||
overflowRecoveryUsed: false,
|
||||
},
|
||||
attempt: 1,
|
||||
|
|
|
|||
|
|
@ -234,7 +234,7 @@ describe("runtime lane watch", () => {
|
|||
triggerEntryId: "trigger",
|
||||
configuration,
|
||||
streamOptions: {},
|
||||
retryPolicy: { maxAttempts: 2, baseDelayMs: 0 },
|
||||
retryPolicy: { maxAttempts: 2, baseDelayMs: 0, maxAgentDelayMs: 30_000 },
|
||||
overflowRecoveryUsed: false,
|
||||
},
|
||||
attempt: 1,
|
||||
|
|
@ -282,7 +282,7 @@ describe("runtime lane watch", () => {
|
|||
triggerEntryId: "trigger",
|
||||
configuration,
|
||||
streamOptions: {},
|
||||
retryPolicy: { maxAttempts: 2, baseDelayMs: 0 },
|
||||
retryPolicy: { maxAttempts: 2, baseDelayMs: 0, maxAgentDelayMs: 30_000 },
|
||||
overflowRecoveryUsed: false,
|
||||
},
|
||||
attempt: 1,
|
||||
|
|
|
|||
|
|
@ -75,7 +75,7 @@ const configuration = {
|
|||
activeToolNames: ["read"],
|
||||
} satisfies LaneConfiguration;
|
||||
|
||||
const retryPolicy = { maxAttempts: 3, baseDelayMs: 100 } as const;
|
||||
const retryPolicy = { maxAttempts: 3, baseDelayMs: 100, maxAgentDelayMs: 30_000 } as const;
|
||||
const generationContext = {
|
||||
stepId: "step",
|
||||
triggerEntryId: "trigger",
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@
|
|||
|
||||
### Fixed
|
||||
|
||||
- Added `RetryPolicy.maxAgentDelayMs` support to cap shared assistant retry backoff for summarization calls ([#8826](https://github.com/earendil-works/pi/issues/8826)).
|
||||
- Fixed quadratic CPU usage when draining buffered `EventStream` events ([#9055](https://github.com/earendil-works/pi/issues/9055)).
|
||||
- Fixed Mistral Medium reasoning requests to use `reasoning_effort` for all reasoning-capable `mistral-medium-*` model IDs instead of the unsupported `prompt_mode` ([#8700](https://github.com/earendil-works/pi/issues/8700)).
|
||||
- Fixed OpenCode and OpenCode Go requests to send `x-opencode-session` from `sessionId` across all supported API adapters ([#9326](https://github.com/earendil-works/pi/issues/9326)).
|
||||
|
|
|
|||
|
|
@ -91,7 +91,8 @@ const RETRYABLE_PROVIDER_ERROR_PATTERN = buildProviderErrorPattern([
|
|||
|
||||
/**
|
||||
* Retry policy: bounded attempts with exponential backoff (`baseDelayMs * 2^(attempt-1)`).
|
||||
* Matches `settings.retry` (`enabled`, `maxRetries`, `baseDelayMs`) in coding-agent; kept
|
||||
* `maxAgentDelayMs` caps each computed delay and defaults to 60 seconds.
|
||||
* Matches `settings.retry` (`enabled`, `maxRetries`, `baseDelayMs`, `maxAgentDelayMs`) in coding-agent; kept
|
||||
* here so the classifier and the policy-driven retry loop live together and stay reusable
|
||||
* by the SDK and other callers.
|
||||
*/
|
||||
|
|
@ -101,6 +102,16 @@ export interface RetryPolicy {
|
|||
maxRetries: number;
|
||||
/** Base delay in ms. Per-attempt delay is `baseDelayMs * 2^(attempt-1)` before jitter. */
|
||||
baseDelayMs: number;
|
||||
/** Optional cap for agent-level retry delays in ms. Defaults to 60 seconds. */
|
||||
maxAgentDelayMs?: number;
|
||||
}
|
||||
|
||||
export const DEFAULT_MAX_AGENT_RETRY_DELAY_MS = 60_000;
|
||||
|
||||
export function retryDelayMs(policy: Pick<RetryPolicy, "baseDelayMs" | "maxAgentDelayMs">, attempt: number): number {
|
||||
const delay = policy.baseDelayMs * 2 ** Math.max(0, attempt - 1);
|
||||
const safeDelay = Number.isSafeInteger(delay) ? delay : Number.MAX_SAFE_INTEGER;
|
||||
return Math.min(safeDelay, policy.maxAgentDelayMs ?? DEFAULT_MAX_AGENT_RETRY_DELAY_MS);
|
||||
}
|
||||
|
||||
/** Optional callbacks emitted by {@link retryAssistantCall} around each retry. */
|
||||
|
|
@ -193,7 +204,7 @@ export async function retryAssistantCall(
|
|||
|
||||
attempt++;
|
||||
lastRetry = { attempt, errorMessage: response.errorMessage || "Unknown error" };
|
||||
const delayMs = policy!.baseDelayMs * 2 ** (attempt - 1);
|
||||
const delayMs = retryDelayMs(policy!, attempt);
|
||||
await callbacks?.onRetryScheduled?.(attempt, maxAttempts, delayMs, lastRetry.errorMessage);
|
||||
|
||||
// Normalize aborts during retry backoff to the same AssistantMessage shape as
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import { describe, expect, it, vi } from "vitest";
|
||||
import { fauxAssistantMessage } from "../src/providers/faux.ts";
|
||||
import { isRetryableAssistantError, type RetryPolicy, retryAssistantCall } from "../src/utils/retry.ts";
|
||||
import { isRetryableAssistantError, type RetryPolicy, retryAssistantCall, retryDelayMs } from "../src/utils/retry.ts";
|
||||
|
||||
const openAIExplicitRetryMessage =
|
||||
"An error occurred while processing your request. You can retry your request, or contact us through our help center at help.openai.com if the error persists. Please include the request ID req_******** in your message.";
|
||||
|
|
@ -89,6 +89,15 @@ describe("provider retry classification", () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe("retryDelayMs", () => {
|
||||
it("caps agent retry delay", () => {
|
||||
// Regression for #8826.
|
||||
expect(retryDelayMs({ baseDelayMs: 2000 }, 6)).toBe(60000);
|
||||
expect(retryDelayMs({ baseDelayMs: 2000, maxAgentDelayMs: 5000 }, 5)).toBe(5000);
|
||||
expect(retryDelayMs({ baseDelayMs: 2000, maxAgentDelayMs: 0 }, 5)).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("retryAssistantCall", () => {
|
||||
const disabled: RetryPolicy = { enabled: false, maxRetries: 3, baseDelayMs: 0 };
|
||||
const enabled: RetryPolicy = { enabled: true, maxRetries: 3, baseDelayMs: 0 };
|
||||
|
|
@ -133,6 +142,23 @@ describe("retryAssistantCall", () => {
|
|||
expect(onRetryFinished).toHaveBeenCalledWith(false, 3, "terminated");
|
||||
});
|
||||
|
||||
it("reports capped retry delays", async () => {
|
||||
// Regression for #8826.
|
||||
let n = 0;
|
||||
const policy: RetryPolicy = { enabled: true, maxRetries: 4, baseDelayMs: 10, maxAgentDelayMs: 15 };
|
||||
const produce = vi.fn(async () => {
|
||||
n++;
|
||||
return n < 5
|
||||
? fauxAssistantMessage("", { stopReason: "error", errorMessage: "terminated" })
|
||||
: fauxAssistantMessage("recovered");
|
||||
});
|
||||
const onRetryScheduled = vi.fn();
|
||||
|
||||
await retryAssistantCall(produce, policy, undefined, { onRetryScheduled });
|
||||
|
||||
expect(onRetryScheduled.mock.calls.map((call) => call[2])).toEqual([10, 15, 15, 15]);
|
||||
});
|
||||
|
||||
it("stops retrying once a call succeeds", async () => {
|
||||
let n = 0;
|
||||
const produce = vi.fn(async () => {
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@
|
|||
|
||||
### Fixed
|
||||
|
||||
- Capped agent-level retry backoff at `retry.maxAgentDelayMs` (60s by default) so long retry runs stay responsive during prolonged transient outages ([#8826](https://github.com/earendil-works/pi/issues/8826)).
|
||||
- Fixed direct RPC `steer` and `follow_up` commands bypassing extension `input` handlers ([#8718](https://github.com/earendil-works/pi/issues/8718)).
|
||||
- Fixed premature missing-model errors after login by waiting for catalog discovery. Radius now defaults to `balanced`, falling back to the first available Radius model when needed.
|
||||
- Fixed fullscreen mode reserving a blank row for custom footers that render zero rows ([#8919](https://github.com/earendil-works/pi/issues/8919)).
|
||||
|
|
|
|||
|
|
@ -143,10 +143,13 @@ Set `PI_SKIP_VERSION_CHECK=1` to disable the Pi version update check. Use `--off
|
|||
| `retry.enabled` | boolean | `true` | Enable automatic agent-level retry on transient errors |
|
||||
| `retry.maxRetries` | number | `3` | Maximum agent-level retry attempts |
|
||||
| `retry.baseDelayMs` | number | `2000` | Base delay for agent-level exponential backoff (2s, 4s, 8s) |
|
||||
| `retry.maxAgentDelayMs` | number | `60000` | Max agent-level retry delay (60s) |
|
||||
| `retry.provider.timeoutMs` | number | SDK default | Provider/SDK request timeout in milliseconds |
|
||||
| `retry.provider.maxRetries` | number | `0` | Provider/SDK retry attempts |
|
||||
| `retry.provider.maxRetryDelayMs` | number | `60000` | Max server-requested delay before failing (60s) |
|
||||
|
||||
Agent-level retries use exponential backoff capped by `retry.maxAgentDelayMs`, so long retry runs stay responsive after prolonged outages.
|
||||
|
||||
When a provider requests a retry delay longer than `retry.provider.maxRetryDelayMs`, the request fails immediately with an informative error instead of waiting silently. Set it to `0` to disable the limit.
|
||||
|
||||
Keep `retry.provider.maxRetries` at `0` unless provider-level retries are explicitly needed. Setting it above `0` can make SDK/provider retries handle out-of-usage-limit errors before Pi sees them, which may block the agent until the provider quota resets in some circumstances.
|
||||
|
|
@ -157,6 +160,7 @@ Keep `retry.provider.maxRetries` at `0` unless provider-level retries are explic
|
|||
"enabled": true,
|
||||
"maxRetries": 3,
|
||||
"baseDelayMs": 2000,
|
||||
"maxAgentDelayMs": 60000,
|
||||
"provider": {
|
||||
"timeoutMs": 3600000,
|
||||
"maxRetries": 0,
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ import type {
|
|||
PrepareNextTurnContext,
|
||||
ThinkingLevel,
|
||||
} from "@earendil-works/pi-agent-core";
|
||||
import { contentText } from "@earendil-works/pi-ai";
|
||||
import { contentText, retryDelayMs } from "@earendil-works/pi-ai";
|
||||
import type {
|
||||
AssistantMessage,
|
||||
AuthResult,
|
||||
|
|
@ -299,10 +299,6 @@ function estimateMessagesTokens(messages: AgentMessage[]): number {
|
|||
return tokens;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Constants
|
||||
// ============================================================================
|
||||
|
||||
// ============================================================================
|
||||
// AgentSession Class
|
||||
// ============================================================================
|
||||
|
|
@ -2930,7 +2926,7 @@ export class AgentSession {
|
|||
return false;
|
||||
}
|
||||
|
||||
const delayMs = settings.baseDelayMs * 2 ** (this._retryAttempt - 1);
|
||||
const delayMs = retryDelayMs(settings, this._retryAttempt);
|
||||
|
||||
this._emit({
|
||||
type: "auto_retry_start",
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import type { ThinkingLevel } from "@earendil-works/pi-agent-core";
|
||||
import type { Transport } from "@earendil-works/pi-ai";
|
||||
import { DEFAULT_MAX_AGENT_RETRY_DELAY_MS, type Transport } from "@earendil-works/pi-ai";
|
||||
import type { TuiMode as RendererTuiMode, ScrollViewScrollbar, TerminalCapabilities } from "@earendil-works/pi-tui";
|
||||
import { randomUUID } from "crypto";
|
||||
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
|
||||
|
|
@ -31,6 +31,7 @@ export interface RetrySettings {
|
|||
enabled?: boolean; // default: true
|
||||
maxRetries?: number; // default: 3
|
||||
baseDelayMs?: number; // default: 2000 (exponential backoff: 2s, 4s, 8s)
|
||||
maxAgentDelayMs?: number; // default: 60000
|
||||
provider?: ProviderRetrySettings;
|
||||
}
|
||||
|
||||
|
|
@ -879,11 +880,12 @@ export class SettingsManager {
|
|||
this.save();
|
||||
}
|
||||
|
||||
getRetrySettings(): { enabled: boolean; maxRetries: number; baseDelayMs: number } {
|
||||
getRetrySettings(): { enabled: boolean; maxRetries: number; baseDelayMs: number; maxAgentDelayMs: number } {
|
||||
return {
|
||||
enabled: this.getRetryEnabled(),
|
||||
maxRetries: this.settings.retry?.maxRetries ?? 3,
|
||||
baseDelayMs: this.settings.retry?.baseDelayMs ?? 2000,
|
||||
maxAgentDelayMs: this.settings.retry?.maxAgentDelayMs ?? DEFAULT_MAX_AGENT_RETRY_DELAY_MS,
|
||||
};
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -71,10 +71,12 @@ describe("AgentSession retry", () => {
|
|||
async function createSession(options?: {
|
||||
failCount?: number;
|
||||
maxRetries?: number;
|
||||
maxAgentDelayMs?: number;
|
||||
delayAssistantMessageEndMs?: number;
|
||||
}) {
|
||||
const failCount = options?.failCount ?? 1;
|
||||
const maxRetries = options?.maxRetries ?? 3;
|
||||
const maxAgentDelayMs = options?.maxAgentDelayMs ?? 60000;
|
||||
const delayAssistantMessageEndMs = options?.delayAssistantMessageEndMs ?? 0;
|
||||
let callCount = 0;
|
||||
|
||||
|
|
@ -108,7 +110,7 @@ describe("AgentSession retry", () => {
|
|||
const authStorage = AuthStorage.create(join(tempDir, "auth.json"));
|
||||
const modelRegistry = await createModelRegistry(authStorage, tempDir);
|
||||
await authStorage.modify("anthropic", async () => ({ type: "api_key", key: "test-key" }));
|
||||
settingsManager.applyOverrides({ retry: { enabled: true, maxRetries, baseDelayMs: 1 } });
|
||||
settingsManager.applyOverrides({ retry: { enabled: true, maxRetries, baseDelayMs: 1, maxAgentDelayMs } });
|
||||
|
||||
session = new AgentSession({
|
||||
agent,
|
||||
|
|
@ -165,6 +167,19 @@ describe("AgentSession retry", () => {
|
|||
expect(created.session.isRetrying).toBe(false);
|
||||
});
|
||||
|
||||
it("caps agent retry delay", async () => {
|
||||
// Regression for #8826.
|
||||
const created = await createSession({ failCount: 4, maxRetries: 5, maxAgentDelayMs: 5 });
|
||||
const delays: number[] = [];
|
||||
created.session.subscribe((event) => {
|
||||
if (event.type === "auto_retry_start") delays.push(event.delayMs);
|
||||
});
|
||||
|
||||
await created.session.prompt("Test");
|
||||
|
||||
expect(delays).toEqual([1, 2, 4, 5]);
|
||||
});
|
||||
|
||||
it("prompt waits for retry completion even when assistant message_end handling is delayed", async () => {
|
||||
const created = await createSession({ failCount: 1, delayAssistantMessageEndMs: 40 });
|
||||
|
||||
|
|
|
|||
|
|
@ -354,6 +354,22 @@ describe("SettingsManager", () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe("retry settings", () => {
|
||||
it("defaults and overrides agent retry delay cap", () => {
|
||||
expect(SettingsManager.inMemory().getRetrySettings()).toEqual({
|
||||
enabled: true,
|
||||
maxRetries: 3,
|
||||
baseDelayMs: 2000,
|
||||
maxAgentDelayMs: 60000,
|
||||
});
|
||||
expect(
|
||||
SettingsManager.inMemory({
|
||||
retry: { enabled: true, maxRetries: 10, baseDelayMs: 500, maxAgentDelayMs: 5000 },
|
||||
}).getRetrySettings(),
|
||||
).toEqual({ enabled: true, maxRetries: 10, baseDelayMs: 500, maxAgentDelayMs: 5000 });
|
||||
});
|
||||
});
|
||||
|
||||
describe("httpIdleTimeoutMs", () => {
|
||||
it("should default to 5 minutes", () => {
|
||||
const manager = SettingsManager.create(projectDir, agentDir);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue