sscoble/pi-mono
1
0
Fork 0
mirror of https://github.com/badlogic/pi-mono.git synced 2026-11-05 23:37:25 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

feat(agent): simplify durable drive runtime

Browse source
This commit is contained in:
Mario Zechner 2026-08-26 00:15:08 +02:00
parent eb1185d93e
commit df2a3409e8
28 changed files with 2576 additions and 4691 deletions
Download patch file Download diff file Expand all files Collapse all files

463
packages/agent/docs/harness.md
View file

@ -51,7 +51,6 @@
- [4.6 Abort and cancellation reconciliation](#46-abort-and-cancellation-reconciliation)
- [4.7 Close — a controlled crash](#47-close--a-controlled-crash)
- [4.8 Faults](#48-faults)
- [4.9 External finalization](#49-external-finalization)
- [Part 5 — Public surface](#part-5--public-surface)
- [5.1 The lane surface](#51-the-lane-surface)
- [5.2 The harness](#52-the-harness)
@ -1278,220 +1277,106 @@ interface OperationMeta {
## 3.2 Operation state — the durable restart point
`operationState(operationId)` holds one total `OperationState` directly. Every durable transition replaces the whole value; the terminal transaction deletes it (§3.13). There is no finished member of the union — an ended operation has no state at all, and its outcome lives in `pi.lane.lastResult`.
`operationState(operationId)` holds one total `OperationState` directly. Every durable transition replaces the whole value; the terminal transaction deletes it (§3.13). There is no finished member of the union.
That value is authoritative after process loss, but it is not the finer instruction pointer of a live async procedure. For example, a fresh assistant procedure commits `effect_pending` and retains its JavaScript continuation through the request and settlement. If that continuation is lost, the same durable state becomes the restart point for unknown-outcome recovery (§4.1, §4.5).
The state has one namespaced `at` discriminator. There is no nested operation-kind, phase, generation-status, deferred-status, or structural-status dispatch hierarchy:
```ts
type OperationState = RunState | CompactionState | NavigationState;
type OperationState =
| RunStartingOperation // run.starting
| RunCheckpointOperation // run.checkpoint
| RunAssistantReadyOperation // run.assistant.ready
| RunAssistantEffectPendingOperation // run.assistant.effect_pending
| RunAssistantRetryWaitOperation // run.assistant.retry_wait
| RunToolsOperation // run.tools
| RunDeferredSuspendedOperation // run.deferred.suspended
| RunDeferredEffectPendingOperation // run.deferred.effect_pending
| RunCompactionDecidingOperation // run.compaction.deciding
| RunCompactionReadyOperation // run.compaction.ready
| RunCompactionEffectPendingOperation // run.compaction.effect_pending
| RunCompactionRetryWaitOperation // run.compaction.retry_wait
| RunFailureDrainOperation // run.failure_drain
| CompactionDecidingOperation // compaction.deciding
| CompactionReadyOperation // compaction.ready
| CompactionEffectPendingOperation // compaction.effect_pending
| CompactionRetryWaitOperation // compaction.retry_wait
| NavigationReadyToCommitOperation // navigation.ready_to_commit
| NavigationSummaryDecidingOperation // navigation.summary.deciding
| NavigationSummaryReadyOperation // navigation.summary.ready
| NavigationSummaryEffectPendingOperation // navigation.summary.effect_pending
| NavigationSummaryRetryWaitOperation; // navigation.summary.retry_wait
```
/** Process-local owned projection assembled from two durable values. */
interface Operation {
meta: OperationMeta;
state: OperationState;
}
Every leaf carries orthogonal `control`. Every `run.*` leaf also carries captured run settings, inbox ids, and `latestAssistantEntryId` through `RunScope`. Structural and navigation families similarly factor their shared durable data with intersections. `ToolBatch`/`ToolCall` remain a nested child collection state machine because parallel tool children genuinely update sibling statuses concurrently.
```ts
type Control =
| { status: "running" }
| { status: "cancel_requested"; requestedAt: number;
/** Drained queue ids. Their `pendingEntry(id)` values survive the drain
and are deleted only by the terminal transaction (§3.11, §3.13). */
drainedSteer: string[]; drainedFollowUp: string[] };
interface RunState {
kind: "run";
interface RunScope {
control: Control;
/** Captured atomically at acceptance; setters affect later operations. */
settings: {
compaction: CompactionSettings;
steeringMode: QueueMode;
followUpMode: QueueMode;
toolExecution: "sequential" | "parallel";
};
phase: RunPhase;
inbox: Inbox;
/** Newest durable assistant generation/fetch response in this operation. */
inbox: { steer: string[]; followUp: string[]; writes: string[] };
latestAssistantEntryId: string | null;
}
interface CheckpointPhase {
kind: "checkpoint";
continuation: Continuation;
/** Durable correlation source for the next generation step. */
interface RunCheckpointOperation extends RunScope {
at: "run.checkpoint";
continuation:
| { kind: "need_assistant"; overflowRecoveryUsed: boolean }
| { kind: "may_finish"; includeFinalAssistant: boolean };
triggerEntryId: string;
/** Threshold compaction is attempted at most once per trigger boundary. */
thresholdCheckedTriggerEntryId?: string;
/** Generate before draining another queued input after one-at-a-time drain. */
skipInboxOnce?: boolean;
}
type RunPhase =
| { kind: "starting" }
| CheckpointPhase
| { kind: "assistant"; generation: Generation }
| { kind: "tools"; batch: ToolBatch }
| { kind: "compaction"; reason: "threshold" | "overflow";
structural: StructuralDecision; resumeAfter: CheckpointPhase }
| { kind: "deferred"; deferred: Deferred }
| { kind: "failure_drain"; error: OperationError; provenance:
| { kind: "response"; entryId: string }
| { kind: "structural"; taskId: string }
| { kind: "configuration" } };
type Continuation =
| { kind: "need_assistant"; overflowRecoveryUsed: boolean }
| { kind: "may_finish"; includeFinalAssistant: boolean };
interface Inbox {
/** Reserved entry ids. Payloads — and, for writes, the entry type and
customType — live at `pendingEntry(id)` (§1.3, §2.2). */
steer: string[];
followUp: string[];
writes: string[];
interface RunAssistantReadyOperation extends RunScope {
at: "run.assistant.ready";
generationContext: GenerationContext;
nextAttempt: number;
}
interface RunAssistantEffectPendingOperation extends RunScope {
at: "run.assistant.effect_pending";
generationContext: GenerationContext;
attempt: number;
responseEntryId: string;
usageId: string;
intendedOutputLimit: number;
contextWindow: number;
}
interface RunAssistantRetryWaitOperation extends RunScope {
at: "run.assistant.retry_wait";
generationContext: GenerationContext;
nextAttempt: number;
notBefore: number;
errorMessage: string;
}
interface OperationError { code: string; message: string; details?: JsonValue }
```
Execution configuration failures use stable operation-error codes rather than identity suspension: `model_unavailable` carries `{ provider, modelId }`, and `configured_tools_unavailable` carries `{ tools: string[] }`. These errors are non-retryable because retrying the same absent implementation cannot change the process registry. They are distinct from a missing requested tool, which becomes an ordinary `isError` tool-result message (§3.8).
A queue item is one entry id; everything else about it — payload, write type, `customType` — is dereferenced from its `pendingEntry(id)` value.
`latestAssistantEntryId` updates in the same settlement transaction as every assistant generation or deferred-fetch response. It lets finish and resume construct results/events without a branch scan. A tool batch retains its producing turn id while tool work remains active.
Any transition that appends conversational input or tool results and requires another assistant writes a checkpoint with `need_assistant(false)` and the appended entry as `triggerEntryId`. A `may_finish` checkpoint sets `triggerEntryId` to the entry that caused the boundary: the settled response for a `stop`/genuine-`length` settlement (§3.7), the newest result entry for an all-terminating tool batch (§3.8) — so threshold dedup (§3.12) and every later consumer always name an existing entry. An unprojected custom write preserves the current checkpoint, including trigger and overflow flag. Entering threshold compaction first copies the checkpoint to `resumeAfter` with `thresholdCheckedTriggerEntryId = triggerEntryId`; decline, empty preparation, success, and crash therefore cannot recheck the same boundary.
### Generation
```ts
interface NormalizedRetryPolicy { maxAttempts: number; baseDelayMs: number }
interface GenerationContext {
stepId: string;
triggerEntryId: string;
/** Inline snapshot of the lane configuration at step start. */
configuration: LaneConfiguration;
streamOptions: AgentHarnessStreamOptions;
retryPolicy: NormalizedRetryPolicy;
/** Copied from the producing checkpoint's need_assistant continuation so a
settlement classified after crash-restore still knows whether overflow
recovery was already spent (§3.7, §3.9). */
overflowRecoveryUsed: boolean;
interface RunToolsOperation extends RunScope {
at: "run.tools";
batch: ToolBatch;
}
type Generation =
| { status: "ready"; context: GenerationContext; nextAttempt: number }
| { status: "effect_pending"; context: GenerationContext; attempt: number;
responseEntryId: string; usageId: string;
intendedOutputLimit: number; contextWindow: number }
| { status: "retry_wait"; context: GenerationContext; nextAttempt: number;
notBefore: number; errorMessage: string };
```
The context snapshots configuration, stream options, and retry policy **inline**; `LaneConfiguration` is small. Recovery therefore retains the exact provider/model and active-tool identities selected for the step even when their current implementations are unavailable. No frame count, cursor, or list identity is stored in the state: the procedure constructs `pendingAssistantFrames(operationId, responseEntryId)` from the effect-pending state (§3.7). For each attempt, `before_request` runs from generation `ready` (an elapsed retry wait first returns to `ready`). Its curated patch is composed with the context's captured base stream options, then `intendedOutputLimit` and `contextWindow` are calculated and persisted in the `effect_pending` intent before dispatch. A pre-intent crash may rerun the hook. Harness-owned `before_payload`/`after_response` callbacks are mounted only after intent and cannot be replaced through stream options.
### Tool batch
```ts
interface ToolBatch {
assistantEntryId: string;
/** Producing generation/fetch snapshot; active tool names come from here. */
configuration: LaneConfiguration;
/** The assistant generation step id; recovered tool events use it as turnId. */
turnId: string;
calls: ToolCall[];
interface RunDeferredSuspendedOperation extends RunScope, DeferredScope {
at: "run.deferred.suspended";
}
type ToolCall =
| { status: "planned"; sourceIndex: number; resultEntryId: string }
| { status: "effect_pending"; sourceIndex: number; resultEntryId: string;
replay: "never" | "safe" }
| { status: "outcome_ready"; sourceIndex: number; resultEntryId: string;
terminate: boolean }
| { status: "completed"; sourceIndex: number; resultEntryId: string;
terminate: boolean };
```
The source call comes from `assistantEntryId` plus `sourceIndex`. `sourceIndex` is the zero-based index in the assistant message's complete `content` array, not an ordinal in a filtered tool-call list; the indexed block must be a tool call. Large effective arguments live once in the `operationToolArgs(operationId, stepId, sourceIndex)` value — the producing generation's `stepId` disambiguates batches across turns — written at clearance (§3.8) and located by that deterministic key. Persist them unconditionally because `prepareArguments`, not only `before_tool`, may change them.
`outcome_ready` means execution and post-effect hooks are finished and the complete finalized result is durable in `pendingEntry(resultEntryId)`, but an earlier source call prevents tree placement. Such a call never executes again. Completed calls form a source-ordered prefix. In parallel mode, the suffix may mix `planned`, `effect_pending`, and `outcome_ready`; source-ordered materialization alone extends the completed prefix. In sequential mode, the suffix contains at most one `effect_pending` or `outcome_ready` call before planned calls.
### Deferred
```ts
type Deferred =
| { status: "suspended"; stepId: string; sourceEntryId: string; poll: number;
configuration: LaneConfiguration; streamOptions: AgentHarnessStreamOptions }
| { status: "effect_pending"; stepId: string; sourceEntryId: string; poll: number;
responseEntryId: string; usageId: string;
configuration: LaneConfiguration; streamOptions: AgentHarnessStreamOptions };
```
One `drive({ pollDeferred: true }, context)` performs at most one `Models.streamDeferred(handle, { wait: 0 })` poll; `resume(context)` is the convenience call that supplies that permit. `Models.streamDeferred` preserves the provider event stream required for frame persistence, while `Models.fetchDeferred` remains its result-only convenience and is not the harness runtime path. Suspended `poll` is the number of completed polls; a fresh intent uses `poll + 1`, and that 1-based value is `before_request.attempt` and the poll turn-id suffix. A poll starts from the original generation's copied base stream options, forces `deferred:false`, runs `before_request`, mounts `before_payload`/`after_response`, then commits its fresh intent and dispatches like assistant generation. Current global stream settings do not affect it. There is no polling retry cap, backoff, or internal loop. A `deferred` response has a valid handle only when `message.deferred` exists, its id is non-empty, and its `{ provider, modelId, api }` equals the captured request model identity/API. An invalid initial handle is normalized to a durable `error` response and response-provenance `failure_drain`. A later pending response becomes the next source. Response, usage, `latestAssistantEntryId`, and failure state commit atomically.
A deferred poll that returns an event stream persists frames exactly like assistant generation, at `pendingAssistantFrames(operationId, responseEntryId)` for the poll's reserved response id (§3.7). Every row below that inserts a response entry additionally deletes that response's frame list in the same transaction. Frames never change poll-number rules.
The complete transition table — every row is one `commit()`; classification order (§3.7) applies to every poll settlement, cancellation first:
| From | Trigger | Transaction | To |
| ----------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------- |
| assistant `effect_pending` | settlement classifies `deferred` with a valid handle | §3.7's deferred row | suspended, `poll: 0`, `sourceEntryId: R` |
| suspended, poll _k_ | captured model is unavailable when a drive supplies `pollDeferred: true` | `TX[ S(failure_drain{error:{code:"model_unavailable", ...}, provenance:{kind:"configuration"}}) ]` — the handle is abandoned without fabricating a fetch response or usage | failure_drain |
| suspended, poll _k_ | captured model resolves and a drive supplies `pollDeferred: true`: the poll's `before_request` settlement commits its intent, consuming the pass's single poll permit | mint fresh R′ and U′, then `TX[ S(deferred{effect_pending, poll k+1, responseEntryId R′, usageId U′}) ]` | effect_pending, poll _k_+1 |
| effect_pending, poll _k_+1 | fetch returns **pending** | `TX[ insert response entry R′, insert usage U′, upsert pi.branch.tip = R′, S(latestAssistantEntryId=R′, deferred{suspended, sourceEntryId R′, poll k+1}) ]` — the pending response becomes the next source and the operation re-suspends; no second poll this invocation | suspended, poll _k_+1 |
| effect_pending | fetch returns **ready** with tool calls | `TX[ insert response R′, insert usage U′, upsert pi.branch.tip = R′, S(latestAssistantEntryId=R′, tools{plan with reserved result ids}) ]` — result ids minted as followers of R′ (§1.2) | tools |
| effect_pending | fetch returns **ready** without tool calls | `TX[ insert response R′, insert usage U′, upsert pi.branch.tip = R′, S(latestAssistantEntryId=R′, checkpoint{may_finish, includeFinalAssistant:true}) ]` | checkpoint |
| effect_pending | fetch settles as a provider `error` | `TX[ insert response R′, insert usage U′, upsert pi.branch.tip = R′, S(latestAssistantEntryId=R′, failure_drain{error, provenance:response R′}) ]` — polls have no retry path | failure_drain |
| effect_pending, restored, running control | crash left the poll's outcome unknown; a drive with a poll permit finds the captured model unavailable | `TX[ delete list pi.pending.assistant_frame/O:R′, S(failure_drain{error:{code:"model_unavailable", ...}, provenance:{kind:"configuration"}}) ]` — abandon the old reserved response/usage strings without fabricating settlement | failure_drain |
| effect_pending, restored, running control | crash left the poll's outcome unknown; the next drive with a poll permit resolves the captured model and replaces it | mint fresh R″/U″ and commit a fresh intent at the **same** poll number — an unknown-outcome poll never completed, so `poll` does not increment; the old reserved id strings are abandoned, never materialized, and the intent transaction deletes the abandoned old frame list; the replacement poll starts a fresh list under its fresh response id | effect_pending, poll _k_+1 |
| effect_pending, cancelled control | reconciliation, live or restored (§4.5, §4.6) | synthetic settlement under the **existing** reserved ids, preserving partial content reduced from committed frames: `TX[ insert synthetic aborted response R′, insert zero usage U′, upsert pi.branch.tip = R′, delete frame list O:R′, S(latestAssistantEntryId=R′, cancelled checkpoint{may_finish}) ]` | cancelled checkpoint → aborted finish |
| suspended, cancelled control | reconciliation | no fetch starts; best-effort `cancel_deferred` targets the newest source (§4.6), and the operation finishes through the aborted terminal transaction | terminal |
### Structural work
```ts
type StructuralDecision = { taskId: string } & (
| { status: "deciding" }
| { status: "generating"; generation: SummaryGeneration }
);
interface SummaryContext {
taskId: string;
resultEntryId: string;
kind: "compaction" | "branch_summary";
configuration: LaneConfiguration;
streamOptions: AgentHarnessStreamOptions;
retryPolicy: NormalizedRetryPolicy;
reason?: "manual" | "threshold" | "overflow";
interface RunDeferredEffectPendingOperation extends RunScope, DeferredScope {
at: "run.deferred.effect_pending";
responseEntryId: string;
usageId: string;
}
type SummaryGeneration =
| { status: "ready"; context: SummaryContext; nextAttempt: number }
| { status: "effect_pending"; context: SummaryContext; attempt: number;
/** Current nested request intent; absent between requests. */
request?: { index: number; usageId: string };
usageIds: string[] }
| { status: "retry_wait"; context: SummaryContext; nextAttempt: number;
notBefore: number; errorMessage: string };
interface CompactionState {
kind: "compaction";
control: Control;
customInstructions?: string;
structural: StructuralDecision;
}
type NavigationState =
| { kind: "navigation"; control: Control; targetId: string | null; label?: string;
summarize: false; phase: { kind: "ready_to_commit" } }
| { kind: "navigation"; control: Control; targetId: string; label?: string;
customInstructions?: string; summarize: true;
phase: { kind: "summary"; structural: StructuralDecision } };
```
Structural preparation is built from an observed source tip and settings snapshot, normalized (`Set<string>` file-operation fields become sorted arrays), and written once to the `operationPreparation(operationId, taskId)` value before the decision hook, in the same transaction as the `deciding` state (§3.9). State carries only `taskId`; the deterministic constructor locates the value, and hooks/generators hydrate arrays back to the source preparation types. Reopen never rebuilds it from current settings, so the provider sees the same summary input the hook approved.
The canonical complete declarations live in `src/harness/session/types.ts`. Large stable payloads remain at referenced sibling addresses. State carries only the ids and bounded policy data required to dispatch and recover.
One structural attempt may make one or two sequential provider requests using the existing compaction implementation. Its request callback first commits `request:{index,usageId}`, invokes that request through `drive.gate.admit(...)`, then atomically writes usage and clears/advances the request field. Intermediate content remains process-local; any activated orphaned `effect_pending` attempt is treated as wholly uncertain and starts a later attempt under the captured policy rather than continuing request two. Never suspend between two requests of one live structural attempt: if its captured model identity disappears after request one, fail the attempt in-band so retry/failure follows the captured policy and no process-local intermediate text is parked waiting for registration. A durable `generating` decision prevents its decision hook from rerunning.
The value is authoritative after process loss, but it is not the finer instruction pointer of a live async procedure. A live assistant procedure commits `run.assistant.effect_pending` and retains its JavaScript continuation through request settlement. If that continuation is lost, the same durable leaf becomes the restart point for unknown-outcome recovery.
## 3.3 Lane state and the restore projection
@ -1513,7 +1398,7 @@ Attachment does not dereference transcript, queue payloads, deferred sources, as
> Compute the next total state in memory, then atomically commit every entry insert, usage insert, and value/list write that makes that state true.
A transition plans from the authoritative owned projection supplied by the Session mutation line, verifies the semantic state it extends, and changes only the fields it owns. In particular, settlement preserves newer inbox/control fields, and the terminal transaction clears `currentOperationId` while preserving concurrently accepted `pendingNextRun`. A procedure that awaited external work rechecks its expected phase and effect identity against that current projection before committing. Cancellation routes to reconciliation; a missing or different operation means ownership was lost; any other unexpected phase or identity is an invariant defect (§4.3, §4.9). The callback reader is used only for referenced payloads and operation-owned cleanup scans. Every edge below is exactly one `commit()`.
A transition plans from the authoritative owned projection supplied by the Session mutation line and changes only the fields it owns. Settlement preserves newer inbox/control fields, and the terminal transaction clears `currentOperationId` while preserving concurrently accepted `pendingNextRun`. The live procedure is the only top-level state-advance writer, so its continuation does not revalidate operation identity or `at`; cancellation and inbox are the only supported orthogonal changes. The callback reader is used only for referenced payloads and operation-owned cleanup scans. Every edge below is exactly one `commit()`.
## 3.5 The graph
@ -1632,7 +1517,7 @@ The provider loop never awaits storage per frame; existing assistant event deliv
At stream settlement, the procedure stops frame admission and awaits the latest frame-write promise before `after_response`; lane FIFO means its completion implies completion of every earlier append. There is **no** timer, batcher, coalescer, active/waiting state, or flush API — public or internal. A failed frame append faults the harness before `after_response` starts; the complete final response stays process-local and never commits after a storage fault.
Each append mutation verifies against the current owned projection that the same operation, attempt, and reserved response id still own the lane when it executes, and exact `Drive` identity is checked again at commit admission; a terminal transaction that won first fences a stale append without recreating operation state (§4.9). Frames are auxiliary: a missing list is valid, frames never prove request admission or completion, never select a restart point, and a crash after the final frame append but before settlement still restores `effect_pending` — a complete-looking draft is not a settled response. Structural summary streams persist no frames (§3.9). There is no frame-persistence event, and reconnect replays no historical updates (§5.4, §5.5).
Each append mutation verifies that the current leaf is assistant/deferred effect-pending under the same reserved response id. This is a child-progress fence: an append queued before settlement may commit first, while one that executes after the leaf changes declines without recreating the list. The lane-owned Drive is never replaced, so no top-level owner or exact-Drive check exists. Frames are auxiliary: a missing list is valid, frames never prove request admission or completion, never select a restart point, and a crash after the final frame append but before settlement still restores `effect_pending` — a complete-looking draft is not a settled response. Structural summary streams persist no frames (§3.9). There is no frame-persistence event, and reconnect replays no historical updates (§5.4, §5.5).
### Classification order
@ -1694,7 +1579,7 @@ Modes:
Blocked and invalid calls skip intent/execution but still stage a synthetic outcome. A missing tool implementation is the ordinary unknown-tool case: stage an `isError:true` `ToolResultMessage` whose text says the named tool is unavailable, then continue the batch and later assistant turn. The harness constructs that message directly and omits `details`; it must not invent a value for the tool's application-defined typed details contract. A crash before staging reruns ordinary clearance, including `before_tool` under its replay contract. A crash after staging never reruns the hook or tool.
Calls are tracked internally by `sourceIndex`. Hooks and events see provider `toolCallId` and tool name. `AgentHarnessToolInvocation.invocationId` equals the reserved `resultEntryId`, remains stable across safe replay, and scopes durable memos under `operationToolMemo(O, invocationId, name)` addresses. Memo names must be non-empty and contain no `:`; `setMemo(name, undefined)` deletes. Memo operations synchronously enqueue on the Session mutation line before returning their promises, and tools must await writes. Each job verifies the same effect-pending invocation when it executes, protecting against authorized external finalization. A pre-return write is FIFO-ordered before staging and then deleted by staging; a post-return call rejects after capability expiry. No separate write drain exists. Flue-style `step.do(name, effect)` awaits these operations: a committed value returns on replay, while a crash before its memo commit may rerun the effect. There is no nested per-step replay state and no exactly-once external-effect promise.
Calls are tracked internally by `sourceIndex`. Hooks and events see provider `toolCallId` and tool name. `AgentHarnessToolInvocation.invocationId` equals the reserved `resultEntryId`, remains stable across safe replay, and scopes durable memos under `operationToolMemo(O, invocationId, name)` addresses. Memo names must be non-empty and contain no `:`; `setMemo(name, undefined)` deletes. Memo operations synchronously enqueue on the Session mutation line before returning their promises, and tools must await writes. Each job verifies the same effect-pending invocation when it executes, so a queued memo write cannot outlive outcome staging. A pre-return write is FIFO-ordered before staging and then deleted by staging; a post-return call rejects after capability expiry. No separate write drain exists. Flue-style `step.do(name, effect)` awaits these operations: a committed value returns on replay, while a crash before its memo commit may rerun the effect. There is no nested per-step replay state and no exactly-once external-effect promise.
## 3.9 Summary generation — compaction and navigation summaries
@ -1711,7 +1596,7 @@ Both operations generate a summary through the same `deciding → generating →
| `threshold` | context-size check at a checkpoint | back to the stored `resumeAfter` |
| `overflow` | a request that did not fit | `failure_drain` |
"Auto compaction" is the in-run row: `threshold` and `overflow`. Non-empty preparation and the transition into `deciding` commit together (`upsert pi.op.preparation/O:{taskId}` plus the structural state and, for threshold, marked `resumeAfter`). Preparation returning `undefined` never creates `StructuralDecision`: threshold atomically marks the checkpoint checked and continues; overflow atomically enters response-provenance `failure_drain` using the normalized overflow response. Neither path emits structural lifecycle. Empty standalone preparation is rejected before acceptance.
"Auto compaction" is the in-run row: `threshold` and `overflow`. Non-empty preparation and the transition into `deciding` commit together (`upsert pi.op.preparation/O:{taskId}` plus the structural state and, for threshold, marked `resumeAfter`). Preparation returning `undefined` never enters a structural deciding leaf: threshold atomically marks the checkpoint checked and continues; overflow atomically enters response-provenance `failure_drain` using the normalized overflow response. Neither path emits structural lifecycle. Empty standalone preparation is rejected before acceptance.
| From | Trigger | Transaction |
| -------------------------------- | ------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
@ -1895,7 +1780,7 @@ type LaneLastResult =
));
```
A normal run finish copies `RunState.latestAssistantEntryId` and records `runCompletion: "assistant"` when `may_finish.includeFinalAssistant` is true. An all-terminating tool batch records `runCompletion: "terminated_tools"` and omits the final assistant. Failed and aborted run outcomes include the newest settled assistant when non-null and omit the field otherwise. Structural operations omit `runCompletion` and the final assistant. Navigation records its source tip and, when it publishes one, the summary entry id, so a later matching `drive` can hydrate the complete public result after `pi.op.meta` has been deleted. Only terminal transitions construct a `LaneLastResult`.
A normal run finish copies `RunScope.latestAssistantEntryId` and records `runCompletion: "assistant"` when `may_finish.includeFinalAssistant` is true. An all-terminating tool batch records `runCompletion: "terminated_tools"` and omits the final assistant. Failed and aborted run outcomes include the newest settled assistant when non-null and omit the field otherwise. Structural operations omit `runCompletion` and the final assistant. Navigation records its source tip and, when it publishes one, the summary entry id, so a later matching `drive` can hydrate the complete public result after `pi.op.meta` has been deleted. Only terminal transitions construct a `LaneLastResult`.
Every terminal transaction, for every operation kind and outcome, has one shape:
@ -1948,139 +1833,44 @@ The invariant this section carries (restated in Part 9): operation-owned values
## 4.1 The live operation task
An open operation has durable state whether or not this process is executing it. A **live operation task** is only the process-local async continuation currently executing that operation. It is recorded because durable state cannot answer three process-local questions: whether this process already owns execution, whether an `effect_pending` state still has a live continuation, and what `abort(context)` should signal.
Public operation status is a small process-relative observation, not a prediction of future identity availability:
- `running`: this process has a live drive owner;
- `open`: durable work exists with no local owner;
- `aborting`: durable control is `cancel_requested` and cancellation reconciliation is next;
- idle: `current: null`; it is not an open-operation status.
There is at most one process-local record per lane. It owns one `drive` pass, not the durable operation's whole lifetime: a pass may settle or reach a durable wait. Durable operation phase and control remain the recovery authority. Model/tool availability is checked only when a procedure reaches the corresponding effect-admission boundary; absence becomes an in-band durable error/result rather than a public operation status.
An open operation has durable state whether or not this process is executing it. A `Drive` is the lane-owned process-local continuation for one pass. It answers only whether this lane already has a live continuation, supplies the effect gate, and exposes one shared completion to callers.
```ts
class Drive {
readonly operationId: string;
readonly completion: Promise<DriveOutcome>; // shared by installer and joiners
readonly completion: Promise<DriveOutcome>;
readonly gate: Gate;
readonly context: Context; // installer cancellation removed
readonly installerSignal: AbortSignal | undefined;
readonly context: Context; // installing invocation cancellation removed
readonly waitForRetry: boolean;
deferredPermits: number;
finalizedOutcome?: DriveOutcome;
passEnded: boolean;
// private GateControl, first-call-wins settle/fail closures, and hasSettled()
}
class AbortRequested extends Error {
readonly cancellation: Promise<void>;
constructor(cancellation: Promise<void>) {
super("Abort requested");
this.cancellation = cancellation;
}
}
class OperationEnded extends Error {}
class DriveAbandoned extends Error {}
```
This is not another durable state machine. `Drive` is one installed process-local pass: shared completion, exact ownership identity, effect admission, invocation Context, and pass-local wait policy. Current durable state, retry data, responses awaiting settlement, structural intermediate content, and parallel tool promises remain ordinary procedure-local variables. Streaming drafts and running-tool display values remain lane fields used only for snapshots.
The first matching `drive` caller installs the Drive on the Session mutation line. Every later matching caller observes the same `Drive.completion`. The first caller is not an owner; after installation all callers are observation peers and the Lane owns execution.
`pi.op.state/{operationId}` is the operation's **durable restart point**, not the live JavaScript instruction pointer. While a task is alive, its call stack is the finer program counter:
Each caller races only its own observation with `context.abortSignal`. A signal that wins before installation starts nothing. A signal after installation rejects that caller's invocation but does not remove, replace, or cancel the Drive. Durable operation cancellation exists only through `requestAbort(operationId, context)`.
One Drive is the sole top-level state-advance writer. Inbox methods mutate only inbox fields, `requestAbort` mutates only control, and close seals mutation admission. Therefore a live procedure's operation identity and `at` leaf cannot disappear or change concurrently. Procedures do not repeatedly verify operation existence, operation id, operation kind, Drive identity, or expected `at`. After awaiting external work they enter the Session mutation line and receive the latest authoritative `Lane.state`, preserving concurrent control and inbox changes.
Parallel tool children are the exception: sibling call statuses genuinely race, so call identity/status checks and source-ready-prefix checks remain.
The task runs direct async procedures. There is no graph interpreter or action scheduler. The Lane supplies two concrete mutation operations:
- `advanceOperation` declines ordinary progress when current control is cancelled, pairs the next operation-state write with process-local projection publication, and otherwise exposes the procedure's writes/events;
- `mutateOperation` performs effect settlement and tool-child transitions against current control, and also owns the universal terminal suffix when the procedure returns a terminal decision.
A pass ends at a terminal result or durable wait. `activeDrive` is then cleared by that pass; no live pass is replaced in-process. A process crash or close destroys/detaches the continuation, and a later attachment rebuilds `Lane.state` from durable values before another pass starts.
Normal procedures remain straight line:
```text
live task:
JavaScript continuation currently awaiting this exact request
durable restart point assistant.effect_pending
process lost:
JavaScript continuation gone
durable restart point assistant.effect_pending → unknown-outcome recovery
prepare
→ commit effect intent
→ admit and await external effect
→ commit output + next state
```
That asymmetry is unavoidable around an external effect. Making the task explicit prevents normal execution from mistaking a live pending request for an orphaned one.
The task runs direct async procedures. There is no generic planner, action interpreter, effect-plan graph, or storage-loading transition helper:
```ts
async function executeDrivePass(
lane: Lane, drive: Drive, options: DriveOptions,
): Promise<DriveOutcome> {
try {
let current = lane.state.operation;
if (current === null || current.meta.operationId !== drive.operationId)
return hydrateMatchingLastResult(lane, drive.operationId);
if (current.state.control.status === "cancel_requested")
return reconcileCancellation(current);
await runBeforeDrive(current);
while (true) {
current = lane.state.operation;
if (current === null || current.meta.operationId !== drive.operationId)
return hydrateMatchingLastResult(lane, drive.operationId);
if (current.state.control.status === "cancel_requested")
return reconcileCancellation(current);
if (mustWaitForRetry(current, options)) return retryWait(current);
if (isDeferredSuspended(current) && !options.pollDeferred)
return deferredWait(current);
if (current.state.kind === "run" && current.state.phase.kind === "starting")
await settleBeforeRun(current);
else if (hasOrphanedEffect(current))
await recoverOrphanedEffect(current, options);
else {
const result = await continueOrdinaryOperation(current, options);
if (result) return result;
}
}
} catch (error) {
if (error instanceof AbortRequested) {
await error.cancellation;
const current = lane.state.operation;
if (current === null || current.meta.operationId !== drive.operationId)
return hydrateMatchingLastResult(lane, drive.operationId);
return reconcileCancellation(current);
}
if (error instanceof OperationEnded)
return hydrateMatchingLastResult(lane, drive.operationId);
throw error;
}
}
```
The pseudo-code shows ownership and catch boundaries, not required helper names. `waitForRetry: false` returns the durable `notBefore`; `true` starts the existing abort-aware retry timer. `pollDeferred: true` grants at most the existing one-poll permit. `AbortRequested` is the one internal control-flow signal used when cancellation wins the effect gate. A procedure holding live local results catches it before the outer boundary when those values affect reconciliation. In particular, the parallel-tool procedure retains its promise map, stages each finalized started call as it settles, and materializes only the source-ready prefix; an assistant procedure parked after receiving a response retains that response long enough to settle it under cancelled control.
Ordinary procedures are straight-line across their own durable restart points. Assistant generation is representative:
```text
run before_request
→ commit assistant.effect_pending intent
→ invoke and await the provider
→ commit response + usage + next state
```
The procedure does not return to a dispatcher after its intent commit. Structural requests and deferred polls use the same shape. Parallel tool execution retains its local promises until every started call has either staged a finalized outcome or lost ownership; a separate Session mutation transition materializes ready outcomes in source order.
Normal execution handles checkpoints, ready/retry states, planned or locally owned tools, outcome-ready materialization, structural decisions, deferred suspension, failure drain, and navigation commit. An unowned `effect_pending` state must not reach it. Such a state is handled only by orphan recovery (§4.5); reaching it from ordinary dispatch is an invariant defect.
### Task installation, joining, and removal
`accept` commits the first durable state and returns without installing a `Drive`. A durably accepted but locally unowned operation is ordinary. `drive({ operationId }, context)` owns one pass. On the Session mutation line it inspects the current owned projection, including its latest result, then applies exactly one rule:
- current operation absent and `pi.lane.lastResult.operationId === operationId`: return its already-settled outcome;
- current operation absent with another/no last id, or current operation has another id: return `OperationMismatch` without starting work;
- the same operation already has a `Drive`: join that pass's completion;
- the same operation has no Drive: construct and install one before releasing the Session mutation line, then start the pass after `Session.mutate` returns.
The first caller to install a pass supplies that pass's retry-wait and deferred-poll policy. Same-operation joiners observe the shared completion; a convenience caller whose policy requires more progress may call `drive` again after that pass returns. Installer cancellation abandons that shared pass, so installer and current joiners reject with `DriveAbandoned` and may drive the still-open durable operation again. A joiner's own cancellation ends only that joiner's observation. This arbitration lets `prompt` literally compose `accept` and `drive` even if an alarm reaches the accepted operation first, while preserving one process-local driver. Every drive is fenced by its expected operation id, so a stale wake for A never drives later operation B.
A newly installed real pass checks cancellation, then invokes `before_drive` once before recovery or ordinary work. Joiners do not rerun it. A later pass after retry wait, deferred suspension, close/reopen, or process restart invokes it again. Durable phase and local ownership alone determine dispatch: `starting` requires `before_run`; an unowned `effect_pending` phase is orphaned; every other phase follows its ordinary procedure. No fresh/continue/restore marker exists.
Every task has one outer `finally` path: after its last durable commit, wait decision, or terminal observation and final local events, it synchronously removes itself only when `lane.activeDrive === drive`. Normal completion settles only after that exact cleanup. Invocation abandonment and close/fault fail completion and remove the exact Drive immediately so non-cooperative work cannot block recovery or shutdown; every late planner and progress write also checks exact Drive identity. External finalization instead closes the gate in committed materialization and settles completion immediately after terminal event delivery without waiting for detached work. The exact Drive remains installed until both that settlement and pass ending have occurred; whichever side arrives second removes it. This preserves `LaneBusy` and same-id joining throughout event delivery and detached cleanup (§4.9). Once `finalizedOutcome` is set, the pass continuation may neither settle nor fail the shared completion; only the finalizer settles it after event delivery.
Convenience methods add policy, not another execution path. `prompt`/`skill`/`compact`/`navigateTree` call `accept` and then drive the returned id; they wait locally through retry waits and return on terminal or deferred suspension. `resume(context)` inspects the current id and drives it with one deferred-poll permit. `abort(context)` calls `requestAbort`, then ensures cancellation reconciliation has a drive pass; it still returns once cancellation is durable. A scheduler calls the expected-id primitives directly and may persist the admission or set an alarm from `waiting.notBefore`. Normal applications do not manage operation ids when resuming.
`AgentHarness.create(options, context)` restores the minimal projection and reports open operations but installs no tasks and starts no hooks, providers, tools, timers, or application callbacks.
Recovery dispatches directly from the flat `state.at` leaf. Cancellation reconciliation runs before ordinary dispatch and never starts new ordinary effects.
## 4.2 Effect gate
@ -2096,7 +1886,7 @@ interface Gate {
interface GateControl {
beginAbort(cancellation: Promise<void>): void;
signalAbort(): void;
close(error: HarnessClosed | HarnessFault | OperationEnded | DriveAbandoned): void;
close(error: HarnessClosed | HarnessFault): void;
}
```
@ -2113,7 +1903,7 @@ type GateState =
`Gate.admit(invoke)` performs the only check and immediately returns `invoke()`. If abort is being committed it throws `AbortRequested(cancellation)`; if closed it throws the closing error. The gate owns the cooperative `AbortController`, exposed through `gate.signal`.
`requestAbort(operationId, context)` is the durable cancellation primitive. With a matching live Drive it creates the abort-mutation promise and calls `drive.beginAbort(promise)` synchronously before the lane mutation. A committed marker resolves that promise and then calls `drive.signalAbort()`. Ownership loss resolves it, closes the gate with `OperationEnded`, and returns `OperationMismatch`. Commit fault rejects it and closes with `HarnessFault`. With no Drive, requestAbort commits or observes the same durable marker but starts no pass.
`requestAbort(operationId, context)` is the durable cancellation primitive. With a matching live Drive it creates the abort-mutation promise and calls `drive.beginAbort(promise)` synchronously before the lane mutation. A committed marker resolves that promise and then calls `drive.signalAbort()`. An operation-id mismatch resolves it and returns `OperationMismatch`. Commit fault rejects it and closes with `HarnessFault`. With no Drive, requestAbort commits or observes the same durable marker but starts no pass.
**The admitted-operation boundary is load-bearing.** Preparation finishes first; then the gate check and operation invocation are one synchronous expression:
@ -2175,7 +1965,7 @@ Lane commands, lane acquisition, progress writes, Branch creation/appends, metad
`Session.mutate()` is a trusted sharp edge. Its callback must use the supplied mutator for bounded reads and its sole commit. Calling a public Session writer inside that callback queues the nested write behind the active callback; awaiting the nested write therefore deadlocks. Plugins must not perform nested public writes or unbounded work while holding the line.
A drive-owned command uses the current owned Lane projection for control flow and rechecks exact `lane.activeDrive === drive` immediately before commit admission. Settlement therefore preserves newer inbox/control fields and cannot write after owner abandonment. Providers, tools, hooks, timers, event delivery, idle waits, and Drive completion remain outside the line.
A Drive procedure uses the current owned Lane projection for control flow. The Lane pairs every operation-state write with publication of the matching process-local projection. Settlement therefore preserves newer inbox/control fields. Providers, tools, hooks, timers, event delivery, idle waits, and Drive completion remain outside the line.
Raw Branch mutation while a Harness owns the corresponding AgentLane can stale the Harness projection and is a trusted-programming defect; AgentLane methods are the operation-aware surface during ownership.
@ -2245,7 +2035,7 @@ For a live operation, the first matching `requestAbort()` performs this order:
`cancel_requested` or observes that the expected id no longer owns the lane;
3. for runs, atomically move current steer/follow-up ids to `control.drained*`
and empty those inbox lists; their `pendingEntry(id)` values remain;
4. after a cancellation commit, resolve the promise and call `drive.signalAbort()`; after an ownership mismatch, resolve it and close the Drive gate with `OperationEnded`;
4. after a cancellation commit, resolve the promise and call `drive.signalAbort()`; after an operation-id mismatch, resolve it and return `OperationMismatch`;
5. return `AbortRequestResult` once cancellation is durable, or `OperationMismatch`
when the fence failed; a matching live pass continues reconciliation.
```
@ -2290,20 +2080,19 @@ For structural operations the atomic publication commit decides the race: cancel
## 4.7 Close — a controlled crash
Close is not abort and writes no cancellation or terminal state. It seals new `beginMutation()` admission and waits for every acquired mutation to reach `end()`:
Close is not abort and writes no cancellation or terminal state. It seals new Session mutation admission and waits for mutations admitted before the seal to finish:
```text
seal harness admission
→ close every live Drive gate
→ pull cooperative signals
→ reject local operation promises
→ let commits admitted before the seal drain
→ close storage and release the writer lease
seal harness and Lane admission
→ reject caller observations through the harness-close boundary
→ keep detached pass promises observed
→ drain admitted Session mutations
→ close storage
```
A harness-wide admission barrier orders close against every operation and public commit. A commit admitted first may finish and close waits for it; close first prevents admission. A provider/tool result produced after sealing cannot commit. Tool checkpoint replacements and assistant frame appends already enqueued are ordinary admitted session work and may drain under the barrier; a request after sealing cannot commit, and live output newer than the last committed checkpoint or frame may be lost. Durable state therefore stops at the last committed restart point, often `effect_pending` with a committed frame prefix or bounded checkpoint, exactly as after process death. No synthetic response or cancellation marker is written.
A provider/tool result produced after sealing cannot commit because its next Lane mutation rejects with `HarnessClosed`. The Drive is not replaced and durable operation state is not changed. Reopening therefore sees the same restart point as process loss.
Reopening reports the open operation. A later matching `drive(options, context)`—direct or through `resume(context)`—applies orphan recovery when the durable phase requires it. No close-specific recovery path exists, and the durable aborted-implies-cancelled invariant remains true because close admits no locally aborted settlement under running control.
Whether a host also signals cooperative provider/tool work is local resource cleanup. It must not write cancellation, synthesize settlement, remove a durable operation, or create an ownership-loss recovery path.
## 4.8 Faults
@ -2311,14 +2100,6 @@ A failed admitted storage commit faults the whole harness. Fault closes Drive ga
Close rejects active drive and convenience-operation promises with `HarnessClosed`; already-resolved admissions remain durable, while calls not yet accepted return `Err(Closed)`. Surfaces without a `Result` channel reject with `HarnessClosed` on and after close. Provider, tool, and isolated hook failures remain per-lane and in-band. A throw/rejection from trusted deterministic application computation (`systemPrompt`, `toolContext`, `toProviderMessages`, or an `entryProjector`) faults the harness. `AgentTool.prepareArguments` is the deliberate exception normalized to a synthetic tool error.
## 4.9 External finalization
An authorized administrative operation may commit the terminal transaction while a task still exists. In-process finalization must use the live owning `Lane`'s command path, on the same Session mutation line, so its terminal commit publishes the idle projection before notifying the owner. Bypassing that path and mutating reserved control values behind the live `Lane` is a programming defect. This specification adds no public `forceFinalize()` method: authorization, SLA policy, and initiation belong to repository or hosting administration, while terminal cleanup and live-task notification remain package-internal. Live admitted work receives `OperationEnded` when the finalizer closes `drive.gate`. Any queued or later planner observes ownership loss in the current projection, returns `lost_ownership`, and commits nothing.
With a live in-process Drive, the finalizer itself records `drive.finalizedOutcome` and closes the gate in committed materialization, publishes and awaits the end observation, then settles shared completion without waiting for detached work; the detached task stops without publishing or writing. With no local Drive—for example, finalization by a replacement process—a later matching `drive()` hydrates from `pi.lane.lastResult`. Neither path recreates operation values/lists or commits a competing terminal result. Scalar and list invocation writes are fenced the same way: invocation memo writes, tool checkpoint replacements, and assistant frame appends each verify the current owned projection when their lane job executes and exact Drive identity at commit admission, so a write enqueued before finalization is deleted by the terminal cleanup (which removes all operation-owned values/lists) and a write after it declines without committing or recreating state. An unowned operation needs no task to stop; a later matching `drive()` returns the settled result from `pi.lane.lastResult`, while `resume(context)` sees an idle lane and returns `NothingToResume`.
At most one terminal transaction commits because every candidate verifies the current owned projection on the Session mutation line before writing. An in-process finalizer records the matching result in `drive.finalizedOutcome` and closes its gate with `OperationEnded` in committed materialization, publishes and awaits the terminal event, then settles shared completion without waiting for non-cooperative provider/tool work. It retains the Drive until both terminal event delivery/settlement and the detached task's exact-object `finally` have occurred. Acceptance returns `LaneBusy`, and same-id drive callers join that retained completion, until the later boundary removes it. The task publishes no duplicate end event and cannot write after ownership loss. A replacement-process finalizer first acquires ownership after close/crash, so no live Drive exists to notify.
# Part 5 — Public surface
## 5.1 The lane surface
@ -2456,7 +2237,7 @@ interface WatchHandle<T> { snapshot: T; start(listener: EventListener): void; un
The convenience surface adds only invocation-lifetime policy: `prompt`/`skill`/`compact`/`navigateTree` call `accept` then drive the returned id with local retry waiting; `resume(context)` inspects and drives the current id with one deferred-poll permit; `abort(context)` requests durable cancellation and ensures reconciliation is driven. A serving layer may instead store `OperationAdmission`, return to its client, schedule from `waiting.notBefore`, and invoke the expected-id primitives later.
`getLastResult(context)` is the post-crash reconciliation path: an application that accepted an operation, lost its process, and reopened reads `laneLastResult(lane)` for the outcome its promise never delivered (§3.13). `inspectExecution(context)` enters a no-write lane job and reports the current small owned projection, local owner status, configured model identity, optional captured step identity, and latest result coherently. It does not resolve either identity through `Models` and performs no transcript/payload/frame reads. This lets a scheduler reconcile its own durable submission without an accept/settle race. Every state publication into that projection is serialized with the committing lane mutation. `pi.lane.lastResult` is still overwritten by the next terminal operation; permanent per-submission receipts remain a serving-layer concern. It is also how a caller learns the outcome of an operation finalized externally (§4.9).
`getLastResult(context)` is the post-crash reconciliation path: an application that accepted an operation, lost its process, and reopened reads `laneLastResult(lane)` for the outcome its promise never delivered (§3.13). `inspectExecution(context)` enters a no-write lane job and reports the current small owned projection, local owner status, configured model identity, optional captured step identity, and latest result coherently. It does not resolve either identity through `Models` and performs no transcript/payload/frame reads. This lets a scheduler reconcile its own durable submission without an accept/settle race. Every state publication into that projection is serialized with the committing lane mutation. `pi.lane.lastResult` is still overwritten by the next terminal operation; permanent per-submission receipts remain a serving-layer concern.
`waitForIdle(context)` registers on the Session mutation line and resolves when all earlier admitted lane jobs have settled, `currentOperationId` is null, and no process-local idle callback owns the lane. Later operations may start immediately after it resolves. Multiple waiters resolve together; close/fault rejects pending waiters.
@ -3402,7 +3183,7 @@ Telemetry attributes may contain declared ids, names, counts, durations, statuse
**This part is informative.** Nothing in it binds the shipping backends: Memory, JSONL, and SQLite never partition and never delete entries or usage rows (§1.2), and no core rule references this part for its correctness. It exists to show that the identity choices in §1.2 are sufficient for the one backend that would eventually retire old data — a possible Postgres deployment with TTL retention. It is a bridge we cross when we get there; this sketch is the current best guess, not a contract.
- **The id is the partition key.** UUIDv7 sorts bytewise in time order, so the bulk tables — entries, usage ledger — use `PARTITION BY RANGE (id)` on the uuid id column, with period-boundary UUIDs (zeroed tails) as bounds. No partition column exists anywhere; §1.2's time prefix is the whole mechanism. Scalar values, `branch_meta`, stats, leases, and sessions stay in a hot unpartitioned catalog. `branch_entries` partitions by `entry_id` with the same bounds, so dropping a period cleans the branch index for free; `branch_meta` stays hot, and base pointers dangling into a dropped period are trimmed lazily on first access.
- **Pre-pass repair.** Before a period P is dropped, an online repairer makes live state stop referencing it: reparent edges crossing into P onto the nearest retained ancestor, found by an indexed uuid-range query; null any dormant `pi.branch.tip` decoding into P via a value-seq CAS; force-expire open operations still referencing P through values only — the terminal transaction of §3.13 writing `pi.lane.lastResult`, no synthetic entries, with any live task stopping through external finalization (§4.9); delete `pi.entry.label` values whose keys decode into P with one uuid-range delete.
- **Pre-pass repair.** Before a period P is dropped, an online repairer makes live state stop referencing it: reparent edges crossing into P onto the nearest retained ancestor, found by an indexed uuid-range query; null any dormant `pi.branch.tip` decoding into P via a value-seq CAS; force-expire open operations still referencing P through values only — the terminal transaction of §3.13 writing `pi.lane.lastResult`, no synthetic entries, after exclusive administrative ownership has been acquired; delete `pi.entry.label` values whose keys decode into P with one uuid-range delete.
- **The commit barrier.** Repair races ordinary commits, so the final step is atomic against all of them: `BEGIN; LOCK entries, scalar_values IN ACCESS EXCLUSIVE MODE; <delta repair for anything committed since the online pass>; ALTER TABLE … DETACH PARTITION p; COMMIT;` — plain `DETACH`, not `CONCURRENTLY`, precisely because it is transactional under the lock; the `DROP TABLE` happens later, unhurried. The barrier makes repair-plus-detach one linearization point: every commit sees either the fully attached period or a fully repaired store without it.
- **The default partition.** A `DEFAULT` partition absorbs stray inserts whose ids predate every attached partition — an ancient `pendingNextRun` item consumed years after its mint still places under its reserved id and simply lands there. Nothing errors and nothing is lost; the default partition stays small and is never dropped.
- **Value access under an external repairer.** A backend that admits an external repairer must perform value reads and CAS checks inside the commit transaction itself, so a repairer holding the barrier cannot interleave between a harness's read and its dependent write. The shipping backends need no such rule: single-writer sessions have no external repairer.
@ -3529,8 +3310,8 @@ These rows preserve requirements, not frozen package boundaries. Create their ha
| R3 | **Generation recovery and retry** | Activation-only unknown-effect recovery that reduces committed frames into the synthetic zero-usage partial error under the reserved ids and then classifies ordinarily (retry or failure drain), non-retryable configuration failure when a later ready/retry attempt cannot resolve its captured model/tools, ordinary stop/error/deferred classification, provider-compliant `aborted`, failure-drain foundation, and both retry-wait policies: explicit drive returns `waiting/notBefore`, while convenience drive may own the abort-aware timer. Define deterministic timer and recovery-transition tests. Overflow classification remains explicitly unimplemented until R9. | Every generation state before/after reopen; accept/close/reopen/drive; recovery with no frames, partial frames, and authoritative end-frame content; synthetic settlement always commits under reserved ids with the interruption warning and deletes the frame list; interrupted partial tool calls never execute; `after_response` never runs for synthetic settlement; caps/backoff; no timer under `waitForRetry:false`; due and future `notBefore`; convenience/explicit-drive durable equivalence; timer abort-first/start-first; deterministic transition coverage; stop/error/aborted/deferred classification; non-retryable configuration unavailability. |
| R4 | **Tools** | Add the harness-specific tool-update checkpoint options/callback while leaving pi-ai's `AgentToolUpdateCallback` unchanged, then compose §5.7's neutral tool phases with bound context, invocation-scoped memos, `pi.op.tool_args`, scalar `pendingToolOutput(operationId, invocationId)` checkpoints, `outcome_ready` staging in `pi.pending.entry`, completion-order outcome durability, source-ordered materialization, orphan recovery/replay, sequential/parallel promises, hook-blocked/genuine-length outcomes, hooks/events/usage, and deterministic transition coverage. Missing requested tools stage direct `ToolResultMessage` errors with no `details`; missing safe-replay implementations synthesize interruption rather than suspending. The built-in bash tool emits live updates at 100 ms and requests distinct checkpoints at most every two seconds. | Stable invocation id/memos across safe replay; every planned/effect-pending/outcome-ready/completed state; parallel out-of-order staging without replay; synchronous update callback with the latest delivery and latest checkpoint write awaited before `after_tool`; `tool_end` before staging; every requested checkpoint enqueued synchronously with only the latest promise reference retained and no write dropped or coalesced; tool-selected cadence/duplicate suppression; late-write fencing after settlement; JSONL checkpoint growth/compaction; safe replay deletes old checkpoint; unsafe recovery with/without checkpoint; missing planned tools become source-ordered error results; absent or no-longer-safe replay implementations synthesize interruption without invented typed details; source-ordered materialization; termination and abort-ready states. |
| R5 | **Inbox, configuration, and writes** | `nextRun`/steer/follow-up via `pendingEntry(id)` values, pending-payload consumption-time dereference, `cancelQueued` triage (`not_found`), durable drain markers, checkpoint consumption with pending-value deletion, immediate total config setters, deferred tree writes, adjustments, and deterministic tests for the checkpoint boundaries introduced here. | Capture/cancel/consume races, repeated cancellation answering `not_found`, invalid queue payload kinds and missing pending values rejected at consumption, one-at-a-time crash after one drain, pending-value/entry exclusivity at every boundary, custom-write continuation, config-step race, writes surviving reopen. |
| R6 | **Abort, close, and failure drain** | Add expected-id `requestAbort` as commit/signalling only. Compose `abort()` from inspection, marker, and same-id drive. Implement orthogonal control, per-phase reconciliation including planned/effect-pending/outcome-ready tools, best-effort deferred cancellation, waiters/run-when-idle, controlled-crash close, complete terminal deletion of memos/checkpoints/staged outcomes, external-finalization stop, and deterministic transition coverage. | Cancellation with/without live pass; idempotency/fencing; abort at every tool state; restored interruption incorporates latest checkpoint without replay; restored cancellation synthesizes aborted responses preserving frame-reduced partial content and deleting the list; outcome-ready values survive and materialize; checkpoint/memo/frame cleanup; gate ordering; deferred cancellation; close races including a non-cooperative started tool that cannot block close and parallel retained promises that are all observed without later starts or unhandled rejection; external finalization fences late scalar/list writes; failure drain. |
| R7 | **Deferred provider redemption** | One poll per drive pass carrying `pollDeferred:true`; `resume()` composes inspection and that drive option. Add deferred consumption-time dereference and captured model resolution; unavailable models durably abandon redemption through configuration-provenance failure rather than waiting. Preserve copied configuration/options, per-poll hooks, source lineage, poll frame persistence with fresh intent after unknown poll deleting the abandoned frame list, ready tools, advancement of R6 cancellation, and deterministic poll-transition coverage. | `pollDeferred:false` returns waiting without work; one permit performs at most one poll; invalid or missing durable source handles; repeated pending, ready/error/aborted, crash positions, poll frame cleanup on every settlement, replacement, and configuration abandonment with no fabricated response/usage, unknown-poll snapshot without permit, convenience/primitive equivalence, no cap/backoff/loop, newest-handle cancellation. |
| R6 | **Abort, close, and failure drain** | Add expected-id `requestAbort` as commit/signalling only. Compose `abort()` from inspection, marker, and same-id drive. Implement orthogonal control, per-state reconciliation including planned/effect-pending/outcome-ready tools, best-effort deferred cancellation, waiters/run-when-idle, controlled-crash close, complete terminal deletion of memos/checkpoints/staged outcomes, and deterministic transition coverage. | Cancellation with/without live pass; idempotency/fencing; abort at every tool state; restored interruption incorporates the latest checkpoint without replay; restored cancellation synthesizes aborted responses preserving frame-reduced partial content and deleting the list; outcome-ready values survive and materialize; checkpoint/memo/frame cleanup; gate ordering; deferred cancellation; close races including a non-cooperative started tool; failure drain. |
| R7 | **Deferred provider redemption** | One poll per drive pass carrying `pollDeferred:true`; `resume()` composes inspection and that drive option. Add deferred consumption-time dereference and captured model resolution; unavailable models durably end redemption through configuration-provenance failure rather than waiting. Preserve copied configuration/options, per-poll hooks, source lineage, poll frame persistence with fresh intent after unknown poll deleting the abandoned frame list, ready tools, advancement of R6 cancellation, and deterministic poll-transition coverage. | `pollDeferred:false` returns waiting without work; one permit performs at most one poll; invalid or missing durable source handles; repeated pending, ready/error/aborted, crash positions, poll frame cleanup on every settlement, replacement, and configuration failure with no fabricated response/usage, unknown-poll snapshot without permit, convenience/primitive equivalence, no cap/backoff/loop, newest-handle cancellation. |
| R8 | **Manual compaction** | Add compaction `OperationRequest` acceptance without task installation and compose `compact()` from accept+drive. Implement unowned preparation with final-command source revalidation, `pi.op.preparation/{opId}:{taskId}`, total structural state with consumption-time dereference, in-band structural model-unavailability failure, hook/generated sources, nested request intents/usage, retained tail, retry/recovery/abort, and deterministic structural-transition coverage. | Explicit accept+drive versus convenience equivalence; empty/stale-preparation race; accept/close/reopen/drive; invalid task/preparation/source relationships; hook decline/result; crash after request one of split-turn generation; every state/crash; no public summary-stream messages and no structural frame persistence. |
| R9 | **Threshold and overflow compaction** | In-run structural decision, durable once-per-trigger threshold marker, continuation preservation, all overflow predicates, atomic response/preparation publication, specified normalization/projection, one overflow recovery flag, bounded second failure, and deterministic tests for added boundaries. | Threshold decline/empty across reopen, all overflow classifier/preparation inputs, no overflow tool plan, genuine length, crash/reopen at every transition. |
| R10 | **Navigation** | Add navigation `OperationRequest` acceptance without task installation and compose `navigateTree()` from accept+drive. Implement navigation source/target/summary consumption-time dereference, validation, summarized decision/generation, one final transaction combining move/summary/tip/label with terminal writes, summary-only navigation hook, and deterministic transition coverage. | Explicit accept+drive versus convenience equivalence; root/current/unknown rejection; invalid durable source/target/summary combinations; accept/close/reopen/drive; summarized/unsummarized paths; final tip at summary; abort race; exact atomic publication including cleanup. |
@ -3569,9 +3350,9 @@ Operations:
17. At most one operation is open per lane. Two is corruption.
18. `overflowRecoveryUsed` is `true` only after overflow compaction. A transition that adds projecting conversational input or tool results and requires an assistant writes `false`; an unprojected custom write preserves it.
19. **The settlement transaction that commits a response with `stopReason: "aborted"` must, in that same transaction, write an operation state with `control.status === "cancel_requested"`.** The invariant is scoped to the committing transaction — later terminal cleanup or forks may remove the state without violating it. Providers must comply with the harness-owned signal contract; violation is corruption.
20. Attachment restores and validates only the small lane/operation projection (§3.3, §4.4). That owned projection is authoritative until close, fault, or process loss. Detailed presentation references are validated by `watch(context)` under the Session mutation line; drive payload references are validated by their consuming procedure. Missing or contradictory required data faults that consumer, while optional frame/checkpoint absence is legal. Live phase/effect-identity and exact-`Drive` checks remain concurrency fences. `pi.lane.lastResult` never determines an open operation's next procedure.
21. At most one terminal transaction ever commits per operation. In-process finalization closes the live Drive gate with `OperationEnded` and records `drive.finalizedOutcome` in committed materialization, then settles after terminal event delivery without waiting for detached work; once that outcome is recorded, the pass continuation may neither settle nor fail shared completion. The exact Drive is removed only after both finalizer-owned settlement and pass ending, so no caller hydrates before terminal event delivery. Queued/later planners return `lost_ownership` and commit nothing. With no local Drive, later matching callers hydrate from `pi.lane.lastResult` (§4.9).
22. At most one `Drive` exists per lane. Acceptance and taskless `requestAbort` never install one. A matching `drive` installs it before releasing the Session mutation line; another matching drive joins that pass, and a stale id starts nothing. Every drive-owned write rechecks exact Drive identity synchronously at commit admission. Normal `finally` removes only itself; abandonment and close/fault remove immediately; external finalization settles after terminal event delivery and retains ownership until both settlement and `finally` have occurred. Each newly installed pass invokes `before_drive` once after the cancellation check; joiners do not. `starting + cancel_requested` invokes neither `before_drive` nor `before_run`.
20. Attachment restores and validates only the small lane/operation projection (§3.3, §4.4). That owned projection is authoritative until close, fault, or process loss. Detailed presentation references are validated by `watch(context)` under the Session mutation line; drive payload references are validated by their consuming procedure. Missing or contradictory required data faults that consumer, while optional frame/checkpoint absence is legal. Top-level operation state has one live writer; only parallel tool-call status and queued progress/memo writes require child-state fencing. `pi.lane.lastResult` never determines an open operation's next procedure.
21. At most one terminal transaction commits per operation. The one lane-owned Drive is the sole top-level state-advance writer, and every terminal candidate serializes on the Session mutation line. Administrative mutation of a live Lane's reserved control values is unsupported; offline administration first acquires exclusive Session ownership.
22. At most one `Drive` exists per lane. Acceptance and taskless `requestAbort` never install one. A matching `drive` installs it before releasing the Session mutation line; another matching drive joins that pass, and a stale id starts nothing. Caller cancellation ends only that caller's observation. A live Drive is never replaced in-process. Close/fault seal mutation admission and reject observations without writing operation state. Each newly installed pass invokes `before_drive` once after the cancellation check; joiners do not. `run.starting` under cancelled control invokes neither `before_drive` nor `before_run`.
23. The §4.2 `Gate.admit()` catalog is complete. Every listed hook/provider/tool/timer integration calls `admit(() => operation())` after preparation; no unlisted code calls it. Admitted asynchronous provider setup/delegation owns `drive.gate.signal`.
24. `drive` and `requestAbort` are fenced by expected operation id. They may affect only that current operation; `drive` may also return its matching latest terminal result. A stale wake for A cannot drive or cancel B.
25. No public drive option encodes a wall-clock budget or partial-progress return. An admitted effect settles normally or is recovered from durable state after task loss; host scheduling and process termination remain outside the harness contract.
@ -3581,9 +3362,9 @@ Operations:
29. Every outcome-ready call has exactly one matching finalized `pi.pending.entry`, no immutable result entry, no invocation memos, and no tool-output checkpoint. Outcome-ready and completed calls never execute again.
30. A tool progress checkpoint is an optional bounded complete `AgentToolResult` snapshot, selected with `checkpoint:true`. It never proves completion. Every selected checkpoint synchronously enqueues one invocation-fenced scalar replacement; no write is dropped or coalesced, only the latest write promise reference is retained, and awaiting it implies completion of every earlier write. Staging or terminal cleanup deletes the value and fences late recreation.
31. Scalar assistant/deferred state is the sole restart authority for streamed partials. One effect-pending response id constructs exactly one `pendingAssistantFrames(operationId, responseEntryId)` address; every element is an exported pi-ai `AssistantMessageFrame`; frame order is a subsequence of provider event order because already-covered queued events produce no frame; terminal `done`/`error` events are never stored; frames never establish provider completion or suppress unknown-outcome recovery.
32. Every final or synthetic response settlement — normal, recovery, cancellation, or external finalization — atomically deletes its exact frame list. Idle forks contain no frame lists. A restored partial may appear in `streamingMessage` but never in `transcript` before settlement.
32. Every final or synthetic response settlement — normal, recovery, or cancellation — atomically deletes its exact frame list. Idle forks contain no frame lists. A restored partial may appear in `streamingMessage` but never in `transcript` before settlement.
33. The provider loop never awaits storage per frame; frame appends are enqueued synchronously in provider-event order, and awaiting the latest frame-write promise at stream settlement implies every accepted append completed.
34. Successful attachment publishes only complete lane projections and an open-operation inventory. It resolves no model/tool identity and starts no work. A later drive fences the authoritative owned projection and exact `Drive` identity; storage reads only dereference payloads named by that projection.
34. Successful attachment publishes only complete lane projections and an open-operation inventory. It resolves no model/tool identity and starts no work. A later drive uses the authoritative owned projection; storage reads only dereference payloads named by that projection.
35. Every event-producing committing harness lane job publishes its owned projection and calls `emitBatch` with its complete event batch in the exact continuation that observes commit, as the callback's final action; this includes AgentLane appends, lane and metadata setters, acceptance, and AgentLane acquisition/attachment. The mutation never awaits delivery, but the public operation does. A lane watch registers buffering and clones live presentation synchronously, then performs bounded durable reads while holding the line. Snapshot plus buffered events has no gap or duplicate and replays no pre-registration lifecycle. `emitBatch` binds recipients and the emitting Context immediately; a delayed watcher receives the object-identical source Context, never its start Context.
36. Shared Harness/AgentLane/Session/Branch receivers retain no invocation Context and expose no receiver-level telemetry default. Concurrent calls preserve independent telemetry and cancellation lineage. Context and its values are neither durable operation data nor serialized business arguments. RPC cancel/disconnect reaches only the matching invocation through `context.abortSignal` and never becomes durable cancellation.
37. Process-local model/tool registry absence never becomes durable waiting state or an acceptance error. Pre-intent request-configuration absence fails in-band without fabricating a response/usage; missing requested tools stage `isError` tool-result messages with no invented details; uncertain effects settle under their existing recovery rules first.
@ -3591,7 +3372,7 @@ Operations:
## 9.2 Race catalog
Each durable mutation race has exactly two durable histories. Process-local ownership races additionally have the one-driver/join or fence outcome shown below. Test every listed order with test-only commit gating and controlled hooks, providers, tools, and timers.
Each durable mutation race has exactly two durable histories. Matching callers install or join one lane-owned Drive; stale operation ids are rejected. Test every listed order with test-only commit gating and controlled hooks, providers, tools, and timers.
| Race | Orders |
| ------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
@ -3603,9 +3384,7 @@ Each durable mutation race has exactly two durable histories. Process-local owne
| `abort` vs started tool outcome staging | abort first → real result stages under cancelled control; outcome first → finalized result is preserved and later materializes |
| checkpoint vs tool settlement | every accepted checkpoint was enqueued before settlement closed admission; settlement awaits the latest write, then staging deletes the scalar; a late update is fenced without committing |
| assistant frame append vs response settlement | settlement awaits the latest frame write, then its transaction deletes the list; a crash between leaves the committed frame prefix under `effect_pending` |
| assistant frame append vs external finalization | append first → terminal cleanup deletes the list; finalization first → the fenced append declines without committing |
| live update event vs its queued frame/checkpoint commit | either finishes first; events are observation, and reconnect uses only committed frames/checkpoints |
| external finalization vs memo/checkpoint mutation | mutation first → terminal cleanup deletes it; finalization first → owned-projection/exact-Drive check declines without committing |
| later tool B settles vs earlier tool A | B stages outcome-ready immediately; tree placement waits for A |
| `abort` vs `before_run_end` follow-up | follow-up dropped; or committed and the run continues |
| `cancelQueued` vs checkpoint consumption | `cancelled`; or `already_consumed` |
@ -3620,8 +3399,7 @@ Each durable mutation race has exactly two durable histories. Process-local owne
| watcher registration vs state publication | watcher first → old snapshot plus the complete buffered event batch; publication/`emitBatch` first → new snapshot without that old batch |
| close vs attachment | create completes and publishes a fully open harness; or close/fault rejects attachment without a partial harness |
| snapshot capture vs resume | capture first yields pre-resume snapshot plus events; resume publication first yields post-transition snapshot |
| external finalization vs attachment/inspection | finalization is visible as idle/latest result; or attachment sees the still-open operation and later Session mutation inspection observes publication |
| concurrent invocation contexts | each call/event/session write retains its own telemetry parent and abort signal; joiner cancellation ends only that observer; installer cancellation abandons the shared pass for current joiners but writes no durable cancellation |
| concurrent invocation contexts | each call/event/session write retains its own telemetry parent and abort signal; cancellation ends only that caller observation and writes no durable cancellation |
| `close` vs settlement | settlement abandoned, state stays `effect_pending`; or it committed before the flag was set |
## 9.3 Test tiers
@ -3632,7 +3410,7 @@ For each recovery prefix: close, reopen, drive, and compare against uninterrupte
One corruption assertion constructs an `aborted` response with running control directly and requires the consuming transition to reject it as an invariant defect. Provider conformance separately proves implementations emit `aborted` only for the supplied signal.
**Tier B — writer conformance.** Run the public harness against the instrumented-storage decorator: a spy wrapping `Storage.commit()` that records every transaction's writes in order. Assert exact write order and content against the Part 3 transaction tables and §5.5 ordering. Faux provider/tool/hook spies interleave starts/events with commits. This tier catches: effects before intent; failure to await the latest update delivery or latest checkpoint write before `after_tool`; a provider loop awaiting storage per frame; frame appends out of provider-event order or persisted for `done`/`error`; a settlement transaction missing its frame-list delete; `tool_end` emitted after rather than before outcome staging; missing response/usage settlement; checkpoint or frame writes after ownership loss; outcomes not staged before replay becomes impossible; out-of-order tree placement; result ids reserved late; invocation memos or staged/checkpoint/frame values leaked by outcome/terminal cleanup.
**Tier B — writer conformance.** Run the public harness against the instrumented-storage decorator: a spy wrapping `Storage.commit()` that records every transaction's writes in order. Assert exact write order and content against the Part 3 transaction tables and §5.5 ordering. Faux provider/tool/hook spies interleave starts/events with commits. This tier catches: effects before intent; failure to await the latest update delivery or latest checkpoint write before `after_tool`; a provider loop awaiting storage per frame; frame appends out of provider-event order or persisted for `done`/`error`; a settlement transaction missing its frame-list delete; `tool_end` emitted after rather than before outcome staging; missing response/usage settlement; checkpoint or frame writes after their child state settled; outcomes not staged before replay becomes impossible; out-of-order tree placement; result ids reserved late; invocation memos or staged/checkpoint/frame values leaked by outcome/terminal cleanup.
**Tier C — deterministic interleavings.** Every race in §9.2 is exercised in both orders with test-only gated commits and controlled hooks, providers, tools, and timers.
@ -3669,11 +3447,11 @@ One corruption assertion constructs an `aborted` response with running control d
| **Attachment** | Minimal lane/operation projection restore plus open inventory; it starts no execution. |
| **Invocation Context** | Immutable explicit telemetry/cancellation lineage for one call. It is never durable state or a shared receiver default. |
| **Acceptance** | The atomic creation of one durable operation by `accept`; it starts no driver. |
| **Drive pass** | One process-local ownership span created or joined by `drive`, ending at settlement or a durable wait. |
| **Drive pass** | One lane-owned process-local continuation installed or observed by `drive`, ending at settlement or a durable wait. |
| **Effect** | Anything not pure computation: commit, provider request, tool, hook, timer. |
| **Repeat-sensitive effect** | One whose repetition is observable outside the harness. |
| **Operation state** | The complete durable state of one open operation — the `operationState(operationId)` value and restart point after task loss. |
| **Drive** | The one installed process-local lane pass: exact ownership identity, shared completion, Gate, invocation Context, and wait policy. |
| **Drive** | The one installed lane-owned pass: shared completion, Gate, cancellation-free execution Context, and wait policy. |
| **Effect gate** | Process-local synchronous arbitration of ordinary operation admission against a cancellation request, paired with the operation's cooperative signal. |
| **Reserved id** | An id minted before content exists. Tool-result ids progress from a string in state to staged pending content to an immutable entry. |
| **Follower id** | An id minted with its leader's 48-bit timestamp so a call/result group shares one time prefix (§1.2). |
@ -3688,7 +3466,6 @@ One corruption assertion constructs an `aborted` response with running control d
| **Invocation memo** | Tool-invocation-scoped durable current value used for replay-safe memoization; valid only while the call is effect-pending. |
| **Terminal transaction** | The commit that deletes an operation's values/lists, writes `pi.lane.lastResult`, and clears `currentOperationId`. |
| **Segment** | A branch-index range that references an older branch instead of copying it. |
| **External finalization** | A terminal transaction committed outside the live task. An in-process finalizer closes the gate in committed materialization, publishes once, then settles the live Drive from `finalizedOutcome` after event delivery without waiting for detached work; the exact owner remains installed until both settlement and pass ending, preventing early hydration; with no local Drive, later callers hydrate from `pi.lane.lastResult` (§4.9). |
| **Precise rewrite** | The administrative copy-retained-and-swap rebuild of a session store — the sole sanctioned path that removes entries or usage rows (§2.9). |
# Appendix B — Coding-agent v3-format compatibility

344
packages/agent/docs/runtime-simplification.md Normal file
View file

@ -0,0 +1,344 @@
# Existing AgentHarness runtime simplification
## Scope
Rework the real implementation under `packages/agent/src/harness/runtime/` and the canonical durable types in `packages/agent/src/harness/session/types.ts`. This is not the isolated scratch spike.
Public drive remains disabled until the execution graph is total. Format 4 is still work in progress, so the durable type replacement requires no migration or compatibility representation.
## Core model
A Lane is one process-local actor over one durable lane projection.
- `Lane.state` is authoritative for tip, configuration, current operation, control, inbox IDs, and latest result.
- Every supported mutation commits through the Session mutation line and publishes the matching `Lane.state` before releasing it.
- One lane-owned Drive is the sole writer that advances an operation state.
- Same-operation callers observe the same Drive. No caller owns it.
- Invocation cancellation stops only that caller's observation after Drive installation.
- `requestAbort` is the only durable operation cancellation.
- Close seals mutation admission. It neither mutates operation state nor replaces a Drive.
- Process loss destroys all live continuations; recovery starts from the durable state after attachment.
- Session reads during execution dereference content named by `Lane.state`; they never rediscover control state.
## Durable state
`OperationState` has one discriminator, `at`, with 22 direct leaves:
- `run.starting`
- `run.checkpoint`
- `run.assistant.ready`
- `run.assistant.effect_pending`
- `run.assistant.retry_wait`
- `run.tools`
- `run.deferred.suspended`
- `run.deferred.effect_pending`
- `run.compaction.deciding`
- `run.compaction.ready`
- `run.compaction.effect_pending`
- `run.compaction.retry_wait`
- `run.failure_drain`
- `compaction.deciding`
- `compaction.ready`
- `compaction.effect_pending`
- `compaction.retry_wait`
- `navigation.ready_to_commit`
- `navigation.summary.deciding`
- `navigation.summary.ready`
- `navigation.summary.effect_pending`
- `navigation.summary.retry_wait`
Shared run, structural, and navigation data is factored with intersections. `Control` remains orthogonal. `ToolBatch` remains a child state machine because parallel tool children genuinely mutate sibling call statuses concurrently.
## State and content boundary
Procedures never read these addresses to decide execution:
- `laneState`
- `operationMeta`
- `operationState`
- `branchTip`
- `laneConfig`
- `laneLastResult`
They use the current `Lane.state` supplied by the mutation line.
Storage reads remain only for content and cleanup:
- prompt, assistant, deferred-source, final-assistant, and completed-tool entries;
- compaction-bounded branch context;
- `pendingEntry` payloads;
- assistant-frame lists;
- tool arguments, memos, and checkpoints;
- structural preparations;
- staged tool outcomes;
- operation-owned prefix scans required for terminal cleanup.
## Concurrency model
### Operation advancement
While a Drive procedure is alive, no supported concurrent actor can change or remove its operation state:
- inbox methods change only inbox fields;
- `requestAbort` changes only `control`;
- close prevents later mutation admission;
- another same-operation caller joins the existing Drive;
- another operation cannot be accepted while the lane is busy;
- process crash removes the continuation itself.
Consequently, ordinary procedure transitions do not recheck operation existence, operation ID, operation kind, `at`, attempt identity, or nested status. Those are procedure preconditions established by dispatch, not supported races.
The remaining real races are:
1. cancellation arriving before effect admission or before settlement;
2. inbox arrival before checkpoint routing or terminal finish;
3. parallel tool children staging and materializing sibling outcomes;
4. queued frame/checkpoint/memo writes racing effect settlement;
5. retry timers racing cancellation or close;
6. deferred permit consumption;
7. accept/claim serialization on the Session line.
### Cancellation boundaries
Cancellation is checked in three places only:
1. the drive loop before ordinary dispatch;
2. the gate immediately before an external effect starts;
3. effect settlement, which sees current `control` and commits the appropriate cancelled result.
An ordinary transition helper may centrally decline progress when current control is `cancel_requested`. Procedures do not repeat that branch manually.
### Close
Closing the harness:
1. marks harness/lane admission closed;
2. seals and drains the Session mutation line;
3. rejects client observations through a harness-close observation promise;
4. keeps every detached pass promise observed so a later rejection is never unhandled;
5. closes the Session after admitted mutations drain.
Close performs no durable write, installs no replacement Drive, and creates no ownership-loss state. A late effect may return, but its mutation is rejected with `HarnessClosed`.
Whether close also signals process-local provider/tool work is a resource-cleanup policy, not durable state-machine behavior. Do not couple it to Drive replacement or recovery.
## Small concrete mutation API
Replace repeated `LaneCommand` ceremony in procedures with two concrete Lane operations. They are not a scheduler, graph, or action interpreter.
### `advanceOperation`
Used for ordinary non-terminal progress.
- Enters the Session mutation line.
- Receives the current authoritative Lane projection and operation state.
- If control is cancelled, returns to the drive loop without invoking the semantic planner.
- The planner supplies procedure-specific writes, the next complete `OperationState`, materialization, and events.
- The helper appends the `operationState` write and publishes the matching process-local operation projection.
- It does not verify an expected state or `at` value.
### `mutateOperation`
Used after an admitted provider/tool/structural effect and for genuine parallel child transitions.
- Enters the Session mutation line even when control is cancelled.
- Supplies current control/inbox fields and the process-local effect result to the semantic settlement planner.
- Atomically commits payload, usage, tip movement, cleanup, and the classified next state.
- Appends the canonical operation-state write and publishes the matching projection.
- When the planner returns a terminal decision, appends `laneLastResult`, idle `laneState`, and the idle process-local projection.
The caller supplies the typed outcome, cleanup/publication writes, last result, and event. Terminal business decisions remain visible in the owning procedure.
## Drive lifecycle
Delete the installer-owned model.
- Remove `installerSignal`.
- Remove `DriveAbandoned`.
- Remove `LostOwnership` and `lost_ownership` from procedure results.
- Remove exact-object ABA fencing and `commandDriveOwned`.
- Remove `finalizedOutcome` and planned external-finalization owner retention unless the Flue investigation establishes a concrete requirement.
- Keep `activeDrive` only as the lane's install/join slot.
- Installation transfers work to the lane. Every caller, including the installer, observes completion with its own invocation Context.
- A caller signal abort before installation installs nothing. After installation it only rejects that caller's observation.
- The Drive is removed after its pass settles or reaches a durable wait. No live pass is replaced in-process.
`requestAbort` retains the two-part gate ordering:
```text
beginAbort before cancellation mutation
commit cancel_requested
signalAbort after commit
```
This prevents a new effect from entering while the durable marker is being committed.
## Checks that remain
Do not remove validation of external or referenced content:
- required entry existence and role;
- pending payload kind;
- deferred handle identity;
- provider stream protocol ordering;
- response stop-reason invariants;
- UUIDv7 follower timestamp parsing;
- configured model/tool availability;
- tool-call source index and staged result identity;
- parallel tool call status and ready-prefix placement;
- progress/memo invocation identity;
- retry timestamp and deferred permit arithmetic.
These validate data or genuine child concurrency. They are not defensive revalidation of the operation state machine.
## Staged implementation plan
### Stage 1 — Canonical flat durable types
Files:
- `src/harness/session/types.ts`
- compile-only consumers in `src/harness/runtime/`, restore, conformance helpers, and focused tests
- `docs/harness.md`
- `docs/work-packages/05-direct-durable-drive.md`
Actions:
- Replace nested operation state declarations with the flat `at` union.
- Preserve every durable datum without compatibility aliases.
- Update pattern matching mechanically without changing behavior.
- Keep ToolBatch/ToolCall nested.
- Update normative documentation in the same change.
Exit: `npm run check`; existing focused runtime tests pass; no `phase.kind`, generation `status`, deferred `status`, or structural decision `status` remains in canonical operation state.
### Stage 2 — Add canonical transition operations
Files:
- `src/harness/runtime/lane.ts`
- `src/harness/runtime/types.ts`
- focused Lane tests
Actions:
- Add `advanceOperation` and `mutateOperation`, including the terminal-decision suffix.
- Pair every durable operation-state write with process-local projection publication in one implementation.
- Centralize ordinary cancellation diversion.
- Trust the dispatcher-established current leaf; perform no expected-state check.
- Keep procedure-specific writes and event builders visible at call sites.
Exit: focused tests prove durable state and `Lane.state` remain identical after every helper commit.
### Stage 3 — Convert starting, checkpoint, and assistant
Files:
- `runtime/drive/checkpoint.ts`
- `runtime/drive/generation.ts`
- `runtime/drive/recovery.ts`
- `runtime/progress.ts`
Actions:
- Remove repeated operation/null/kind/state checks and `same*` predicates.
- Convert ordinary progress to `advanceOperation`.
- Convert assistant settlement to `mutateOperation`.
- Keep only checkpoint inbox/finish races and progress-channel ownership checks.
- Fold assistant recovery into the effect-pending handler if that is smaller.
Exit: no control-state storage reads; no repeated assistant state verification; focused generation tests pass.
### Stage 4 — Convert deferred and tools
Files:
- `runtime/drive/deferred.ts`
- `runtime/drive/tools.ts`
- `runtime/progress.ts`
Actions:
- Share assistant/deferred response-entry, usage, and tool-plan construction where their semantics match.
- Preserve deferred permit and handle checks.
- Remove top-level operation-state revalidation.
- Preserve per-call tool status merging, completion-order staging, source-order placement, memo fencing, and progress fencing.
- Use one tool-batch procedure for live, recovery, and cancellation modes instead of separate ownership-result paths.
Exit: tool status checks exist only for genuine sibling concurrency; focused deferred/tool/progress tests pass.
### Stage 5 — Remove ownership-loss machinery
Files:
- `runtime/types.ts`
- `runtime/lane.ts`
- `execution/effect-gate.ts`
- all existing `runtime/drive/*.ts`
- associated focused tests
Actions:
- Remove `LostOwnership`, `commandDriveOwned`, exact Drive checks, `installerSignal`, `DriveAbandoned`, and `finalizedOutcome`.
- Make all drive callers observation peers.
- Add observation-only Context cancellation to Drive completion waiting.
- Keep `activeDrive` only for install/join arbitration.
- Remove ABA/replacement tests and replace them with install/join/observation-cancellation tests.
Exit: grep finds no ownership-loss or installer-abandonment vocabulary in production runtime.
### Stage 6 — Close and fault
Files:
- `runtime/harness.ts`
- `runtime/lane.ts`
- Drive observation helper
- lifecycle tests
Actions:
- Seal mutation admission and drain admitted mutations.
- Reject client observations on close/fault without replacing a Drive or changing durable operation state.
- Observe detached pass failures.
- Verify a late effect cannot commit after close.
- Decide separately whether to signal local effects for resource cleanup.
Exit: close and process loss leave the same durable restart point; no close path writes cancellation or synthetic settlement.
### Stage 7 — Finish WP05 on the simpler substrate
Files:
- `runtime/drive/structural.ts`
- `runtime/drive/reconcile.ts`
- `runtime/drive.ts`
- `runtime/lane.ts` public surfaces
Actions:
- Implement structural generation directly over flat leaves and the transition operations.
- Implement cancellation reconciliation as one flat-state switch.
- Implement the total drive switch as one `state.at` switch.
- Wire public claim/join/observation and convenience methods only after every leaf is total.
External finalization is excluded unless the Flue investigation identifies a concrete, current caller that cannot be expressed through close, explicit abort, recovery, or offline administration.
## Validation
After every code stage:
```bash
npm run check
```
Run each modified focused test file from the package root. Do not run the full Vitest suite directly. Public drive remains disabled until the final stage.
Final audit:
```bash
rg 'lost_ownership|LostOwnership|DriveAbandoned|commandDriveOwned|installerSignal|finalizedOutcome' packages/agent/src/harness
rg 'operationState\(|laneState\(|branchTip\(|laneConfig\(' packages/agent/src/harness/runtime/drive
```
The second audit may match write constructors, but no reader call may use those control addresses.

2005
packages/agent/docs/work-packages/05-direct-durable-drive.md
View file

File diff suppressed because it is too large Load diff

8
packages/agent/src/harness/execution/effect-gate.ts
View file

@ -9,14 +9,6 @@ export class AbortRequested extends Error {
}
}
/** Invocation cancellation or owner abandonment. Never durable cancellation. */
export class DriveAbandoned extends Error {
constructor() {
super("Drive pass was abandoned by its installing invocation");
this.name = "DriveAbandoned";
}
}
/** Procedure-facing synchronous admission capability for one drive pass. */
export interface Gate {
readonly signal: AbortSignal;

377
packages/agent/src/harness/runtime/drive/checkpoint.ts
View file

@ -3,49 +3,27 @@ import type { HarnessEvent, TerminalOperationOutcome } from "../../agent-harness
import { insertEntry } from "../../session/commit.ts";
import { buildSessionContext } from "../../session/context.ts";
import { SessionInvariantError } from "../../session/session.ts";
import type {
CheckpointPhase,
Entry,
LaneLastResult,
MessageEntry,
NewEntry,
RunState,
SettledAssistantMessage,
} from "../../session/types.ts";
import {
branchTip,
deleteValue,
laneLastResult as laneLastResultValue,
laneState as laneStateValue,
operationState as operationStateValue,
pendingEntry,
setValue,
} from "../../session/values.ts";
checkpointDataOf,
type Entry,
type LaneLastResult,
type MessageEntry,
type NewEntry,
type RunAssistantReadyOperation,
type RunCheckpointOperation,
type RunStartingOperation,
runScopeOf,
type SettledAssistantMessage,
} from "../../session/types.ts";
import { branchTip, deleteValue, pendingEntry, setValue } from "../../session/values.ts";
import type { Lane } from "../lane.ts";
import type { Drive, LaneCommand, LostOwnership, ProcedureResult } from "../types.ts";
import type { Drive, ProcedureResult } from "../types.ts";
import { operationCleanupWrites } from "./terminal.ts";
type ReadResult<T> = { kind: "value"; value: T } | { kind: "state_changed" } | LostOwnership;
function isSettledAssistant(message: AgentMessage): message is SettledAssistantMessage {
return message.role === "assistant" && message.stopReason !== "pending";
}
function sameCheckpoint(current: CheckpointPhase, expected: CheckpointPhase): boolean {
return (
current.triggerEntryId === expected.triggerEntryId &&
current.thresholdCheckedTriggerEntryId === expected.thresholdCheckedTriggerEntryId &&
current.skipInboxOnce === expected.skipInboxOnce &&
current.continuation.kind === expected.continuation.kind &&
(current.continuation.kind !== "need_assistant" ||
(expected.continuation.kind === "need_assistant" &&
current.continuation.overflowRecoveryUsed === expected.continuation.overflowRecoveryUsed)) &&
(current.continuation.kind !== "may_finish" ||
(expected.continuation.kind === "may_finish" &&
current.continuation.includeFinalAssistant === expected.continuation.includeFinalAssistant))
);
}
function lifecycleEvents(entry: Entry, lane: string, runId: string): HarnessEvent[] {
return entry.type === "message"
? [
@ -59,21 +37,11 @@ function lifecycleEvents(entry: Entry, lane: string, runId: string): HarnessEven
async function readRunContext<TContext extends object | undefined>(
lane: Lane<TContext>,
drive: Drive,
phase: CheckpointPhase,
): Promise<ReadResult<AgentMessage[]>> {
const entries = await lane.commandDriveOwned<ReadResult<Entry[]>>(
drive,
async (state, reader): Promise<LaneCommand<ReadResult<Entry[]>>> => {
const operation = state.operation;
if (operation === null || operation.state.kind !== "run") {
throw new SessionInvariantError("Checkpoint context reached a non-run operation");
}
if (operation.state.control.status === "cancel_requested") {
return { kind: "return", result: { kind: "state_changed" } as const };
}
if (operation.state.phase.kind !== "checkpoint" || !sameCheckpoint(operation.state.phase, phase)) {
throw new SessionInvariantError("Checkpoint context lost its durable boundary");
}
checkpoint: RunCheckpointOperation,
): Promise<AgentMessage[] | undefined> {
const entries = await lane.advanceOperation(
checkpoint,
async (state, _current, _meta, reader) => {
if (state.tipId === null) throw new SessionInvariantError("Run checkpoint has no Branch tip");
const path = (
await reader.scanBranch(
@ -81,59 +49,42 @@ async function readRunContext<TContext extends object | undefined>(
drive.context,
)
).reverse();
return { kind: "return", result: { kind: "value", value: path } as const };
return { kind: "return", result: path };
},
drive.context,
);
if (entries.kind !== "value") return entries;
return {
kind: "value",
value: await buildSessionContext(
entries.value,
{ entryProjectors: lane.readConfig().entryProjectors },
drive.context,
),
};
return entries === undefined
? undefined
: buildSessionContext(entries, { entryProjectors: lane.readConfig().entryProjectors }, drive.context);
}
/** Consume before_run and commit the initial checkpoint. */
export async function startRun<TContext extends object | undefined>(
lane: Lane<TContext>,
drive: Drive,
run: RunState,
run: RunStartingOperation,
): Promise<ProcedureResult> {
if (run.phase.kind !== "starting") throw new SessionInvariantError("startRun requires the starting phase");
const prompt = await lane.commandDriveOwned<ReadResult<AgentMessage[]>>(
drive,
async (state, reader): Promise<LaneCommand<ReadResult<AgentMessage[]>>> => {
const operation = state.operation;
if (operation === null || operation.state.kind !== "run" || operation.meta.intent.kind !== "run") {
throw new SessionInvariantError("before_run reached a non-run operation");
}
if (operation.state.control.status === "cancel_requested") {
return { kind: "return", result: { kind: "state_changed" } as const };
}
if (operation.state.phase.kind !== "starting") {
throw new SessionInvariantError("before_run lost the starting boundary");
}
const entries = await reader.getEntries(operation.meta.intent.promptEntryIds, drive.context);
const messages = operation.meta.intent.promptEntryIds.map((id) => {
const prompt = await lane.advanceOperation(
run,
async (_state, _current, meta, reader) => {
if (meta.intent.kind !== "run") throw new SessionInvariantError("Run operation has non-run intent");
const entries = await reader.getEntries(meta.intent.promptEntryIds, drive.context);
const messages = meta.intent.promptEntryIds.map((id) => {
const entry = entries.get(id);
if (entry?.type !== "message") {
throw new SessionInvariantError(`Run prompt entry ${id} is missing its message`);
}
return entry.message;
});
return { kind: "return", result: { kind: "value", value: messages } as const };
return { kind: "return", result: messages };
},
drive.context,
);
if (prompt.kind === "lost_ownership") return prompt;
if (prompt.kind === "state_changed") return { kind: "continue" };
if (prompt === undefined) return { kind: "continue" };
const hook = await lane.hooks.runWithGate(
"before_run",
{ lane: lane.name, runId: drive.operationId, prompt: prompt.value, resources: lane.readConfig().resources },
{ lane: lane.name, runId: drive.operationId, prompt, resources: lane.readConfig().resources },
drive.gate,
drive.context,
);
@ -145,19 +96,9 @@ export async function startRun<TContext extends object | undefined>(
}
const reserved = injected.map((message) => ({ id: lane.session.idGenerator.next(), message }));
return lane.commandDriveOwned<ProcedureResult>(
drive,
(state) => {
const operation = state.operation;
if (operation === null || operation.state.kind !== "run") {
throw new SessionInvariantError("before_run settlement reached a non-run operation");
}
if (operation.state.control.status === "cancel_requested") {
return { kind: "return", result: { kind: "continue" } as const };
}
if (operation.state.phase.kind !== "starting") {
throw new SessionInvariantError("before_run settlement lost the starting boundary");
}
const result = await lane.advanceOperation(
run,
(state, current) => {
let parentId = state.tipId;
const entries = reserved.map(({ id, message }) => {
const entry: NewEntry<MessageEntry> = { id, parentId, type: "message", message };
@ -166,26 +107,20 @@ export async function startRun<TContext extends object | undefined>(
});
const triggerEntryId = parentId;
if (triggerEntryId === null) throw new SessionInvariantError("Run start has no trigger entry");
const nextState: RunState = {
...operation.state,
phase: {
kind: "checkpoint",
continuation: { kind: "need_assistant", overflowRecoveryUsed: false },
triggerEntryId,
},
const nextState: RunCheckpointOperation = {
...runScopeOf(current),
at: "run.checkpoint",
continuation: { kind: "need_assistant", overflowRecoveryUsed: false },
triggerEntryId,
};
return {
kind: "commit",
writes: [
...entries.map((entry) => insertEntry(entry)),
...(entries.length === 0 ? [] : [setValue(branchTip(lane.name), triggerEntryId)]),
setValue(operationStateValue(drive.operationId), nextState),
],
next: {
...state,
tipId: triggerEntryId,
operation: { meta: operation.meta, state: nextState },
},
operationState: nextState,
lane: { tipId: triggerEntryId },
materialize: () => ({ kind: "continue" }) as const,
events: (commit) =>
entries.flatMap((entry, index) =>
@ -199,27 +134,17 @@ export async function startRun<TContext extends object | undefined>(
},
drive.context,
);
return result ?? { kind: "continue" };
}
async function applyPendingWrites<TContext extends object | undefined>(
lane: Lane<TContext>,
drive: Drive,
phase: CheckpointPhase,
checkpoint: RunCheckpointOperation,
): Promise<ProcedureResult> {
return lane.commandDriveOwned<ProcedureResult>(
drive,
async (state, reader) => {
const operation = state.operation;
if (operation === null || operation.state.kind !== "run") {
throw new SessionInvariantError("Pending-write drain reached a non-run operation");
}
const current = operation.state;
if (current.control.status === "cancel_requested") {
return { kind: "return", result: { kind: "continue" } as const };
}
if (current.phase.kind !== "checkpoint" || !sameCheckpoint(current.phase, phase)) {
throw new SessionInvariantError("Pending-write drain lost its checkpoint boundary");
}
const result = await lane.advanceOperation(
checkpoint,
async (state, current, _meta, reader) => {
const ids = current.inbox.writes;
if (ids.length === 0) return { kind: "return", result: { kind: "continue" } as const };
const pending = await Promise.all(
@ -248,29 +173,26 @@ async function applyPendingWrites<TContext extends object | undefined>(
}
return entry;
});
const nextPhase: CheckpointPhase =
const scope = { ...runScopeOf(current), inbox: { ...current.inbox, writes: [] } };
const nextState: RunCheckpointOperation =
triggerEntryId === undefined
? current.phase
? { ...scope, at: "run.checkpoint", ...checkpointDataOf(current) }
: {
kind: "checkpoint",
...scope,
at: "run.checkpoint",
continuation: { kind: "need_assistant", overflowRecoveryUsed: false },
triggerEntryId,
skipInboxOnce: true,
};
const nextState: RunState = {
...current,
phase: nextPhase,
inbox: { ...current.inbox, writes: [] },
};
return {
kind: "commit",
writes: [
...entries.map((entry) => insertEntry(entry)),
...ids.map((id) => deleteValue(pendingEntry(id))),
setValue(branchTip(lane.name), parentId),
setValue(operationStateValue(drive.operationId), nextState),
],
next: { ...state, tipId: parentId, operation: { meta: operation.meta, state: nextState } },
operationState: nextState,
lane: { tipId: parentId },
materialize: () => ({ kind: "continue" }) as const,
events: (commit) =>
entries.flatMap((entry, index) =>
@ -284,28 +206,18 @@ async function applyPendingWrites<TContext extends object | undefined>(
},
drive.context,
);
return result ?? { kind: "continue" };
}
async function consumeQueuedMessages<TContext extends object | undefined>(
lane: Lane<TContext>,
drive: Drive,
phase: CheckpointPhase,
checkpoint: RunCheckpointOperation,
queue: "steer" | "followUp",
): Promise<ProcedureResult> {
return lane.commandDriveOwned<ProcedureResult>(
drive,
async (state, reader) => {
const operation = state.operation;
if (operation === null || operation.state.kind !== "run") {
throw new SessionInvariantError("Queue drain reached a non-run operation");
}
const current = operation.state;
if (current.control.status === "cancel_requested") {
return { kind: "return", result: { kind: "continue" } as const };
}
if (current.phase.kind !== "checkpoint" || !sameCheckpoint(current.phase, phase)) {
throw new SessionInvariantError("Queue drain lost its checkpoint boundary");
}
const result = await lane.advanceOperation(
checkpoint,
async (state, current, _meta, reader) => {
const allIds = current.inbox[queue];
const mode = queue === "steer" ? current.settings.steeringMode : current.settings.followUpMode;
const ids = mode === "all" ? allIds : allIds.slice(0, 1);
@ -339,14 +251,12 @@ async function consumeQueuedMessages<TContext extends object | undefined>(
});
const triggerEntryId = entries.at(-1)?.id;
if (triggerEntryId === undefined) throw new SessionInvariantError("Queue drain produced no trigger entry");
const nextState: RunState = {
...current,
phase: {
kind: "checkpoint",
continuation: { kind: "need_assistant", overflowRecoveryUsed: false },
triggerEntryId,
skipInboxOnce: true,
},
const nextState: RunCheckpointOperation = {
...runScopeOf(current),
at: "run.checkpoint",
continuation: { kind: "need_assistant", overflowRecoveryUsed: false },
triggerEntryId,
skipInboxOnce: true,
inbox: { ...current.inbox, [queue]: allIds.slice(ids.length) },
};
return {
@ -355,13 +265,9 @@ async function consumeQueuedMessages<TContext extends object | undefined>(
...entries.map((entry) => insertEntry(entry)),
...ids.map((id) => deleteValue(pendingEntry(id))),
setValue(branchTip(lane.name), triggerEntryId),
setValue(operationStateValue(drive.operationId), nextState),
],
next: {
...state,
tipId: triggerEntryId,
operation: { meta: operation.meta, state: nextState },
},
operationState: nextState,
lane: { tipId: triggerEntryId },
materialize: () => ({ kind: "continue" }) as const,
events: (commit) => [
...entries.flatMap((entry, index) =>
@ -383,19 +289,19 @@ async function consumeQueuedMessages<TContext extends object | undefined>(
},
drive.context,
);
return result ?? { kind: "continue" };
}
async function finishRun<TContext extends object | undefined>(
lane: Lane<TContext>,
drive: Drive,
phase: CheckpointPhase,
checkpoint: RunCheckpointOperation,
): Promise<ProcedureResult> {
const context = await readRunContext(lane, drive, phase);
if (context.kind === "lost_ownership") return context;
if (context.kind === "state_changed") return { kind: "continue" };
const context = await readRunContext(lane, drive, checkpoint);
if (context === undefined) return { kind: "continue" };
const hook = await lane.hooks.runWithGate(
"before_run_end",
{ lane: lane.name, runId: drive.operationId, messages: context.value },
{ lane: lane.name, runId: drive.operationId, messages: context },
drive.gate,
drive.context,
);
@ -404,22 +310,11 @@ async function finishRun<TContext extends object | undefined>(
const followUpMessage: AgentMessage | undefined =
followUp === undefined ? undefined : { role: "user", content: followUp, timestamp: Date.now() };
return lane.commandDriveOwned<ProcedureResult>(
drive,
async (state, reader) => {
const operation = state.operation;
if (operation === null || operation.state.kind !== "run") {
throw new SessionInvariantError("Run finish reached a non-run operation");
}
const current = operation.state;
if (current.control.status === "cancel_requested") {
return { kind: "return", result: { kind: "continue" } as const };
}
if (current.phase.kind !== "checkpoint" || !sameCheckpoint(current.phase, phase)) {
throw new SessionInvariantError("Run finish lost its checkpoint boundary");
}
const result = await lane.advanceOperation<RunCheckpointOperation, ProcedureResult>(
checkpoint,
async (state, current, _meta, reader) => {
if (
current.phase.continuation.kind !== "may_finish" ||
current.continuation.kind !== "may_finish" ||
current.inbox.steer.length !== 0 ||
current.inbox.followUp.length !== 0 ||
current.inbox.writes.length !== 0
@ -427,13 +322,11 @@ async function finishRun<TContext extends object | undefined>(
return { kind: "return", result: { kind: "continue" } as const };
}
if (followUpId !== undefined && followUpMessage !== undefined) {
const nextState: RunState = {
...current,
phase: {
kind: "checkpoint",
continuation: { kind: "need_assistant", overflowRecoveryUsed: false },
triggerEntryId: followUpId,
},
const nextState: RunCheckpointOperation = {
...runScopeOf(current),
at: "run.checkpoint",
continuation: { kind: "need_assistant", overflowRecoveryUsed: false },
triggerEntryId: followUpId,
};
const entry: NewEntry<MessageEntry> = {
id: followUpId,
@ -443,16 +336,9 @@ async function finishRun<TContext extends object | undefined>(
};
return {
kind: "commit",
writes: [
insertEntry(entry),
setValue(branchTip(lane.name), followUpId),
setValue(operationStateValue(drive.operationId), nextState),
],
next: {
...state,
tipId: followUpId,
operation: { meta: operation.meta, state: nextState },
},
writes: [insertEntry(entry), setValue(branchTip(lane.name), followUpId)],
operationState: nextState,
lane: { tipId: followUpId },
materialize: () => ({ kind: "continue" }) as const,
events: (commit) =>
lifecycleEvents(
@ -464,8 +350,8 @@ async function finishRun<TContext extends object | undefined>(
}
if (state.tipId === null) throw new SessionInvariantError("Completed run has no tip");
const finalId = current.phase.continuation.includeFinalAssistant ? current.latestAssistantEntryId : null;
if (current.phase.continuation.includeFinalAssistant && finalId === null) {
const finalId = current.continuation.includeFinalAssistant ? current.latestAssistantEntryId : null;
if (current.continuation.includeFinalAssistant && finalId === null) {
throw new SessionInvariantError("Completed run is missing its final assistant");
}
let finalEntry: { id: string; message: SettledAssistantMessage } | undefined;
@ -499,16 +385,9 @@ async function finishRun<TContext extends object | undefined>(
};
const cleanup = await operationCleanupWrites(reader, drive.operationId, current, drive.context);
return {
kind: "commit",
writes: [
...cleanup,
setValue(laneLastResultValue(lane.name), lastResult),
setValue(laneStateValue(lane.name), {
currentOperationId: null,
pendingNextRun: state.pendingNextRun,
}),
],
next: { ...state, lastResult, operation: null },
kind: "finish",
writes: cleanup,
lastResult,
materialize: () => ({ kind: "settled", outcome }) as const,
events: () => [
{
@ -524,85 +403,63 @@ async function finishRun<TContext extends object | undefined>(
},
drive.context,
);
return result ?? { kind: "continue" };
}
/** Advance one durable run checkpoint by one visible transition. */
export async function runCheckpoint<TContext extends object | undefined>(
lane: Lane<TContext>,
drive: Drive,
run: RunState,
phase: CheckpointPhase,
run: RunCheckpointOperation,
): Promise<ProcedureResult> {
if (run.phase.kind !== "checkpoint" || !sameCheckpoint(run.phase, phase)) {
throw new SessionInvariantError("runCheckpoint requires its current checkpoint phase");
if (run.skipInboxOnce !== true && run.inbox.writes.length !== 0) {
return applyPendingWrites(lane, drive, run);
}
if (run.control.status === "cancel_requested") return { kind: "continue" };
if (phase.skipInboxOnce !== true && run.inbox.writes.length !== 0) {
return applyPendingWrites(lane, drive, phase);
if (run.skipInboxOnce !== true && run.inbox.steer.length !== 0) {
return consumeQueuedMessages(lane, drive, run, "steer");
}
if (phase.skipInboxOnce !== true && run.inbox.steer.length !== 0) {
return consumeQueuedMessages(lane, drive, phase, "steer");
}
if (phase.continuation.kind === "need_assistant") {
if (run.continuation.kind === "need_assistant") {
const overflowRecoveryUsed = run.continuation.overflowRecoveryUsed;
const config = lane.readConfig();
const retryPolicy = config.retryPolicy.enabled
? { maxAttempts: config.retryPolicy.maxRetries + 1, baseDelayMs: config.retryPolicy.baseDelayMs }
: { maxAttempts: 1, baseDelayMs: config.retryPolicy.baseDelayMs };
const stepId = lane.session.idGenerator.next();
return lane.commandDriveOwned<ProcedureResult>(
drive,
(state) => {
const operation = state.operation;
if (operation === null || operation.state.kind !== "run") {
throw new SessionInvariantError("Generation start reached a non-run operation");
}
const current = operation.state;
if (current.control.status === "cancel_requested") {
return { kind: "return", result: { kind: "continue" } as const };
}
const result = await lane.advanceOperation(
run,
(state, current) => {
if (
current.phase.kind !== "checkpoint" ||
!sameCheckpoint(current.phase, phase) ||
current.phase.continuation.kind !== "need_assistant"
) {
throw new SessionInvariantError("Generation start lost its checkpoint boundary");
}
if (
current.phase.skipInboxOnce !== true &&
current.skipInboxOnce !== true &&
(current.inbox.writes.length !== 0 || current.inbox.steer.length !== 0)
) {
return { kind: "return", result: { kind: "continue" } as const };
}
const nextState: RunState = {
...current,
phase: {
kind: "assistant",
generation: {
status: "ready",
context: {
stepId,
triggerEntryId: current.phase.triggerEntryId,
configuration: state.configuration,
streamOptions: config.streamOptions,
retryPolicy,
overflowRecoveryUsed: current.phase.continuation.overflowRecoveryUsed,
},
nextAttempt: 1,
},
const nextState: RunAssistantReadyOperation = {
...runScopeOf(current),
at: "run.assistant.ready",
generationContext: {
stepId,
triggerEntryId: current.triggerEntryId,
configuration: state.configuration,
streamOptions: config.streamOptions,
retryPolicy,
overflowRecoveryUsed,
},
nextAttempt: 1,
};
return {
kind: "commit",
writes: [setValue(operationStateValue(drive.operationId), nextState)],
next: { ...state, operation: { meta: operation.meta, state: nextState } },
writes: [],
operationState: nextState,
materialize: () => ({ kind: "continue" }) as const,
};
},
drive.context,
);
return result ?? { kind: "continue" };
}
if (run.inbox.followUp.length !== 0) {
return consumeQueuedMessages(lane, drive, phase, "followUp");
return consumeQueuedMessages(lane, drive, run, "followUp");
}
return finishRun(lane, drive, phase);
return finishRun(lane, drive, run);
}

405
packages/agent/src/harness/runtime/drive/deferred.ts
View file

@ -5,55 +5,31 @@ import {
type DeferredHandle,
type Model,
} from "@earendil-works/pi-ai";
import type { HarnessEvent } from "../../agent-harness.ts";
import { withAbortSignal } from "../../context.ts";
import { AbortRequested } from "../../execution/effect-gate.ts";
import { applyStreamOptionsPatch } from "../../hooks.ts";
import { insertEntry, insertUsage } from "../../session/commit.ts";
import { SessionInvariantError } from "../../session/session.ts";
import type {
Deferred,
MessageEntry,
NewEntry,
OperationError,
RunState,
SettledAssistantMessage,
ToolCall,
UsageRow,
} from "../../session/types.ts";
import {
branchTip,
deleteList,
operationState as operationStateValue,
pendingAssistantFrames,
setValue,
} from "../../session/values.ts";
type LaneConfiguration,
type OperationError,
type OperationState,
type RunDeferredEffectPendingOperation,
type RunDeferredSuspendedOperation,
type RunScope,
runScopeOf,
type SettledAssistantMessage,
} from "../../session/types.ts";
import { deleteList, pendingAssistantFrames } from "../../session/values.ts";
import type { AgentHarnessStreamOptions } from "../../types.ts";
import type { Lane } from "../lane.ts";
import { openFrameProgress } from "../progress.ts";
import type { Drive, LaneCommand, LostOwnership, ProcedureResult } from "../types.ts";
import type { Drive, ProcedureResult } from "../types.ts";
import { settleResponse } from "./response.ts";
type LocalResult = { kind: "committed" } | { kind: "state_changed" } | LostOwnership;
type SourceResult = { kind: "value"; handle: DeferredHandle } | { kind: "state_changed" } | LostOwnership;
function sameDeferred(current: Deferred, expected: Deferred): boolean {
if (
current.status !== expected.status ||
current.stepId !== expected.stepId ||
current.sourceEntryId !== expected.sourceEntryId ||
current.poll !== expected.poll
) {
return false;
}
return (
current.status === "suspended" ||
(expected.status === "effect_pending" &&
current.responseEntryId === expected.responseEntryId &&
current.usageId === expected.usageId)
);
}
function configurationError(identity: Deferred["configuration"]["model"]): OperationError {
type DeferredLeaf = RunDeferredSuspendedOperation | RunDeferredEffectPendingOperation;
/** Deferred effect payload without the run-wide scope fields. */
type EffectPendingFields = Omit<RunDeferredEffectPendingOperation, keyof RunScope>;
function configurationError(identity: LaneConfiguration["model"]): OperationError {
return {
code: "model_unavailable",
message: "The configured model is unavailable in this process",
@ -61,31 +37,6 @@ function configurationError(identity: Deferred["configuration"]["model"]): Opera
};
}
function providerError(message: SettledAssistantMessage): OperationError {
return {
code: "assistant_error",
message: message.errorMessage ?? `Deferred request ended with ${message.stopReason}`,
};
}
function normalizeError(message: SettledAssistantMessage, errorMessage: string): SettledAssistantMessage {
return { ...message, stopReason: "error", errorMessage };
}
function normalizeAborted(message: SettledAssistantMessage): SettledAssistantMessage {
return {
...message,
stopReason: "aborted",
errorMessage: message.errorMessage ?? "Deferred request was cancelled",
};
}
function uuidV7Timestamp(id: string): number {
const timestamp = Number.parseInt(id.slice(0, 8) + id.slice(9, 13), 16);
if (!Number.isSafeInteger(timestamp)) throw new SessionInvariantError(`Invalid reserved UUIDv7 ${id}`);
return timestamp;
}
function isUpdateEvent(
event: AssistantMessageEvent,
): event is Exclude<AssistantMessageEvent, { type: "start" | "done" | "error" }> {
@ -95,23 +46,12 @@ function isUpdateEvent(
async function readSourceHandle<TContext extends object | undefined>(
lane: Lane<TContext>,
drive: Drive,
expected: Deferred,
): Promise<SourceResult> {
return lane.commandDriveOwned<SourceResult>(
drive,
async (state, reader): Promise<LaneCommand<SourceResult>> => {
const operation = state.operation;
if (operation === null || operation.state.kind !== "run") {
throw new SessionInvariantError("Deferred source reached a non-run operation");
}
const run = operation.state;
if (run.control.status === "cancel_requested") {
return { kind: "return", result: { kind: "state_changed" } as const };
}
if (run.phase.kind !== "deferred" || !sameDeferred(run.phase.deferred, expected)) {
throw new SessionInvariantError("Deferred source lost its durable boundary");
}
const source = (await reader.getEntries([expected.sourceEntryId], drive.context)).get(expected.sourceEntryId);
deferred: DeferredLeaf,
): Promise<DeferredHandle | undefined> {
return lane.advanceOperation(
deferred,
async (_state, _current, _meta, reader) => {
const source = (await reader.getEntries([deferred.sourceEntryId], drive.context)).get(deferred.sourceEntryId);
if (
source?.type !== "message" ||
source.message.role !== "assistant" ||
@ -119,105 +59,78 @@ async function readSourceHandle<TContext extends object | undefined>(
source.message.deferred === undefined
) {
throw new SessionInvariantError(
`Deferred source ${expected.sourceEntryId} is missing its assistant handle`,
`Deferred source ${deferred.sourceEntryId} is missing its assistant handle`,
);
}
const handle = source.message.deferred;
const identity = expected.configuration.model;
const identity = deferred.configuration.model;
if (
handle.id.length === 0 ||
handle.provider !== identity.provider ||
handle.modelId !== identity.modelId ||
handle.api !== source.message.api
) {
throw new SessionInvariantError(`Deferred source ${expected.sourceEntryId} has an invalid handle`);
throw new SessionInvariantError(`Deferred source ${deferred.sourceEntryId} has an invalid handle`);
}
return { kind: "return", result: { kind: "value", handle } as const };
return { kind: "return", result: handle };
},
drive.context,
);
}
function enterDeferredConfigurationFailure<TContext extends object | undefined>(
async function enterDeferredConfigurationFailure<TContext extends object | undefined>(
lane: Lane<TContext>,
drive: Drive,
expected: Deferred,
deferred: DeferredLeaf,
): Promise<ProcedureResult> {
return lane.commandDriveOwned<ProcedureResult>(
drive,
(state) => {
const operation = state.operation;
if (operation === null || operation.state.kind !== "run") {
throw new SessionInvariantError("Deferred configuration failure reached a non-run operation");
}
const run = operation.state;
if (run.control.status === "cancel_requested") {
return { kind: "return", result: { kind: "continue" } as const };
}
if (run.phase.kind !== "deferred" || !sameDeferred(run.phase.deferred, expected)) {
throw new SessionInvariantError("Deferred configuration failure lost its durable boundary");
}
const nextState: RunState = {
...run,
phase: {
kind: "failure_drain",
error: configurationError(expected.configuration.model),
provenance: { kind: "configuration" },
},
const result = await lane.advanceOperation(
deferred,
(_state, current) => {
const nextState: OperationState = {
...runScopeOf(current),
at: "run.failure_drain",
error: configurationError(deferred.configuration.model),
provenance: { kind: "configuration" },
};
return {
kind: "commit",
writes: [
...(expected.status === "effect_pending"
? [deleteList(pendingAssistantFrames(drive.operationId, expected.responseEntryId))]
: []),
setValue(operationStateValue(drive.operationId), nextState),
],
next: { ...state, operation: { meta: operation.meta, state: nextState } },
writes:
deferred.at === "run.deferred.effect_pending"
? [deleteList(pendingAssistantFrames(drive.operationId, deferred.responseEntryId))]
: [],
operationState: nextState,
materialize: () => ({ kind: "continue" }) as const,
};
},
drive.context,
);
return result ?? { kind: "continue" };
}
async function commitPollIntent<TContext extends object | undefined>(
lane: Lane<TContext>,
drive: Drive,
expected: Deferred,
next: Extract<Deferred, { status: "effect_pending" }>,
deferred: DeferredLeaf,
next: EffectPendingFields,
recovery: boolean,
): Promise<LocalResult> {
return lane.commandDriveOwned<LocalResult>(
drive,
(state) => {
const operation = state.operation;
if (operation === null || operation.state.kind !== "run") {
throw new SessionInvariantError("Deferred poll intent reached a non-run operation");
}
const run = operation.state;
if (run.control.status === "cancel_requested") {
return { kind: "return", result: { kind: "state_changed" } as const };
}
): Promise<RunDeferredEffectPendingOperation | undefined> {
return lane.advanceOperation(
deferred,
(_state, current) => {
if (drive.deferredPermits === 0) {
throw new SessionInvariantError("Deferred poll intent lost its pass-local permit");
}
if (run.phase.kind !== "deferred" || !sameDeferred(run.phase.deferred, expected)) {
throw new SessionInvariantError("Deferred poll intent lost its durable boundary");
}
const nextState: RunState = { ...run, phase: { kind: "deferred", deferred: next } };
const nextState: RunDeferredEffectPendingOperation = { ...runScopeOf(current), ...next };
return {
kind: "commit",
writes: [
...(expected.status === "effect_pending"
? [deleteList(pendingAssistantFrames(drive.operationId, expected.responseEntryId))]
: []),
setValue(operationStateValue(drive.operationId), nextState),
],
next: { ...state, operation: { meta: operation.meta, state: nextState } },
writes:
deferred.at === "run.deferred.effect_pending"
? [deleteList(pendingAssistantFrames(drive.operationId, deferred.responseEntryId))]
: [],
operationState: nextState,
materialize: () => {
drive.deferredPermits--;
return { kind: "committed" } as const;
return nextState;
},
events: () => [
{
@ -355,180 +268,14 @@ async function streamDeferredResponse<TContext extends object | undefined>(
return final;
}
async function settleDeferredResponse<TContext extends object | undefined>(
lane: Lane<TContext>,
drive: Drive,
expected: Extract<Deferred, { status: "effect_pending" }>,
response: SettledAssistantMessage,
recovery: boolean,
): Promise<ProcedureResult> {
return lane.commandDriveOwned<ProcedureResult>(
drive,
(state) => {
const operation = state.operation;
if (operation === null || operation.state.kind !== "run") {
throw new SessionInvariantError("Deferred settlement reached a non-run operation");
}
const run = operation.state;
if (run.phase.kind !== "deferred" || !sameDeferred(run.phase.deferred, expected)) {
throw new SessionInvariantError("Deferred settlement lost its effect-pending boundary");
}
let committed = response;
let nextPhase: RunState["phase"];
if (run.control.status === "cancel_requested") {
committed = normalizeAborted(response);
nextPhase = {
kind: "checkpoint",
continuation: { kind: "may_finish", includeFinalAssistant: true },
triggerEntryId: expected.responseEntryId,
};
} else if (response.stopReason === "aborted") {
throw new SessionInvariantError("Deferred response is aborted while durable control is running");
} else if (response.stopReason === "deferred") {
nextPhase = {
kind: "deferred",
deferred: {
status: "suspended",
stepId: expected.stepId,
sourceEntryId: expected.responseEntryId,
poll: expected.poll,
configuration: expected.configuration,
streamOptions: expected.streamOptions,
},
};
} else if (response.stopReason === "error") {
nextPhase = {
kind: "failure_drain",
error: providerError(response),
provenance: { kind: "response", entryId: expected.responseEntryId },
};
} else {
const calls = response.content.flatMap((content, sourceIndex) =>
content.type === "toolCall" ? [{ content, sourceIndex }] : [],
);
if (calls.length !== 0) {
const timestamp = uuidV7Timestamp(expected.responseEntryId);
const planned: ToolCall[] = calls.map(({ sourceIndex }) => ({
status: "planned",
sourceIndex,
resultEntryId: lane.session.idGenerator.next(timestamp),
}));
nextPhase = {
kind: "tools",
batch: {
assistantEntryId: expected.responseEntryId,
configuration: expected.configuration,
turnId: `${expected.stepId}:poll:${expected.poll}`,
calls: planned,
},
};
} else if (response.stopReason === "toolUse") {
committed = normalizeError(response, "Provider reported tool use without any tool calls");
nextPhase = {
kind: "failure_drain",
error: providerError(committed),
provenance: { kind: "response", entryId: expected.responseEntryId },
};
} else {
nextPhase = {
kind: "checkpoint",
continuation: { kind: "may_finish", includeFinalAssistant: true },
triggerEntryId: expected.responseEntryId,
};
}
}
const nextState: RunState = {
...run,
phase: nextPhase,
latestAssistantEntryId: expected.responseEntryId,
};
const responseEntry: NewEntry<MessageEntry> = {
id: expected.responseEntryId,
parentId: state.tipId,
type: "message",
message: committed,
};
const usageRow: Omit<UsageRow, "seq"> = {
id: expected.usageId,
usage: committed.usage,
entryId: expected.responseEntryId,
adjustment: false,
};
return {
kind: "commit",
writes: [
insertEntry(responseEntry),
insertUsage(usageRow),
setValue(branchTip(lane.name), expected.responseEntryId),
deleteList(pendingAssistantFrames(drive.operationId, expected.responseEntryId)),
setValue(operationStateValue(drive.operationId), nextState),
],
next: {
...state,
tipId: expected.responseEntryId,
operation: { meta: operation.meta, state: nextState },
},
materialize: () => ({ kind: "continue" }) as const,
events: (commit) => {
const entry: MessageEntry = {
...responseEntry,
seq: commit.seqs[0]!,
timestamp: commit.timestamp,
};
const events: HarnessEvent[] = [
{ type: "entry_added", lane: lane.name, entry, ...(recovery ? { recovery: true } : {}) },
{
type: "usage",
lane: lane.name,
row: { ...usageRow, seq: commit.seqs[1]! },
totals: commit.stats.usage,
},
];
if (nextPhase.kind !== "tools") {
events.push({
type: "turn_end",
lane: lane.name,
runId: drive.operationId,
turnId: `${expected.stepId}:poll:${expected.poll}`,
message: committed,
toolResults: [],
...(recovery ? { recovery: true } : {}),
});
}
if (nextPhase.kind === "deferred" && committed.deferred !== undefined) {
events.push({
type: "run_suspend",
lane: lane.name,
runId: drive.operationId,
reason: "deferred",
deferred: committed.deferred,
...(recovery ? { recovery: true } : {}),
});
}
return events;
},
};
},
drive.context,
);
}
async function pollDeferred<TContext extends object | undefined>(
lane: Lane<TContext>,
drive: Drive,
run: RunState,
expected: Deferred,
expected: DeferredLeaf,
recovery: boolean,
): Promise<ProcedureResult> {
if (run.phase.kind !== "deferred" || !sameDeferred(run.phase.deferred, expected)) {
throw new SessionInvariantError("Deferred procedure requires its current durable phase");
}
if (run.control.status === "cancel_requested") return { kind: "continue" };
const source = await readSourceHandle(lane, drive, expected);
if (source.kind === "lost_ownership") return source;
if (source.kind === "state_changed") return { kind: "continue" };
if (source === undefined) return { kind: "continue" };
if (drive.deferredPermits === 0) {
return {
kind: "waiting",
@ -536,7 +283,7 @@ async function pollDeferred<TContext extends object | undefined>(
kind: "waiting",
operationId: drive.operationId,
reason: "deferred",
deferred: source.handle,
deferred: source,
},
};
}
@ -552,7 +299,7 @@ async function pollDeferred<TContext extends object | undefined>(
runId: drive.operationId,
model,
step: "deferred",
attempt: expected.status === "suspended" ? expected.poll + 1 : expected.poll,
attempt: expected.at === "run.deferred.suspended" ? expected.poll + 1 : expected.poll,
streamOptions: baseOptions,
},
drive.gate,
@ -564,12 +311,12 @@ async function pollDeferred<TContext extends object | undefined>(
: applyStreamOptionsPatch(baseOptions, beforeRequest.streamOptions)),
deferred: false,
};
const poll = expected.status === "suspended" ? expected.poll + 1 : expected.poll;
const poll = expected.at === "run.deferred.suspended" ? expected.poll + 1 : expected.poll;
const at = Date.now();
const responseEntryId = lane.session.idGenerator.next(at);
const usageId = lane.session.idGenerator.next(at);
const effectPending: Extract<Deferred, { status: "effect_pending" }> = {
status: "effect_pending",
const effectPending: EffectPendingFields = {
at: "run.deferred.effect_pending",
stepId: expected.stepId,
sourceEntryId: expected.sourceEntryId,
poll,
@ -579,49 +326,45 @@ async function pollDeferred<TContext extends object | undefined>(
streamOptions: expected.streamOptions,
};
const intent = await commitPollIntent(lane, drive, expected, effectPending, recovery);
if (intent.kind === "lost_ownership") return intent;
if (intent.kind === "state_changed") return { kind: "continue" };
if (intent === undefined) return { kind: "continue" };
const response = await streamDeferredResponse(
lane,
drive,
model,
source.handle,
source,
streamOptions,
responseEntryId,
intent.responseEntryId,
recovery,
);
return settleDeferredResponse(lane, drive, effectPending, response, recovery);
return settleResponse(lane, drive, intent, response, recovery ? { recovery: true } : {});
}
/** Poll one durably suspended deferred response when this pass carries a permit. */
export function runDeferredSuspended<TContext extends object | undefined>(
lane: Lane<TContext>,
drive: Drive,
run: RunState,
deferred: Extract<Deferred, { status: "suspended" }>,
deferred: RunDeferredSuspendedOperation,
): Promise<ProcedureResult> {
return pollDeferred(lane, drive, run, deferred, false);
return pollDeferred(lane, drive, deferred, false);
}
/** Replace one orphaned unknown-outcome poll under fresh ids when this pass carries a permit. */
export function recoverDeferredPoll<TContext extends object | undefined>(
lane: Lane<TContext>,
drive: Drive,
run: RunState,
deferred: Extract<Deferred, { status: "effect_pending" }>,
deferred: RunDeferredEffectPendingOperation,
): Promise<ProcedureResult> {
return pollDeferred(lane, drive, run, deferred, true);
return pollDeferred(lane, drive, deferred, true);
}
/** Advance or report the wait for one deferred run phase. */
export function runDeferred<TContext extends object | undefined>(
lane: Lane<TContext>,
drive: Drive,
run: RunState,
deferred: Deferred,
deferred: RunDeferredSuspendedOperation | RunDeferredEffectPendingOperation,
): Promise<ProcedureResult> {
return deferred.status === "suspended"
? runDeferredSuspended(lane, drive, run, deferred)
: recoverDeferredPoll(lane, drive, run, deferred);
return deferred.at === "run.deferred.suspended"
? runDeferredSuspended(lane, drive, deferred)
: recoverDeferredPoll(lane, drive, deferred);
}

551
packages/agent/src/harness/runtime/drive/generation.ts
View file

@ -2,44 +2,33 @@ import {
type AssistantMessage,
type AssistantMessageEvent,
AssistantMessageFrameEncoder,
isRetryableAssistantError,
type Tool,
} from "@earendil-works/pi-ai";
import type { AgentMessage } from "../../../types.ts";
import type { HarnessEvent } from "../../agent-harness.ts";
import { type Context, withAbortSignal } from "../../context.ts";
import { streamHarnessAssistant } from "../../execution/assistant.ts";
import { applyStreamOptionsPatch } from "../../hooks.ts";
import { insertEntry, insertUsage } from "../../session/commit.ts";
import { buildSessionContext } from "../../session/context.ts";
import { SessionInvariantError } from "../../session/session.ts";
import type {
Entry,
Generation,
GenerationContext,
JsonValue,
MessageEntry,
NewEntry,
OperationError,
RunState,
SettledAssistantMessage,
ToolCall,
UsageRow,
} from "../../session/types.ts";
import {
branchTip,
deleteList,
operationState as operationStateValue,
pendingAssistantFrames,
setValue,
} from "../../session/values.ts";
type JsonValue,
type OperationError,
type OperationState,
type RunAssistantEffectPendingOperation,
type RunAssistantReadyOperation,
type RunAssistantRetryWaitOperation,
type RunScope,
runScopeOf,
type SettledAssistantMessage,
} from "../../session/types.ts";
import type { AgentHarnessStreamOptions } from "../../types.ts";
import type { Lane } from "../lane.ts";
import { openFrameProgress } from "../progress.ts";
import type { Drive, LaneCommand, LostOwnership, ProcedureResult } from "../types.ts";
import type { Drive, ProcedureResult } from "../types.ts";
import { settleResponse } from "./response.ts";
type LocalResult = { kind: "committed" } | { kind: "state_changed" } | LostOwnership;
type ReadResult<T> = { kind: "value"; value: T } | { kind: "state_changed" } | LostOwnership;
/** Assistant effect-pending payload without the run-wide scope fields. */
type AssistantEffectPending = Omit<RunAssistantEffectPendingOperation, keyof RunScope>;
const ZERO_USAGE = {
input: 0,
@ -50,20 +39,6 @@ const ZERO_USAGE = {
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
};
function sameReadyGeneration(current: Generation, expected: Extract<Generation, { status: "ready" }>): boolean {
return (
current.status === "ready" &&
current.context.stepId === expected.context.stepId &&
current.nextAttempt === expected.nextAttempt
);
}
function uuidV7Timestamp(id: string): number {
const timestamp = Number.parseInt(id.slice(0, 8) + id.slice(9, 13), 16);
if (!Number.isSafeInteger(timestamp)) throw new SessionInvariantError(`Invalid reserved UUIDv7 ${id}`);
return timestamp;
}
function configurationError(
code: "model_unavailable" | "configured_tools_unavailable",
details: JsonValue,
@ -78,35 +53,6 @@ function configurationError(
};
}
function providerError(message: SettledAssistantMessage): OperationError {
return {
code: "assistant_error",
message: message.errorMessage ?? `Assistant request ended with ${message.stopReason}`,
};
}
function normalizeError(message: SettledAssistantMessage, errorMessage: string): SettledAssistantMessage {
return { ...message, stopReason: "error", errorMessage };
}
function normalizeAborted(message: SettledAssistantMessage): SettledAssistantMessage {
return {
...message,
stopReason: "aborted",
errorMessage: message.errorMessage ?? "Assistant request was cancelled",
};
}
function retryDelay(baseDelayMs: number, attempt: number): number {
const delay = baseDelayMs * 2 ** Math.max(0, attempt - 1);
return Number.isSafeInteger(delay) ? delay : Number.MAX_SAFE_INTEGER;
}
function saturatingAdd(left: number, right: number): number {
const sum = left + right;
return Number.isSafeInteger(sum) ? sum : Number.MAX_SAFE_INTEGER;
}
function waitUntil(notBefore: number, signal: AbortSignal): Promise<void> {
return new Promise<void>((resolve, reject) => {
let timer: ReturnType<typeof setTimeout> | undefined;
@ -133,30 +79,6 @@ function waitUntil(notBefore: number, signal: AbortSignal): Promise<void> {
});
}
function sameRetryWait(current: Generation, expected: Extract<Generation, { status: "retry_wait" }>): boolean {
return (
current.status === "retry_wait" &&
current.context.stepId === expected.context.stepId &&
current.nextAttempt === expected.nextAttempt &&
current.notBefore === expected.notBefore
);
}
function deferredHandleIsValid(
message: SettledAssistantMessage,
identity: GenerationContext["configuration"]["model"],
): boolean {
const handle = message.deferred;
return (
message.stopReason === "deferred" &&
handle !== undefined &&
handle.id.length !== 0 &&
handle.provider === identity.provider &&
handle.modelId === identity.modelId &&
handle.api === message.api
);
}
async function resolveSystemPrompt<TContext extends object | undefined>(
lane: Lane<TContext>,
context: Context,
@ -172,24 +94,11 @@ async function resolveSystemPrompt<TContext extends object | undefined>(
async function readRequestMessages<TContext extends object | undefined>(
lane: Lane<TContext>,
drive: Drive,
generation: Extract<Generation, { status: "ready" }>,
): Promise<ReadResult<AgentMessage[]>> {
const path = await lane.commandDriveOwned<ReadResult<Entry[]>>(
drive,
async (state, reader): Promise<LaneCommand<ReadResult<Entry[]>>> => {
const operation = state.operation;
if (operation === null || operation.state.kind !== "run") {
throw new SessionInvariantError("Generation context reached a non-run operation");
}
if (operation.state.control.status === "cancel_requested") {
return { kind: "return", result: { kind: "state_changed" } as const };
}
if (
operation.state.phase.kind !== "assistant" ||
!sameReadyGeneration(operation.state.phase.generation, generation)
) {
throw new SessionInvariantError("Generation context lost its ready boundary");
}
generation: RunAssistantReadyOperation,
): Promise<AgentMessage[] | undefined> {
const path = await lane.advanceOperation(
generation,
async (state, _current, _meta, reader) => {
if (state.tipId === null) throw new SessionInvariantError("Assistant generation has no Branch tip");
const entries = (
await reader.scanBranch(
@ -197,55 +106,38 @@ async function readRequestMessages<TContext extends object | undefined>(
drive.context,
)
).reverse();
return { kind: "return", result: { kind: "value", value: entries } as const };
return { kind: "return", result: entries };
},
drive.context,
);
if (path.kind !== "value") return path;
return {
kind: "value",
value: await buildSessionContext(
path.value,
{ entryProjectors: lane.readConfig().entryProjectors },
drive.context,
),
};
return path === undefined
? undefined
: buildSessionContext(path, { entryProjectors: lane.readConfig().entryProjectors }, drive.context);
}
async function commitGenerationIntent<TContext extends object | undefined>(
lane: Lane<TContext>,
drive: Drive,
expected: Extract<Generation, { status: "ready" }>,
generation: Extract<Generation, { status: "effect_pending" }>,
): Promise<LocalResult> {
return lane.commandDriveOwned<LocalResult>(
drive,
(state) => {
const operation = state.operation;
if (operation === null || operation.state.kind !== "run") {
throw new SessionInvariantError("Generation intent reached a non-run operation");
}
const run = operation.state;
if (run.control.status === "cancel_requested") {
return { kind: "return", result: { kind: "state_changed" } as const };
}
if (run.phase.kind !== "assistant" || !sameReadyGeneration(run.phase.generation, expected)) {
throw new SessionInvariantError("Generation intent lost its ready boundary");
}
const nextState: RunState = { ...run, phase: { kind: "assistant", generation } };
ready: RunAssistantReadyOperation,
pending: AssistantEffectPending,
): Promise<RunAssistantEffectPendingOperation | undefined> {
return lane.advanceOperation(
ready,
(_state, current) => {
const nextState: RunAssistantEffectPendingOperation = { ...runScopeOf(current), ...pending };
return {
kind: "commit",
writes: [setValue(operationStateValue(drive.operationId), nextState)],
next: { ...state, operation: { meta: operation.meta, state: nextState } },
materialize: () => ({ kind: "committed" }) as const,
writes: [],
operationState: nextState,
materialize: () => nextState,
events: () =>
expected.nextAttempt === 1
ready.nextAttempt === 1
? [
{
type: "turn_start",
lane: lane.name,
runId: drive.operationId,
turnId: expected.context.stepId,
turnId: ready.generationContext.stepId,
},
]
: [],
@ -256,293 +148,39 @@ async function commitGenerationIntent<TContext extends object | undefined>(
}
/** Commit a non-retryable request-configuration failure before reserving response ids. */
export function enterConfigurationFailure<TContext extends object | undefined>(
export async function enterConfigurationFailure<TContext extends object | undefined>(
lane: Lane<TContext>,
drive: Drive,
expected: Extract<Generation, { status: "ready" }>,
ready: RunAssistantReadyOperation,
error: OperationError,
): Promise<ProcedureResult> {
return lane.commandDriveOwned<ProcedureResult>(
drive,
(state) => {
const operation = state.operation;
if (operation === null || operation.state.kind !== "run") {
throw new SessionInvariantError("Configuration failure reached a non-run operation");
}
const run = operation.state;
if (run.control.status === "cancel_requested") {
return { kind: "return", result: { kind: "continue" } as const };
}
if (run.phase.kind !== "assistant" || !sameReadyGeneration(run.phase.generation, expected)) {
throw new SessionInvariantError("Configuration failure lost its ready boundary");
}
const nextState: RunState = {
...run,
phase: { kind: "failure_drain", error, provenance: { kind: "configuration" } },
const result = await lane.advanceOperation(
ready,
(_state, current) => {
const nextState: OperationState = {
...runScopeOf(current),
at: "run.failure_drain",
error,
provenance: { kind: "configuration" },
};
return {
kind: "commit",
writes: [setValue(operationStateValue(drive.operationId), nextState)],
next: { ...state, operation: { meta: operation.meta, state: nextState } },
writes: [],
operationState: nextState,
materialize: () => ({ kind: "continue" }) as const,
};
},
drive.context,
);
}
interface SettlementIdentity {
responseEntryId: string;
usageId: string;
stepId: string;
}
/** Classify and atomically settle one complete assistant response under its reserved ids. */
export function settleAssistant<TContext extends object | undefined>(
lane: Lane<TContext>,
drive: Drive,
identity: SettlementIdentity,
response: SettledAssistantMessage,
options: { recovery?: true } = {},
): Promise<ProcedureResult> {
return lane.commandDriveOwned<ProcedureResult>(
drive,
(state) => {
const operation = state.operation;
if (operation === null || operation.state.kind !== "run") {
throw new SessionInvariantError("Assistant settlement reached a non-run operation");
}
const run = operation.state;
if (run.phase.kind !== "assistant" || run.phase.generation.status !== "effect_pending") {
throw new SessionInvariantError("Assistant settlement lost its effect-pending boundary");
}
const generation = run.phase.generation;
if (
generation.context.stepId !== identity.stepId ||
generation.responseEntryId !== identity.responseEntryId ||
generation.usageId !== identity.usageId
) {
throw new SessionInvariantError("Assistant settlement ids contradict the durable request intent");
}
const modelIdentity = generation.context.configuration.model;
let committed = response;
let nextPhase: RunState["phase"];
if (run.control.status === "cancel_requested") {
committed = normalizeAborted(response);
nextPhase = {
kind: "checkpoint",
continuation: { kind: "may_finish", includeFinalAssistant: true },
triggerEntryId: identity.responseEntryId,
};
} else if (response.stopReason === "aborted") {
throw new SessionInvariantError("Assistant response is aborted while durable control is running");
} else if (response.stopReason === "deferred") {
if (deferredHandleIsValid(response, modelIdentity)) {
nextPhase = {
kind: "deferred",
deferred: {
status: "suspended",
stepId: generation.context.stepId,
sourceEntryId: identity.responseEntryId,
poll: 0,
configuration: generation.context.configuration,
streamOptions: generation.context.streamOptions,
},
};
} else {
committed = normalizeError(response, "Provider returned an invalid deferred handle");
nextPhase = {
kind: "failure_drain",
error: providerError(committed),
provenance: { kind: "response", entryId: identity.responseEntryId },
};
}
} else if (
response.stopReason === "error" &&
(options.recovery === true || isRetryableAssistantError(response)) &&
generation.attempt < generation.context.retryPolicy.maxAttempts
) {
const delayMs = retryDelay(generation.context.retryPolicy.baseDelayMs, generation.attempt);
nextPhase = {
kind: "assistant",
generation: {
status: "retry_wait",
context: generation.context,
nextAttempt: generation.attempt + 1,
notBefore: saturatingAdd(Date.now(), delayMs),
errorMessage: response.errorMessage ?? "Assistant request failed",
},
};
} else if (response.stopReason === "error") {
nextPhase = {
kind: "failure_drain",
error: providerError(response),
provenance: { kind: "response", entryId: identity.responseEntryId },
};
} else {
const calls = response.content.flatMap((content, sourceIndex) =>
content.type === "toolCall" ? [{ content, sourceIndex }] : [],
);
if (calls.length !== 0) {
const timestamp = uuidV7Timestamp(identity.responseEntryId);
const planned: ToolCall[] = calls.map(({ sourceIndex }) => ({
status: "planned",
sourceIndex,
resultEntryId: lane.session.idGenerator.next(timestamp),
}));
nextPhase = {
kind: "tools",
batch: {
assistantEntryId: identity.responseEntryId,
configuration: generation.context.configuration,
turnId: generation.context.stepId,
calls: planned,
},
};
} else if (response.stopReason === "toolUse") {
committed = normalizeError(response, "Provider reported tool use without any tool calls");
nextPhase = {
kind: "failure_drain",
error: providerError(committed),
provenance: { kind: "response", entryId: identity.responseEntryId },
};
} else {
nextPhase = {
kind: "checkpoint",
continuation: { kind: "may_finish", includeFinalAssistant: true },
triggerEntryId: identity.responseEntryId,
};
}
}
const nextState: RunState = {
...run,
phase: nextPhase,
latestAssistantEntryId: identity.responseEntryId,
};
const responseEntry: NewEntry<MessageEntry> = {
id: identity.responseEntryId,
parentId: state.tipId,
type: "message",
message: committed,
};
const usageRow: Omit<UsageRow, "seq"> = {
id: identity.usageId,
usage: committed.usage,
entryId: identity.responseEntryId,
adjustment: false,
};
return {
kind: "commit",
writes: [
insertEntry(responseEntry),
insertUsage(usageRow),
setValue(branchTip(lane.name), identity.responseEntryId),
deleteList(pendingAssistantFrames(drive.operationId, identity.responseEntryId)),
setValue(operationStateValue(drive.operationId), nextState),
],
next: {
...state,
tipId: identity.responseEntryId,
operation: { meta: operation.meta, state: nextState },
},
materialize: () => ({ kind: "continue" }) as const,
events: (commit) => {
const entry: MessageEntry = {
...responseEntry,
seq: commit.seqs[0]!,
timestamp: commit.timestamp,
};
const events: HarnessEvent[] = [
{ type: "entry_added", lane: lane.name, entry, ...options },
{
type: "usage",
lane: lane.name,
row: { ...usageRow, seq: commit.seqs[1]! },
totals: commit.stats.usage,
},
];
if (
options.recovery !== true &&
generation.attempt > 1 &&
!(nextPhase.kind === "assistant" && nextPhase.generation.status === "retry_wait")
) {
const success = committed.stopReason !== "error" && committed.stopReason !== "aborted";
events.push({
type: "retry_end",
lane: lane.name,
runId: drive.operationId,
step: identity.stepId,
attempt: generation.attempt,
success,
...(success
? {}
: {
finalError:
committed.errorMessage ?? `Assistant request ended with ${committed.stopReason}`,
}),
});
}
if (
options.recovery !== true &&
nextPhase.kind === "assistant" &&
nextPhase.generation.status === "retry_wait"
) {
events.push({
type: "retry_scheduled",
lane: lane.name,
runId: drive.operationId,
step: identity.stepId,
attempt: nextPhase.generation.nextAttempt,
maxAttempts: nextPhase.generation.context.retryPolicy.maxAttempts,
delayMs: retryDelay(generation.context.retryPolicy.baseDelayMs, generation.attempt),
errorMessage: nextPhase.generation.errorMessage,
});
}
if (
options.recovery !== true &&
nextPhase.kind !== "tools" &&
!(nextPhase.kind === "assistant" && nextPhase.generation.status === "retry_wait")
) {
events.push({
type: "turn_end",
lane: lane.name,
runId: drive.operationId,
turnId: identity.stepId,
message: committed,
toolResults: [],
...options,
});
}
if (options.recovery !== true && nextPhase.kind === "deferred" && committed.deferred !== undefined) {
events.push({
type: "run_suspend",
lane: lane.name,
runId: drive.operationId,
reason: "deferred",
deferred: committed.deferred,
});
}
return events;
},
};
},
drive.context,
);
return result ?? { kind: "continue" };
}
/** Advance one durable assistant retry wait according to this pass's local wait policy. */
export async function runRetryWait<TContext extends object | undefined>(
lane: Lane<TContext>,
drive: Drive,
run: RunState,
generation: Extract<Generation, { status: "retry_wait" }>,
generation: RunAssistantRetryWaitOperation,
): Promise<ProcedureResult> {
if (run.phase.kind !== "assistant" || !sameRetryWait(run.phase.generation, generation)) {
throw new SessionInvariantError("runRetryWait requires its current retry wait");
}
if (run.control.status === "cancel_requested") return { kind: "continue" };
if (Date.now() < generation.notBefore) {
if (!drive.waitForRetry) {
return {
@ -558,42 +196,26 @@ export async function runRetryWait<TContext extends object | undefined>(
await drive.gate.admit(() => waitUntil(generation.notBefore, drive.gate.signal));
}
return lane.commandDriveOwned<ProcedureResult>(
drive,
(state) => {
const operation = state.operation;
if (operation === null || operation.state.kind !== "run") {
throw new SessionInvariantError("Retry wait reached a non-run operation");
}
const current = operation.state;
if (current.control.status === "cancel_requested") {
return { kind: "return", result: { kind: "continue" } as const };
}
if (current.phase.kind !== "assistant" || !sameRetryWait(current.phase.generation, generation)) {
throw new SessionInvariantError("Retry wait lost its durable boundary");
}
const nextState: RunState = {
...current,
phase: {
kind: "assistant",
generation: {
status: "ready",
context: generation.context,
nextAttempt: generation.nextAttempt,
},
},
const result = await lane.advanceOperation(
generation,
(_state, current) => {
const nextState: RunAssistantReadyOperation = {
...runScopeOf(current),
at: "run.assistant.ready",
generationContext: generation.generationContext,
nextAttempt: generation.nextAttempt,
};
return {
kind: "commit",
writes: [setValue(operationStateValue(drive.operationId), nextState)],
next: { ...state, operation: { meta: operation.meta, state: nextState } },
writes: [],
operationState: nextState,
materialize: () => ({ kind: "continue" }) as const,
events: () => [
{
type: "retry_start",
lane: lane.name,
runId: drive.operationId,
step: generation.context.stepId,
step: generation.generationContext.stepId,
attempt: generation.nextAttempt,
},
],
@ -601,32 +223,27 @@ export async function runRetryWait<TContext extends object | undefined>(
},
drive.context,
);
return result ?? { kind: "continue" };
}
/** Execute one ready assistant generation or advance its durable retry wait. */
export async function runGeneration<TContext extends object | undefined>(
lane: Lane<TContext>,
drive: Drive,
run: RunState,
generation: Generation,
generation: RunAssistantReadyOperation | RunAssistantRetryWaitOperation,
): Promise<ProcedureResult> {
if (generation.status === "retry_wait") return runRetryWait(lane, drive, run, generation);
if (generation.status !== "ready") {
throw new SessionInvariantError("runGeneration requires a ready generation or retry wait");
}
if (run.phase.kind !== "assistant" || !sameReadyGeneration(run.phase.generation, generation)) {
throw new SessionInvariantError("runGeneration requires its current ready phase");
}
if (run.control.status === "cancel_requested") return { kind: "continue" };
if (generation.at === "run.assistant.retry_wait") return runRetryWait(lane, drive, generation);
const identity = generation.context.configuration.model;
const identity = generation.generationContext.configuration.model;
const model = lane.models.getModel(identity.provider, identity.modelId);
if (model === undefined) {
return enterConfigurationFailure(lane, drive, generation, configurationError("model_unavailable", identity));
}
const config = lane.readConfig();
const toolsByName = new Map(config.tools.map((tool) => [tool.name, tool]));
const missingTools = generation.context.configuration.activeToolNames.filter((name) => !toolsByName.has(name));
const missingTools = generation.generationContext.configuration.activeToolNames.filter(
(name) => !toolsByName.has(name),
);
if (missingTools.length !== 0) {
return enterConfigurationFailure(
lane,
@ -635,7 +252,7 @@ export async function runGeneration<TContext extends object | undefined>(
configurationError("configured_tools_unavailable", { tools: missingTools }),
);
}
const tools: Tool[] = generation.context.configuration.activeToolNames.map((name) => {
const tools: Tool[] = generation.generationContext.configuration.activeToolNames.map((name) => {
const tool = toolsByName.get(name);
if (tool === undefined) throw new SessionInvariantError(`Configured tool ${name} disappeared during resolution`);
return {
@ -647,8 +264,7 @@ export async function runGeneration<TContext extends object | undefined>(
});
const messages = await readRequestMessages(lane, drive, generation);
if (messages.kind === "lost_ownership") return messages;
if (messages.kind === "state_changed") return { kind: "continue" };
if (messages === undefined) return { kind: "continue" };
const systemPrompt = await resolveSystemPrompt(lane, drive.context);
const beforeRequest = await lane.hooks.runWithGate(
"before_request",
@ -658,21 +274,21 @@ export async function runGeneration<TContext extends object | undefined>(
model,
step: "assistant",
attempt: generation.nextAttempt,
streamOptions: generation.context.streamOptions,
streamOptions: generation.generationContext.streamOptions,
},
drive.gate,
drive.context,
);
const streamOptions: AgentHarnessStreamOptions =
beforeRequest?.streamOptions === undefined
? generation.context.streamOptions
: applyStreamOptionsPatch(generation.context.streamOptions, beforeRequest.streamOptions);
? generation.generationContext.streamOptions
: applyStreamOptionsPatch(generation.generationContext.streamOptions, beforeRequest.streamOptions);
const at = Date.now();
const responseEntryId = lane.session.idGenerator.next(at);
const usageId = lane.session.idGenerator.next(at);
const effectPending: Extract<Generation, { status: "effect_pending" }> = {
status: "effect_pending",
context: generation.context,
const effectPending: AssistantEffectPending = {
at: "run.assistant.effect_pending",
generationContext: generation.generationContext,
attempt: generation.nextAttempt,
responseEntryId,
usageId,
@ -680,10 +296,9 @@ export async function runGeneration<TContext extends object | undefined>(
contextWindow: model.contextWindow,
};
const intent = await commitGenerationIntent(lane, drive, generation, effectPending);
if (intent.kind === "lost_ownership") return intent;
if (intent.kind === "state_changed") return { kind: "continue" };
if (intent === undefined) return { kind: "continue" };
const progress = openFrameProgress(lane, drive, responseEntryId);
const progress = openFrameProgress(lane, drive, intent.responseEntryId);
const frameEncoder = new AssistantMessageFrameEncoder();
let drained = false;
const drainProgress = async () => {
@ -695,12 +310,12 @@ export async function runGeneration<TContext extends object | undefined>(
let response: SettledAssistantMessage;
try {
response = await streamHarnessAssistant(
messages.value,
messages,
{
model,
systemPrompt,
tools,
thinkingLevel: generation.context.configuration.thinkingLevel,
thinkingLevel: generation.generationContext.configuration.thinkingLevel,
streamOptions,
transformContext: async (requestContext, context) => {
const result = await lane.hooks.runWithGate(
@ -792,11 +407,11 @@ export async function runGeneration<TContext extends object | undefined>(
await drainProgress();
}
return settleAssistant(lane, drive, { responseEntryId, usageId, stepId: generation.context.stepId }, response);
return settleResponse(lane, drive, intent, response);
}
export function interruptedAssistantMessage(
generation: Extract<Generation, { status: "effect_pending" }>,
generation: RunAssistantEffectPendingOperation,
partial: AssistantMessage | undefined,
): SettledAssistantMessage {
const warning =
@ -806,8 +421,8 @@ export function interruptedAssistantMessage(
role: "assistant",
content: [],
api: "unknown",
provider: generation.context.configuration.model.provider,
model: generation.context.configuration.model.modelId,
provider: generation.generationContext.configuration.model.provider,
model: generation.generationContext.configuration.model.modelId,
usage: ZERO_USAGE,
stopReason: "error",
errorMessage: warning,

66
packages/agent/src/harness/runtime/drive/recovery.ts
View file

@ -1,39 +1,21 @@
import { type AssistantMessageFrame, reduceAssistantMessageFrames } from "@earendil-works/pi-ai";
import { SessionInvariantError } from "../../session/session.ts";
import type { Generation, RunState } from "../../session/types.ts";
import type { RunAssistantEffectPendingOperation } from "../../session/types.ts";
import { pendingAssistantFrames } from "../../session/values.ts";
import type { Lane } from "../lane.ts";
import type { Drive, ProcedureResult } from "../types.ts";
import { interruptedAssistantMessage, settleAssistant } from "./generation.ts";
import { interruptedAssistantMessage } from "./generation.ts";
import { settleResponse } from "./response.ts";
/** Settle an orphaned assistant request from its bounded committed frame prefix without another provider call. */
export async function recoverAssistantGeneration<TContext extends object | undefined>(
lane: Lane<TContext>,
drive: Drive,
run: RunState,
generation: Extract<Generation, { status: "effect_pending" }>,
generation: RunAssistantEffectPendingOperation,
): Promise<ProcedureResult> {
if (
run.phase.kind !== "assistant" ||
run.phase.generation.status !== "effect_pending" ||
run.phase.generation.responseEntryId !== generation.responseEntryId
) {
throw new SessionInvariantError("Assistant recovery requires its current effect-pending generation");
}
const read = await lane.commandDriveOwned<{ kind: "frames"; frames: AssistantMessageFrame[] }>(
drive,
async (state, reader) => {
const operation = state.operation;
if (
operation === null ||
operation.state.kind !== "run" ||
operation.state.phase.kind !== "assistant" ||
operation.state.phase.generation.status !== "effect_pending" ||
operation.state.phase.generation.responseEntryId !== generation.responseEntryId
) {
throw new SessionInvariantError("Assistant recovery lost its effect-pending generation");
}
const frames: AssistantMessageFrame[] = [];
const frames = await lane.advanceOperation(
generation,
async (_state, _current, _meta, reader) => {
const result: AssistantMessageFrame[] = [];
let cursor: { seq: number } | undefined;
for (;;) {
const page = await reader.readList(
@ -41,28 +23,16 @@ export async function recoverAssistantGeneration<TContext extends object | undef
{ order: "asc", limit: 1_000, ...(cursor === undefined ? {} : { cursor }) },
drive.context,
);
frames.push(...page.map(({ value }) => value));
result.push(...page.map(({ value }) => value));
if (page.length < 1_000) break;
cursor = { seq: page[page.length - 1]!.seq };
}
return { kind: "return", result: { kind: "frames", frames } as const };
return { kind: "return", result };
},
drive.context,
);
if (read.kind === "lost_ownership") return read;
const frames = read.frames;
const current = lane.state.operation;
if (current?.meta.operationId !== drive.operationId || !lane.isDriveActive(drive)) {
return { kind: "lost_ownership" };
}
if (
current.state.kind !== "run" ||
current.state.phase.kind !== "assistant" ||
current.state.phase.generation.status !== "effect_pending" ||
current.state.phase.generation.responseEntryId !== generation.responseEntryId
) {
throw new SessionInvariantError("Assistant recovery lost its effect-pending generation");
}
if (frames === undefined) return { kind: "continue" };
const message = interruptedAssistantMessage(generation, reduceAssistantMessageFrames(frames));
await lane.emitBatch(
[
@ -84,15 +54,5 @@ export async function recoverAssistantGeneration<TContext extends object | undef
],
drive.context,
);
return settleAssistant(
lane,
drive,
{
responseEntryId: generation.responseEntryId,
usageId: generation.usageId,
stepId: generation.context.stepId,
},
message,
{ recovery: true },
);
return settleResponse(lane, drive, generation, message, { recovery: true });
}

316
packages/agent/src/harness/runtime/drive/response.ts Normal file
View file

@ -0,0 +1,316 @@
import { isRetryableAssistantError } from "@earendil-works/pi-ai";
import type { HarnessEvent } from "../../agent-harness.ts";
import { insertEntry, insertUsage } from "../../session/commit.ts";
import { SessionInvariantError } from "../../session/session.ts";
import {
type MessageEntry,
type NewEntry,
type OperationError,
type OperationState,
type RunAssistantEffectPendingOperation,
type RunDeferredEffectPendingOperation,
runScopeOf,
type SettledAssistantMessage,
type ToolCall,
type UsageRow,
} from "../../session/types.ts";
import { branchTip, deleteList, pendingAssistantFrames, setValue } from "../../session/values.ts";
import type { Lane } from "../lane.ts";
import type { Drive, ProcedureResult } from "../types.ts";
type ResponseIntent = RunAssistantEffectPendingOperation | RunDeferredEffectPendingOperation;
function uuidV7Timestamp(id: string): number {
const timestamp = Number.parseInt(id.slice(0, 8) + id.slice(9, 13), 16);
if (!Number.isSafeInteger(timestamp)) throw new SessionInvariantError(`Invalid reserved UUIDv7 ${id}`);
return timestamp;
}
function providerError(source: "assistant" | "deferred", message: SettledAssistantMessage): OperationError {
return {
code: "assistant_error",
message:
message.errorMessage ??
`${source === "assistant" ? "Assistant" : "Deferred"} request ended with ${message.stopReason}`,
};
}
function normalizeError(message: SettledAssistantMessage, errorMessage: string): SettledAssistantMessage {
return { ...message, stopReason: "error", errorMessage };
}
function normalizeAborted(source: "assistant" | "deferred", message: SettledAssistantMessage): SettledAssistantMessage {
return {
...message,
stopReason: "aborted",
errorMessage:
message.errorMessage ?? `${source === "assistant" ? "Assistant" : "Deferred"} request was cancelled`,
};
}
function retryDelay(baseDelayMs: number, attempt: number): number {
const delay = baseDelayMs * 2 ** Math.max(0, attempt - 1);
return Number.isSafeInteger(delay) ? delay : Number.MAX_SAFE_INTEGER;
}
function saturatingAdd(left: number, right: number): number {
const sum = left + right;
return Number.isSafeInteger(sum) ? sum : Number.MAX_SAFE_INTEGER;
}
function deferredHandleIsValid(
message: SettledAssistantMessage,
generation: RunAssistantEffectPendingOperation,
): boolean {
const handle = message.deferred;
const identity = generation.generationContext.configuration.model;
return (
message.stopReason === "deferred" &&
handle !== undefined &&
handle.id.length !== 0 &&
handle.provider === identity.provider &&
handle.modelId === identity.modelId &&
handle.api === message.api
);
}
/** Classify and atomically settle one assistant-generation or deferred-poll response. */
export function settleResponse<TContext extends object | undefined>(
lane: Lane<TContext>,
drive: Drive,
intent: ResponseIntent,
response: SettledAssistantMessage,
options: { recovery?: true } = {},
): Promise<ProcedureResult> {
return lane.mutateOperation(
intent,
(state, current) => {
const source = current.at === "run.assistant.effect_pending" ? "assistant" : "deferred";
const responseEntryId = intent.responseEntryId;
const configuration =
current.at === "run.assistant.effect_pending"
? current.generationContext.configuration
: current.configuration;
const turnId =
current.at === "run.assistant.effect_pending"
? current.generationContext.stepId
: `${current.stepId}:poll:${current.poll}`;
const scope = { ...runScopeOf(current), latestAssistantEntryId: responseEntryId };
let committed = response;
let settled: OperationState;
if (current.control.status === "cancel_requested") {
committed = normalizeAborted(source, response);
settled = {
...scope,
at: "run.checkpoint",
continuation: { kind: "may_finish", includeFinalAssistant: true },
triggerEntryId: responseEntryId,
};
} else if (response.stopReason === "aborted") {
throw new SessionInvariantError(
`${source === "assistant" ? "Assistant" : "Deferred"} response is aborted while durable control is running`,
);
} else if (response.stopReason === "deferred") {
if (current.at === "run.assistant.effect_pending") {
if (deferredHandleIsValid(response, current)) {
settled = {
...scope,
at: "run.deferred.suspended",
stepId: current.generationContext.stepId,
sourceEntryId: responseEntryId,
poll: 0,
configuration,
streamOptions: current.generationContext.streamOptions,
};
} else {
committed = normalizeError(response, "Provider returned an invalid deferred handle");
settled = {
...scope,
at: "run.failure_drain",
error: providerError(source, committed),
provenance: { kind: "response", entryId: responseEntryId },
};
}
} else {
settled = {
...scope,
at: "run.deferred.suspended",
stepId: current.stepId,
sourceEntryId: responseEntryId,
poll: current.poll,
configuration,
streamOptions: current.streamOptions,
};
}
} else if (response.stopReason === "error") {
if (
current.at === "run.assistant.effect_pending" &&
(options.recovery === true || isRetryableAssistantError(response)) &&
current.attempt < current.generationContext.retryPolicy.maxAttempts
) {
const delayMs = retryDelay(current.generationContext.retryPolicy.baseDelayMs, current.attempt);
settled = {
...scope,
at: "run.assistant.retry_wait",
generationContext: current.generationContext,
nextAttempt: current.attempt + 1,
notBefore: saturatingAdd(Date.now(), delayMs),
errorMessage: response.errorMessage ?? "Assistant request failed",
};
} else {
settled = {
...scope,
at: "run.failure_drain",
error: providerError(source, response),
provenance: { kind: "response", entryId: responseEntryId },
};
}
} else {
const calls = response.content.flatMap((content, sourceIndex) =>
content.type === "toolCall" ? [{ sourceIndex }] : [],
);
if (calls.length !== 0) {
const timestamp = uuidV7Timestamp(responseEntryId);
const planned: ToolCall[] = calls.map(({ sourceIndex }) => ({
status: "planned",
sourceIndex,
resultEntryId: lane.session.idGenerator.next(timestamp),
}));
settled = {
...scope,
at: "run.tools",
batch: { assistantEntryId: responseEntryId, configuration, turnId, calls: planned },
};
} else if (response.stopReason === "toolUse") {
committed = normalizeError(response, "Provider reported tool use without any tool calls");
settled = {
...scope,
at: "run.failure_drain",
error: providerError(source, committed),
provenance: { kind: "response", entryId: responseEntryId },
};
} else {
settled = {
...scope,
at: "run.checkpoint",
continuation: { kind: "may_finish", includeFinalAssistant: true },
triggerEntryId: responseEntryId,
};
}
}
const responseEntry: NewEntry<MessageEntry> = {
id: responseEntryId,
parentId: state.tipId,
type: "message",
message: committed,
};
const usageRow: Omit<UsageRow, "seq"> = {
id: intent.usageId,
usage: committed.usage,
entryId: responseEntryId,
adjustment: false,
};
return {
kind: "commit",
writes: [
insertEntry(responseEntry),
insertUsage(usageRow),
setValue(branchTip(lane.name), responseEntryId),
deleteList(pendingAssistantFrames(drive.operationId, responseEntryId)),
],
operationState: settled,
lane: { tipId: responseEntryId },
materialize: () => ({ kind: "continue" }) as const,
events: (commit) => {
const entry: MessageEntry = {
...responseEntry,
seq: commit.seqs[0]!,
timestamp: commit.timestamp,
};
const events: HarnessEvent[] = [
{ type: "entry_added", lane: lane.name, entry, ...options },
{
type: "usage",
lane: lane.name,
row: { ...usageRow, seq: commit.seqs[1]! },
totals: commit.stats.usage,
},
];
if (current.at === "run.assistant.effect_pending") {
if (options.recovery !== true && current.attempt > 1 && settled.at !== "run.assistant.retry_wait") {
const success = committed.stopReason !== "error" && committed.stopReason !== "aborted";
events.push({
type: "retry_end",
lane: lane.name,
runId: drive.operationId,
step: turnId,
attempt: current.attempt,
success,
...(success
? {}
: {
finalError:
committed.errorMessage ?? `Assistant request ended with ${committed.stopReason}`,
}),
});
}
if (options.recovery !== true && settled.at === "run.assistant.retry_wait") {
events.push({
type: "retry_scheduled",
lane: lane.name,
runId: drive.operationId,
step: turnId,
attempt: settled.nextAttempt,
maxAttempts: settled.generationContext.retryPolicy.maxAttempts,
delayMs: retryDelay(current.generationContext.retryPolicy.baseDelayMs, current.attempt),
errorMessage: settled.errorMessage,
});
}
if (
options.recovery !== true &&
settled.at !== "run.tools" &&
settled.at !== "run.assistant.retry_wait"
) {
events.push({
type: "turn_end",
lane: lane.name,
runId: drive.operationId,
turnId,
message: committed,
toolResults: [],
});
}
} else if (settled.at !== "run.tools") {
events.push({
type: "turn_end",
lane: lane.name,
runId: drive.operationId,
turnId,
message: committed,
toolResults: [],
...options,
});
}
if (
(current.at === "run.deferred.effect_pending" || options.recovery !== true) &&
settled.at === "run.deferred.suspended" &&
committed.deferred !== undefined
) {
events.push({
type: "run_suspend",
lane: lane.name,
runId: drive.operationId,
reason: "deferred",
deferred: committed.deferred,
...(current.at === "run.deferred.effect_pending" ? options : {}),
});
}
return events;
},
};
},
drive.context,
);
}

37
packages/agent/src/harness/runtime/drive/terminal.ts
View file

@ -2,15 +2,16 @@ import type { AgentMessage } from "../../../types.ts";
import type { TerminalOperationOutcome } from "../../agent-harness.ts";
import type { Context } from "../../context.ts";
import { SessionInvariantError } from "../../session/session.ts";
import type {
BranchSummaryEntry,
CompactionEntry,
LaneLastResult,
MessageEntry,
OperationState,
SessionReader,
SettledAssistantMessage,
Write,
import {
type BranchSummaryEntry,
type CompactionEntry,
isRunOperationState,
type LaneLastResult,
type MessageEntry,
type OperationState,
type SessionReader,
type SettledAssistantMessage,
type Write,
} from "../../session/types.ts";
import {
deleteList,
@ -40,12 +41,12 @@ export async function operationCleanupWrites(
]);
const pendingIds = new Set<string>();
if (state.kind === "run") {
if (isRunOperationState(state)) {
for (const id of state.inbox.steer) pendingIds.add(id);
for (const id of state.inbox.followUp) pendingIds.add(id);
for (const id of state.inbox.writes) pendingIds.add(id);
if (state.phase.kind === "tools") {
for (const call of state.phase.batch.calls) {
if (state.at === "run.tools") {
for (const call of state.batch.calls) {
if (call.status === "outcome_ready") pendingIds.add(call.resultEntryId);
}
}
@ -56,16 +57,8 @@ export async function operationCleanupWrites(
}
let frameDelete: Write | undefined;
if (state.kind === "run" && state.phase.kind === "assistant") {
const generation = state.phase.generation;
if (generation.status === "effect_pending") {
frameDelete = deleteList(pendingAssistantFrames(operationId, generation.responseEntryId));
}
} else if (state.kind === "run" && state.phase.kind === "deferred") {
const deferred = state.phase.deferred;
if (deferred.status === "effect_pending") {
frameDelete = deleteList(pendingAssistantFrames(operationId, deferred.responseEntryId));
}
if (state.at === "run.assistant.effect_pending" || state.at === "run.deferred.effect_pending") {
frameDelete = deleteList(pendingAssistantFrames(operationId, state.responseEntryId));
}
return [

597
packages/agent/src/harness/runtime/drive/tools.ts
View file

@ -13,21 +13,23 @@ import {
} from "../../execution/tools.ts";
import { insertEntry, insertUsage } from "../../session/commit.ts";
import { SessionInvariantError } from "../../session/session.ts";
import type {
JsonValue,
MessageEntry,
NewEntry,
RunState,
ToolBatch,
ToolCall,
UsageRow,
Write,
import {
type JsonValue,
type MessageEntry,
type NewEntry,
type OperationState,
type RunCheckpointOperation,
type RunToolsOperation,
runScopeOf,
type ToolBatch,
type ToolCall,
type UsageRow,
type Write,
} from "../../session/types.ts";
import {
branchTip,
deleteValue,
laneConfig,
operationState as operationStateValue,
operationToolArgs,
operationToolArgsPrefix,
operationToolMemo,
@ -39,10 +41,10 @@ import {
import type { AgentHarnessTool, AgentHarnessToolInvocation } from "../../types.ts";
import type { Lane } from "../lane.ts";
import { openToolProgress } from "../progress.ts";
import type { Drive, LaneState, LostOwnership, ProcedureResult } from "../types.ts";
import type { Drive, LaneState, ProcedureResult } from "../types.ts";
type LocalResult = { kind: "committed" } | { kind: "state_changed" } | LostOwnership;
type PlacementResult = { kind: "none" } | { kind: "committed"; complete: boolean } | LostOwnership;
type LocalResult = { kind: "committed" } | { kind: "state_changed" };
type PlacementResult = { kind: "none" } | { kind: "committed"; complete: boolean };
type BatchSource = {
kind: "source";
@ -72,20 +74,16 @@ class ToolInvocationEnded extends Error {
}
}
function batchMatches(current: ToolBatch, expected: ToolBatch): boolean {
return current.assistantEntryId === expected.assistantEntryId && current.turnId === expected.turnId;
}
function currentBatch<TContext extends object | undefined>(
lane: Lane<TContext>,
drive: Drive,
expected: ToolBatch,
): { run: RunState; batch: ToolBatch } | undefined {
): { run: RunToolsOperation; batch: ToolBatch } | undefined {
const operation = lane.state.operation;
if (operation?.meta.operationId !== drive.operationId || operation.state.kind !== "run") return undefined;
const run = operation.state;
if (run.phase.kind !== "tools" || !batchMatches(run.phase.batch, expected)) return undefined;
return { run, batch: run.phase.batch };
if (operation?.state.at !== "run.tools") return undefined;
return { run: operation.state, batch: operation.state.batch };
}
function toolOperation<TContext extends object | undefined>(lane: Lane<TContext>): RunToolsOperation {
return lane.state.operation!.state as RunToolsOperation;
}
function findCall(batch: ToolBatch, sourceIndex: number, resultEntryId: string): ToolCall | undefined {
@ -103,8 +101,8 @@ function replaceCall(batch: ToolBatch, replacement: ToolCall): ToolBatch {
};
}
function withBatch(run: RunState, batch: ToolBatch): RunState {
return { ...run, phase: { kind: "tools", batch } };
function withBatch(run: RunToolsOperation, batch: ToolBatch): RunToolsOperation {
return { ...runScopeOf(run), at: "run.tools", batch };
}
function validateMemoName(name: string): void {
@ -128,37 +126,24 @@ async function readBatchSource<TContext extends object | undefined>(
lane: Lane<TContext>,
drive: Drive,
batch: ToolBatch,
): Promise<BatchSource | LostOwnership> {
return lane.commandDriveOwned<BatchSource>(
drive,
async (state, reader) => {
const operation = state.operation;
if (
operation === null ||
operation.state.kind !== "run" ||
operation.state.phase.kind !== "tools" ||
!batchMatches(operation.state.phase.batch, batch)
) {
throw new SessionInvariantError("Tool procedure lost its batch source");
): Promise<BatchSource> {
return lane.command<BatchSource>(async (_state, reader) => {
const entry = (await reader.getEntries([batch.assistantEntryId], drive.context)).get(batch.assistantEntryId);
if (entry?.type !== "message" || entry.message.role !== "assistant") {
throw new SessionInvariantError("Tool batch assistant entry is invalid");
}
const calls = new Map<number, AgentToolCall>();
for (const call of batch.calls) {
const block = entry.message.content[call.sourceIndex];
if (block?.type !== "toolCall") {
throw new SessionInvariantError(
`Tool call source index ${call.sourceIndex} does not name a tool-call block`,
);
}
const entry = (await reader.getEntries([batch.assistantEntryId], drive.context)).get(batch.assistantEntryId);
if (entry?.type !== "message" || entry.message.role !== "assistant") {
throw new SessionInvariantError("Tool batch assistant entry is invalid");
}
const calls = new Map<number, AgentToolCall>();
for (const call of batch.calls) {
const block = entry.message.content[call.sourceIndex];
if (block?.type !== "toolCall") {
throw new SessionInvariantError(
`Tool call source index ${call.sourceIndex} does not name a tool-call block`,
);
}
calls.set(call.sourceIndex, block);
}
return { kind: "return", result: { kind: "source", assistant: entry.message, calls } };
},
drive.context,
);
calls.set(call.sourceIndex, block);
}
return { kind: "return", result: { kind: "source", assistant: entry.message, calls } };
}, drive.context);
}
function invocationCapability<TContext extends object | undefined>(
@ -170,15 +155,8 @@ function invocationCapability<TContext extends object | undefined>(
let active = true;
const ownsEffect = (state: LaneState): boolean => {
const operation = state.operation;
if (
operation?.meta.operationId !== drive.operationId ||
operation.state.kind !== "run" ||
operation.state.phase.kind !== "tools" ||
!batchMatches(operation.state.phase.batch, batch)
) {
return false;
}
return findCall(operation.state.phase.batch, call.sourceIndex, call.resultEntryId)?.status === "effect_pending";
if (operation?.state.at !== "run.tools") return false;
return findCall(operation.state.batch, call.sourceIndex, call.resultEntryId)?.status === "effect_pending";
};
const ended = (): ToolInvocationEnded => new ToolInvocationEnded();
return {
@ -190,23 +168,19 @@ function invocationCapability<TContext extends object | undefined>(
validateMemoName(name);
if (!active) return Promise.reject(ended());
return lane
.commandDriveOwned<{ kind: "value"; value: JsonValue | undefined } | { kind: "ended" }>(
drive,
async (state, reader) => {
if (!ownsEffect(state)) return { kind: "return", result: { kind: "ended" } as const };
const stored = await reader.getValue(
operationToolMemo(drive.operationId, call.resultEntryId, name),
drive.context,
);
return {
kind: "return",
result: { kind: "value", value: stored?.value } as const,
};
},
drive.context,
)
.command<{ kind: "value"; value: JsonValue | undefined } | { kind: "ended" }>(async (state, reader) => {
if (!ownsEffect(state)) return { kind: "return", result: { kind: "ended" } as const };
const stored = await reader.getValue(
operationToolMemo(drive.operationId, call.resultEntryId, name),
drive.context,
);
return {
kind: "return",
result: { kind: "value", value: stored?.value } as const,
};
}, drive.context)
.then((result) => {
if (result.kind === "lost_ownership" || result.kind === "ended") throw ended();
if (result.kind === "ended") throw ended();
return result.value;
});
},
@ -214,22 +188,18 @@ function invocationCapability<TContext extends object | undefined>(
validateMemoName(name);
if (!active) return Promise.reject(ended());
return lane
.commandDriveOwned<{ kind: "written" } | { kind: "ended" }>(
drive,
(state) => {
if (!ownsEffect(state)) return { kind: "return", result: { kind: "ended" } as const };
const address = operationToolMemo(drive.operationId, call.resultEntryId, name);
return {
kind: "commit",
writes: [value === undefined ? deleteValue(address) : setValue(address, value)],
next: state,
materialize: () => ({ kind: "written" }) as const,
};
},
drive.context,
)
.command<{ kind: "written" } | { kind: "ended" }>((state) => {
if (!ownsEffect(state)) return { kind: "return", result: { kind: "ended" } as const };
const address = operationToolMemo(drive.operationId, call.resultEntryId, name);
return {
kind: "commit",
writes: [value === undefined ? deleteValue(address) : setValue(address, value)],
next: state,
materialize: () => ({ kind: "written" }) as const,
};
}, drive.context)
.then((result) => {
if (result.kind === "lost_ownership" || result.kind === "ended") throw ended();
if (result.kind === "ended") throw ended();
});
},
},
@ -288,20 +258,10 @@ async function commitIntent<TContext extends object | undefined>(
args: Record<string, JsonValue>,
replay: "never" | "safe",
): Promise<LocalResult> {
return lane.commandDriveOwned<LocalResult>(
drive,
(state) => {
const operation = state.operation;
if (
operation === null ||
operation.state.kind !== "run" ||
operation.state.phase.kind !== "tools" ||
!batchMatches(operation.state.phase.batch, batch)
) {
return { kind: "return", result: { kind: "state_changed" } as const };
}
const run = operation.state;
const currentBatchState = operation.state.phase.batch;
return lane.mutateOperation<RunToolsOperation, LocalResult>(
toolOperation(lane),
(_state, run) => {
const currentBatchState = run.batch;
if (run.control.status === "cancel_requested") {
return { kind: "return", result: { kind: "state_changed" } as const };
}
@ -318,11 +278,8 @@ async function commitIntent<TContext extends object | undefined>(
const nextRun = withBatch(run, replaceCall(currentBatchState, effectPending));
return {
kind: "commit",
writes: [
setValue(operationToolArgs(drive.operationId, batch.turnId, current.sourceIndex), args),
setValue(operationStateValue(drive.operationId), nextRun),
],
next: { ...state, operation: { meta: operation.meta, state: nextRun } },
writes: [setValue(operationToolArgs(drive.operationId, batch.turnId, current.sourceIndex), args)],
operationState: nextRun,
materialize: () => ({ kind: "committed" }) as const,
};
},
@ -333,25 +290,14 @@ async function commitIntent<TContext extends object | undefined>(
async function stageOutcome<TContext extends object | undefined>(
lane: Lane<TContext>,
drive: Drive,
batch: ToolBatch,
call: ToolCall,
message: ToolResultMessage<unknown>,
terminate: boolean,
): Promise<LocalResult> {
return lane.commandDriveOwned<LocalResult>(
drive,
async (state, reader) => {
const operation = state.operation;
if (
operation === null ||
operation.state.kind !== "run" ||
operation.state.phase.kind !== "tools" ||
!batchMatches(operation.state.phase.batch, batch)
) {
return { kind: "return", result: { kind: "state_changed" } as const };
}
const run = operation.state;
const currentBatchState = operation.state.phase.batch;
return lane.mutateOperation<RunToolsOperation, LocalResult>(
toolOperation(lane),
async (_state, run, _meta, reader) => {
const currentBatchState = run.batch;
const current = findCall(currentBatchState, call.sourceIndex, call.resultEntryId);
if (
current === undefined ||
@ -377,9 +323,8 @@ async function stageOutcome<TContext extends object | undefined>(
setValue(pendingEntry(current.resultEntryId), { type: "message", payload: message }),
deleteValue(pendingToolOutput(drive.operationId, current.resultEntryId)),
...memos.map(({ address }) => deleteValue(address)),
setValue(operationStateValue(drive.operationId), nextRun),
],
next: { ...state, operation: { meta: operation.meta, state: nextRun } },
operationState: nextRun,
materialize: () => ({ kind: "committed" }) as const,
};
},
@ -392,20 +337,11 @@ async function clearReplayCheckpoint<TContext extends object | undefined>(
drive: Drive,
batch: ToolBatch,
call: Extract<ToolCall, { status: "effect_pending" }>,
): Promise<{ kind: "args"; args: Record<string, JsonValue> } | { kind: "state_changed" } | LostOwnership> {
return lane.commandDriveOwned<{ kind: "args"; args: Record<string, JsonValue> } | { kind: "state_changed" }>(
drive,
): Promise<{ kind: "args"; args: Record<string, JsonValue> } | { kind: "state_changed" }> {
return lane.command<{ kind: "args"; args: Record<string, JsonValue> } | { kind: "state_changed" }>(
async (state, reader) => {
const operation = state.operation;
if (
operation === null ||
operation.state.kind !== "run" ||
operation.state.phase.kind !== "tools" ||
!batchMatches(operation.state.phase.batch, batch)
) {
return { kind: "return", result: { kind: "state_changed" } as const };
}
const current = findCall(operation.state.phase.batch, call.sourceIndex, call.resultEntryId);
const run = state.operation!.state as RunToolsOperation;
const current = findCall(run.batch, call.sourceIndex, call.resultEntryId);
if (current?.status !== "effect_pending") {
return { kind: "return", result: { kind: "state_changed" } as const };
}
@ -430,26 +366,12 @@ async function clearReplayCheckpoint<TContext extends object | undefined>(
async function readCheckpoint<TContext extends object | undefined>(
lane: Lane<TContext>,
drive: Drive,
batch: ToolBatch,
call: Extract<ToolCall, { status: "effect_pending" }>,
): Promise<
{ kind: "checkpoint"; value: AgentToolResult<unknown> | undefined } | { kind: "state_changed" } | LostOwnership
> {
return lane.commandDriveOwned<
{ kind: "checkpoint"; value: AgentToolResult<unknown> | undefined } | { kind: "state_changed" }
>(
drive,
): Promise<{ kind: "checkpoint"; value: AgentToolResult<unknown> | undefined } | { kind: "state_changed" }> {
return lane.command<{ kind: "checkpoint"; value: AgentToolResult<unknown> | undefined } | { kind: "state_changed" }>(
async (state, reader) => {
const operation = state.operation;
if (
operation === null ||
operation.state.kind !== "run" ||
operation.state.phase.kind !== "tools" ||
!batchMatches(operation.state.phase.batch, batch)
) {
return { kind: "return", result: { kind: "state_changed" } as const };
}
const current = findCall(operation.state.phase.batch, call.sourceIndex, call.resultEntryId);
const run = state.operation!.state as RunToolsOperation;
const current = findCall(run.batch, call.sourceIndex, call.resultEntryId);
if (current?.status !== "effect_pending") {
return { kind: "return", result: { kind: "state_changed" } as const };
}
@ -545,7 +467,6 @@ async function executeAndStage<TContext extends object | undefined>(
return stageOutcome(
lane,
drive,
batch,
call,
recovery ? interruptedMessage(cleared.toolCall, undefined) : abortedMessage(cleared.toolCall),
false,
@ -582,7 +503,7 @@ async function executeAndStage<TContext extends object | undefined>(
if (!(error instanceof AbortRequested)) throw error;
patch = undefined;
}
const current = currentBatch(lane, drive, batch);
const current = currentBatch(lane);
const cancelled = current?.run.control.status === "cancel_requested";
const finalized = finalizeToolCall(cleared, executed, cancelled ? { ...patch, terminate: false } : patch);
await lane.emitBatch(
@ -602,102 +523,81 @@ async function executeAndStage<TContext extends object | undefined>(
],
drive.context,
);
return stageOutcome(
lane,
drive,
batch,
call,
createToolResultMessage(finalized),
cancelled ? false : finalized.terminate,
);
return stageOutcome(lane, drive, call, createToolResultMessage(finalized), cancelled ? false : finalized.terminate);
}
async function readPlacement<TContext extends object | undefined>(
lane: Lane<TContext>,
drive: Drive,
batch: ToolBatch,
): Promise<PlacementRead | { kind: "none" } | LostOwnership> {
return lane.commandDriveOwned<PlacementRead | { kind: "none" }>(
drive,
async (state, reader) => {
const operation = state.operation;
): Promise<PlacementRead | { kind: "none" }> {
return lane.command<PlacementRead | { kind: "none" }>(async (state, reader) => {
const operation = state.operation;
if (operation?.state.at !== "run.tools") return { kind: "return", result: { kind: "none" } as const };
const current = operation.state.batch;
let first = current.calls.findIndex((call) => call.status !== "completed");
if (first === -1) return { kind: "return", result: { kind: "none" } as const };
const ready: Array<Extract<ToolCall, { status: "outcome_ready" }>> = [];
while (first < current.calls.length) {
const call = current.calls[first]!;
if (call.status !== "outcome_ready") break;
ready.push(call);
first += 1;
}
if (ready.length === 0) return { kind: "return", result: { kind: "none" } as const };
const assistantEntry = (await reader.getEntries([current.assistantEntryId], drive.context)).get(
current.assistantEntryId,
);
if (assistantEntry?.type !== "message" || assistantEntry.message.role !== "assistant") {
throw new SessionInvariantError("Tool placement assistant entry is invalid");
}
const items: PlacementItem[] = [];
for (const call of ready) {
const stored = await reader.getValue(pendingEntry(call.resultEntryId), drive.context);
if (stored?.value.type !== "message" || !isToolResultMessage(stored.value.payload)) {
throw new SessionInvariantError(`Tool call ${call.resultEntryId} is missing its staged result`);
}
const block = assistantEntry.message.content[call.sourceIndex];
if (
operation === null ||
operation.state.kind !== "run" ||
operation.state.phase.kind !== "tools" ||
!batchMatches(operation.state.phase.batch, batch)
block?.type !== "toolCall" ||
stored.value.payload.toolCallId !== block.id ||
stored.value.payload.toolName !== block.name
) {
return { kind: "return", result: { kind: "none" } as const };
throw new SessionInvariantError(`Tool call ${call.resultEntryId} has a mismatched staged result`);
}
const current = operation.state.phase.batch;
let first = current.calls.findIndex((call) => call.status !== "completed");
if (first === -1) return { kind: "return", result: { kind: "none" } as const };
const ready: Array<Extract<ToolCall, { status: "outcome_ready" }>> = [];
while (first < current.calls.length) {
const call = current.calls[first]!;
if (call.status !== "outcome_ready") break;
ready.push(call);
first += 1;
}
if (ready.length === 0) return { kind: "return", result: { kind: "none" } as const };
items.push({ call, message: stored.value.payload });
}
const assistantEntry = (await reader.getEntries([current.assistantEntryId], drive.context)).get(
current.assistantEntryId,
);
if (assistantEntry?.type !== "message" || assistantEntry.message.role !== "assistant") {
throw new SessionInvariantError("Tool placement assistant entry is invalid");
}
const items: PlacementItem[] = [];
for (const call of ready) {
const stored = await reader.getValue(pendingEntry(call.resultEntryId), drive.context);
if (stored?.value.type !== "message" || !isToolResultMessage(stored.value.payload)) {
throw new SessionInvariantError(`Tool call ${call.resultEntryId} is missing its staged result`);
let turnResults: ToolResultMessage<unknown>[] | undefined;
if (first === current.calls.length) {
const placedIds = current.calls
.filter((call) => call.status === "completed")
.map((call) => call.resultEntryId);
const placed = await reader.getEntries(placedIds, drive.context);
const staged = new Map(items.map((item) => [item.call.resultEntryId, item.message]));
turnResults = current.calls.map((call) => {
const message =
staged.get(call.resultEntryId) ??
(() => {
const entry = placed.get(call.resultEntryId);
return entry?.type === "message" && isToolResultMessage(entry.message) ? entry.message : undefined;
})();
if (message === undefined) {
throw new SessionInvariantError(`Completed tool call ${call.resultEntryId} is missing its result entry`);
}
const block = assistantEntry.message.content[call.sourceIndex];
if (
block?.type !== "toolCall" ||
stored.value.payload.toolCallId !== block.id ||
stored.value.payload.toolName !== block.name
) {
throw new SessionInvariantError(`Tool call ${call.resultEntryId} has a mismatched staged result`);
}
items.push({ call, message: stored.value.payload });
}
let turnResults: ToolResultMessage<unknown>[] | undefined;
if (first === current.calls.length) {
const placedIds = current.calls
.filter((call) => call.status === "completed")
.map((call) => call.resultEntryId);
const placed = await reader.getEntries(placedIds, drive.context);
const staged = new Map(items.map((item) => [item.call.resultEntryId, item.message]));
turnResults = current.calls.map((call) => {
const message =
staged.get(call.resultEntryId) ??
(() => {
const entry = placed.get(call.resultEntryId);
return entry?.type === "message" && isToolResultMessage(entry.message) ? entry.message : undefined;
})();
if (message === undefined) {
throw new SessionInvariantError(
`Completed tool call ${call.resultEntryId} is missing its result entry`,
);
}
return message;
});
}
return {
kind: "return",
result: {
kind: "placement",
items,
assistant: assistantEntry.message,
...(turnResults === undefined ? {} : { turnResults }),
},
};
},
drive.context,
);
return message;
});
}
return {
kind: "return",
result: {
kind: "placement",
items,
assistant: assistantEntry.message,
...(turnResults === undefined ? {} : { turnResults }),
},
};
}, drive.context);
}
async function commitPlacement<TContext extends object | undefined>(
@ -709,37 +609,10 @@ async function commitPlacement<TContext extends object | undefined>(
const usageIds = read.items.map((item) =>
item.message.usage === undefined ? undefined : lane.session.idGenerator.next(),
);
return lane.commandDriveOwned<PlacementResult>(
drive,
async (state, reader) => {
const operation = state.operation;
if (
operation === null ||
operation.state.kind !== "run" ||
operation.state.phase.kind !== "tools" ||
!batchMatches(operation.state.phase.batch, batch)
) {
return { kind: "return", result: { kind: "none" } as const };
}
const run = operation.state;
const current = operation.state.phase.batch;
const first = current.calls.findIndex((call) => call.status !== "completed");
if (first === -1) return { kind: "return", result: { kind: "none" } as const };
for (const [offset, item] of read.items.entries()) {
const call = current.calls[first + offset];
if (
call?.status !== "outcome_ready" ||
call.sourceIndex !== item.call.sourceIndex ||
call.resultEntryId !== item.call.resultEntryId
) {
return { kind: "return", result: { kind: "none" } as const };
}
const stored = await reader.getValue(pendingEntry(call.resultEntryId), drive.context);
if (stored?.value.type !== "message" || !isToolResultMessage(stored.value.payload)) {
throw new SessionInvariantError(`Tool call ${call.resultEntryId} is missing its staged result`);
}
}
return lane.mutateOperation<RunToolsOperation, PlacementResult>(
toolOperation(lane),
async (state, run, _meta, reader) => {
const current = run.batch;
const writes: Write[] = [];
const eventEntries: Array<{ entry: NewEntry<MessageEntry>; seqIndex: number }> = [];
const eventUsage: Array<{ row: Omit<UsageRow, "seq">; seqIndex: number }> = [];
@ -801,19 +674,18 @@ async function commitPlacement<TContext extends object | undefined>(
}
writes.push(setValue(branchTip(lane.name), parentId));
let nextRun: RunState;
let nextRun: OperationState;
if (complete) {
const allTerminate = completedCalls.every((call) => call.status === "completed" && call.terminate);
nextRun = {
...run,
phase: {
kind: "checkpoint",
continuation: allTerminate
? { kind: "may_finish", includeFinalAssistant: false }
: { kind: "need_assistant", overflowRecoveryUsed: false },
triggerEntryId: parentId!,
},
const checkpoint: RunCheckpointOperation = {
...runScopeOf(run),
at: "run.checkpoint",
continuation: allTerminate
? { kind: "may_finish", includeFinalAssistant: false }
: { kind: "need_assistant", overflowRecoveryUsed: false },
triggerEntryId: parentId!,
};
nextRun = checkpoint;
const args = await reader.scanValues(
operationToolArgsPrefix(drive.operationId, batch.turnId),
drive.context,
@ -822,16 +694,11 @@ async function commitPlacement<TContext extends object | undefined>(
} else {
nextRun = withBatch(run, { ...current, calls: completedCalls });
}
writes.push(setValue(operationStateValue(drive.operationId), nextRun));
return {
kind: "commit",
writes,
next: {
...state,
tipId: parentId,
configuration: nextConfiguration,
operation: { meta: operation.meta, state: nextRun },
},
operationState: nextRun,
lane: { tipId: parentId, configuration: nextConfiguration },
materialize: () => ({ kind: "committed", complete }) as const,
events: (commit) => {
const events: HarnessEvent[] = [];
@ -874,8 +741,8 @@ async function materializeReady<TContext extends object | undefined>(
batch: ToolBatch,
recovery: boolean,
): Promise<PlacementResult> {
const read = await readPlacement(lane, drive, batch);
if (read.kind === "lost_ownership" || read.kind === "none") return read;
const read = await readPlacement(lane, drive);
if (read.kind === "none") return read;
await lane.emitBatch(
read.items.flatMap(({ call, message }) => [
{
@ -928,7 +795,7 @@ async function beginFreshCall<TContext extends object | undefined>(
): Promise<{ job: Promise<LocalResult> }> {
const toolCall = toolCallFor(sources, call);
if (sources.assistant.stopReason === "length") {
return { job: stageOutcome(lane, drive, batch, call, truncatedMessage(toolCall), false) };
return { job: stageOutcome(lane, drive, call, truncatedMessage(toolCall), false) };
}
const prepared = prepareToolCall(toolCall, tools);
if ("kind" in prepared) {
@ -939,7 +806,7 @@ async function beginFreshCall<TContext extends object | undefined>(
terminate: prepared.terminate,
};
return {
job: stageOutcome(lane, drive, batch, call, createToolResultMessage(finalized), finalized.terminate),
job: stageOutcome(lane, drive, call, createToolResultMessage(finalized), finalized.terminate),
};
}
@ -960,7 +827,7 @@ async function beginFreshCall<TContext extends object | undefined>(
} catch (error) {
if (!(error instanceof AbortRequested)) throw error;
await error.cancellation;
return { job: stageOutcome(lane, drive, batch, call, abortedMessage(toolCall), false) };
return { job: stageOutcome(lane, drive, call, abortedMessage(toolCall), false) };
}
const cleared = applyBeforeToolDecision(prepared, decision);
if ("kind" in cleared) {
@ -971,17 +838,12 @@ async function beginFreshCall<TContext extends object | undefined>(
terminate: cleared.terminate,
};
return {
job: stageOutcome(lane, drive, batch, call, createToolResultMessage(finalized), finalized.terminate),
job: stageOutcome(lane, drive, call, createToolResultMessage(finalized), finalized.terminate),
};
}
const intent = await commitIntent(lane, drive, batch, call, cleared.args, cleared.tool.replay ?? "never");
if (intent.kind !== "committed") {
return {
job:
intent.kind === "lost_ownership"
? Promise.resolve(intent)
: stageOutcome(lane, drive, batch, call, abortedMessage(toolCall), false),
};
return { job: stageOutcome(lane, drive, call, abortedMessage(toolCall), false) };
}
const effectPending: Extract<ToolCall, { status: "effect_pending" }> = {
status: "effect_pending",
@ -1012,80 +874,62 @@ async function beginRecoveryCall<TContext extends object | undefined>(
const cleared: ClearedToolCall<TContext> = { toolCall, tool, args: replay.args };
return { job: executeAndStage(lane, drive, batch, call, cleared, toolContext, true) };
}
const checkpoint = await readCheckpoint(lane, drive, batch, call);
const checkpoint = await readCheckpoint(lane, drive, call);
if (checkpoint.kind !== "checkpoint") return { job: Promise.resolve(checkpoint) };
return {
job: stageOutcome(lane, drive, batch, call, interruptedMessage(toolCall, checkpoint.value), false),
job: stageOutcome(lane, drive, call, interruptedMessage(toolCall, checkpoint.value), false),
};
}
async function runCancelled<TContext extends object | undefined>(
lane: Lane<TContext>,
drive: Drive,
batch: ToolBatch,
sources: BatchSource,
recovery: boolean,
): Promise<ProcedureResult> {
for (let transition = 0; transition <= batch.calls.length * 2 + 1; transition += 1) {
const placement = await materializeReady(lane, drive, batch, recovery);
if (placement.kind === "lost_ownership") return placement;
const current = currentBatch(lane, drive, batch);
if (current === undefined) return { kind: "continue" };
const call = current.batch.calls.find((candidate) => candidate.status !== "completed");
if (call === undefined) throw new SessionInvariantError("Tool batch remained open after every call completed");
if (call.status === "outcome_ready") {
throw new SessionInvariantError("Ready tool outcome was not materialized");
}
const toolCall = toolCallFor(sources, call);
let staged: LocalResult;
if (call.status === "planned") {
staged = await stageOutcome(lane, drive, batch, call, abortedMessage(toolCall), false);
} else {
const checkpoint = await readCheckpoint(lane, drive, batch, call);
if (checkpoint.kind === "lost_ownership") return checkpoint;
if (checkpoint.kind === "state_changed") continue;
staged = await stageOutcome(lane, drive, batch, call, interruptedMessage(toolCall, checkpoint.value), false);
}
if (staged.kind === "lost_ownership") return staged;
}
throw new SessionInvariantError("Cancelled tool batch exceeded its bounded transition count");
}
async function runSequential<TContext extends object | undefined>(
lane: Lane<TContext>,
drive: Drive,
batch: ToolBatch,
sources: BatchSource,
tools: AgentHarnessTool<TContext>[],
toolsByName: Map<string, AgentHarnessTool<TContext>>,
toolContext: TContext,
execution:
| {
tools: AgentHarnessTool<TContext>[];
toolsByName: Map<string, AgentHarnessTool<TContext>>;
toolContext: TContext;
}
| undefined,
recovery: boolean,
): Promise<ProcedureResult> {
for (let transition = 0; transition <= batch.calls.length * 2 + 1; transition += 1) {
const placement = await materializeReady(lane, drive, batch, recovery);
if (placement.kind === "lost_ownership") return placement;
const current = currentBatch(lane, drive, batch);
await materializeReady(lane, drive, batch, recovery);
const current = currentBatch(lane);
if (current === undefined) return { kind: "continue" };
const call = current.batch.calls.find((candidate) => candidate.status !== "completed");
if (call === undefined) throw new SessionInvariantError("Tool batch remained open after every call completed");
if (call.status === "outcome_ready") {
throw new SessionInvariantError("Ready tool outcome was not materialized");
if (call.status === "outcome_ready") throw new SessionInvariantError("Ready tool outcome was not materialized");
if (current.run.control.status === "cancel_requested") {
const toolCall = toolCallFor(sources, call);
if (call.status === "planned") {
await stageOutcome(lane, drive, call, abortedMessage(toolCall), false);
} else {
const checkpoint = await readCheckpoint(lane, drive, call);
if (checkpoint.kind === "state_changed") continue;
await stageOutcome(lane, drive, call, interruptedMessage(toolCall, checkpoint.value), false);
}
continue;
}
if (execution === undefined) throw new SessionInvariantError("Running tool batch is missing execution context");
const started =
call.status === "planned"
? await beginFreshCall(lane, drive, batch, sources, call, tools, toolContext, recovery)
? await beginFreshCall(lane, drive, batch, sources, call, execution.tools, execution.toolContext, recovery)
: await beginRecoveryCall(
lane,
drive,
batch,
sources,
call,
toolsByName,
toolContext,
current.run.control.status === "cancel_requested",
execution.toolsByName,
execution.toolContext,
false,
);
const result = await started.job;
if (result.kind === "lost_ownership") return result;
await started.job;
}
throw new SessionInvariantError("Sequential tool batch exceeded its bounded transition count");
}
@ -1107,11 +951,11 @@ async function runParallel<TContext extends object | undefined>(
return scheduled;
};
const jobs: Promise<LocalResult>[] = [];
const snapshot = currentBatch(lane, drive, batch);
const snapshot = currentBatch(lane);
if (snapshot === undefined) return { kind: "continue" };
for (const call of snapshot.batch.calls) {
if (call.status === "completed" || call.status === "outcome_ready") continue;
const current = currentBatch(lane, drive, batch);
const current = currentBatch(lane);
if (current === undefined) break;
const started =
call.status === "planned"
@ -1133,10 +977,8 @@ async function runParallel<TContext extends object | undefined>(
void job.catch(() => {});
jobs.push(job);
}
const results = await Promise.all(jobs);
if (results.some((result) => result.kind === "lost_ownership")) return { kind: "lost_ownership" };
const placement = await scheduleMaterialization();
if (placement.kind === "lost_ownership") return placement;
await Promise.all(jobs);
await scheduleMaterialization();
return { kind: "continue" };
}
@ -1144,12 +986,9 @@ async function runParallel<TContext extends object | undefined>(
export async function runTools<TContext extends object | undefined>(
lane: Lane<TContext>,
drive: Drive,
run: RunState,
batch: ToolBatch,
run: RunToolsOperation,
): Promise<ProcedureResult> {
if (run.phase.kind !== "tools" || !batchMatches(run.phase.batch, batch)) {
throw new SessionInvariantError("runTools requires its current tool batch");
}
const batch = run.batch;
const recovery = batch.calls.some((call) => call.status === "effect_pending" || call.status === "outcome_ready");
if (recovery) {
await lane.emitBatch(
@ -1165,25 +1004,19 @@ export async function runTools<TContext extends object | undefined>(
drive.context,
);
}
const initialPlacement = await materializeReady(lane, drive, batch, recovery);
if (initialPlacement.kind === "lost_ownership") return initialPlacement;
if (currentBatch(lane, drive, batch) === undefined) return { kind: "continue" };
await materializeReady(lane, drive, batch, recovery);
if (currentBatch(lane) === undefined) return { kind: "continue" };
const sources = await readBatchSource(lane, drive, batch);
if (sources.kind === "lost_ownership") return sources;
const beforeResolution = currentBatch(lane, drive, batch);
if (beforeResolution === undefined) return { kind: "continue" };
if (beforeResolution.run.control.status === "cancel_requested") {
return runCancelled(lane, drive, batch, sources, recovery);
if (toolOperation(lane).control.status === "cancel_requested") {
return runSequential(lane, drive, batch, sources, undefined, recovery);
}
const config = lane.readConfig();
const active = new Set(batch.configuration.activeToolNames);
const tools = config.tools.filter((tool) => active.has(tool.name));
const toolsByName = new Map(tools.map((tool) => [tool.name, tool]));
const toolContext = await resolveToolContext(lane, drive);
const current = currentBatch(lane, drive, batch);
if (current === undefined) return { kind: "continue" };
return current.run.settings.toolExecution === "sequential"
? runSequential(lane, drive, batch, sources, tools, toolsByName, toolContext, recovery)
return toolOperation(lane).settings.toolExecution === "sequential"
? runSequential(lane, drive, batch, sources, { tools, toolsByName, toolContext }, recovery)
: runParallel(lane, drive, batch, sources, tools, toolsByName, toolContext, recovery);
}

387
packages/agent/src/harness/runtime/lane.ts
View file

@ -33,21 +33,25 @@ import {
} from "../result.ts";
import { insertEntry } from "../session/commit.ts";
import { SessionInvariantError, SessionPendingAssistantMessageError } from "../session/session.ts";
import type {
BranchScan,
Entry,
JsonValue,
LaneLastResult,
Operation,
PendingEntry,
Session,
SessionReader,
StructuralDecision,
import {
type BranchScan,
type Entry,
isRunOperationState,
type JsonValue,
type LaneLastResult,
type Operation,
type OperationMeta,
type OperationState,
type PendingEntry,
type RunStartingOperation,
type Session,
type SessionReader,
} from "../session/types.ts";
import {
branchTip,
deleteValue,
laneConfig,
laneLastResult as laneLastResultValue,
laneState as laneStateValue,
operationMeta as operationMetaValue,
operationState as operationStateValue,
@ -63,7 +67,7 @@ import {
type Drive,
type LaneCommand,
type LaneState,
type LostOwnership,
type OperationCommand,
SliceNotImplemented,
} from "./types.ts";
@ -80,28 +84,29 @@ function isPromiseLike(value: unknown): value is PromiseLike<unknown> {
return "then" in value && typeof value.then === "function";
}
function structuralModel(structural: StructuralDecision): ModelIdentity | undefined {
return structural.status === "generating" ? structural.generation.context.configuration.model : undefined;
}
function capturedModel(operation: Operation): ModelIdentity | undefined {
const { state } = operation;
if (state.kind === "compaction") return structuralModel(state.structural);
if (state.kind === "navigation") {
return state.summarize ? structuralModel(state.phase.structural) : undefined;
}
switch (state.phase.kind) {
case "assistant":
return state.phase.generation.context.configuration.model;
case "tools":
return state.phase.batch.configuration.model;
case "compaction":
return structuralModel(state.phase.structural);
case "deferred":
return state.phase.deferred.configuration.model;
case "starting":
case "checkpoint":
case "failure_drain":
switch (state.at) {
case "run.assistant.ready":
case "run.assistant.effect_pending":
case "run.assistant.retry_wait":
return state.generationContext.configuration.model;
case "run.tools":
return state.batch.configuration.model;
case "run.deferred.suspended":
case "run.deferred.effect_pending":
return state.configuration.model;
case "run.compaction.ready":
case "run.compaction.effect_pending":
case "run.compaction.retry_wait":
case "compaction.ready":
case "compaction.effect_pending":
case "compaction.retry_wait":
case "navigation.summary.ready":
case "navigation.summary.effect_pending":
case "navigation.summary.retry_wait":
return state.summaryContext.configuration.model;
default:
return undefined;
}
}
@ -158,10 +163,6 @@ export class Lane<TContext extends object | undefined> implements AgentLane {
return this.config();
}
isDriveActive(drive: Drive): boolean {
return this.activeDrive === drive;
}
mismatch(expected: string, currentOperationId: string | null, last: LaneLastResult | undefined): OperationMismatch {
return new OperationMismatch({
lane: this.name,
@ -186,10 +187,8 @@ export class Lane<TContext extends object | undefined> implements AgentLane {
*
* Planner, commit, and materialization errors fault the harness before releasing the Session line. Close/fault gates
* are checked both before queueing and when the callback starts: close-first rejects, while a callback admitted
* before close may finish its commit, publish memory, and resolve without another open check. Drive-owned commands
* additionally need a final fence because their exact owner may be removed outside the Session line. Never invoke
* providers, tools, hooks,
* timers, event handlers, or wait for task completion here; perform those after `command()` returns.
* before close may finish its commit, publish memory, and resolve without another open check. Never invoke providers,
* tools, hooks, timers, event handlers, or wait for task completion here; perform those after `command()` returns.
*/
async command<TResult>(
plan: (state: LaneState, reader: SessionReader) => LaneCommand<TResult> | Promise<LaneCommand<TResult>>,
@ -233,64 +232,73 @@ export class Lane<TContext extends object | undefined> implements AgentLane {
return outcome.result;
}
/**
* Run one drive-owned command with an exact-object fence at commit admission.
* Durable control state comes from the authoritative projection; storage reads are only for referenced payloads.
* Unlike an ordinary command, owner abandonment can occur outside the Session line while the planner awaits. The final
* open check preserves close/fault as lifecycle errors before the exact-owner check can report lost ownership.
*/
async commandDriveOwned<TResult>(
drive: Drive,
plan: (state: LaneState, reader: SessionReader) => LaneCommand<TResult> | Promise<LaneCommand<TResult>>,
/** Commit one transition for the operation owned by the current Drive continuation. */
mutateOperation<TState extends OperationState, TResult>(
_current: TState,
plan: (
state: LaneState,
current: TState,
meta: OperationMeta,
reader: SessionReader,
) => OperationCommand<TResult> | Promise<OperationCommand<TResult>>,
context: Context,
): Promise<TResult | LostOwnership> {
this.assertOpen();
let outcome: LaneCommandOutcome<TResult | LostOwnership>;
try {
outcome = await this.session.mutate(async (mutator) => {
this.assertOpen();
const operation = this.state.operation;
if (operation?.meta.operationId !== drive.operationId || this.activeDrive !== drive) {
return { kind: "return", result: { kind: "lost_ownership" } as const };
}
try {
const decision = await plan(this.state, mutator);
switch (decision.kind) {
case "return":
return { kind: "return", result: decision.result };
case "reject":
return { kind: "reject", error: decision.error };
case "commit": {
// Keep these checks adjacent to commit admission. Owner abandonment may occur
// outside the line; close/fault must still win classification over ownership loss.
this.assertOpen();
if (this.activeDrive !== drive) {
return { kind: "return", result: { kind: "lost_ownership" } as const };
}
const commitPromise = mutator.commit(decision.writes, context);
const commit = await commitPromise;
this.state = decision.next;
const result = decision.materialize(commit);
if (isPromiseLike(result)) {
throw new TypeError("Lane command materialize() must be synchronous");
}
const events = decision.events?.(commit) ?? [];
const delivery = events.length === 0 ? undefined : this.emitBatch(events, context);
return { kind: "return", result, ...(delivery === undefined ? {} : { delivery }) };
}
}
} catch (error) {
if (this.closedError !== undefined) throw this.closedError;
throw this.onFault(error, context);
}
}, context);
} catch (error) {
if (this.closedError !== undefined) throw this.closedError;
throw error;
}
if (outcome.kind === "reject") throw outcome.error;
await outcome.delivery;
return outcome.result;
): Promise<TResult> {
return this.command(async (state, reader) => {
const operation = state.operation!;
const decision = await plan(state, operation.state as TState, operation.meta, reader);
if (decision.kind === "commit") {
return {
kind: "commit",
writes: [
...decision.writes,
setValue(operationStateValue(operation.meta.operationId), decision.operationState),
],
next: {
...state,
...decision.lane,
operation: { meta: operation.meta, state: decision.operationState },
},
materialize: decision.materialize,
...(decision.events === undefined ? {} : { events: decision.events }),
};
}
if (decision.kind !== "finish") return decision;
return {
kind: "commit",
writes: [
...decision.writes,
setValue(laneLastResultValue(this.name), decision.lastResult),
setValue(laneStateValue(this.name), {
currentOperationId: null,
pendingNextRun: state.pendingNextRun,
}),
],
next: { ...state, ...decision.lane, lastResult: decision.lastResult, operation: null },
materialize: decision.materialize,
...(decision.events === undefined ? {} : { events: decision.events }),
};
}, context);
}
/** Ordinary progress declines when explicit durable cancellation has already won. */
advanceOperation<TState extends OperationState, TResult>(
current: TState,
plan: (
state: LaneState,
current: TState,
meta: OperationMeta,
reader: SessionReader,
) => OperationCommand<TResult> | Promise<OperationCommand<TResult>>,
context: Context,
): Promise<TResult | undefined> {
return this.mutateOperation<TState, TResult | undefined>(
current,
(state, latest, meta, reader) =>
latest.control.status === "cancel_requested"
? { kind: "return", result: undefined }
: plan(state, latest, meta, reader),
context,
);
}
async accept(request: OperationRequest, context: Context): Promise<OperationAdmissionResult> {
@ -428,16 +436,15 @@ export class Lane<TContext extends object | undefined> implements AgentLane {
startedAt,
intent: { kind: "run" as const, promptEntryIds: prompt.map(({ id }) => id) },
};
const operationState = {
kind: "run" as const,
control: { status: "running" as const },
const operationState: RunStartingOperation = {
at: "run.starting",
control: { status: "running" },
settings: {
compaction: acceptanceConfig.compaction,
steeringMode: acceptanceConfig.steeringMode,
followUpMode: acceptanceConfig.followUpMode,
toolExecution: acceptanceConfig.toolExecution,
},
phase: { kind: "starting" as const },
inbox: { steer: [], followUp: [], writes: [] },
latestAssistantEntryId: null,
};
@ -683,17 +690,15 @@ export class Lane<TContext extends object | undefined> implements AgentLane {
);
const nextRun = await queuedItems(captured.pendingNextRun, "Pending next-run entry");
const operation = captured.operation;
const steer =
operation?.state.kind === "run" ? await queuedItems(operation.state.inbox.steer, "Steer entry") : [];
const runState = operation !== null && isRunOperationState(operation.state) ? operation.state : undefined;
const steer = runState === undefined ? [] : await queuedItems(runState.inbox.steer, "Steer entry");
const followUp =
operation?.state.kind === "run"
? await queuedItems(operation.state.inbox.followUp, "Follow-up entry")
: [];
runState === undefined ? [] : await queuedItems(runState.inbox.followUp, "Follow-up entry");
// TODO does a client really need pending writes? We wouldn't visualize those, any other uses for them?
const pendingWrites =
operation?.state.kind === "run"
runState !== undefined
? await Promise.all(
operation.state.inbox.writes.map(async (entryId) => {
runState.inbox.writes.map(async (entryId) => {
const stored = await reader.getValue(pendingEntry(entryId), context);
if (stored === undefined) {
throw new SessionInvariantError(`Pending write ${entryId} is missing its payload`);
@ -734,103 +739,81 @@ export class Lane<TContext extends object | undefined> implements AgentLane {
}
return reduceAssistantMessageFrames(frames);
};
const readRetry = (generation: {
status: string;
nextAttempt?: number;
notBefore?: number;
context: { retryPolicy: { maxAttempts: number } };
}) => {
if (generation.status !== "retry_wait") return undefined;
if (generation.nextAttempt === undefined || generation.notBefore === undefined) {
throw new SessionInvariantError("Retry wait is missing retry metadata");
}
return {
attempt: generation.nextAttempt,
maxAttempts: generation.context.retryPolicy.maxAttempts,
nextAttemptAt: generation.notBefore,
};
};
const readStructuralRetry = (structural: StructuralDecision) => {
if (structural.status !== "generating") return undefined;
return readRetry(structural.generation);
};
if (operation.state.kind === "run") {
switch (operation.state.phase.kind) {
case "assistant":
retry = readRetry(operation.state.phase.generation);
if (operation.state.phase.generation.status === "effect_pending") {
streamingMessage = await readStreamingMessage(
operation.state.phase.generation.responseEntryId,
);
}
break;
case "deferred": {
const durable = operation.state.phase.deferred;
const source = (await reader.getEntries([durable.sourceEntryId], context)).get(
durable.sourceEntryId,
);
if (
source?.type !== "message" ||
source.message.role !== "assistant" ||
source.message.deferred === undefined
) {
throw new SessionInvariantError("Deferred source is missing its assistant handle");
}
deferred = { handle: source.message.deferred, poll: durable.poll };
if (durable.status === "effect_pending") {
streamingMessage = await readStreamingMessage(durable.responseEntryId);
}
break;
const state = operation.state;
switch (state.at) {
case "run.assistant.retry_wait":
retry = {
attempt: state.nextAttempt,
maxAttempts: state.generationContext.retryPolicy.maxAttempts,
nextAttemptAt: state.notBefore,
};
break;
case "run.assistant.effect_pending":
streamingMessage = await readStreamingMessage(state.responseEntryId);
break;
case "run.deferred.suspended":
case "run.deferred.effect_pending": {
const source = (await reader.getEntries([state.sourceEntryId], context)).get(state.sourceEntryId);
if (
source?.type !== "message" ||
source.message.role !== "assistant" ||
source.message.deferred === undefined
) {
throw new SessionInvariantError("Deferred source is missing its assistant handle");
}
case "tools": {
const { batch } = operation.state.phase;
const assistant = (await reader.getEntries([batch.assistantEntryId], context)).get(
batch.assistantEntryId,
);
if (assistant?.type !== "message" || assistant.message.role !== "assistant") {
throw new SessionInvariantError("Tool batch assistant entry is invalid");
}
for (const call of batch.calls) {
if (call.status !== "effect_pending") continue;
const block = assistant.message.content[call.sourceIndex];
if (block?.type !== "toolCall") {
throw new SessionInvariantError(
`Tool call source index ${call.sourceIndex} does not name a tool-call block`,
);
}
const args = await reader.getValue(
operationToolArgs(operation.meta.operationId, batch.turnId, call.sourceIndex),
context,
);
if (args === undefined) {
throw new SessionInvariantError(`Tool call ${block.id} is missing persisted arguments`);
}
const checkpoint = await reader.getValue(
pendingToolOutput(operation.meta.operationId, call.resultEntryId),
context,
);
runningTools.push({
toolCallId: block.id,
toolName: block.name,
args: args.value,
...(checkpoint === undefined ? {} : { partialResult: checkpoint.value }),
});
}
break;
deferred = { handle: source.message.deferred, poll: state.poll };
if (state.at === "run.deferred.effect_pending") {
streamingMessage = await readStreamingMessage(state.responseEntryId);
}
case "compaction":
retry = readStructuralRetry(operation.state.phase.structural);
break;
case "starting":
case "checkpoint":
case "failure_drain":
break;
break;
}
} else if (operation.state.kind === "compaction") {
retry = readStructuralRetry(operation.state.structural);
} else if (operation.state.summarize) {
retry = readStructuralRetry(operation.state.phase.structural);
case "run.tools": {
const { batch } = state;
const assistant = (await reader.getEntries([batch.assistantEntryId], context)).get(
batch.assistantEntryId,
);
if (assistant?.type !== "message" || assistant.message.role !== "assistant") {
throw new SessionInvariantError("Tool batch assistant entry is invalid");
}
for (const call of batch.calls) {
if (call.status !== "effect_pending") continue;
const block = assistant.message.content[call.sourceIndex];
if (block?.type !== "toolCall") {
throw new SessionInvariantError(
`Tool call source index ${call.sourceIndex} does not name a tool-call block`,
);
}
const args = await reader.getValue(
operationToolArgs(operation.meta.operationId, batch.turnId, call.sourceIndex),
context,
);
if (args === undefined) {
throw new SessionInvariantError(`Tool call ${block.id} is missing persisted arguments`);
}
const checkpoint = await reader.getValue(
pendingToolOutput(operation.meta.operationId, call.resultEntryId),
context,
);
runningTools.push({
toolCallId: block.id,
toolName: block.name,
args: args.value,
...(checkpoint === undefined ? {} : { partialResult: checkpoint.value }),
});
}
break;
}
case "run.compaction.retry_wait":
case "compaction.retry_wait":
case "navigation.summary.retry_wait":
retry = {
attempt: state.nextAttempt,
maxAttempts: state.summaryContext.retryPolicy.maxAttempts,
nextAttemptAt: state.notBefore,
};
break;
default:
break;
}
const drained =
@ -977,7 +960,7 @@ export class Lane<TContext extends object | undefined> implements AgentLane {
}
const operation = state.operation;
if (operation.state.kind !== "run") {
if (!isRunOperationState(operation.state)) {
return {
kind: "reject",
error: new Error(`Cannot append while structural operation ${operation.meta.operationId} is active`),

50
packages/agent/src/harness/runtime/progress.ts
View file

@ -32,19 +32,15 @@ function openProgress<TContext extends object | undefined, T>(
write(item) {
if (sealed) return;
const write = lane
.commandDriveOwned(
drive,
(projection) => {
if (!stillOwns(projection)) return { kind: "return", result: undefined };
return {
kind: "commit",
writes: [commitWrite(item)],
next: projection,
materialize: () => undefined,
};
},
drive.context,
)
.command((projection) => {
if (!stillOwns(projection)) return { kind: "return", result: undefined };
return {
kind: "commit",
writes: [commitWrite(item)],
next: projection,
materialize: () => undefined,
};
}, drive.context)
.then(() => undefined);
latest = write;
void write.catch(() => {});
@ -71,18 +67,12 @@ export function openFrameProgress<TContext extends object | undefined>(
(frame) => appendList(address, frame),
() => deleteList(address),
(state) => {
const operation = state.operation;
if (operation?.meta.operationId !== drive.operationId || operation.state.kind !== "run") return false;
const run = operation.state;
if (run.phase.kind === "assistant") {
const generation = run.phase.generation;
return generation.status === "effect_pending" && generation.responseEntryId === responseEntryId;
}
if (run.phase.kind === "deferred") {
const deferred = run.phase.deferred;
return deferred.status === "effect_pending" && deferred.responseEntryId === responseEntryId;
}
return false;
const run = state.operation?.state;
if (run === undefined) return false;
return (
(run.at === "run.assistant.effect_pending" || run.at === "run.deferred.effect_pending") &&
run.responseEntryId === responseEntryId
);
},
);
}
@ -102,14 +92,8 @@ export function openToolProgress<TContext extends object | undefined>(
() => deleteValue(address),
(state) => {
const operation = state.operation;
if (
operation?.meta.operationId !== drive.operationId ||
operation.state.kind !== "run" ||
operation.state.phase.kind !== "tools"
) {
return false;
}
const batch = operation.state.phase.batch;
if (operation?.state.at !== "run.tools") return false;
const batch = operation.state.batch;
return (
batch.turnId === turnId &&
batch.calls.some(

4
packages/agent/src/harness/runtime/restore.ts
View file

@ -78,9 +78,9 @@ export async function restoreLaneState(reader: SessionReader, lane: string, cont
`Operation ${operationId} belongs to lane ${JSON.stringify(meta.value.lane)}, not ${JSON.stringify(lane)}`,
);
}
if (meta.value.intent.kind !== state.value.kind) {
if (!state.value.at.startsWith(`${meta.value.intent.kind}.`)) {
throw new SessionInvariantError(
`Operation ${operationId} intent ${meta.value.intent.kind} does not match state ${state.value.kind}`,
`Operation ${operationId} intent ${meta.value.intent.kind} does not match state ${state.value.at}`,
);
}
operation = { meta: meta.value, state: state.value };

50
packages/agent/src/harness/runtime/types.ts
View file

@ -11,7 +11,14 @@ import type {
import type { CompactionSettings } from "../compaction/compaction.ts";
import { type Context, withoutAbortSignal } from "../context.ts";
import { createGate, type Gate, type GateControl } from "../execution/effect-gate.ts";
import type { CommitResult, LaneConfiguration, LaneLastResult, Operation, Write } from "../session/types.ts";
import type {
CommitResult,
LaneConfiguration,
LaneLastResult,
Operation,
OperationState,
Write,
} from "../session/types.ts";
import type { AgentHarnessStreamOptions, AgentHarnessTool } from "../types.ts";
export class SliceNotImplemented extends Error {
@ -48,15 +55,34 @@ export interface LaneState {
type Synchronous<TResult> = TResult extends PromiseLike<unknown> ? never : TResult;
interface CommitDecision<TResult> {
kind: "commit";
writes: Write[];
materialize(commit: CommitResult): Synchronous<TResult>;
events?(commit: CommitResult): HarnessEvent[];
}
/** One effect-free decision made on a lane's serialized mutation line. */
export type LaneCommand<TResult> =
| {
kind: "commit";
writes: Write[];
next: LaneState;
materialize(commit: CommitResult): Synchronous<TResult>;
events?(commit: CommitResult): HarnessEvent[];
}
| (CommitDecision<TResult> & { next: LaneState })
| { kind: "return"; result: TResult }
| { kind: "reject"; error: Error };
/** A durable operation transition. The Lane pairs the state write with projection publication. */
type LanePatch = Partial<Pick<LaneState, "tipId" | "configuration" | "pendingNextRun" | "lastResult">>;
interface FinishDecision<TResult> {
kind: "finish";
writes: Write[];
lastResult: LaneLastResult;
lane?: LanePatch;
materialize(commit: CommitResult): Synchronous<TResult>;
events?(commit: CommitResult): HarnessEvent[];
}
export type OperationCommand<TResult> =
| (CommitDecision<TResult> & { operationState: OperationState; lane?: LanePatch })
| FinishDecision<TResult>
| { kind: "return"; result: TResult }
| { kind: "reject"; error: Error };
@ -66,10 +92,8 @@ export class Drive {
readonly completion: Promise<DriveOutcome>;
readonly gate: Gate;
readonly context: Context;
readonly installerSignal: AbortSignal | undefined;
readonly waitForRetry: boolean;
deferredPermits: number;
finalizedOutcome: DriveOutcome | undefined;
private readonly control: GateControl;
private readonly resolveCompletion: (outcome: DriveOutcome) => void;
@ -78,7 +102,6 @@ export class Drive {
constructor(options: DriveOptions, context: Context) {
this.operationId = options.operationId;
this.installerSignal = context.abortSignal;
this.context = withoutAbortSignal(context);
this.waitForRetry = options.waitForRetry ?? false;
this.deferredPermits = options.pollDeferred === true ? 1 : 0;
@ -121,10 +144,7 @@ export class Drive {
}
}
export type LostOwnership = { kind: "lost_ownership" };
export type ProcedureResult =
| { kind: "continue" }
| { kind: "waiting"; outcome: DriveOutcome }
| { kind: "settled"; outcome: TerminalOperationOutcome }
| LostOwnership;
| { kind: "settled"; outcome: TerminalOperationOutcome };

4
packages/agent/src/harness/session/testing/conformance/session-repo.ts
View file

@ -390,9 +390,9 @@ export function createSessionRepoForkBehaviorConformance<TMetadata extends Sessi
intent: { kind: "compaction" },
}),
setValue(operationState(OPERATION_ID), {
kind: "compaction",
at: "compaction.deciding",
control: { status: "running" },
structural: { taskId: OPERATION_ID, status: "deciding" },
taskId: OPERATION_ID,
}),
setValue(operationToolArgs(OPERATION_ID, ROOT_ID, 0), {
argument: true,

370
packages/agent/src/harness/session/types.ts
View file

@ -107,8 +107,8 @@ export type Continuation =
| { kind: "need_assistant"; overflowRecoveryUsed: boolean }
| { kind: "may_finish"; includeFinalAssistant: boolean };
export interface CheckpointPhase {
kind: "checkpoint";
/** Checkpoint payload; the flat leaf literal replaces the old nested phase tag. */
export interface CheckpointData {
continuation: Continuation;
triggerEntryId: string;
thresholdCheckedTriggerEntryId?: string;
@ -135,25 +135,6 @@ export interface GenerationContext {
overflowRecoveryUsed: boolean;
}
export type Generation =
| { status: "ready"; context: GenerationContext; nextAttempt: number }
| {
status: "effect_pending";
context: GenerationContext;
attempt: number;
responseEntryId: string;
usageId: string;
intendedOutputLimit: number;
contextWindow: number;
}
| {
status: "retry_wait";
context: GenerationContext;
nextAttempt: number;
notBefore: number;
errorMessage: string;
};
interface ToolCallSource {
/** Zero-based index in the assistant message's complete content array, not a filtered tool-call ordinal. */
sourceIndex: number;
@ -175,26 +156,6 @@ export interface ToolBatch {
calls: ToolCall[];
}
export type Deferred =
| {
status: "suspended";
stepId: string;
sourceEntryId: string;
poll: number;
configuration: LaneConfiguration;
streamOptions: AgentHarnessStreamOptions;
}
| {
status: "effect_pending";
stepId: string;
sourceEntryId: string;
poll: number;
responseEntryId: string;
usageId: string;
configuration: LaneConfiguration;
streamOptions: AgentHarnessStreamOptions;
};
export interface SummaryContext {
taskId: string;
resultEntryId: string;
@ -205,90 +166,267 @@ export interface SummaryContext {
reason?: "manual" | "threshold" | "overflow";
}
export type SummaryGeneration =
| { status: "ready"; context: SummaryContext; nextAttempt: number }
| {
status: "effect_pending";
context: SummaryContext;
attempt: number;
request?: { index: number; usageId: string };
usageIds: string[];
}
| {
status: "retry_wait";
context: SummaryContext;
nextAttempt: number;
notBefore: number;
errorMessage: string;
};
/*
* Durable operation state is one flat union: a single `at` discriminator with one
* namespaced literal per dispatcher leaf. There is no nested phase/status hierarchy.
* ToolBatch/ToolCall remain the nested child collection state machine, and
* cancellation stays orthogonal via `Control`.
*/
export type StructuralDecision = { taskId: string } & (
| { status: "deciding" }
| { status: "generating"; generation: SummaryGeneration }
);
export type RunPhase =
| { kind: "starting" }
| CheckpointPhase
| { kind: "assistant"; generation: Generation }
| { kind: "tools"; batch: ToolBatch }
| {
kind: "compaction";
reason: "threshold" | "overflow";
structural: StructuralDecision;
resumeAfter: CheckpointPhase;
}
| { kind: "deferred"; deferred: Deferred }
| {
kind: "failure_drain";
error: OperationError;
provenance:
| { kind: "response"; entryId: string }
| { kind: "structural"; taskId: string }
| { kind: "configuration" };
};
export interface RunState {
kind: "run";
/** Every leaf carries the orthogonal cancellation shape. */
export interface Cancellable {
control: Control;
settings: {
compaction: CompactionSettings;
steeringMode: QueueMode;
followUpMode: QueueMode;
toolExecution: "sequential" | "parallel";
};
phase: RunPhase;
}
export interface RunSettings {
compaction: CompactionSettings;
steeringMode: QueueMode;
followUpMode: QueueMode;
toolExecution: "sequential" | "parallel";
}
/** Run-wide durable data shared by every `run.*` leaf. */
export interface RunScope extends Cancellable {
settings: RunSettings;
inbox: Inbox;
latestAssistantEntryId: string | null;
}
export interface CompactionState {
kind: "compaction";
control: Control;
customInstructions?: string;
structural: StructuralDecision;
/** Failure origin. Payload datum only; never used for operation dispatch. */
export type FailureProvenance =
| { kind: "response"; entryId: string }
| { kind: "structural"; taskId: string }
| { kind: "configuration" };
/** Shared backoff data for any `*.retry_wait` leaf. */
export interface RetryWait {
nextAttempt: number;
notBefore: number;
errorMessage: string;
}
export type NavigationState =
| {
kind: "navigation";
control: Control;
targetId: string | null;
label?: string;
summarize: false;
phase: { kind: "ready_to_commit" };
}
| {
kind: "navigation";
control: Control;
targetId: string;
label?: string;
customInstructions?: string;
summarize: true;
phase: { kind: "summary"; structural: StructuralDecision };
};
/** Assistant generation input shared by every `run.assistant.*` leaf. */
export interface AssistantGenerationScope {
generationContext: GenerationContext;
}
export type OperationState = RunState | CompactionState | NavigationState;
/**
* Summary generation input shared by every structural generating leaf. The structural
* task id lives in `summaryContext.taskId`; only deciding leaves carry it explicitly
* (via StructuralTask).
*/
export interface SummaryGenerationScope {
summaryContext: SummaryContext;
}
export interface SummaryGenerationReady extends SummaryGenerationScope {
nextAttempt: number;
}
export interface SummaryGenerationEffectPending extends SummaryGenerationScope {
attempt: number;
request?: { index: number; usageId: string };
usageIds: string[];
}
export interface SummaryGenerationRetryWait extends SummaryGenerationScope, RetryWait {}
/** Explicit structural task id for deciding leaves that have no SummaryContext yet. */
export interface StructuralTask {
taskId: string;
}
/** Deferred-suspension durable data shared by both `run.deferred.*` leaves. */
export interface DeferredScope extends RunScope {
stepId: string;
sourceEntryId: string;
poll: number;
configuration: LaneConfiguration;
streamOptions: AgentHarnessStreamOptions;
}
/** In-run compaction data: why it started and which checkpoint resumes after it. */
export interface RunCompactionScope extends RunScope {
reason: "threshold" | "overflow";
resumeAfter: CheckpointData;
}
/** Standalone compaction shared data. */
export interface CompactionScope extends Cancellable {
customInstructions?: string;
}
/** Navigation shared data. */
export interface NavigationScope extends Cancellable {
label?: string;
}
/** Summarized navigation requires a concrete target entry. */
export interface NavigationSummaryScope extends NavigationScope {
targetId: string;
customInstructions?: string;
}
export interface RunStartingOperation extends RunScope {
at: "run.starting";
}
export interface RunCheckpointOperation extends RunScope, CheckpointData {
at: "run.checkpoint";
}
export interface RunAssistantReadyOperation extends RunScope, AssistantGenerationScope {
at: "run.assistant.ready";
nextAttempt: number;
}
export interface RunAssistantEffectPendingOperation extends RunScope, AssistantGenerationScope {
at: "run.assistant.effect_pending";
attempt: number;
responseEntryId: string;
usageId: string;
intendedOutputLimit: number;
contextWindow: number;
}
export interface RunAssistantRetryWaitOperation extends RunScope, AssistantGenerationScope, RetryWait {
at: "run.assistant.retry_wait";
}
export interface RunToolsOperation extends RunScope {
at: "run.tools";
/** Unchanged child collection state machine. */
batch: ToolBatch;
}
export interface RunDeferredSuspendedOperation extends DeferredScope {
at: "run.deferred.suspended";
}
export interface RunDeferredEffectPendingOperation extends DeferredScope {
at: "run.deferred.effect_pending";
responseEntryId: string;
usageId: string;
}
export interface RunCompactionDecidingOperation extends RunCompactionScope, StructuralTask {
at: "run.compaction.deciding";
}
export interface RunCompactionReadyOperation extends RunCompactionScope, SummaryGenerationReady {
at: "run.compaction.ready";
}
export interface RunCompactionEffectPendingOperation extends RunCompactionScope, SummaryGenerationEffectPending {
at: "run.compaction.effect_pending";
}
export interface RunCompactionRetryWaitOperation extends RunCompactionScope, SummaryGenerationRetryWait {
at: "run.compaction.retry_wait";
}
export interface RunFailureDrainOperation extends RunScope {
at: "run.failure_drain";
error: OperationError;
provenance: FailureProvenance;
}
export interface CompactionDecidingOperation extends CompactionScope, StructuralTask {
at: "compaction.deciding";
}
export interface CompactionReadyOperation extends CompactionScope, SummaryGenerationReady {
at: "compaction.ready";
}
export interface CompactionEffectPendingOperation extends CompactionScope, SummaryGenerationEffectPending {
at: "compaction.effect_pending";
}
export interface CompactionRetryWaitOperation extends CompactionScope, SummaryGenerationRetryWait {
at: "compaction.retry_wait";
}
export interface NavigationReadyToCommitOperation extends NavigationScope {
at: "navigation.ready_to_commit";
/** Unsummarized navigation may target the branch root (null). */
targetId: string | null;
}
export interface NavigationSummaryDecidingOperation extends NavigationSummaryScope, StructuralTask {
at: "navigation.summary.deciding";
}
export interface NavigationSummaryReadyOperation extends NavigationSummaryScope, SummaryGenerationReady {
at: "navigation.summary.ready";
}
export interface NavigationSummaryEffectPendingOperation
extends NavigationSummaryScope,
SummaryGenerationEffectPending {
at: "navigation.summary.effect_pending";
}
export interface NavigationSummaryRetryWaitOperation extends NavigationSummaryScope, SummaryGenerationRetryWait {
at: "navigation.summary.retry_wait";
}
/** Flat durable operation state: exactly one leaf per dispatcher target. */
export type OperationState =
| RunStartingOperation
| RunCheckpointOperation
| RunAssistantReadyOperation
| RunAssistantEffectPendingOperation
| RunAssistantRetryWaitOperation
| RunToolsOperation
| RunDeferredSuspendedOperation
| RunDeferredEffectPendingOperation
| RunCompactionDecidingOperation
| RunCompactionReadyOperation
| RunCompactionEffectPendingOperation
| RunCompactionRetryWaitOperation
| RunFailureDrainOperation
| CompactionDecidingOperation
| CompactionReadyOperation
| CompactionEffectPendingOperation
| CompactionRetryWaitOperation
| NavigationReadyToCommitOperation
| NavigationSummaryDecidingOperation
| NavigationSummaryReadyOperation
| NavigationSummaryEffectPendingOperation
| NavigationSummaryRetryWaitOperation;
export type OperationAt = OperationState["at"];
export type RunOperationState = Extract<OperationState, { at: `run.${string}` }>;
export type CompactionOperationState = Extract<OperationState, { at: `compaction.${string}` }>;
export type NavigationOperationState = Extract<OperationState, { at: `navigation.${string}` }>;
export function isRunOperationState(state: OperationState): state is RunOperationState {
return state.at.startsWith("run.");
}
/** Copy only the run-wide scope fields when constructing a successor leaf. */
export function runScopeOf(state: RunOperationState): RunScope {
return {
control: state.control,
settings: state.settings,
inbox: state.inbox,
latestAssistantEntryId: state.latestAssistantEntryId,
};
}
/** Copy only the checkpoint payload fields from a checkpoint-bearing state. */
export function checkpointDataOf(state: CheckpointData): CheckpointData {
return {
continuation: state.continuation,
triggerEntryId: state.triggerEntryId,
...(state.thresholdCheckedTriggerEntryId === undefined
? {}
: { thresholdCheckedTriggerEntryId: state.thresholdCheckedTriggerEntryId }),
...(state.skipInboxOnce === undefined ? {} : { skipInboxOnce: state.skipInboxOnce }),
};
}
export type Operation = { meta: OperationMeta; state: OperationState };
export interface LaneState {

7
packages/agent/test/harness/execution-primitives.test.ts
View file

@ -8,7 +8,7 @@ import {
withTelemetryContext,
} from "../../src/harness/context.ts";
import { HarnessEventBus } from "../../src/harness/events.ts";
import { AbortRequested, createGate, DriveAbandoned } from "../../src/harness/execution/effect-gate.ts";
import { AbortRequested, createGate } from "../../src/harness/execution/effect-gate.ts";
import { HookRegistry } from "../../src/harness/hooks.ts";
import { InMemoryTelemetryContext } from "../../src/index.ts";
@ -48,11 +48,6 @@ describe("createGate", () => {
expect(() => gate.admit(() => undefined)).toThrow(error);
expect(gate.signal.aborted).toBe(true);
});
it("keeps invocation abandonment distinct from durable abort control flow", () => {
expect(new DriveAbandoned()).not.toBeInstanceOf(AbortRequested);
expect(new DriveAbandoned().name).toBe("DriveAbandoned");
});
});
describe("HookRegistry", () => {

5
packages/agent/test/harness/runtime/accept.test.ts
View file

@ -113,14 +113,13 @@ describe("runtime atomic run acceptance", () => {
const admission = unwrap(await lane.accept(request, BACKGROUND_CONTEXT));
const operation = lane.state.operation;
if (operation?.state.kind !== "run") throw new Error("Expected accepted run");
if (operation?.state.at !== "run.starting") throw new Error("Expected accepted run");
const entryId = operation.meta.intent.kind === "run" ? operation.meta.intent.promptEntryIds[0] : undefined;
if (entryId === undefined) throw new Error("Expected prompt entry");
const entry = await session.getEntry(entryId, BACKGROUND_CONTEXT);
expect(admission).toMatchObject({ operationId: operation.meta.operationId, kind: "run" });
expect(entry).toMatchObject({ type: "message", message: { role: "user", content: expectedContent } });
expect(operation.state.phase).toEqual({ kind: "starting" });
expect(operation.state.settings).toEqual({
compaction: DEFAULT_COMPACTION_SETTINGS,
steeringMode: "all",
@ -215,7 +214,7 @@ describe("runtime atomic run acceptance", () => {
unwrap(await lane.accept({ kind: "prompt", prompt: "" }, BACKGROUND_CONTEXT));
const operation = lane.state.operation;
if (operation?.meta.intent.kind !== "run" || operation.state.kind !== "run") {
if (operation?.meta.intent.kind !== "run" || operation.state.at !== "run.starting") {
throw new Error("Expected accepted run");
}
expect(operation.meta.intent.promptEntryIds).toEqual([]);

197
packages/agent/test/harness/runtime/drive-generation.test.ts
View file

@ -21,13 +21,16 @@ import { type Config, Drive } from "../../../src/harness/runtime/types.ts";
import { MemoryStorage } from "../../../src/harness/session/memory.ts";
import { StorageBackedSession } from "../../../src/harness/session/session.ts";
import { InstrumentedStorage } from "../../../src/harness/session/testing/instrumented-storage.ts";
import type {
Generation,
LaneConfiguration,
MessageEntry,
RunState,
Session,
Write,
import {
isRunOperationState,
type LaneConfiguration,
type MessageEntry,
type RunAssistantEffectPendingOperation,
type RunAssistantReadyOperation,
type RunOperationState,
runScopeOf,
type Session,
type Write,
} from "../../../src/harness/session/types.ts";
import * as storedValues from "../../../src/harness/session/values.ts";
import { deferred } from "./test-utils.ts";
@ -144,26 +147,29 @@ async function createFixture(backend: MemoryStorage = new MemoryStorage({ now: (
return { lane, drive, session, storage, models, faux, hooks, events, config };
}
function currentRun(lane: Lane<undefined>): RunState {
function currentRun(lane: Lane<undefined>): RunOperationState {
const operation = lane.state.operation;
if (operation?.state.kind !== "run") throw new Error("fixture has no run");
if (operation === null || !isRunOperationState(operation.state)) throw new Error("fixture has no run");
return operation.state;
}
function readyGeneration(lane: Lane<undefined>): Extract<Generation, { status: "ready" }> {
function readyGeneration(lane: Lane<undefined>): RunAssistantReadyOperation {
const run = currentRun(lane);
if (run.phase.kind !== "assistant" || run.phase.generation.status !== "ready") {
throw new Error("fixture has no ready generation");
}
return run.phase.generation;
if (run.at !== "run.assistant.ready") throw new Error("fixture has no ready generation");
return run;
}
async function advanceToReady(fixture: Fixture): Promise<Extract<Generation, { status: "ready" }>> {
let run = currentRun(fixture.lane);
await startRun(fixture.lane, fixture.drive, run);
run = currentRun(fixture.lane);
if (run.phase.kind !== "checkpoint") throw new Error("run did not reach checkpoint");
await runCheckpoint(fixture.lane, fixture.drive, run, run.phase);
async function startFixtureRun(fixture: Fixture) {
const run = currentRun(fixture.lane);
if (run.at !== "run.starting") throw new Error("run did not start at its initial boundary");
return startRun(fixture.lane, fixture.drive, run);
}
async function advanceToReady(fixture: Fixture): Promise<RunAssistantReadyOperation> {
await startFixtureRun(fixture);
const run = currentRun(fixture.lane);
if (run.at !== "run.checkpoint") throw new Error("run did not reach checkpoint");
await runCheckpoint(fixture.lane, fixture.drive, run);
return readyGeneration(fixture.lane);
}
@ -179,16 +185,8 @@ function writesOperationState(writes: readonly Write[], status: string): boolean
write.namespace === "pi.op.state" &&
typeof write.value === "object" &&
write.value !== null &&
"phase" in write.value &&
typeof write.value.phase === "object" &&
write.value.phase !== null &&
"kind" in write.value.phase &&
write.value.phase.kind === "assistant" &&
"generation" in write.value.phase &&
typeof write.value.phase.generation === "object" &&
write.value.phase.generation !== null &&
"status" in write.value.phase.generation &&
write.value.phase.generation.status === status,
"at" in write.value &&
write.value.at === `run.assistant.${status}`,
);
}
@ -202,21 +200,19 @@ describe("runtime generation checkpoint", () => {
fixture.hooks.on("before_run", () => ({
messages: [{ role: "user", content: "injected", timestamp: 2 }],
}));
const starting = currentRun(fixture.lane);
expect(await startRun(fixture.lane, fixture.drive, starting)).toEqual({ kind: "continue" });
expect(await startFixtureRun(fixture)).toEqual({ kind: "continue" });
let run = currentRun(fixture.lane);
if (run.phase.kind !== "checkpoint") throw new Error("missing checkpoint");
expect(run.phase.triggerEntryId).toBe(fixture.lane.state.tipId);
if (run.at !== "run.checkpoint") throw new Error("missing checkpoint");
expect(run.triggerEntryId).toBe(fixture.lane.state.tipId);
expect(fixture.events.map((event) => event.type)).toContain("entry_added");
await expectProjectionRestores(fixture);
expect(await runCheckpoint(fixture.lane, fixture.drive, run, run.phase)).toEqual({ kind: "continue" });
expect(await runCheckpoint(fixture.lane, fixture.drive, run)).toEqual({ kind: "continue" });
run = currentRun(fixture.lane);
if (run.phase.kind !== "assistant" || run.phase.generation.status !== "ready") {
if (run.at !== "run.assistant.ready") {
throw new Error("missing ready generation");
}
expect(run.phase.generation.context).toMatchObject({
expect(run.generationContext).toMatchObject({
configuration: fixture.lane.state.configuration,
streamOptions: {},
retryPolicy: { maxAttempts: 4, baseDelayMs: 1 },
@ -227,13 +223,13 @@ describe("runtime generation checkpoint", () => {
it("drains mixed pending writes while separating leaf and generation trigger", async () => {
const fixture = await createFixture();
await startRun(fixture.lane, fixture.drive, currentRun(fixture.lane));
await startFixtureRun(fixture);
const messageId = "01950000-0000-7000-8000-000000000010";
const customId = "01950000-0000-7000-8000-000000000011";
await fixture.lane.command((state) => {
const operation = state.operation;
if (operation?.state.kind !== "run") throw new Error("missing run");
const nextState: RunState = {
if (operation === null || !isRunOperationState(operation.state)) throw new Error("missing run");
const nextState: RunOperationState = {
...operation.state,
inbox: { ...operation.state.inbox, writes: [messageId, customId] },
};
@ -256,15 +252,15 @@ describe("runtime generation checkpoint", () => {
};
}, BACKGROUND_CONTEXT);
const run = currentRun(fixture.lane);
if (run.phase.kind !== "checkpoint") throw new Error("missing checkpoint");
if (run.at !== "run.checkpoint") throw new Error("missing checkpoint");
await runCheckpoint(fixture.lane, fixture.drive, run, run.phase);
await runCheckpoint(fixture.lane, fixture.drive, run);
const drained = currentRun(fixture.lane);
if (drained.phase.kind !== "checkpoint") throw new Error("missing drained checkpoint");
if (drained.at !== "run.checkpoint") throw new Error("missing drained checkpoint");
expect(fixture.lane.state.tipId).toBe(customId);
expect(drained.phase.triggerEntryId).toBe(messageId);
expect(drained.phase).toMatchObject({
expect(drained.triggerEntryId).toBe(messageId);
expect(drained).toMatchObject({
continuation: { kind: "need_assistant", overflowRecoveryUsed: false },
skipInboxOnce: true,
});
@ -288,21 +284,18 @@ describe("runtime assistant generation", () => {
fixture.faux.setResponses([
async () => {
const state = await fixture.session.getValue(storedValues.operationState(operationId), BACKGROUND_CONTEXT);
effectPendingAtProvider =
state?.value.kind === "run" &&
state.value.phase.kind === "assistant" &&
state.value.phase.generation.status === "effect_pending";
effectPendingAtProvider = state?.value.at === "run.assistant.effect_pending";
return fauxAssistantMessage("answer", { timestamp: 5 });
},
]);
fixture.storage.clearCommitAttempts();
const generating = runGeneration(fixture.lane, fixture.drive, currentRun(fixture.lane), ready);
const generating = runGeneration(fixture.lane, fixture.drive, ready);
await hookStarted.promise;
const steerId = "01950000-0000-7000-8000-000000000020";
await fixture.lane.command((state) => {
const operation = state.operation;
if (operation?.state.kind !== "run") throw new Error("missing run");
const nextState: RunState = {
if (operation === null || !isRunOperationState(operation.state)) throw new Error("missing run");
const nextState: RunOperationState = {
...operation.state,
inbox: { ...operation.state.inbox, steer: [steerId] },
};
@ -325,7 +318,7 @@ describe("runtime assistant generation", () => {
expect(effectPendingAtProvider).toBe(true);
const settled = currentRun(fixture.lane);
expect(settled.inbox.steer).toEqual([steerId]);
if (settled.phase.kind !== "checkpoint") throw new Error("response did not reach checkpoint");
if (settled.at !== "run.checkpoint") throw new Error("response did not reach checkpoint");
const responseId = settled.latestAssistantEntryId;
if (responseId === null) throw new Error("missing response id");
const entry = await fixture.session.getEntry(responseId, BACKGROUND_CONTEXT);
@ -401,7 +394,7 @@ describe("runtime assistant generation", () => {
backend.blockNextFrame = true;
fixture.storage.clearCommitAttempts();
const generating = runGeneration(fixture.lane, fixture.drive, currentRun(fixture.lane), ready);
const generating = runGeneration(fixture.lane, fixture.drive, ready);
await backend.frameStarted.promise;
await expect
.poll(() => fixture.events.filter((event) => event.type === "message_update").length)
@ -433,13 +426,13 @@ describe("runtime assistant generation", () => {
const ready = await advanceToReady(fixture);
fixture.storage.clearCommitAttempts();
expect(await runGeneration(fixture.lane, fixture.drive, currentRun(fixture.lane), ready)).toEqual({
expect(await runGeneration(fixture.lane, fixture.drive, ready)).toEqual({
kind: "continue",
});
const run = currentRun(fixture.lane);
expect(run.phase).toMatchObject({
kind: "failure_drain",
expect(run).toMatchObject({
at: "run.failure_drain",
error: { code: "configured_tools_unavailable", details: { tools: ["missing"] } },
provenance: { kind: "configuration" },
});
@ -459,12 +452,12 @@ describe("runtime assistant generation", () => {
fixture.models.deleteProvider(fixture.faux.provider.id);
fixture.storage.clearCommitAttempts();
expect(await runGeneration(fixture.lane, fixture.drive, currentRun(fixture.lane), ready)).toEqual({
expect(await runGeneration(fixture.lane, fixture.drive, ready)).toEqual({
kind: "continue",
});
expect(currentRun(fixture.lane).phase).toMatchObject({
kind: "failure_drain",
expect(currentRun(fixture.lane)).toMatchObject({
at: "run.failure_drain",
error: { code: "model_unavailable" },
provenance: { kind: "configuration" },
});
@ -478,7 +471,7 @@ describe("runtime assistant generation", () => {
await expectProjectionRestores(fixture);
});
it("declines intent when cancellation or exact-owner replacement wins preparation", async () => {
it("declines intent when cancellation wins preparation", async () => {
const cancelled = await createFixture();
const cancelledReady = await advanceToReady(cancelled);
const cancelHookStarted = deferred();
@ -489,12 +482,12 @@ describe("runtime assistant generation", () => {
return undefined;
});
cancelled.storage.clearCommitAttempts();
const cancelling = runGeneration(cancelled.lane, cancelled.drive, currentRun(cancelled.lane), cancelledReady);
const cancelling = runGeneration(cancelled.lane, cancelled.drive, cancelledReady);
await cancelHookStarted.promise;
await cancelled.lane.command((state) => {
const operation = state.operation;
if (operation?.state.kind !== "run") throw new Error("missing run");
const nextState: RunState = {
if (operation === null || !isRunOperationState(operation.state)) throw new Error("missing run");
const nextState: RunOperationState = {
...operation.state,
control: { status: "cancel_requested", requestedAt: 10, drainedSteer: [], drainedFollowUp: [] },
};
@ -511,24 +504,6 @@ describe("runtime assistant generation", () => {
expect(
cancelled.storage.getCommitAttempts().some((writes) => writesOperationState(writes, "effect_pending")),
).toBe(false);
const replaced = await createFixture();
const replacedReady = await advanceToReady(replaced);
const ownerHookStarted = deferred();
const releaseOwnerHook = deferred();
replaced.hooks.on("before_request", async () => {
ownerHookStarted.resolve();
await releaseOwnerHook.promise;
return undefined;
});
replaced.storage.clearCommitAttempts();
const losing = runGeneration(replaced.lane, replaced.drive, currentRun(replaced.lane), replacedReady);
await ownerHookStarted.promise;
replaced.lane.activeDrive = new Drive({ operationId }, BACKGROUND_CONTEXT);
releaseOwnerHook.resolve();
expect(await losing).toEqual({ kind: "lost_ownership" });
expect(replaced.faux.state.callCount).toBe(0);
expect(replaced.storage.getCommitAttempts()).toEqual([]);
});
it("recovers an orphan with no frames into a zero-usage retry wait", async () => {
@ -536,9 +511,10 @@ describe("runtime assistant generation", () => {
const ready = await advanceToReady(fixture);
const responseEntryId = "01950000-0000-7000-8000-000000000028";
const usageId = "01950000-0000-7000-8000-000000000029";
const pending: Extract<Generation, { status: "effect_pending" }> = {
status: "effect_pending",
context: { ...ready.context, retryPolicy: { maxAttempts: 2, baseDelayMs: 1 } },
const pending: RunAssistantEffectPendingOperation = {
...runScopeOf(ready),
at: "run.assistant.effect_pending",
generationContext: { ...ready.generationContext, retryPolicy: { maxAttempts: 2, baseDelayMs: 1 } },
attempt: 1,
responseEntryId,
usageId,
@ -547,33 +523,29 @@ describe("runtime assistant generation", () => {
};
await fixture.lane.command((state) => {
const operation = state.operation;
if (operation?.state.kind !== "run") throw new Error("missing run");
const nextState: RunState = {
...operation.state,
phase: { kind: "assistant", generation: pending },
};
if (operation === null || !isRunOperationState(operation.state)) throw new Error("missing run");
return {
kind: "commit",
writes: [storedValues.setValue(storedValues.operationState(operationId), nextState)],
next: { ...state, operation: { meta: operation.meta, state: nextState } },
writes: [storedValues.setValue(storedValues.operationState(operationId), pending)],
next: { ...state, operation: { meta: operation.meta, state: pending } },
materialize: () => undefined,
};
}, BACKGROUND_CONTEXT);
await recoverAssistantGeneration(fixture.lane, fixture.drive, currentRun(fixture.lane), pending);
await recoverAssistantGeneration(fixture.lane, fixture.drive, pending);
const entry = (await fixture.session.getEntry(responseEntryId, BACKGROUND_CONTEXT)) as MessageEntry;
expect(entry.message).toMatchObject({
api: "unknown",
provider: ready.context.configuration.model.provider,
model: ready.context.configuration.model.modelId,
provider: ready.generationContext.configuration.model.provider,
model: ready.generationContext.configuration.model.modelId,
content: [],
stopReason: "error",
usage: { totalTokens: 0 },
});
expect(currentRun(fixture.lane).phase).toMatchObject({
kind: "assistant",
generation: { status: "retry_wait", nextAttempt: 2 },
expect(currentRun(fixture.lane)).toMatchObject({
at: "run.assistant.retry_wait",
nextAttempt: 2,
});
expect(fixture.faux.state.callCount).toBe(0);
await expectProjectionRestores(fixture);
@ -588,9 +560,10 @@ describe("runtime assistant generation", () => {
...fauxAssistantMessage([], { timestamp: 6 }),
stopReason: "pending",
};
const pending: Extract<Generation, { status: "effect_pending" }> = {
status: "effect_pending",
context: { ...ready.context, retryPolicy: { maxAttempts: 1, baseDelayMs: 1 } },
const pending: RunAssistantEffectPendingOperation = {
...runScopeOf(ready),
at: "run.assistant.effect_pending",
generationContext: { ...ready.generationContext, retryPolicy: { maxAttempts: 1, baseDelayMs: 1 } },
attempt: 1,
responseEntryId,
usageId,
@ -599,15 +572,11 @@ describe("runtime assistant generation", () => {
};
await fixture.lane.command((state) => {
const operation = state.operation;
if (operation?.state.kind !== "run") throw new Error("missing run");
const nextState: RunState = {
...operation.state,
phase: { kind: "assistant", generation: pending },
};
if (operation === null || !isRunOperationState(operation.state)) throw new Error("missing run");
return {
kind: "commit",
writes: [
storedValues.setValue(storedValues.operationState(operationId), nextState),
storedValues.setValue(storedValues.operationState(operationId), pending),
storedValues.appendList(storedValues.pendingAssistantFrames(operationId, responseEntryId), {
type: "start",
partial,
@ -628,7 +597,7 @@ describe("runtime assistant generation", () => {
content: "corrected",
}),
],
next: { ...state, operation: { meta: operation.meta, state: nextState } },
next: { ...state, operation: { meta: operation.meta, state: pending } },
materialize: () => undefined,
};
}, BACKGROUND_CONTEXT);
@ -638,7 +607,7 @@ describe("runtime assistant generation", () => {
return undefined;
});
expect(await recoverAssistantGeneration(fixture.lane, fixture.drive, currentRun(fixture.lane), pending)).toEqual({
expect(await recoverAssistantGeneration(fixture.lane, fixture.drive, pending)).toEqual({
kind: "continue",
});
@ -651,7 +620,7 @@ describe("runtime assistant generation", () => {
stopReason: "error",
usage: { input: 0, output: 0 },
});
expect(currentRun(fixture.lane).phase).toMatchObject({ kind: "failure_drain" });
expect(currentRun(fixture.lane).at).toBe("run.failure_drain");
expect(
await fixture.session.readList(
storedValues.pendingAssistantFrames(operationId, responseEntryId),
@ -669,11 +638,11 @@ describe("runtime assistant generation", () => {
const fixture = await createFixture();
const ready = await advanceToReady(fixture);
fixture.faux.setResponses([fauxAssistantMessage("done", { timestamp: 7 })]);
await runGeneration(fixture.lane, fixture.drive, currentRun(fixture.lane), ready);
await runGeneration(fixture.lane, fixture.drive, ready);
const run = currentRun(fixture.lane);
if (run.phase.kind !== "checkpoint") throw new Error("missing terminal checkpoint");
if (run.at !== "run.checkpoint") throw new Error("missing terminal checkpoint");
const result = await runCheckpoint(fixture.lane, fixture.drive, run, run.phase);
const result = await runCheckpoint(fixture.lane, fixture.drive, run);
expect(result).toMatchObject({
kind: "settled",

183
packages/agent/test/harness/runtime/drive-retry-deferred.test.ts
View file

@ -23,13 +23,18 @@ import { MemoryStorage } from "../../../src/harness/session/memory.ts";
import { StorageBackedSession } from "../../../src/harness/session/session.ts";
import { GatingStorage } from "../../../src/harness/session/testing/gating-storage.ts";
import { InstrumentedStorage } from "../../../src/harness/session/testing/instrumented-storage.ts";
import type {
Deferred,
Generation,
LaneConfiguration,
RunState,
Session,
Write,
import {
isRunOperationState,
type LaneConfiguration,
type OperationState,
type RunAssistantReadyOperation,
type RunAssistantRetryWaitOperation,
type RunDeferredEffectPendingOperation,
type RunDeferredSuspendedOperation,
type RunOperationState,
runScopeOf,
type Session,
type Write,
} from "../../../src/harness/session/types.ts";
import * as storedValues from "../../../src/harness/session/values.ts";
@ -133,29 +138,30 @@ async function createFixture(
return { backend, gating, storage, session, lane, drive, models, faux, hooks, events, config };
}
function currentRun(fixture: Fixture): RunState {
function currentRun(fixture: Fixture): RunOperationState {
const operation = fixture.lane.state.operation;
if (operation?.state.kind !== "run") throw new Error("fixture has no run");
if (operation === null || !isRunOperationState(operation.state)) throw new Error("fixture has no run");
return operation.state;
}
async function advanceToReady(fixture: Fixture): Promise<Extract<Generation, { status: "ready" }>> {
async function advanceToReady(fixture: Fixture): Promise<RunAssistantReadyOperation> {
const starting = currentRun(fixture);
if (starting.at !== "run.starting") throw new Error("run did not start at its initial boundary");
await startRun(fixture.lane, fixture.drive, starting);
let run = currentRun(fixture);
await startRun(fixture.lane, fixture.drive, run);
if (run.at !== "run.checkpoint") throw new Error("run did not reach checkpoint");
await runCheckpoint(fixture.lane, fixture.drive, run);
run = currentRun(fixture);
if (run.phase.kind !== "checkpoint") throw new Error("run did not reach checkpoint");
await runCheckpoint(fixture.lane, fixture.drive, run, run.phase);
run = currentRun(fixture);
if (run.phase.kind !== "assistant" || run.phase.generation.status !== "ready") {
if (run.at !== "run.assistant.ready") {
throw new Error("run did not reach ready generation");
}
return run.phase.generation;
return run;
}
async function replaceRunState(fixture: Fixture, nextState: RunState, extraWrites: Write[] = []): Promise<void> {
async function replaceRunState(fixture: Fixture, nextState: OperationState, extraWrites: Write[] = []): Promise<void> {
await fixture.lane.command((state) => {
const operation = state.operation;
if (operation?.state.kind !== "run") throw new Error("fixture has no run");
if (operation === null || !isRunOperationState(operation.state)) throw new Error("fixture has no run");
return {
kind: "commit",
writes: [...extraWrites, storedValues.setValue(storedValues.operationState(operationId), nextState)],
@ -168,23 +174,24 @@ async function replaceRunState(fixture: Fixture, nextState: RunState, extraWrite
async function submitDeferred(
fixture: Fixture,
response: AssistantMessage = fauxAssistantMessage("done", { timestamp: 20 }),
): Promise<Extract<Deferred, { status: "suspended" }>> {
): Promise<RunDeferredSuspendedOperation> {
fixture.faux.setResponses([response]);
const ready = await advanceToReady(fixture);
await runGeneration(fixture.lane, fixture.drive, currentRun(fixture), ready);
await runGeneration(fixture.lane, fixture.drive, ready);
const run = currentRun(fixture);
if (run.phase.kind !== "deferred" || run.phase.deferred.status !== "suspended") {
if (run.at !== "run.deferred.suspended") {
throw new Error("initial request did not suspend");
}
return run.phase.deferred;
return run;
}
async function installUnknownPoll(
fixture: Fixture,
suspended: Extract<Deferred, { status: "suspended" }>,
): Promise<Extract<Deferred, { status: "effect_pending" }>> {
const effectPending: Extract<Deferred, { status: "effect_pending" }> = {
status: "effect_pending",
suspended: RunDeferredSuspendedOperation,
): Promise<RunDeferredEffectPendingOperation> {
const effectPending: RunDeferredEffectPendingOperation = {
...runScopeOf(currentRun(fixture)),
at: "run.deferred.effect_pending",
stepId: suspended.stepId,
sourceEntryId: suspended.sourceEntryId,
poll: suspended.poll + 1,
@ -193,9 +200,7 @@ async function installUnknownPoll(
configuration: suspended.configuration,
streamOptions: suspended.streamOptions,
};
const run = currentRun(fixture);
const nextState: RunState = { ...run, phase: { kind: "deferred", deferred: effectPending } };
await replaceRunState(fixture, nextState, [
await replaceRunState(fixture, effectPending, [
storedValues.appendList(storedValues.pendingAssistantFrames(operationId, effectPending.responseEntryId), {
type: "text_delta",
contentIndex: 0,
@ -205,10 +210,12 @@ async function installUnknownPoll(
return effectPending;
}
function currentDeferred(fixture: Fixture): Deferred {
function currentDeferred(fixture: Fixture): RunDeferredSuspendedOperation | RunDeferredEffectPendingOperation {
const run = currentRun(fixture);
if (run.phase.kind !== "deferred") throw new Error("fixture has no deferred phase");
return run.phase.deferred;
if (run.at !== "run.deferred.suspended" && run.at !== "run.deferred.effect_pending") {
throw new Error("fixture has no deferred phase");
}
return run;
}
function installDrive(fixture: Fixture, options: { waitForRetry?: boolean; pollDeferred?: boolean }): Drive {
@ -234,16 +241,13 @@ describe("runtime assistant retry wait", () => {
fauxAssistantMessage([], { stopReason: "error", errorMessage: "503 service unavailable", timestamp: 10 }),
]);
expect(await runGeneration(fixture.lane, fixture.drive, currentRun(fixture), ready)).toEqual({
expect(await runGeneration(fixture.lane, fixture.drive, ready)).toEqual({
kind: "continue",
});
expect(currentRun(fixture).phase).toMatchObject({
kind: "assistant",
generation: {
status: "retry_wait",
nextAttempt: 2,
errorMessage: "503 service unavailable",
},
expect(currentRun(fixture)).toMatchObject({
at: "run.assistant.retry_wait",
nextAttempt: 2,
errorMessage: "503 service unavailable",
});
expect(fixture.events.at(-1)).toMatchObject({ type: "retry_scheduled", attempt: 2 });
await expectProjectionRestores(fixture);
@ -254,20 +258,19 @@ describe("runtime assistant retry wait", () => {
vi.setSystemTime(1_000);
const fixture = await createFixture({ deferredSubmission: false });
const ready = await advanceToReady(fixture);
const retryWait: Extract<Generation, { status: "retry_wait" }> = {
status: "retry_wait",
context: ready.context,
const retryWait: RunAssistantRetryWaitOperation = {
...runScopeOf(ready),
at: "run.assistant.retry_wait",
generationContext: ready.generationContext,
nextAttempt: 2,
notBefore: 1_100,
errorMessage: "retry",
};
const run = currentRun(fixture);
const nextState: RunState = { ...run, phase: { kind: "assistant", generation: retryWait } };
await replaceRunState(fixture, nextState);
await replaceRunState(fixture, retryWait);
fixture.storage.clearCommitAttempts();
const setTimeoutSpy = vi.spyOn(globalThis, "setTimeout");
expect(await runGeneration(fixture.lane, fixture.drive, currentRun(fixture), retryWait)).toEqual({
expect(await runGeneration(fixture.lane, fixture.drive, retryWait)).toEqual({
kind: "waiting",
outcome: { kind: "waiting", operationId, reason: "retry", notBefore: 1_100 },
});
@ -280,27 +283,27 @@ describe("runtime assistant retry wait", () => {
vi.setSystemTime(2_000);
const fixture = await createFixture({ deferredSubmission: false });
const ready = await advanceToReady(fixture);
const retryWait: Extract<Generation, { status: "retry_wait" }> = {
status: "retry_wait",
context: ready.context,
const retryWait: RunAssistantRetryWaitOperation = {
...runScopeOf(ready),
at: "run.assistant.retry_wait",
generationContext: ready.generationContext,
nextAttempt: 2,
notBefore: 2_000,
errorMessage: "retry",
};
const run = currentRun(fixture);
await replaceRunState(fixture, { ...run, phase: { kind: "assistant", generation: retryWait } });
await replaceRunState(fixture, retryWait);
fixture.storage.clearCommitAttempts();
expect(await runGeneration(fixture.lane, fixture.drive, currentRun(fixture), retryWait)).toEqual({
expect(await runGeneration(fixture.lane, fixture.drive, retryWait)).toEqual({
kind: "continue",
});
const next = currentRun(fixture);
if (next.phase.kind !== "assistant" || next.phase.generation.status !== "ready") {
if (next.at !== "run.assistant.ready") {
throw new Error("retry did not become ready");
}
expect(fixture.events.at(-1)).toMatchObject({ type: "retry_start", attempt: 2 });
fixture.faux.setResponses([fauxAssistantMessage("retried", { timestamp: 30 })]);
await runGeneration(fixture.lane, fixture.drive, next, next.phase.generation);
await runGeneration(fixture.lane, fixture.drive, next);
expect(fixture.events).toContainEqual(expect.objectContaining({ type: "retry_end", attempt: 2, success: true }));
await expectProjectionRestores(fixture);
});
@ -310,26 +313,26 @@ describe("runtime assistant retry wait", () => {
vi.setSystemTime(3_000);
const fixture = await createFixture({ deferredSubmission: false });
const ready = await advanceToReady(fixture);
const retryWait: Extract<Generation, { status: "retry_wait" }> = {
status: "retry_wait",
context: ready.context,
const retryWait: RunAssistantRetryWaitOperation = {
...runScopeOf(ready),
at: "run.assistant.retry_wait",
generationContext: ready.generationContext,
nextAttempt: 2,
notBefore: 3_100,
errorMessage: "retry",
};
const run = currentRun(fixture);
await replaceRunState(fixture, { ...run, phase: { kind: "assistant", generation: retryWait } });
await replaceRunState(fixture, retryWait);
const drive = installDrive(fixture, { waitForRetry: true });
fixture.storage.clearCommitAttempts();
const waiting = runGeneration(fixture.lane, drive, currentRun(fixture), retryWait);
const waiting = runGeneration(fixture.lane, drive, retryWait);
await vi.advanceTimersByTimeAsync(99);
expect(fixture.storage.getCommitAttempts()).toEqual([]);
await vi.advanceTimersByTimeAsync(1);
expect(await waiting).toEqual({ kind: "continue" });
expect(currentRun(fixture).phase).toMatchObject({
kind: "assistant",
generation: { status: "ready", nextAttempt: 2 },
expect(currentRun(fixture)).toMatchObject({
at: "run.assistant.ready",
nextAttempt: 2,
});
});
});
@ -340,7 +343,7 @@ describe("runtime deferred polling", () => {
const suspended = await submitDeferred(fixture);
fixture.storage.clearCommitAttempts();
expect(await runDeferredSuspended(fixture.lane, fixture.drive, currentRun(fixture), suspended)).toEqual({
expect(await runDeferredSuspended(fixture.lane, fixture.drive, suspended)).toEqual({
kind: "waiting",
outcome: {
kind: "waiting",
@ -373,7 +376,7 @@ describe("runtime deferred polling", () => {
const drive = new Drive(driveOptions, BACKGROUND_CONTEXT);
fixture.lane.activeDrive = drive;
expect(await runDeferred(fixture.lane, drive, currentRun(fixture), currentDeferred(fixture))).toEqual({
expect(await runDeferred(fixture.lane, drive, currentDeferred(fixture))).toEqual({
kind: "continue",
});
expect(drive.deferredPermits).toBe(0);
@ -381,11 +384,11 @@ describe("runtime deferred polling", () => {
expect(hookOptions).toMatchObject({ deferred: false });
expect(fetchOptions).toMatchObject({ wait: 0 });
expect(fetchOptions?.signal).toBe(drive.gate.signal);
expect(currentDeferred(fixture)).toMatchObject({ status: "suspended", poll: 1 });
expect(currentDeferred(fixture)).toMatchObject({ at: "run.deferred.suspended", poll: 1 });
expect(fixture.faux.state.deferredFetchCount).toBe(1);
expect(fixture.events.slice(-2).map((event) => event.type)).toEqual(["turn_end", "run_suspend"]);
expect(await runDeferred(fixture.lane, drive, currentRun(fixture), currentDeferred(fixture))).toMatchObject({
expect(await runDeferred(fixture.lane, drive, currentDeferred(fixture))).toMatchObject({
kind: "waiting",
outcome: { reason: "deferred" },
});
@ -404,16 +407,16 @@ describe("runtime deferred polling", () => {
);
const drive = installDrive(fixture, { pollDeferred: true });
expect(await runDeferredSuspended(fixture.lane, drive, currentRun(fixture), suspended)).toEqual({
expect(await runDeferredSuspended(fixture.lane, drive, suspended)).toEqual({
kind: "continue",
});
const run = currentRun(fixture);
if (run.phase.kind !== "tools") throw new Error("deferred tool response did not create a batch");
expect(run.phase.batch).toMatchObject({
if (run.at !== "run.tools") throw new Error("deferred tool response did not create a batch");
expect(run.batch).toMatchObject({
turnId: `${suspended.stepId}:poll:1`,
calls: [{ status: "planned", sourceIndex: 0 }],
});
expect(run.phase.batch.calls[0]?.resultEntryId.slice(0, 13)).toBe(run.latestAssistantEntryId?.slice(0, 13));
expect(run.batch.calls[0]?.resultEntryId.slice(0, 13)).toBe(run.latestAssistantEntryId?.slice(0, 13));
expect(fixture.events.at(-1)).toMatchObject({ type: "usage" });
await expectProjectionRestores(fixture);
});
@ -428,7 +431,7 @@ describe("runtime deferred polling", () => {
if (gating === undefined) throw new Error("fixture is not gated");
const drive = installDrive(fixture, { pollDeferred: true });
gating.arm();
const polling = runDeferredSuspended(fixture.lane, drive, currentRun(fixture), suspended);
const polling = runDeferredSuspended(fixture.lane, drive, suspended);
await gating.waitPending();
expect(drive.deferredPermits).toBe(1);
@ -438,7 +441,7 @@ describe("runtime deferred polling", () => {
await expect.poll(() => fixture.faux.state.deferredFetchCount).toBe(1);
await gating.next(2); // start frame, then response settlement
expect(await polling).toEqual({ kind: "continue" });
expect(currentRun(fixture).phase).toMatchObject({ kind: "failure_drain" });
expect(currentRun(fixture).at).toBe("run.failure_drain");
});
it("leaves suspended state unchanged when the fresh intent is discarded", async () => {
@ -448,7 +451,7 @@ describe("runtime deferred polling", () => {
if (gating === undefined) throw new Error("fixture is not gated");
const drive = installDrive(fixture, { pollDeferred: true });
gating.arm();
const polling = runDeferredSuspended(fixture.lane, drive, currentRun(fixture), suspended);
const polling = runDeferredSuspended(fixture.lane, drive, suspended);
void polling.catch(() => {});
await gating.waitPending();
@ -457,10 +460,7 @@ describe("runtime deferred polling", () => {
expect(drive.deferredPermits).toBe(1);
expect(fixture.faux.state.deferredFetchCount).toBe(0);
const durable = await fixture.backend.getValue(storedValues.operationState(operationId), BACKGROUND_CONTEXT);
expect(durable?.value).toMatchObject({
kind: "run",
phase: { kind: "deferred", deferred: { status: "suspended", poll: 0 } },
});
expect(durable?.value).toMatchObject({ at: "run.deferred.suspended", poll: 0 });
});
it("replaces an unknown poll under fresh ids at the same poll number and deletes old frames", async () => {
@ -470,7 +470,7 @@ describe("runtime deferred polling", () => {
fixture.storage.clearCommitAttempts();
const noPermit = installDrive(fixture, {});
expect(await recoverDeferredPoll(fixture.lane, noPermit, currentRun(fixture), unknown)).toMatchObject({
expect(await recoverDeferredPoll(fixture.lane, noPermit, unknown)).toMatchObject({
kind: "waiting",
outcome: { reason: "deferred" },
});
@ -484,7 +484,7 @@ describe("runtime deferred polling", () => {
expect(fixture.faux.state.deferredFetchCount).toBe(0);
const replacement = installDrive(fixture, { pollDeferred: true });
expect(await recoverDeferredPoll(fixture.lane, replacement, currentRun(fixture), unknown)).toEqual({
expect(await recoverDeferredPoll(fixture.lane, replacement, unknown)).toEqual({
kind: "continue",
});
const intent = fixture.storage
@ -507,15 +507,10 @@ describe("runtime deferred polling", () => {
);
expect(intentState).toMatchObject({
value: {
phase: {
kind: "deferred",
deferred: {
status: "effect_pending",
poll: unknown.poll,
responseEntryId: expect.not.stringMatching(unknown.responseEntryId),
usageId: expect.not.stringMatching(unknown.usageId),
},
},
at: "run.deferred.effect_pending",
poll: unknown.poll,
responseEntryId: expect.not.stringMatching(unknown.responseEntryId),
usageId: expect.not.stringMatching(unknown.usageId),
},
});
expect(await fixture.session.getEntry(unknown.responseEntryId, BACKGROUND_CONTEXT)).toBeUndefined();
@ -529,7 +524,7 @@ describe("runtime deferred polling", () => {
BACKGROUND_CONTEXT,
),
).toEqual([]);
expect(currentRun(fixture).phase).toMatchObject({ kind: "checkpoint" });
expect(currentRun(fixture).at).toBe("run.checkpoint");
expect(fixture.events.some((event) => "recovery" in event && event.recovery === true)).toBe(true);
await expectProjectionRestores(fixture);
});
@ -542,13 +537,13 @@ describe("runtime deferred polling", () => {
fixture.storage.clearCommitAttempts();
const drive = installDrive(fixture, { pollDeferred: true });
expect(await recoverDeferredPoll(fixture.lane, drive, currentRun(fixture), unknown)).toEqual({
expect(await recoverDeferredPoll(fixture.lane, drive, unknown)).toEqual({
kind: "continue",
});
expect(drive.deferredPermits).toBe(1);
expect(fixture.faux.state.deferredFetchCount).toBe(0);
expect(currentRun(fixture).phase).toMatchObject({
kind: "failure_drain",
expect(currentRun(fixture)).toMatchObject({
at: "run.failure_drain",
error: { code: "model_unavailable" },
provenance: { kind: "configuration" },
});

77
packages/agent/test/harness/runtime/drive-terminal.test.ts
View file

@ -6,13 +6,15 @@ import { insertEntry } from "../../../src/harness/session/commit.ts";
import { MemoryStorage } from "../../../src/harness/session/memory.ts";
import { StorageBackedSession } from "../../../src/harness/session/session.ts";
import type {
CompactionState,
CompactionDecidingOperation,
LaneConfiguration,
LaneLastResult,
NavigationState,
NavigationReadyToCommitOperation,
OperationMeta,
OperationState,
RunState,
RunAssistantEffectPendingOperation,
RunScope,
RunToolsOperation,
Session,
Write,
} from "../../../src/harness/session/types.ts";
@ -25,9 +27,8 @@ const configuration: LaneConfiguration = {
activeToolNames: [],
};
function runState(phase: RunState["phase"]): RunState {
function runScope(): RunScope {
return {
kind: "run",
control: { status: "running" },
settings: {
compaction: { enabled: true, reserveTokens: 1_000, keepRecentTokens: 2_000 },
@ -35,7 +36,6 @@ function runState(phase: RunState["phase"]): RunState {
followUpMode: "all",
toolExecution: "parallel",
},
phase,
inbox: { steer: [], followUp: [], writes: [] },
latestAssistantEntryId: null,
};
@ -43,11 +43,11 @@ function runState(phase: RunState["phase"]): RunState {
function meta(operationId: string, state: OperationState): OperationMeta {
const intent: OperationMeta["intent"] =
state.kind === "run"
? { kind: "run", promptEntryIds: [] }
: state.kind === "compaction"
? { kind: "compaction" }
: { kind: "navigation", targetId: state.targetId, summarize: state.summarize };
state.at === "compaction.deciding"
? { kind: "compaction" }
: state.at === "navigation.ready_to_commit"
? { kind: "navigation", targetId: state.targetId, summarize: false }
: { kind: "run", promptEntryIds: [] };
return { operationId, lane: "main", sourceTipId: null, startedAt: 1, intent };
}
@ -101,26 +101,22 @@ describe("runtime terminal cleanup mechanics", () => {
const { session } = await createSession();
const operationId = "run";
const responseEntryId = "response";
const state: RunState = {
...runState({
kind: "assistant",
generation: {
status: "effect_pending",
context: {
stepId: "step",
triggerEntryId: "trigger",
configuration,
streamOptions: {},
retryPolicy: { maxAttempts: 2, baseDelayMs: 1 },
overflowRecoveryUsed: false,
},
attempt: 1,
responseEntryId,
usageId: "usage",
intendedOutputLimit: 100,
contextWindow: 1_000,
},
}),
const state: RunAssistantEffectPendingOperation = {
...runScope(),
at: "run.assistant.effect_pending",
generationContext: {
stepId: "step",
triggerEntryId: "trigger",
configuration,
streamOptions: {},
retryPolicy: { maxAttempts: 2, baseDelayMs: 1 },
overflowRecoveryUsed: false,
},
attempt: 1,
responseEntryId,
usageId: "usage",
intendedOutputLimit: 100,
contextWindow: 1_000,
control: {
status: "cancel_requested",
requestedAt: 2,
@ -182,8 +178,9 @@ describe("runtime terminal cleanup mechanics", () => {
it("deletes staged tool outcomes and leaves completed results alone", async () => {
const { session } = await createSession();
const operationId = "tools";
const state = runState({
kind: "tools",
const state: RunToolsOperation = {
...runScope(),
at: "run.tools",
batch: {
assistantEntryId: "assistant",
configuration,
@ -193,7 +190,7 @@ describe("runtime terminal cleanup mechanics", () => {
{ status: "completed", sourceIndex: 1, resultEntryId: "placed", terminate: false },
],
},
});
};
await seedLeftovers(session, operationId, state);
await commit(session, [
storedValues.setValue(storedValues.pendingEntry("staged"), {
@ -219,20 +216,18 @@ describe("runtime terminal cleanup mechanics", () => {
[
"compaction",
{
kind: "compaction",
at: "compaction.deciding",
control: { status: "running" },
structural: { status: "deciding", taskId: "task" },
} satisfies CompactionState,
taskId: "task",
} satisfies CompactionDecidingOperation,
],
[
"navigation",
{
kind: "navigation",
at: "navigation.ready_to_commit",
control: { status: "running" },
targetId: null,
summarize: false,
phase: { kind: "ready_to_commit" },
} satisfies NavigationState,
} satisfies NavigationReadyToCommitOperation,
],
] as const)("defensively deletes leftover %s operation families", async (operationId, state) => {
const { session } = await createSession();

78
packages/agent/test/harness/runtime/drive-tools.test.ts
View file

@ -18,7 +18,15 @@ import { type Config, Drive } from "../../../src/harness/runtime/types.ts";
import { insertEntry } from "../../../src/harness/session/commit.ts";
import { MemoryStorage } from "../../../src/harness/session/memory.ts";
import { StorageBackedSession } from "../../../src/harness/session/session.ts";
import type { LaneConfiguration, RunState, Session, ToolCall, Write } from "../../../src/harness/session/types.ts";
import {
isRunOperationState,
type LaneConfiguration,
type RunOperationState,
type RunToolsOperation,
type Session,
type ToolCall,
type Write,
} from "../../../src/harness/session/types.ts";
import * as storedValues from "../../../src/harness/session/values.ts";
import type {
AgentHarnessTool,
@ -51,7 +59,7 @@ interface FixtureOptions {
operationId: string;
assistantEntryId: string;
resultEntryIds: string[];
run: RunState;
run: RunToolsOperation;
}) => Write[];
toolContext?: Config<undefined>["toolContext"];
cancelled?: boolean;
@ -133,8 +141,8 @@ async function createFixture(options: FixtureOptions): Promise<Fixture> {
sourceIndex: index,
resultEntryId,
}));
const run: RunState = {
kind: "run",
const run: RunToolsOperation = {
at: "run.tools",
control: options.cancelled
? { status: "cancel_requested", requestedAt: 30, drainedSteer: [], drainedFollowUp: [] }
: { status: "running" },
@ -144,10 +152,7 @@ async function createFixture(options: FixtureOptions): Promise<Fixture> {
followUpMode: "all",
toolExecution: options.mode ?? "parallel",
},
phase: {
kind: "tools",
batch: { assistantEntryId, configuration, turnId: "turn-1", calls },
},
batch: { assistantEntryId, configuration, turnId: "turn-1", calls },
inbox: { steer: [], followUp: [], writes: [] },
latestAssistantEntryId: assistantEntryId,
};
@ -211,22 +216,22 @@ async function createFixture(options: FixtureOptions): Promise<Fixture> {
return { session, lane, drive, hooks, events, assistantEntryId, resultEntryIds, operationId, config };
}
function currentRun(fixture: Fixture): RunState {
function currentRun(fixture: Fixture): RunOperationState {
const operation = fixture.lane.state.operation;
if (operation?.state.kind !== "run") throw new Error("fixture has no run");
if (operation === null || !isRunOperationState(operation.state)) throw new Error("fixture has no run");
return operation.state;
}
function currentCalls(fixture: Fixture): ToolCall[] {
const run = currentRun(fixture);
if (run.phase.kind !== "tools") return [];
return run.phase.batch.calls;
if (run.at !== "run.tools") return [];
return run.batch.calls;
}
async function driveTools(fixture: Fixture) {
const run = currentRun(fixture);
if (run.phase.kind !== "tools") throw new Error("fixture has no tool batch");
return runTools(fixture.lane, fixture.drive, run, run.phase.batch);
if (run.at !== "run.tools") throw new Error("fixture has no tool batch");
return runTools(fixture.lane, fixture.drive, run);
}
async function expectProjectionRestores(fixture: Fixture): Promise<void> {
@ -297,8 +302,8 @@ describe("durable tool batch", () => {
});
expect(contextResolutions).toBe(1);
expect(checkpointSeenByAfterHook).toBe(true);
expect(currentRun(fixture).phase).toMatchObject({
kind: "checkpoint",
expect(currentRun(fixture)).toMatchObject({
at: "run.checkpoint",
continuation: { kind: "need_assistant" },
});
expect(fixture.lane.state.configuration.activeToolNames).toEqual(["first", "second", "introduced"]);
@ -444,8 +449,9 @@ describe("durable tool batch", () => {
expect(beforeTool).not.toHaveBeenCalled();
expect(afterTool).not.toHaveBeenCalled();
expect(currentRun(fixture)).toMatchObject({
at: "run.checkpoint",
control: { status: "cancel_requested" },
phase: { kind: "checkpoint", continuation: { kind: "need_assistant" } },
continuation: { kind: "need_assistant" },
});
const pending = await fixture.session.getEntry(fixture.resultEntryIds[1]!, BACKGROUND_CONTEXT);
expect(
@ -492,8 +498,8 @@ describe("durable tool batch", () => {
await driveTools(fixture);
expect(execute).not.toHaveBeenCalled();
expect(context).not.toHaveBeenCalled();
expect(currentRun(fixture).phase).toMatchObject({
kind: "checkpoint",
expect(currentRun(fixture)).toMatchObject({
at: "run.checkpoint",
continuation: { kind: "may_finish", includeFinalAssistant: false },
});
expect(fixture.lane.state.configuration.activeToolNames).toEqual(["ready", "later"]);
@ -574,32 +580,6 @@ describe("durable tool batch", () => {
expect(afterStarted).toBe(true);
});
it("fences outcome staging after exact Drive ownership changes", async () => {
const finish = deferred<void>();
const started = deferred<void>();
const fixture = await createFixture({
calls: [{ name: "owned" }],
tools: [
tool("owned", async () => {
started.resolve();
await finish.promise;
return { content: [{ type: "text", text: "done" }], details: {} };
}),
],
mode: "sequential",
});
const running = driveTools(fixture);
await started.promise;
fixture.lane.activeDrive = new Drive({ operationId: fixture.operationId }, BACKGROUND_CONTEXT);
finish.resolve();
expect(await running).toEqual({ kind: "lost_ownership" });
expect(currentCalls(fixture)[0]?.status).toBe("effect_pending");
expect(
await fixture.session.getValue(storedValues.pendingEntry(fixture.resultEntryIds[0]!), BACKGROUND_CONTEXT),
).toBeUndefined();
});
it("does not stage a cancelled outcome before cancellation is durable", async () => {
const execute = vi.fn(async () => ({ content: [], details: {} }));
const fixture = await createFixture({
@ -618,8 +598,8 @@ describe("durable tool batch", () => {
).toBeUndefined();
await fixture.lane.command((state) => {
const operation = state.operation;
if (operation?.state.kind !== "run") throw new Error("missing run");
const nextRun: RunState = {
if (operation === null || !isRunOperationState(operation.state)) throw new Error("missing run");
const nextRun: RunOperationState = {
...operation.state,
control: { status: "cancel_requested", requestedAt: 40, drainedSteer: [], drainedFollowUp: [] },
};
@ -672,8 +652,8 @@ describe("durable tool batch", () => {
fixture.drive.beginAbort(cancellation.promise);
await fixture.lane.command((state) => {
const operation = state.operation;
if (operation?.state.kind !== "run") throw new Error("missing run");
const nextRun: RunState = {
if (operation === null || !isRunOperationState(operation.state)) throw new Error("missing run");
const nextRun: RunOperationState = {
...operation.state,
control: { status: "cancel_requested", requestedAt: 40, drainedSteer: [], drainedFollowUp: [] },
};

82
packages/agent/test/harness/runtime/lane.test.ts
View file

@ -8,7 +8,6 @@ import { HookRegistry } from "../../../src/harness/hooks.ts";
import { convertToLlm } from "../../../src/harness/messages.ts";
import { Lane } from "../../../src/harness/runtime/lane.ts";
import { restoreLane } from "../../../src/harness/runtime/restore.ts";
import { Drive } from "../../../src/harness/runtime/types.ts";
import { StorageBackedSession } from "../../../src/harness/session/session.ts";
import type { LaneConfiguration, Session } from "../../../src/harness/session/types.ts";
import * as storedValues from "../../../src/harness/session/values.ts";
@ -247,87 +246,6 @@ describe("runtime Lane commands", () => {
);
});
it("declines an owned commit when its exact drive is replaced while planning", async () => {
const { lane, session } = await createLane();
const admission = await lane.accept({ kind: "prompt", prompt: "start" }, BACKGROUND_CONTEXT);
if (!admission.ok) throw admission.error;
const drive = new Drive({ operationId: admission.value.operationId }, BACKGROUND_CONTEXT);
lane.activeDrive = drive;
const plannerStarted = deferred();
const releasePlanner = deferred();
const command = lane.commandDriveOwned(
drive,
async (state) => {
plannerStarted.resolve();
await releasePlanner.promise;
const configuration: LaneConfiguration = { ...state.configuration, thinkingLevel: "high" };
return {
kind: "commit",
writes: [storedValues.setValue(storedValues.laneConfig("main"), configuration)],
next: { ...state, configuration },
materialize: () => "committed" as const,
};
},
BACKGROUND_CONTEXT,
);
await plannerStarted.promise;
lane.activeDrive = new Drive({ operationId: admission.value.operationId }, BACKGROUND_CONTEXT);
releasePlanner.resolve();
expect(await command).toEqual({ kind: "lost_ownership" });
expect(lane.state.configuration.thinkingLevel).toBe("off");
expect((await session.getValue(storedValues.laneConfig("main"), BACKGROUND_CONTEXT))?.value.thinkingLevel).toBe(
"off",
);
});
it("finishes a commit admitted before exact drive replacement", async () => {
const { lane, session, storage } = await createLane();
const admission = await lane.accept({ kind: "prompt", prompt: "start" }, BACKGROUND_CONTEXT);
if (!admission.ok) throw admission.error;
const drive = new Drive({ operationId: admission.value.operationId }, BACKGROUND_CONTEXT);
lane.activeDrive = drive;
const commitStarted = deferred();
const releaseCommit = deferred();
storage.beforeNextCommit = async () => {
commitStarted.resolve();
await releaseCommit.promise;
};
const command = lane.commandDriveOwned(
drive,
(state) => {
const configuration: LaneConfiguration = { ...state.configuration, thinkingLevel: "high" };
return {
kind: "commit",
writes: [storedValues.setValue(storedValues.laneConfig("main"), configuration)],
next: { ...state, configuration },
materialize: () => "committed" as const,
};
},
BACKGROUND_CONTEXT,
);
await commitStarted.promise;
const replacement = new Drive({ operationId: admission.value.operationId }, BACKGROUND_CONTEXT);
let publishedBeforeReplacement = false;
const replacing = lane.command((state) => {
publishedBeforeReplacement = state === lane.state && state.configuration.thinkingLevel === "high";
lane.activeDrive = replacement;
return { kind: "return", result: undefined };
}, BACKGROUND_CONTEXT);
releaseCommit.resolve();
expect(await command).toBe("committed");
await replacing;
expect(publishedBeforeReplacement).toBe(true);
expect(lane.activeDrive).toBe(replacement);
expect(lane.state.configuration.thinkingLevel).toBe("high");
expect((await session.getValue(storedValues.laneConfig("main"), BACKGROUND_CONTEXT))?.value.thinkingLevel).toBe(
"high",
);
});
it("rejects work after sealing while an admitted commit finishes", async () => {
const { lane, storage } = await createLane();
const commitStarted = deferred();

180
packages/agent/test/harness/runtime/progress.test.ts
View file

@ -9,7 +9,15 @@ import { openFrameProgress, openToolProgress } from "../../../src/harness/runtim
import { type Config, Drive, type LaneState as RuntimeLaneState } from "../../../src/harness/runtime/types.ts";
import { MemoryStorage } from "../../../src/harness/session/memory.ts";
import { StorageBackedSession } from "../../../src/harness/session/session.ts";
import type { RunState, Session, Write } from "../../../src/harness/session/types.ts";
import {
isRunOperationState,
type OperationState,
type RunAssistantEffectPendingOperation,
type RunCheckpointOperation,
runScopeOf,
type Session,
type Write,
} from "../../../src/harness/session/types.ts";
import * as storedValues from "../../../src/harness/session/values.ts";
import { deferred } from "./test-utils.ts";
@ -55,24 +63,42 @@ function unusedWatch<T>(): WatchHandle<T> {
throw new Error("watch is not used by progress tests");
}
function runState(phase: RunState["phase"]): RunState {
function runScope() {
return {
kind: "run",
control: { status: "running" },
control: { status: "running" as const },
settings: {
compaction: DEFAULT_COMPACTION_SETTINGS,
steeringMode: "all",
followUpMode: "all",
toolExecution: "parallel",
steeringMode: "all" as const,
followUpMode: "all" as const,
toolExecution: "parallel" as const,
},
phase,
inbox: { steer: [], followUp: [], writes: [] },
latestAssistantEntryId: null,
};
}
function assistantEffectPending(responseEntryId: string): RunAssistantEffectPendingOperation {
return {
...runScope(),
at: "run.assistant.effect_pending",
generationContext: {
stepId: "step",
triggerEntryId: "trigger",
configuration,
streamOptions: {},
retryPolicy: { maxAttempts: 2, baseDelayMs: 1 },
overflowRecoveryUsed: false,
},
attempt: 1,
responseEntryId,
usageId: "usage",
intendedOutputLimit: 100,
contextWindow: 1_000,
};
}
async function createFixture(
state: RunState,
state: OperationState,
storage = new FailingStorage(),
): Promise<{ lane: Lane<undefined>; drive: Drive; storage: FailingStorage }> {
const session = new StorageBackedSession(
@ -145,7 +171,7 @@ describe("runtime active drive ownership", () => {
expect("beginAbort" in failed.gate).toBe(false);
});
it("owns installer cancellation, pass context, policy, and gate control", () => {
it("strips invocation cancellation from pass context and owns policy and gate control", () => {
const controller = new AbortController();
const drive = new Drive(
{ operationId, waitForRetry: true, pollDeferred: true },
@ -153,7 +179,6 @@ describe("runtime active drive ownership", () => {
);
expect(drive.operationId).toBe(operationId);
expect(drive.installerSignal).toBe(controller.signal);
expect(drive.context.abortSignal).toBeUndefined();
expect(drive.waitForRetry).toBe(true);
expect(drive.deferredPermits).toBe(1);
@ -168,27 +193,7 @@ describe("runtime active drive ownership", () => {
describe("runtime progress channels", () => {
it("enqueues assistant frames in order, seals admission, and exposes the settlement delete", async () => {
const responseEntryId = "response";
const { lane, drive } = await createFixture(
runState({
kind: "assistant",
generation: {
status: "effect_pending",
context: {
stepId: "step",
triggerEntryId: "trigger",
configuration,
streamOptions: {},
retryPolicy: { maxAttempts: 2, baseDelayMs: 1 },
overflowRecoveryUsed: false,
},
attempt: 1,
responseEntryId,
usageId: "usage",
intendedOutputLimit: 100,
contextWindow: 1_000,
},
}),
);
const { lane, drive } = await createFixture(assistantEffectPending(responseEntryId));
const progress = openFrameProgress(lane, drive, responseEntryId);
const frames: AssistantMessageFrame[] = [
{ type: "text_delta", contentIndex: 0, delta: "a" },
@ -217,27 +222,7 @@ describe("runtime progress channels", () => {
it("declines a queued frame after the authoritative projection leaves its phase without control reads", async () => {
const responseEntryId = "response";
const { lane, drive, storage } = await createFixture(
runState({
kind: "assistant",
generation: {
status: "effect_pending",
context: {
stepId: "step",
triggerEntryId: "trigger",
configuration,
streamOptions: {},
retryPolicy: { maxAttempts: 2, baseDelayMs: 1 },
overflowRecoveryUsed: false,
},
attempt: 1,
responseEntryId,
usageId: "usage",
intendedOutputLimit: 100,
contextWindow: 1_000,
},
}),
);
const { lane, drive, storage } = await createFixture(assistantEffectPending(responseEntryId));
const progress = openFrameProgress(lane, drive, responseEntryId);
const commitStarted = deferred();
const releaseCommit = deferred();
@ -248,14 +233,12 @@ describe("runtime progress channels", () => {
const getValue = vi.spyOn(storage, "getValue");
const moving = lane.command((state) => {
const operation = state.operation;
if (operation?.state.kind !== "run") throw new Error("missing run");
const nextState: RunState = {
...operation.state,
phase: {
kind: "checkpoint",
continuation: { kind: "may_finish", includeFinalAssistant: true },
triggerEntryId: responseEntryId,
},
if (operation === null || !isRunOperationState(operation.state)) throw new Error("missing run");
const nextState: RunCheckpointOperation = {
...runScopeOf(operation.state),
at: "run.checkpoint",
continuation: { kind: "may_finish", includeFinalAssistant: true },
triggerEntryId: responseEntryId,
};
return {
kind: "commit",
@ -282,27 +265,7 @@ describe("runtime progress channels", () => {
it("declines a queued frame after terminal projection publication", async () => {
const responseEntryId = "response";
const { lane, drive, storage } = await createFixture(
runState({
kind: "assistant",
generation: {
status: "effect_pending",
context: {
stepId: "step",
triggerEntryId: "trigger",
configuration,
streamOptions: {},
retryPolicy: { maxAttempts: 2, baseDelayMs: 1 },
overflowRecoveryUsed: false,
},
attempt: 1,
responseEntryId,
usageId: "usage",
intendedOutputLimit: 100,
contextWindow: 1_000,
},
}),
);
const { lane, drive, storage } = await createFixture(assistantEffectPending(responseEntryId));
const progress = openFrameProgress(lane, drive, responseEntryId);
const commitStarted = deferred();
const releaseCommit = deferred();
@ -343,19 +306,18 @@ describe("runtime progress channels", () => {
expect(getValue).not.toHaveBeenCalled();
});
it("replaces tool checkpoints and fences writes after owner identity changes", async () => {
it("replaces tool checkpoints in invocation order", async () => {
const invocationId = "result";
const { lane, drive } = await createFixture(
runState({
kind: "tools",
batch: {
assistantEntryId: "assistant",
configuration,
turnId: "turn",
calls: [{ status: "effect_pending", sourceIndex: 0, resultEntryId: invocationId, replay: "safe" }],
},
}),
);
const { lane, drive } = await createFixture({
...runScope(),
at: "run.tools",
batch: {
assistantEntryId: "assistant",
configuration,
turnId: "turn",
calls: [{ status: "effect_pending", sourceIndex: 0, resultEntryId: invocationId, replay: "safe" }],
},
} satisfies OperationState);
const progress = openToolProgress(lane, drive, "turn", 0, invocationId);
progress.write({ content: [{ type: "text", text: "first" }], details: {} });
progress.write({ content: [{ type: "text", text: "second" }], details: {} });
@ -364,39 +326,11 @@ describe("runtime progress channels", () => {
(await lane.session.getValue(storedValues.pendingToolOutput(operationId, invocationId), BACKGROUND_CONTEXT))
?.value.content,
).toEqual([{ type: "text", text: "second" }]);
lane.activeDrive = new Drive({ operationId }, BACKGROUND_CONTEXT);
progress.write({ content: [{ type: "text", text: "stale" }], details: {} });
await progress.drain();
expect(
(await lane.session.getValue(storedValues.pendingToolOutput(operationId, invocationId), BACKGROUND_CONTEXT))
?.value.content,
).toEqual([{ type: "text", text: "second" }]);
});
it("retains the rejecting write promise so drain propagates commit failure", async () => {
const responseEntryId = "response";
const { lane, drive, storage } = await createFixture(
runState({
kind: "assistant",
generation: {
status: "effect_pending",
context: {
stepId: "step",
triggerEntryId: "trigger",
configuration,
streamOptions: {},
retryPolicy: { maxAttempts: 2, baseDelayMs: 1 },
overflowRecoveryUsed: false,
},
attempt: 1,
responseEntryId,
usageId: "usage",
intendedOutputLimit: 100,
contextWindow: 1_000,
},
}),
);
const { lane, drive, storage } = await createFixture(assistantEffectPending(responseEntryId));
const failure = new Error("frame commit failed");
storage.failure = failure;
const progress = openFrameProgress(lane, drive, responseEntryId);

18
packages/agent/test/harness/runtime/restore.test.ts
View file

@ -3,7 +3,12 @@ import { DEFAULT_COMPACTION_SETTINGS } from "../../../src/harness/compaction/com
import { BACKGROUND_CONTEXT } from "../../../src/harness/context.ts";
import { restoreLane, restoreSession } from "../../../src/harness/runtime/restore.ts";
import { MemorySessionRepo, MemoryStorage } from "../../../src/harness/session/memory.ts";
import type { LaneConfiguration, OperationMeta, RunState, Session } from "../../../src/harness/session/types.ts";
import type {
LaneConfiguration,
OperationMeta,
RunCheckpointOperation,
Session,
} from "../../../src/harness/session/types.ts";
import * as storedValues from "../../../src/harness/session/values.ts";
const repos: MemorySessionRepo[] = [];
@ -35,9 +40,9 @@ async function createSession(): Promise<Session> {
return session;
}
function runState(triggerEntryId: string): RunState {
function runState(triggerEntryId: string): RunCheckpointOperation {
return {
kind: "run",
at: "run.checkpoint",
control: { status: "running" },
settings: {
compaction: DEFAULT_COMPACTION_SETTINGS,
@ -45,11 +50,8 @@ function runState(triggerEntryId: string): RunState {
followUpMode: "all",
toolExecution: "parallel",
},
phase: {
kind: "checkpoint",
continuation: { kind: "need_assistant", overflowRecoveryUsed: false },
triggerEntryId,
},
continuation: { kind: "need_assistant", overflowRecoveryUsed: false },
triggerEntryId,
inbox: { steer: [], followUp: [], writes: [] },
latestAssistantEntryId: null,
};

155
packages/agent/test/harness/runtime/watch.test.ts
View file

@ -10,8 +10,8 @@ import { MemoryStorage } from "../../../src/harness/session/memory.ts";
import { StorageBackedSession } from "../../../src/harness/session/session.ts";
import type {
OperationMeta,
RunPhase,
RunState,
OperationState,
RunScope,
Session,
StorageBranchScan,
Write,
@ -54,9 +54,8 @@ async function commit(session: Session, writes: Write[]): Promise<void> {
await session.mutate((mutator) => mutator.commit(writes, BACKGROUND_CONTEXT), BACKGROUND_CONTEXT);
}
function runState(phase: RunPhase, control: RunState["control"] = { status: "running" }): RunState {
function runScope(control: RunScope["control"] = { status: "running" }): RunScope {
return {
kind: "run",
control,
settings: {
compaction: DEFAULT_COMPACTION_SETTINGS,
@ -64,7 +63,6 @@ function runState(phase: RunPhase, control: RunState["control"] = { status: "run
followUpMode: "all",
toolExecution: "parallel",
},
phase,
inbox: { steer: [], followUp: [], writes: [] },
latestAssistantEntryId: null,
};
@ -161,26 +159,21 @@ describe("runtime lane watch", () => {
payload: { role: "user" as const, content, timestamp: 1 },
});
const operationId = session.idGenerator.next();
const state = runState(
{
kind: "deferred",
deferred: {
status: "suspended",
stepId: "step",
sourceEntryId: "source",
poll: 3,
configuration,
streamOptions: {},
},
},
{
const state: OperationState = {
...runScope({
status: "cancel_requested",
requestedAt: 2,
drainedSteer: [ids.drainedSteer],
drainedFollowUp: [ids.drainedFollow],
},
);
state.inbox = { steer: [ids.steer], followUp: [ids.follow], writes: [ids.write] };
}),
at: "run.deferred.suspended",
stepId: "step",
sourceEntryId: "source",
poll: 3,
configuration,
streamOptions: {},
inbox: { steer: [ids.steer], followUp: [ids.follow], writes: [ids.write] },
};
const meta: OperationMeta = {
operationId,
lane: "main",
@ -238,28 +231,23 @@ describe("runtime lane watch", () => {
startedAt: 1,
intent: { kind: "run", promptEntryIds: [] },
}),
storedValues.setValue(
storedValues.operationState(operationId),
runState({
kind: "assistant",
generation: {
status: "effect_pending",
context: {
stepId: "step",
triggerEntryId: "trigger",
configuration,
streamOptions: {},
retryPolicy: { maxAttempts: 2, baseDelayMs: 0 },
overflowRecoveryUsed: false,
},
attempt: 1,
responseEntryId: "response-without-frames",
usageId: "usage",
intendedOutputLimit: 100,
contextWindow: 1000,
},
}),
),
storedValues.setValue(storedValues.operationState(operationId), {
...runScope(),
at: "run.assistant.effect_pending",
generationContext: {
stepId: "step",
triggerEntryId: "trigger",
configuration,
streamOptions: {},
retryPolicy: { maxAttempts: 2, baseDelayMs: 0 },
overflowRecoveryUsed: false,
},
attempt: 1,
responseEntryId: "response-without-frames",
usageId: "usage",
intendedOutputLimit: 100,
contextWindow: 1000,
}),
storedValues.setValue(storedValues.laneState("main"), {
currentOperationId: operationId,
pendingNextRun: [],
@ -290,28 +278,23 @@ describe("runtime lane watch", () => {
startedAt: 1,
intent: { kind: "run", promptEntryIds: [] },
}),
storedValues.setValue(
storedValues.operationState(frameOperationId),
runState({
kind: "assistant",
generation: {
status: "effect_pending",
context: {
stepId: "step",
triggerEntryId: "trigger",
configuration,
streamOptions: {},
retryPolicy: { maxAttempts: 2, baseDelayMs: 0 },
overflowRecoveryUsed: false,
},
attempt: 1,
responseEntryId: "response",
usageId: "usage",
intendedOutputLimit: 100,
contextWindow: 1000,
},
}),
),
storedValues.setValue(storedValues.operationState(frameOperationId), {
...runScope(),
at: "run.assistant.effect_pending",
generationContext: {
stepId: "step",
triggerEntryId: "trigger",
configuration,
streamOptions: {},
retryPolicy: { maxAttempts: 2, baseDelayMs: 0 },
overflowRecoveryUsed: false,
},
attempt: 1,
responseEntryId: "response",
usageId: "usage",
intendedOutputLimit: 100,
contextWindow: 1000,
}),
...frames.map((frame) =>
storedValues.appendList(storedValues.pendingAssistantFrames(frameOperationId, "response"), frame),
),
@ -341,28 +324,26 @@ describe("runtime lane watch", () => {
startedAt: 1,
intent: { kind: "run", promptEntryIds: [] },
}),
storedValues.setValue(
storedValues.operationState(toolOperationId),
runState({
kind: "tools",
batch: {
assistantEntryId: "assistant",
configuration,
turnId: "turn",
calls: [
{ status: "planned", sourceIndex: 1, resultEntryId: "planned" },
{ status: "effect_pending", sourceIndex: 1, resultEntryId: "result", replay: "safe" },
{
status: "effect_pending",
sourceIndex: 2,
resultEntryId: "without-checkpoint",
replay: "never",
},
{ status: "outcome_ready", sourceIndex: 1, resultEntryId: "ready", terminate: false },
],
},
}),
),
storedValues.setValue(storedValues.operationState(toolOperationId), {
...runScope(),
at: "run.tools",
batch: {
assistantEntryId: "assistant",
configuration,
turnId: "turn",
calls: [
{ status: "planned", sourceIndex: 1, resultEntryId: "planned" },
{ status: "effect_pending", sourceIndex: 1, resultEntryId: "result", replay: "safe" },
{
status: "effect_pending",
sourceIndex: 2,
resultEntryId: "without-checkpoint",
replay: "never",
},
{ status: "outcome_ready", sourceIndex: 1, resultEntryId: "ready", terminate: false },
],
},
}),
storedValues.setValue(storedValues.operationToolArgs(toolOperationId, "turn", 1), { path: "file" }),
storedValues.setValue(storedValues.operationToolArgs(toolOperationId, "turn", 2), { path: "output" }),
storedValues.setValue(storedValues.pendingToolOutput(toolOperationId, "result"), {

251
packages/agent/test/harness/types.test.ts
View file

@ -13,19 +13,17 @@ import type {
Branch,
BranchScan,
CancelQueuedResult,
CheckpointPhase,
CompactionState,
CheckpointData,
Context,
Control,
CustomEntry,
Deferred,
DriveOptions,
DriveOutcome,
DriveResult,
Entry,
EntryProjector,
EntryWrite,
Generation,
FailureProvenance,
GenerationContext,
HarnessEvent,
HookHandler,
@ -36,15 +34,14 @@ import type {
LaneExecutionInfo,
LaneLastResult,
LaneSnapshot,
NavigationState,
NewEntry,
OperationAdmissionResult,
OperationAt,
OperationMeta,
OperationRequest,
OperationState,
RunPhase,
RunResult,
RunState,
RunScope,
SearchQuery,
Session,
SessionCreateOptions,
@ -60,9 +57,7 @@ import type {
SettledAssistantMessage,
Storage,
StorageBranchScan,
StructuralDecision,
SummaryContext,
SummaryGeneration,
ToolCall,
UsageRow,
UsageWrite,
@ -96,34 +91,68 @@ const summaryContext = {
reason: "manual",
} satisfies SummaryContext;
const checkpoint = {
kind: "checkpoint",
continuation: { kind: "need_assistant", overflowRecoveryUsed: false },
triggerEntryId: "trigger",
} satisfies CheckpointPhase;
} satisfies CheckpointData;
const runningControl = { status: "running" } satisfies Control;
const generations = [
{ status: "ready", context: generationContext, nextAttempt: 1 },
{
status: "effect_pending",
context: generationContext,
attempt: 1,
responseEntryId: "response",
usageId: "usage",
intendedOutputLimit: 4096,
contextWindow: 128000,
},
{ status: "retry_wait", context: generationContext, nextAttempt: 2, notBefore: 10, errorMessage: "retry" },
] satisfies Generation[];
const toolCalls = [
{ status: "planned", sourceIndex: 0, resultEntryId: "result-0" },
{ status: "effect_pending", sourceIndex: 1, resultEntryId: "result-1", replay: "safe" },
{ status: "outcome_ready", sourceIndex: 2, resultEntryId: "result-2", terminate: true },
{ status: "completed", sourceIndex: 3, resultEntryId: "result-3", terminate: false },
] satisfies ToolCall[];
const deferredStates = [
const provenances = [
{ kind: "response", entryId: "response" },
{ kind: "structural", taskId: "task" },
{ kind: "configuration" },
] satisfies FailureProvenance[];
const runScope = {
control: runningControl,
settings: {
compaction: { enabled: true, reserveTokens: 1000, keepRecentTokens: 2000 },
steeringMode: "all",
followUpMode: "one-at-a-time",
toolExecution: "parallel",
},
inbox: { steer: [], followUp: [], writes: [] },
latestAssistantEntryId: null,
} satisfies RunScope;
const runState = { ...runScope, at: "run.starting" } satisfies OperationState;
// One value fixture per flat leaf; `satisfies OperationState[]` keeps every discriminant checked.
const operationStates = [
runState,
{ ...runScope, at: "run.checkpoint", ...checkpoint },
{ ...runScope, at: "run.assistant.ready", generationContext, nextAttempt: 1 },
{
status: "suspended",
...runScope,
at: "run.assistant.effect_pending",
generationContext,
attempt: 1,
responseEntryId: "response",
usageId: "usage",
intendedOutputLimit: 4096,
contextWindow: 128000,
},
{
...runScope,
at: "run.assistant.retry_wait",
generationContext,
nextAttempt: 2,
notBefore: 10,
errorMessage: "retry",
},
{
...runScope,
at: "run.tools",
batch: { assistantEntryId: "assistant", configuration, turnId: "turn", calls: toolCalls },
},
{
...runScope,
at: "run.deferred.suspended",
stepId: "step",
sourceEntryId: "source",
poll: 0,
@ -131,7 +160,8 @@ const deferredStates = [
streamOptions: {},
},
{
status: "effect_pending",
...runScope,
at: "run.deferred.effect_pending",
stepId: "step",
sourceEntryId: "source",
poll: 1,
@ -140,99 +170,73 @@ const deferredStates = [
configuration,
streamOptions: {},
},
] satisfies Deferred[];
const summaryGenerations = [
{ status: "ready", context: summaryContext, nextAttempt: 1 },
{ ...runScope, at: "run.compaction.deciding", reason: "threshold", resumeAfter: checkpoint, taskId: "task" },
{
status: "effect_pending",
context: summaryContext,
...runScope,
at: "run.compaction.ready",
reason: "threshold",
resumeAfter: checkpoint,
summaryContext,
nextAttempt: 1,
},
{
...runScope,
at: "run.compaction.effect_pending",
reason: "overflow",
resumeAfter: checkpoint,
summaryContext,
attempt: 1,
request: { index: 0, usageId: "usage" },
usageIds: [],
},
{ status: "retry_wait", context: summaryContext, nextAttempt: 2, notBefore: 10, errorMessage: "retry" },
] satisfies SummaryGeneration[];
const structuralDecisions = [
{ status: "deciding", taskId: "task" },
{ status: "generating", taskId: "task", generation: summaryGenerations[0] },
] satisfies StructuralDecision[];
const runPhases: RunPhase[] = [
{ kind: "starting" },
checkpoint,
{ kind: "assistant", generation: generations[0] },
{
kind: "tools",
batch: { assistantEntryId: "assistant", configuration, turnId: "turn", calls: toolCalls },
},
{
kind: "compaction",
reason: "threshold",
structural: structuralDecisions[0],
...runScope,
at: "run.compaction.retry_wait",
reason: "overflow",
resumeAfter: checkpoint,
summaryContext,
nextAttempt: 2,
notBefore: 10,
errorMessage: "retry",
},
{ kind: "deferred", deferred: deferredStates[0] },
{
kind: "failure_drain",
...runScope,
at: "run.failure_drain",
error: { code: "provider", message: "failed" },
provenance: { kind: "response", entryId: "response" },
provenance: provenances[0]!,
},
{ at: "compaction.deciding", control: runningControl, customInstructions: "compact", taskId: "task" },
{ at: "compaction.ready", control: runningControl, summaryContext, nextAttempt: 1 },
{ at: "compaction.effect_pending", control: runningControl, summaryContext, attempt: 1, usageIds: ["usage"] },
{
kind: "failure_drain",
error: {
code: "model_unavailable",
message: "Model is unavailable",
details: { provider: "provider", modelId: "model" },
},
provenance: { kind: "configuration" },
},
{
kind: "failure_drain",
error: {
code: "configured_tools_unavailable",
message: "Configured tools are unavailable",
details: { tools: ["read"] },
},
provenance: { kind: "configuration" },
},
];
const runState = {
kind: "run",
control: runningControl,
settings: {
compaction: { enabled: true, reserveTokens: 1000, keepRecentTokens: 2000 },
steeringMode: "all",
followUpMode: "one-at-a-time",
toolExecution: "parallel",
},
phase: runPhases[0]!,
inbox: { steer: [], followUp: [], writes: [] },
latestAssistantEntryId: null,
} satisfies RunState;
const compactionState = {
kind: "compaction",
control: runningControl,
customInstructions: "compact",
structural: structuralDecisions[0],
} satisfies CompactionState;
const navigationStates = [
{
kind: "navigation",
at: "compaction.retry_wait",
control: runningControl,
targetId: null,
summarize: false,
phase: { kind: "ready_to_commit" },
summaryContext,
nextAttempt: 2,
notBefore: 10,
errorMessage: "retry",
},
{ at: "navigation.ready_to_commit", control: runningControl, targetId: null, label: "target" },
{ at: "navigation.summary.deciding", control: runningControl, targetId: "target", taskId: "task" },
{ at: "navigation.summary.ready", control: runningControl, targetId: "target", summaryContext, nextAttempt: 1 },
{
kind: "navigation",
at: "navigation.summary.effect_pending",
control: runningControl,
targetId: "target",
summarize: true,
phase: { kind: "summary", structural: structuralDecisions[0] },
summaryContext,
attempt: 1,
usageIds: [],
},
] satisfies NavigationState[];
const operationStates = [runState, compactionState, ...navigationStates] satisfies OperationState[];
{
at: "navigation.summary.retry_wait",
control: runningControl,
targetId: "target",
summaryContext,
nextAttempt: 2,
notBefore: 10,
errorMessage: "retry",
},
] satisfies OperationState[];
const operations = [
{
operationId: "run",
@ -338,21 +342,34 @@ it("covers the complete durable storage and Part 3 discriminants", () => {
>();
expectTypeOf<OperationMeta["intent"]["kind"]>().toEqualTypeOf<"run" | "compaction" | "navigation">();
expectTypeOf<Control["status"]>().toEqualTypeOf<"running" | "cancel_requested">();
expectTypeOf<Generation["status"]>().toEqualTypeOf<"ready" | "effect_pending" | "retry_wait">();
expectTypeOf<ToolCall["status"]>().toEqualTypeOf<"planned" | "effect_pending" | "outcome_ready" | "completed">();
expectTypeOf<Deferred["status"]>().toEqualTypeOf<"suspended" | "effect_pending">();
expectTypeOf<SummaryGeneration["status"]>().toEqualTypeOf<"ready" | "effect_pending" | "retry_wait">();
expectTypeOf<StructuralDecision["status"]>().toEqualTypeOf<"deciding" | "generating">();
expectTypeOf<RunPhase["kind"]>().toEqualTypeOf<
"starting" | "checkpoint" | "assistant" | "tools" | "compaction" | "deferred" | "failure_drain"
expectTypeOf<OperationAt>().toEqualTypeOf<
| "run.starting"
| "run.checkpoint"
| "run.assistant.ready"
| "run.assistant.effect_pending"
| "run.assistant.retry_wait"
| "run.tools"
| "run.deferred.suspended"
| "run.deferred.effect_pending"
| "run.compaction.deciding"
| "run.compaction.ready"
| "run.compaction.effect_pending"
| "run.compaction.retry_wait"
| "run.failure_drain"
| "compaction.deciding"
| "compaction.ready"
| "compaction.effect_pending"
| "compaction.retry_wait"
| "navigation.ready_to_commit"
| "navigation.summary.deciding"
| "navigation.summary.ready"
| "navigation.summary.effect_pending"
| "navigation.summary.retry_wait"
>();
expectTypeOf<OperationState["kind"]>().toEqualTypeOf<"run" | "compaction" | "navigation">();
expectTypeOf<NavigationState["summarize"]>().toEqualTypeOf<boolean>();
expectTypeOf<FailureProvenance["kind"]>().toEqualTypeOf<"response" | "structural" | "configuration">();
expectTypeOf<NewEntry["type"]>().toEqualTypeOf<"message" | "compaction" | "branch_summary" | "custom">();
void transaction;
void generations;
void deferredStates;
void summaryGenerations;
void operationStates;
const compileTimeFailures = () => {
@ -479,9 +496,9 @@ it("covers Part 5 results, events, hooks, snapshots, tools, and stream options",
| "hooks"
| "activeDrive"
| "command"
| "commandDriveOwned"
| "mutateOperation"
| "advanceOperation"
| "readConfig"
| "isDriveActive"
| "mismatch"
>
>().toEqualTypeOf<never>();
@ -492,9 +509,9 @@ it("covers Part 5 results, events, hooks, snapshots, tools, and stream options",
| "models"
| "activeDrive"
| "command"
| "commandDriveOwned"
| "mutateOperation"
| "advanceOperation"
| "readConfig"
| "isDriveActive"
| "mismatch"
>
>().toEqualTypeOf<never>();