diff --git a/.gitattributes b/.gitattributes index d8a52c2ec..5dfbf7c0e 100644 --- a/.gitattributes +++ b/.gitattributes @@ -9,6 +9,9 @@ # Shell scripts should keep LF *.sh text eol=lf +# Archived patch artifacts preserve whitespace exactly +packages/agent/docs/**/*.diff -whitespace + # Common binary assets *.png binary *.jpg binary diff --git a/packages/agent/docs/assistant-durability.md b/packages/agent/docs/assistant-durability.md new file mode 100644 index 000000000..e0bff1672 --- /dev/null +++ b/packages/agent/docs/assistant-durability.md @@ -0,0 +1,355 @@ +# Assistant partial durability — implementation handoff + +This document specifies durable partial assistant messages for ordinary assistant generation and deferred-response polling. It builds on: + +- bound typed value/list addresses from `values.md`; +- `AssistantMessageFrame`, `AssistantMessageFrameEncoder`, and `reduceAssistantMessageFrames()` from `@earendil-works/pi-ai`; +- the assistant intent/effect/settlement state machine in `harness.md`. + +The design persists compact replayable stream frames without making them operation-state authority and without storing a growing full partial message on every update. + +## Goals + +1. Reconstruct the latest committed partial assistant message after process loss. +2. Preserve provider stream order without repeated full-snapshot write amplification. +3. Avoid provider backpressure from storage commits. +4. Reuse pi-ai's canonical frame conversion and reduction semantics. +5. Preserve current public assistant event ordering. +6. Keep operation `effect_pending` state authoritative for recovery. +7. Delete all partial frames atomically with normal or synthetic response settlement. + +## Non-goals + +- Inferring provider completion from frames. +- Persisting terminal `done`/`error` events separately from response settlement. +- Exactly-once provider requests. +- Public frame cursors or a frame-persistence event. +- Generic batching, timers, coalescing, or flush APIs. +- Persisting structural summary-generation streams. +- Persisting arbitrary provider SDK events or repeated full `partial` snapshots. + +## Storage + +Built-in address constructor in `session/values.ts`: + +```ts +export const pendingAssistantFrames = ( + operationId: string, + responseEntryId: string, +) => list( + "pi.pending.assistant_frame", + `${operationId}:${responseEntryId}`, +); +``` + +The procedure binds one exact address for ordinary generation or a deferred poll: + +```ts +const frames = pendingAssistantFrames(operationId, responseEntryId); +``` + +`responseEntryId` is already reserved in assistant/deferred `effect_pending` state. No frame count, cursor, or list identity is stored in operation state, and later list operations receive only `frames`—never another key. + +Each list element is one `AssistantMessageFrame`. The storage transaction's global write sequence orders frames. The list is auxiliary: + +- missing is valid; +- it does not prove request admission, completion, success, or failure; +- it never selects the restart point; +- base restore does not read it. + +## Frame contract + +Create one pi-ai encoder per provider stream and feed it every event in order: + +```ts +const encoder = new AssistantMessageFrameEncoder(); +const frame = encoder.encode(event); +``` + +`partial` is the provider's shared live response-so-far helper, not an event-time snapshot. The encoder keeps per-open-block counters and trims text/thinking delta prefixes already represented by an advanced block-start snapshot. It temporarily buffers only the raw JSON prefix needed to synchronize an already-advanced tool call, then emits one checkpoint and resumes compact deltas. It never clones the growing full message on every event. + +- `start` produces an empty-content metadata frame; +- a non-terminal event produces zero or one frame; +- covered queued deltas produce no frame; +- terminal `done` and `error` produce no frame because final response settlement is separate; +- a setup `error` before `start` is valid and produces no frames; +- text/thinking/tool end frames contain the authoritative completed block value; +- completed tool-call arguments remain unvalidated against a tool schema. + +Do not define a second harness frame codec or reducer. Persistence stores the exported pi-ai value directly; hydration calls `reduceAssistantMessageFrames()`. + +Provider event blocks may interleave. Encoding and reduction rely on `contentIndex`, never block contiguity. Text and ordinary thinking blocks must be empty when `*_start` is published and then append only through matching deltas until end; redacted thinking may be complete at start and emit no deltas. Streaming tool calls start with empty arguments and emit their complete raw JSON through deltas; a provider that starts with complete arguments must emit a cumulative delta prefix that parses to that snapshot at an event boundary before later argument deltas. + +## Fresh stream scheduling + +The simplest scheduling is deliberate: one list-append transaction per converted frame, enqueued without awaiting storage in the provider loop. + +For every `start` or update event: + +```text +encode the event against the per-stream frame encoder +→ when a frame is returned, synchronously enqueue invocation-fenced appendList(frames, frame) +→ attach the ordinary harness-fault observer to the returned promise +→ replace process-local latestFrameWrite promise reference +→ emit and await the existing message_start/message_update event +→ consume the next provider event +``` + +`appendList()` is called synchronously for each returned frame, so all frame mutations enter the Session line in provider-event order. The procedure does not await each write. Every promise is immediately observed for fault propagation before the latest reference replaces its predecessor. Covered queued events allocate no durable frame or write. Encoder state is proportional to open block count plus the unsynchronized prefix of an active tool-call JSON stream; it retains no second full assistant message. Model output limits bound queued frame bytes to bounded output plus frame/transaction overhead. + +The event side retains existing behavior: `AssistantStreamObserver.start/update` awaits `events.emit()` for every event. There is no separate latest event-delivery promise because no assistant event delivery remains outstanding when the provider loop advances. + +When the provider stream settles: + +```text +stop frame admission +→ await latestFrameWrite when present +→ run after_response +→ emit message_end +→ classify and commit final response settlement +``` + +The Session mutation line is FIFO, so completion of `latestFrameWrite` implies every earlier append completed. There is no array of promises, timer, batcher, active/waiting state, coalescer, or public/internal flush method. + +A failed frame append faults the harness before `after_response` starts. The complete final response remains process-local and does not commit after a storage fault. + +## Normal settlement + +Every assistant response settlement that used a frame list deletes that exact list in the same transaction as the immutable response entry, usage, tip, and next operation state: + +```text +TX[ + insert response entry R, + insert usage U, + setValue(branchTip(lane), R), + deleteList(frames), + setValue(operationState(operationId), classified next state) +] +``` + +This applies to: + +- successful assistant responses; +- provider `error`/`aborted` responses; +- valid deferred responses; +- deferred poll responses. + +`after_response` may transform the final response before settlement. Frames preserve provider-stream observation, while the immutable entry remains the post-hook canonical result. + +A crash after the final frame append but before settlement still restores `effect_pending`; frames do not turn a complete-looking draft into a settled response. + +## Unknown-outcome generation recovery + +An orphaned assistant generation `effect_pending` has no surviving provider stream. Activation: + +1. constructs `frames = pendingAssistantFrames(operationId, responseEntryId)` from current typed state and reads bounded pages from that exact address; +2. reduces frame values with `reduceAssistantMessageFrames()`; +3. constructs a harness-owned synthetic assistant response under the already-reserved response ID; +4. preserves reconstructed partial content and safe message identity metadata when frames exist; +5. sets `stopReason: "error"`, zero usage, and an explicit interruption/unknown-outcome `errorMessage`/diagnostic; +6. commits the synthetic response, zero-usage row, frame-list deletion, tip, and ordinary retry/failure state atomically. + +Required warning meaning: + +```text +The provider request was interrupted. The preceding content is the latest +committed partial response; newer live output may be missing, and the external +request outcome is unknown. +``` + +If no start frame committed, the harness constructs the same synthetic error with empty content from the captured model/API identity. + +The synthetic response follows ordinary assistant error classification: + +- attempts remain → insert the error response and enter the ordinary retry-wait/next-attempt path; +- cap reached → insert it and terminal-fail the operation in the same settlement transaction. + +Error responses remain durable transcript history but are omitted from later provider context by the existing projection rule. Partial tool calls inside the interrupted error response never execute. + +Recovery does not run `after_response`: there is no trustworthy complete provider result to transform. + +## Cancellation + +A live cancelled provider stream settles through its ordinary final `aborted` response. All accepted frames are awaited first, and normal settlement deletes the list. + +For restored cancelled assistant/deferred `effect_pending`, cancellation reconciliation: + +- reduces available frames; +- constructs a synthetic `aborted` response preserving the committed partial content; +- uses zero usage and the existing reserved IDs; +- atomically inserts the response and deletes the frame list; +- starts no provider request and runs no response hook. + +Cancellation wins classification even if reduced content appears complete. + +## Deferred polling + +A deferred poll that returns an `AssistantMessageEventStream` uses the same `pendingAssistantFrames(operationId, responseEntryId)` address constructor. + +- normal pending/ready/error settlement deletes the poll's frame list; +- restored cancellation synthesizes an aborted response from frames; +- an unknown restored poll without a permit remains suspended and may expose its durable partial in snapshots; +- when a poll permit replaces the unknown poll with fresh reserved response/usage IDs, that intent transaction deletes the abandoned old frame-list address; +- the replacement poll starts a fresh list under its fresh response ID. + +Frames never change poll-number rules. + +## Structural generation scope + +Structural summary-generation streams remain process-local. They do not emit public assistant-message lifecycle and may span multiple nested provider requests before one structural publication. Existing attempt-level retry and usage recovery remains authoritative. + +Do not store their intermediate text at a `pendingAssistantFrames(...)` address in this slice. If structural partial diagnostics become a requirement, add a separate explicitly scoped consumer rather than silently reusing transcript-assistant semantics. + +## Snapshots and reconnect + +`LaneSnapshot.streamingMessage` means the latest observed partial assistant message, not proof that a provider stream is currently attached. + +Precedence: + +1. newest process-local partial while a live stream is owned; +2. otherwise, for assistant/deferred `effect_pending`, the value reduced from committed frame pages; +3. otherwise absent. + +A restored lane may therefore be `suspended` with a non-undefined `streamingMessage`. The field remains outside `transcript`; only `entry_added` moves a complete response into transcript history and clears the partial. + +Snapshot hydration constructs the exact bound frame address from trusted typed operation state and enforces the assistant consumer's total frame/page budget. It does not scan arbitrary lists or perform a broad semantic restore audit. + +Reconnect replays no historical `message_start` or `message_update` events. The snapshot carries the durable partial. Recovery later emits the ordinary recovery-tagged synthetic message lifecycle for the response it actually settles. + +## Events + +A started generation retains this live event ordering: + +```text +message_start +→ message_update* each listener delivery awaited by provider loop +→ await latest frame write +→ after_response +→ message_end +→ atomic response settlement + frame-list delete +→ entry_added +→ usage +``` + +A request setup failure may produce `error` before `start`; that path emits no `message_start` or frame and proceeds through `after_response`, `message_end`, and ordinary error settlement. Successful `done` and updates before `start` are protocol defects. + +Frame commits emit ordinary storage telemetry only. There is no public frame event and no claim that a `message_update` was durable. `entry_added` remains the only proof that the final assistant entry committed. + +A crash may occur after a live update event but before its asynchronously queued frame append commits. Reconnect then shows the latest committed frame prefix, which may be older than the last live event. + +## Close, fault, and external finalization + +Close is a controlled crash: + +- no synthetic response is written; +- already-enqueued frame commits are ordinary admitted session work and may finish under the close barrier; +- process loss may discard mutations that had not committed; +- reopen restores the latest committed frame prefix under unchanged `effect_pending` state. + +A frame-commit storage failure faults the harness. No later response settlement commits in that process. + +Authorized external finalization deletes the operation-owned frame-list address in its terminal transaction. Every append mutation verifies current operation/response ownership on the Session line: + +- append first → external terminal cleanup deletes the list; +- finalization first → stale append rejects without recreating state. + +## Terminal cleanup, forks, and migrations + +Normal/synthetic response settlement should already delete its exact frame address. The operation terminal transaction also defensively constructs and deletes the current operation-owned frame address when state is assistant/deferred `effect_pending`. + +Idle forks never copy lists in the `pi.pending.assistant_frame` address family. Precise rewrites and migrations page frame lists and preserve element sequences when retaining them. + +A migration changing `AssistantMessageFrame` shape must map every surviving element or explicitly delete the whole list and leave `effect_pending` recovery with no partial. It must never infer completion from legacy frames. + +JSONL retains deleted frame bytes until snapshot compaction. Logical deletion is immediate. + +## Races + +| Race | Required result | +|---|---| +| frame append vs next frame | synchronous lane enqueue preserves provider-event order | +| frame append vs stream settlement | settlement awaits the latest promise; all accepted appends finish first | +| live update event vs frame commit | either may finish first; event is observation and reconnect uses only committed frames | +| append vs external finalization | append first is deleted by cleanup; finalization first fences the append | +| process loss with queued writes | only the committed prefix restores | +| final frame vs response settlement | frame commits first; settlement atomically deletes the list and inserts final entry | +| activation vs snapshot | both reduce the same committed sequence prefix; activation may then settle and clear it | +| unknown generation vs retry | synthetic partial error commits under old reserved IDs before later attempt starts | +| unknown deferred poll vs replacement | old list is deleted with fresh replacement intent; new response ID gets a new list | + +## Invariants + +1. Scalar assistant/deferred state is the sole restart authority. +2. One effect-pending response ID constructs exactly one assistant frame-list address. +3. Every stored element is an exported pi-ai `AssistantMessageFrame`. +4. Terminal `done`/`error` events are never stored as frames. +5. Frame order is a subsequence of provider event order; zero-frame covered events do not disturb order. +6. Awaiting the latest frame-write promise at stream settlement implies all accepted appends completed. +7. Frames never establish provider completion or suppress unknown-outcome recovery. +8. Final or synthetic response settlement atomically deletes the exact frame list. +9. A restored partial may appear in `streamingMessage` but never in `transcript` before settlement. +10. Interrupted partial tool calls never produce a tool plan because the synthetic response stops with `error`/`aborted`. +11. Structural generation never writes a `pendingAssistantFrames(...)` list in this slice. +12. Terminal cleanup, external finalization, and idle forks leave no operation-owned assistant frame list. + +## Required tests + +### Frame integration + +- one per-stream encoder handles shared live partials and synchronous event bursts without duplicate content; +- each event appends zero or one frame, with covered queued deltas appending nothing; +- `done`/`error` append nothing, including pre-generation error; +- interleaved content indexes preserve sequence; +- provider loop does not await individual frame writes; +- frame appends enqueue synchronously before the next provider event; +- only the latest promise reference is retained; +- awaiting the latest promise implies all earlier writes completed; +- bounded output bounds queued frame memory; +- storage failure prevents `after_response` and faults the harness. + +### Settlement and recovery + +- every ordinary response class deletes the frame list atomically; +- crash at every frame/settlement boundary; +- no frames and partial text/thinking/tool-call frames; +- authoritative end-frame content survives; +- interrupted generation commits partial synthetic error then retries or fails at cap; +- interrupted partial tool calls never execute; +- recovery uses zero usage and existing reserved IDs; +- cancellation preserves committed partial content in synthetic aborted response; +- `after_response` never runs for restored synthetic settlement. + +### Deferred, snapshots, and lifecycle + +- deferred poll frame persistence and normal cleanup; +- unknown poll snapshot without permit; +- replacement intent deletes abandoned poll frames; +- live partial takes precedence over durable reduction; +- reopen exposes reduced `streamingMessage` on suspended effect-pending lane; +- no historical update event replay; +- recovery settlement clears partial on `entry_added`; +- structural generation writes no assistant frame list. + +### Storage lifecycle + +- Memory/JSONL/SQLite reduce identical frame sequences; +- frame list is absent after normal, synthetic, cancellation, and external terminal transitions; +- idle fork excludes frames; +- JSONL compaction removes deleted frame bytes; +- migration maps or explicitly discards every legacy frame; +- instrumentation records append/delete ordering without frame content in telemetry. + +## Implementation map + +Expected runtime areas: + +- built-in `pendingAssistantFrames(operationId, responseEntryId)` address constructor in `session/values.ts`; +- bound value/list storage implementation from `values.md`; +- assistant execution observer and generation procedure; +- deferred polling procedure; +- activation and cancellation recovery; +- snapshot hydration; +- terminal cleanup, forks, and migrations; +- instrumented writer and backend conformance tests. + +Implement bound typed value/list addresses first, then frame enqueue/settlement, then recovery/snapshots. Update `harness.md` with this complete lifecycle before implementing runtime assistant parity. diff --git a/packages/agent/docs/harness.md b/packages/agent/docs/harness.md new file mode 100644 index 000000000..c63446576 --- /dev/null +++ b/packages/agent/docs/harness.md @@ -0,0 +1,1468 @@ +# AgentHarness — implementation specification + +Sections are numbered (§N.M) for cross-reference; part-level contents: + +- [Part 0 — Orientation](#part-0--orientation): system model · three stores · worked examples · non-goals · notation/source types · validation boundary · implementation status +- [Part 1 — Storage](#part-1--storage): model · identity · bound values/lists · transactions · queries · usage ledger · backends · rationale +- [Part 2 — The conversation tree](#part-2--the-conversation-tree): entries · placement · Branches/AgentLanes · metadata · branch queries/context · branch index · forks · Session/repository (incl. C1, search) · precise rewrite +- [Part 3 — The operation state machine](#part-3--the-operation-state-machine): operations · state · lane state · transition rule · graph · acceptance · assistant · tools · summaries · navigation · inbox · boundary · terminal results +- [Part 4 — Execution, recovery, abort, close](#part-4--execution-recovery-abort-close): Drive · effect gate · mutation line · attachment · recovery · abort · close · faults +- [Part 5 — Public surface](#part-5--public-surface): lane · harness · Session/Branch · snapshots · events · hooks · execution blocks · telemetry +- [Part 6 — Future: partitioned retention (Postgres)](#part-6--future-partitioned-retention-postgres) +- [Part 7 — Schema evolution](#part-7--schema-evolution) +- [Part 8 — Work packages](#part-8--work-packages) +- [Part 9 — Invariants and tests](#part-9--invariants-and-tests): 38 invariants · race catalog · test tiers +- [Appendix A — Glossary](#appendix-a--glossary) · [Appendix B — Coding-agent v3-format compatibility](#appendix-b--coding-agent-v3-format-compatibility) · [Appendix C — Open questions](#appendix-c--open-questions) + +# Part 0 — Orientation + +## 0.1 What this is + +A durable runtime for agent conversations: it persists conversation and operation state so interrupted work resumes without repeating settled effects. This document is the normative specification; §0.9 marks the parts that are specified but not implemented. Public type declarations live in the source files named in §0.7 — this document repeats a declaration only where its shape itself is a rule. + +## 0.2 System model + +A **session** has four parts: an immutable **entry tree** (message, compaction, branch summary, or application-defined custom entries; branches share the tree, enabling branching, compaction, forking, and parallel work while preserving history); mutable **values and lists** at bound typed addresses (built-ins: session name, entry labels; applications define their own collision-resistant addresses); **Branches and AgentLanes** (a Branch is one named data path with a movable tip; an AgentLane adds total model configuration, queues, and at most one operation; sessions may start with zero of either, and `main` is an ordinary explicit name); and an append-only **usage ledger**. + +The Session layer owns global durable data and Branch capabilities. The **harness** drives lanes through four primitives — `accept` durably creates an operation, `drive` advances an expected operation, `requestAbort` durably requests cancellation, `inspectExecution` atomically reports current and latest-terminal execution — plus conveniences (`prompt`, `resume`, `abort`, …) composing them with process-local waiting policy. A serving layer may instead schedule `drive` calls through alarms, jobs, or another host runtime. The harness also owns harness-wide tool and prompt-resource registries, hooks, passive events, and runtime configuration. + +An **operation** is one accepted unit of lane work: run, compaction, or navigation. Immutable metadata records identity, intent, and starting point; a total current state records phase, control, and recovery data; queued input belongs to the lane. Acceptance and execution ownership are separate: an accepted operation may have no process-local driver. Completion deletes operation-owned state and writes one immutable result record. + +**Context.** Every asynchronous public harness/lane/Session/Branch/repository/storage method takes an explicit trailing `Context`; synchronous registration (`events.on()`, `hooks.on()`) is contextless, and handlers receive a Context when invoked. Context exists because concurrent calls need independent telemetry parentage and an RPC adapter must carry one request's cancellation as `context.abortSignal`. Shared receivers never retain a caller Context or discover one through `AsyncLocalStorage`. Request-ID RPC cancellation is implemented: the client maps its signal to `cancel(requestId)`, and the server derives a request Context with an `AbortController` that aborts on matching cancellation or disconnect. Trace injection/extraction and remote telemetry-parent reconstruction are specified but not implemented (T1, §5.8). Context is process-local invocation authority, never durable data: aborting it does not call `requestAbort()` or write `cancel_requested`. + +**Storage** (Part 1) exposes atomic transactions and queries over three durable forms. `pi.op.meta` is written once per operation; `pi.op.state` is replaced after each transition with the complete current state; tool checkpoints of bounded progress are auxiliary and never prove effect completion. The terminal transaction deletes operation-owned values/lists and writes immutable `pi.result/{operationId}`. No partial transaction is ever visible. + +## 0.3 The three stores + +Everything in Parts 1–5 follows from four rules. + +**1. Three stores, one invariant.** + +```text +entries conversation tree — write-once, append-only +values/lists current mutable state — replaceable values; append-only lists + (append or whole-list delete) +usage ledger cost history — append-only rows +``` + +*Every payload is in an entry, a bound value/list, or the ledger; there is no third place.* An entry is the complete conversation record: placement and payload in one row. A `Value` holds only its current value; a `ValueList` holds immutable elements ordered by write sequence, deletable only whole. Complete content that durably exists before tree placement — queued input, deferred writes, finalized out-of-order tool results — waits in `pi.pending.entry` and becomes an entry in the transaction that places it; tool progress may occupy `pi.pending.tool_output` only while its effect is uncertain; streamed assistant frames occupy `pi.pending.assistant_frame` only while their response is effect-pending (§3.7). Per-backend projections (branch index, search, stats) are rebuildable and carry no authority. + +**2. Atomic transactions** (§1.4): entry/usage inserts and value/list writes committed all-or-none with strictly increasing sequence numbers; no crash state exists inside a transaction; the only write primitive. + +**3. The durable restart point** (§3.2): after every durable transition, the harness replaces `operationState(operationId)` with the *complete, total* current state — never depending on a previous state. After task loss, recovery reads it and starts at the responsible procedure, never replaying a journal or inferring position from what is missing. Small captured values are inline; large stable payloads live at sibling operation-owned addresses or are named by id; the terminal transaction deletes them, leaving exactly the conversation, ledger, and a handful of lane/session values. + +**4. Intent and settlement** (§0.4 trace, §3.7–§3.8): provider requests and real tool calls are wrapped in two commits — intent ("about to do X; output will use ids R and U"), the uncertain effect, then settlement (complete output + next state, plus source-ordered materialization for tools). Hooks follow a replay contract instead: a hook result becomes durable in the transaction that consumes it, and a crash before that transaction may rerun the hook. Every external effect can therefore happen without durable settlement; intents make that explicit where replay policy depends on it, and idempotent hooks accept it as a non-goal. + +## 0.4 Worked example — a Slack thread + +A user posts in a channel with 400 entries of history; the application creates a lane anchored at the channel's tip and calls `lane.prompt(...)`. The normative write order (each `TX[...]` is one atomic commit): acceptance is hook-free and starts no task or effect; the intent mints response/usage ids before anything is sent; streamed events append compact frames without blocking the stream (§3.7); settlement commits response, usage, next state, and frame-list deletion together; tool calls follow intent → effect → outcome settlement, materializing in assistant source order; the terminal transaction deletes operation values/lists and writes `pi.result/O`: + +```text +TX[ insert entry n1 (user msg), upsert pi.branch.tip = n1, + upsert pi.op.meta/O, upsert pi.op.state/O = starting, + upsert pi.lane.state = { currentOperationId: O } ] +… first drive owns real work; before_drive then before_run … +TX[ insert injected messages if any, upsert pi.branch.tip when needed, + upsert pi.op.state/O = checkpoint need_assistant ] +TX[ upsert pi.op.state/O = assistant ready (config snapshot) ] +TX[ upsert pi.op.state/O = effect_pending (reserves response n2, usage u1) ] +… provider streams … ← the uncertain window +TX[ append pi.pending.assistant_frame/O:n2 += frame ] ← zero or one per non-terminal + event, enqueued without awaiting +TX[ insert entry n2, insert usage u1, upsert pi.branch.tip = n2, + delete list pi.pending.assistant_frame/O:n2, + upsert pi.op.state/O = tools (result id n3 reserved) ] +TX[ upsert pi.op.tool_args/O:s1:0, upsert pi.op.state/O = call 0 effect_pending ] +… tool runs; selected bounded updates may replace pi.pending.tool_output/O:n3 … +TX[ upsert pi.pending.entry/n3 = finalized tool result, + delete pi.pending.tool_output/O:n3, upsert pi.op.state/O = call 0 outcome_ready ] +TX[ insert entry n3, delete pi.pending.entry/n3, upsert pi.branch.tip = n3, + upsert pi.op.state/O = checkpoint ] +… second turn: ready · intent · stream · settle (n4, u2) … +TX[ delete pi.op.meta/O, pi.op.state/O, pi.op.tool_args/O:*, + set pi.result/O = { operationId: O, kind: "run", status: "completed", + fromTipId, tipId: n4, startedAt, endedAt }, + upsert pi.lane.state = { currentOperationId: null, + lastOperationId: O, inbox: [] } ] +``` + +Kill the process between any two transactions and restart: the harness reads the lane's required values, sees which committed last, and continues. A death during the provider stream leaves a request that may have been billed and may or may not have produced output — the one genuinely uncertain window; §4.5 states the policy, and the committed frame prefix preserves the latest durable partial for synthetic settlement and reconnect display without proving how the request ended. A second thread in the same channel runs its own lane over the same shared history with no coordination. + +## 0.5 Worked example — a crash mid-tool + +The model returns two tool calls for `lane.prompt("delete the stale migrations and run the test suite")`. The harness commits the batch plan, then the intent for call 0 with its exact arguments and `replay: "never"`. The tool deletes files, emits bounded progress every 100 ms, and requests a durable checkpoint every two seconds. The process dies after one checkpoint commits: + +```text +TX[ insert entry n2 (assistant, 2 calls), insert usage u1, upsert pi.branch.tip = n2, + upsert pi.op.state/O = tools (result ids n3, n4 reserved) ] +TX[ upsert pi.op.tool_args/O:s1:0, upsert pi.op.state/O = call 0 effect_pending, + replay: "never" ] +… tool deletes files; live updates u1 … u19 … +TX[ upsert pi.pending.tool_output/O:n3 = bounded update u1 ] +… live updates u2 … u19 … ← CRASH +``` + +On restart, `pi.op.state` says `calls[0].status = "effect_pending", replay = "never"`, so the deletion is not re-run. A later drive reconciles the orphan per §4.5: latest durable checkpoint content plus an explicit interruption warning, staged as a synthetic error under the reserved id, then materialized normally: + +```text +TX[ upsert pi.pending.entry/n3 = synthetic interrupted result containing u1, + delete pi.pending.tool_output/O:n3, upsert pi.op.state/O = call 0 outcome_ready ] +TX[ insert entry n3, delete pi.pending.entry/n3, upsert pi.branch.tip = n3, + upsert pi.op.state/O = call 0 completed ] +``` + +Every tool call has a result and nothing ran twice; without a committed checkpoint the result contains only the warning. Had the tool declared `replay: "safe"` (a read, a query), the harness would instead have re-executed it with the persisted arguments. + +## 0.6 Non-goals + +- **Exactly-once external effects** — hooks with side effects must be idempotent, keyed by operation id. +- **Provider stream resumption** — the harness never reattaches to a provider stream; committed frames (§3.7) preserve the latest durable partial for recovery and reconnect display, and a settled response is persisted *completely* before anything classifies it. +- **Multiple writable owners** — exactly one host-assigned owner may hold a writable Session at a time; normally that owner is its Session worker, while the server may temporarily own a newly created or forked destination before handing it off. Storage backends do not enforce this host-lifecycle rule. Read-only repository work such as a SQLite source snapshot may overlap the worker (§1.7, §2.7). Lanes cover the workload that looks like multi-writer. +- **Work scheduling** — the harness never creates platform alarms, scans repositories for abandoned sessions, leases hosted submissions, or promises an HTTP receipt; it reports durable waits through `drive` and the serving layer decides when to call again. +- **Replication** — a session lives in one place. +- **Durable write history** — values retain only current state, lists only until whole-list deletion; no API or table exposes replaced values or deleted elements. Test write-order assertions use an instrumented decorator around `commit()` (Part 9); production auditing belongs to telemetry (§5.8). +- **Deletion as a runtime feature** — entries and usage rows are never deleted: compaction changes provider context, not storage; terminal cleanup deletes only values/lists; `retainedTail` copies old messages forward and summaries derive from old content, so compaction is not erasure. Compliance-grade erasure is the administrative precise rewrite (§2.9), the sole sanctioned exception. + +## 0.7 Notation and source types + +- `TX[ a, b, c ]` — one atomic commit with writes in that order. Write vocabulary: `insert entry`, `insert usage`, `setValue`, `deleteValue`, `appendList`, `deleteList`. Traces may abbreviate a bound address as its persisted `namespace/key`; that is never an API signature or second key argument (§1.3). +- Ids are UUIDv7s (§1.2), abbreviated `e_*`/`u_*`/`op_*`; where the time prefix matters, examples show it. +- `S(next)` overwrites `operationState(operationId)` with the next total state; `L(next)` the same for `laneState(lane)`. +- Declarative rules, transition/race tables, invariants, and traces explicitly called normative are normative; examples and sections marked informative are not. **must / must not** emphasize obligations but are not the only normative wording. This clarifies the old shorthand: tables that tests consume are part of the contract. + +Path convention: `src/...` is relative to `packages/agent/`; bare harness paths such as `session/types.ts` or `agent-harness.ts` are relative to `packages/agent/src/harness/`; `docs/...` is relative to `packages/agent/`; paths beginning `packages/` are repository-root relative. Source type provenance: `AgentMessage`, `AgentTool`, `AgentToolResult`, `QueueMode`, `ThinkingLevel` — `packages/agent/src/types.ts`. `Skill`, `PromptTemplate`, `AgentHarnessResources` (`Resources` below), the `AgentHarnessTool*` family, `AgentHarnessStreamOptions`/`Patch` — `packages/agent/src/harness/types.ts`. `Model`, `Models`, `Tool`, `Usage`, `RetryPolicy`, `StopReason`, `AssistantMessage`, `ImageContent`, provider messages, stream options, deferred handles — `packages/ai`; `AiContext` aliases pi-ai's provider request `Context` to distinguish it from the harness invocation `Context`. `AssistantMessageFrame`, `AssistantMessageFrameEncoder`, `reduceAssistantMessageFrames` — `packages/ai` `src/utils/assistant-message-frame.ts`; the harness defines no second frame codec or reducer. `CompactionSettings`, `CompactionPreparation`, `CompactResult`, `BranchPreparation`, `BranchSummaryResult` — `packages/agent/src/harness/compaction/`; existing preparation and split-turn algorithms remain the implementation unless this document changes them. `TelemetryContext` and schema helpers — `packages/telemetry`; agent-owned schemas — `src/harness/telemetry.ts`. `Context`, `ContextKey`, `BACKGROUND_CONTEXT`, derivation helpers — `src/harness/context.ts`. Harness/lane public declarations — `src/harness/agent-harness.ts`; Session/storage declarations — `src/harness/session/types.ts` and `session/values.ts`. + +Public `QueueMode` is `"all" | "one-at-a-time"`. Public `RetryPolicy` is `{ enabled, maxRetries, baseDelayMs, maxAgentDelayMs? }`; operation state stores the normalized `{ maxAttempts, baseDelayMs, maxAgentDelayMs }`. `maxRetries`, `baseDelayMs`, and optional `maxAgentDelayMs` must be finite non-negative safe integers and `maxRetries + 1` must remain safe; disabled retry normalizes to one attempt; omitted `maxAgentDelayMs` defaults to 60 seconds; delay and `notBefore` arithmetic saturate at `Number.MAX_SAFE_INTEGER`. Public `CompactionSettings` is `{ enabled, reserveTokens, keepRecentTokens }`; both token counts must be finite non-negative safe integers. Constructors and setters reject invalid settings before publication. `AgentHarnessStreamOptions` and its patch include `deferred?: boolean | { window?: "15m" | "1h" | "24h" }`; structural requests always force it to false. `SettledAssistantMessage` is `AssistantMessage & { stopReason: Exclude }`. Provider dispatch resolves the durable `{ provider, modelId }` identity through `Models` at request time (which also applies auth); a missing or swapped registry entry fails the request in-band, like an unknown tool. + +## 0.8 Validation boundary + +Internal pi objects are trusted typed values: Session, storage, operation procedures, and in-process extensions neither runtime-validate shapes nor defensively clone. Storage still enforces its operational invariants (atomicity, sequence allocation, unique ids, parent existence); backends serialize/parse as needed; externally edited or shape-corrupt storage is unsupported. Runtime schema validation belongs at untrusted wire boundaries — a future protocol-schema slice defines shared TypeBox schemas for serializable pi-ai/harness data and derives TypeScript types from them, without adding validation to internal paths. Attachment validates only the relationships needed to publish the small lane/operation projection (§3.3, §4.4); detailed state-directed references are consumption-time checks (`watch` verifies the pending/entry discriminants and message-role relationships its snapshot needs; drive verifies transition inputs), and optional assistant-frame lists and tool checkpoints may be absent. + +## 0.9 Implementation status + +WP00–WP07 are complete (Part 8): the operation graph, public lane runtime, and SQLite host-ownership alignment are implemented. Part 9 states the required conformance matrix; it is not a claim that every listed row already has one dedicated test. Known missing behavior and current contract debt, each labeled again at its section: + +- **J1 — JSONL snapshot compaction (§1.7):** specified, not implemented; dead bytes are never reclaimed today. +- **C1 — raw RemoteSession (§2.8):** the specified remote mutation transport contradicts the shipped process-local product; a decision is required before implementing either direction. +- **R12 — `watchSession` (§5.2):** public method throws `SliceNotImplemented`; the sole stubbed Harness method. +- **T1 — telemetry (§5.8):** span vocabulary declared; production starts only the tool-hook span. RPC ingress has request-ID cancellation but no trace propagation. +- **S3 — search (§2.8):** design only; the current `src/search/index.ts` skeleton conflicts with it and has no implementation. +- **R11 — schema migrations (Part 7):** mechanism specified; activation-gated; no migration exists or is required. +- **WP08 — named-branch and streaming forks (§2.7):** in progress on Slice A. Explicit scope and named-branch selection, ancestry validation, configured-lane enforcement, and the closed scalar fork policy are implemented. Lists, sequence/high-water preservation, direct Memory construction, and bounded JSONL/SQLite transfer remain. +- **SQLite branch divergence (§2.6):** the current compaction-bounded algorithm can copy O(history) on an uncompacted branch, contrary to its bounded-prefix goal. +- **H1 — contract/test closure:** public `OperationStatus` includes `"running"` but current observations produce only `"open"`/`"aborting"` (§5.4); the pre-rewrite abort contract bound `operation_abort` before resolving/signalling but current code signals first and binds recipients before releasing the line (§4.6); Part 9 remains the required conformance matrix, not a claim that every row has a dedicated test. +- **Source declaration corrections:** `CommitResult.stats` and `SessionReader.getStats()` are implemented; the old inline declarations omitted them even though other old sections depended on post-commit totals (§1.4, §2.8). The old execution-block declarations also predate the current source shapes: standalone `streamHarnessAssistant` permits an absent `afterResponse`, while the durable Harness caller always supplies it; tool phases directly carry `AgentHarnessTool`, `toolContext`, and invocation capabilities and create the canonical result message after an immediate raw result (§5.7). These are source-shape corrections, not changes to the durable boundaries. +- **Gate close typing (§4.2):** the production contract permits only `HarnessClosed | HarnessFault`; source currently widens the private primitive to `Error`, which isolated tests use. Production calls obey the narrower rule; narrowing the source type remains H1 cleanup. +- **Precise rewrite (§2.9)** and **partitioned Postgres (Part 6):** administrative/future; no implementation. + +Storage format 4 is still WIP (pre-stabilization): shapes may change in place without migrations; do not invent migration obligations for them. The detailed future-work inventory is [`post-wp05-roadmap.md`](post-wp05-roadmap.md). + +--- + +# Part 1 — Storage + +Storage knows nothing about agents, lanes, or conversations. It stores entries and usage rows, updates bound values/lists, and answers a small fixed query set. Parts 2–4 are built entirely on this. + +## 1.1 The model + +Declarations: `session/types.ts`, `session/values.ts`. Semantics: + +```ts +type JsonValue = null | boolean | number | string | JsonValue[] | { [k: string]: JsonValue }; + +/** Write-once complete conversation record: placement and payload in one row. + Created in exactly one transaction, never modified or deleted. Concrete + entry types: §2.1. */ +interface EntryBase { + id: string; // UUIDv7 (§1.2) + parentId: string | null; + seq: number; // storage-assigned at commit + timestamp: number; // Unix ms, storage-assigned at commit + type: "message" | "compaction" | "branch_summary" | "custom"; + customType?: string; // when type === "custom" +} + +/** The only mutable store, addressed by bound typed addresses. */ +function value(namespace: string, key = ""): Value; // kind: "value" +function list(namespace: string, key = ""): ValueList; // kind: "list" +interface StoredValue { address: Value; value: T; seq: number } // seq of last set +interface ListElement { seq: number; value: T } // global write seq of the append + +/** Append-only cost ledger row. Never modified, never deleted (§1.6). */ +interface UsageRow { + id: string; // UUIDv7 (§1.2) + seq: number; + usage: Usage; + entryId?: string; // the entry this cost belongs to, when there is one + adjustment: boolean; // true = caller-supplied reconciliation, not a provider report + details?: JsonValue; +} +``` + +## 1.2 Identity + +Every id — operation, entry, usage, every reserved id — is a **UUIDv7** from the session's id generator (§2.8); legacy imports re-mint to conform (Appendix B). `accept` may receive a caller-supplied operation id so a durable host submission and harness operation share one identity; the caller must mint it by the same contract and never reuse it. Omission mints internally. The first 48 bits are the mint time, so every reference is self-describing and time-sortable; the accepted cost is that ids leak creation time. (Part 6's informative Postgres sketch builds on this prefix.) + +Minting rules: (1) ids are minted with `now()` when their committing operation begins — direct appends place in the same transaction; assistant/tool ids trail placement by at most the request duration; (2) **tool-result ids inherit their assistant id's timestamp** (`idGenerator.next(timestampMs?)`, fresh random tail), so a call-and-results group is time-cohesive under id order even across midnight; (3) synthetic settlements write under already-reserved ids (§4.5) — no special case. + +**Opaque payloads** — custom entry `data`, application values, `details`, message text — may embed entry ids; the harness never tracks those references and they may go stale. Copy content, don't reference it. + +**Absolutes.** Within a session, entries and usage rows are never deleted — the precise rewrite (§2.9) is the sole exception. A missing parent is always corruption. + +## 1.3 Bound values and lists + +The public storage abstraction is a **bound typed address**: `value(namespace, key?)` names one replaceable durable value, `list(namespace, key?)` one append-only durable list of `T`. Namespace and key are bound once; every later read or write receives only the address. There is no global value-type map, token catalog, declaration merging, or separate application-state storage mechanism. Built-in constructors live in `session/values.ts` and are imported directly — no runtime catalog or dependency-injection bundle; core and applications use the same universal constructors. + +Rules: + +- `namespace` must be non-empty; neither component may contain `\u0000`. +- Namespace `pi` and every `pi.*` namespace are reserved for built-ins by contract; every built-in namespace starts with `pi.`. Application use of `pi.*` is a trusted-programming defect; constructors perform no ownership check — exact constructor tests, not runtime privilege checks, enforce the convention. +- An empty key is legal and addresses one session-wide value or list. +- Object identity has no durable meaning; equal `(kind, namespace, key)` triples name the same location. +- Constructing one location with incompatible TypeScript types is a trusted-programming defect. Value and list addresses may not share one `(namespace, key)` in a storage version; storage performs no cross-kind collision check. +- Changing namespace, key grammar, kind, or incompatible value shape requires migration (Part 7). Later operations never accept another key after address construction. + +Complete built-in inventory: + +| Address constructor | Kind | Persisted namespace, key | Value | Meaning | +| ----------------------------------------------- | ----- | -------------------------------------------------------- | -------------------------------- | ------------------------------------ | +| `branchTip(lane)` | value | `pi.branch.tip`, lane | entry id or `null` | where this lane appends next | +| `laneConfig(lane)` | value | `pi.lane.config`, lane | `LaneConfiguration` | total lane configuration | +| `laneState(lane)` | value | `pi.lane.state`, lane | `LaneState` (§3.3) | current/last operation ids and inbox | +| `operationResult(opId)` | value | `pi.result`, operation id | `OperationResultRecord` (§3.13) | immutable terminal observation | +| `operationMeta(opId)` | value | `pi.op.meta`, operation id | `OperationMeta` (§3.1) | acceptance data; written once | +| `operationState(opId)` | value | `pi.op.state`, operation id | `OperationState` (§3.2) | total durable restart point | +| `operationToolArgs(opId, stepId, sourceIndex)` | value | `pi.op.tool_args`, `{opId}:{stepId}:{sourceIndex}` | effective arguments | written once at clearance | +| `operationToolMemo(opId, invocationId, name)` | value | `pi.op.tool_memo`, `{opId}:{invocationId}:{name}` | `JsonValue` | invocation-scoped durable memo | +| `operationPreparation(opId, taskId)` | value | `pi.op.preparation`, `{opId}:{taskId}` | `DurableStructuralPreparation` | structural preparation | +| `pendingEntry(entryId)` | value | `pi.pending.entry`, reserved entry id | `PendingEntry` | complete content awaiting placement | +| `pendingToolOutput(opId, invocationId)` | value | `pi.pending.tool_output`, `{opId}:{invocationId}` | `AgentToolResult` | latest bounded progress checkpoint | +| `pendingAssistantFrames(opId, responseEntryId)` | list | `pi.pending.assistant_frame`, `{opId}:{responseEntryId}` | `AssistantMessageFrame` elements | committed stream-frame prefix | +| `sessionName` | value | `pi.session.name`, empty key | string | session name | +| `entryLabel(entryId)` | value | `pi.entry.label`, entry id | string | entry label | + +Exactly five exported scan-prefix constructors encapsulate lane inventory and operation-cleanup grammar. Their results are valid only as namespace-scoped `scanValues()` inputs, never exact get/set/delete addresses: + +| Prefix constructor | Namespace | Prefix key | +|---|---|---| +| `branchTipInventoryPrefix()` | `pi.branch.tip` | `""` (all lanes) | +| `operationToolArgsPrefix(opId, stepId?)` | `pi.op.tool_args` | `{opId}:` or `{opId}:{stepId}:` | +| `operationToolMemoPrefix(opId, invocationId?)` | `pi.op.tool_memo` | `{opId}:` or `{opId}:{invocationId}:` | +| `operationPreparationPrefix(opId)` | `pi.op.preparation` | `{opId}:` | +| `pendingToolOutputPrefix(opId)` | `pi.pending.tool_output` | `{opId}:` | + +```ts +/** Unplaced content: current mutable state until the placement transaction + writes the complete entry and deletes this value (§2.2). */ +type PendingEntry = + | { type: "message"; payload: AgentMessage } + | { type: "custom"; customType: string; payload?: JsonValue }; + // absent custom payload = a custom entry with no data +``` + +`DurableStructuralPreparation` (`session/types.ts`) is a two-variant union: `kind: "compaction"` with `messagesToSummarize`, `turnPrefixMessages`, `retainedTail`, `isSplitTurn`, `tokensBefore`, optional `previousSummary`, `fileOps`, `settings`; and `kind: "branch_summary"` with `messages`, `fileOps`, `totalTokens`. `fileOps` is `{ read, written, edited: string[] }`. + +Lifetimes: + +```text +pi.lane.* pi.session.* pi.entry.* session-lived semantic values +pi.result immutable lane-lived records, one per terminal operation +pi.op.* operation-lived; deleted no later than the terminal transaction (§3.13) +pi.pending.entry until placement, cancellation, or owning-operation cleanup +pi.pending.tool_output only while its invocation is effect-pending +pi.pending.assistant_frame only while its response is effect-pending +``` + +- `pi.op.meta` and `pi.op.preparation` are written exactly once; `pi.op.tool_args` once per call. Invocation memos die when the invocation reaches `outcome_ready`. Every `pi.op.*` value is deleted no later than the terminal transaction. +- The lane inbox and its pending payloads outlive operations and die only when consumed or cancelled; operation-owned staged tool outcomes die at placement or terminal cleanup (§3.11). +- Tool output is optional auxiliary state: outcome staging deletes it atomically; safe replay deletes it before re-execution; unsafe recovery may consume it into an interrupted result. +- Assistant frames are auxiliary list elements ordered by global write `seq`. A missing list is valid. Frames never prove request admission, completion, or failure and never select a restart point; settlement deletes the exact bound list atomically (§3.7). +- `pi.result` records are written once by terminal transactions, never updated or deleted by the runtime, and never read by recovery. +- Deleting a bound value removes it; JSON `null` stays distinct from absence where the address type permits it. + +## 1.4 Transactions + +A `Write` is an erased storage record for one of six operations — entry insert, usage insert, value set/delete, list append/delete — carrying `(namespace, key)` plus the value where applicable. Raw write shapes are storage internals: all code constructs them through `insertEntry(entry)`, `insertUsage(row)`, `setValue(address, next)`, `deleteValue(address)`, `appendList(address, element)`, and `deleteList(address)`, which check the bound address/value relationship before erasure. Value helpers cannot target list addresses and vice versa; `NoInfer` makes the address authoritative instead of widening `T`. + +```ts +interface CommitResult { + firstSeq: number; seqs: number[]; timestamp: number; + stats: SessionStats; // session totals immediately after this commit +} +``` + +Rules: + +1. A transaction commits **all-or-none**; no observable state has some writes and not others. +2. Writes receive **strictly increasing** `seq` in the order given; gaps are legal within and between transactions; `seq` is monotonic session-wide across all lanes and write kinds. A value `set` stamps the stored value with its assigned `seq`. +3. Writes apply in order within a transaction: an entry may name a parent created earlier in the same transaction; a stored value may reference entry/usage ids created earlier in it. A placement transaction inserts the complete entry and deletes its `pendingEntry(id)` together (§2.2) — both never exist at once. +4. Entry and usage ids share one session-wide id namespace; writing either kind under any existing id is **corruption**, not an update. +5. A value `set` replaces the current value; `delete` removes it; a later `set` recreates it; no history is retained. A `delete` naming an absent key is a no-op, so public deletions such as clearing an unset label stay legal. +6. One list `append` carries one element and never reads existing elements. Elements are immutable after commit and ordered by assigned write `seq`; gaps from unrelated writes are irrelevant. No per-element update, delete, insertion, or truncation exists. +7. A list `delete` removes every element under `(namespace, key)`; deleting an absent list is a no-op; `delete` then `append` in one transaction atomically creates a fresh list. "Append-only" describes elements while the key exists — whole-key deletion is lifecycle cleanup, not element mutation. +8. Transactions on one session are **serialized**: one writer, one queue. + +Session passes typed transactions to storage without a codec, runtime shape validation, or cloning. A failed admitted commit **faults the harness** (§4.8): all effects stop, all calls reject, the process must restart. A partially applied transaction is not tolerated. + +## 1.5 Queries + +One `Storage` instance serves one session; repository discovery and lifecycle are outside it (§2.8). + +```ts +interface Storage { + commit(writes: Write[], context: Context): Promise; + getEntries(ids: string[], context: Context): Promise>; + getValue(address: Value, context: Context): Promise | undefined>; + /** Internal namespace-scoped prefix scan; the bound address key is the prefix. */ + scanValues(prefix: Value, context: Context): Promise[]>; + readList(address: ValueList, options: ListReadOptions | undefined, + context: Context): Promise[]>; + scanBranch(q: StorageBranchScan, context: Context): Promise; // §2.5 + scanBranchStructure(q: StorageBranchScan, context: Context): Promise; + scanEntries(q: EntryScan, context: Context): Promise; // session-wide inventory + scanUsage(q: UsageScan, context: Context): Promise; // ledger read (§1.6) + getStats(context: Context): Promise; // maintained projection + close(context: Context): Promise; +} +``` + +`EntryStructure` is the entry minus payload fields (`id`, `parentId`, `seq`, `timestamp`, `type`, `customType`). `EntryScan`/`UsageScan` filter by `type`/`customType` (entries only), `fromSeq`/`toSeq`, `order: "asc" | "desc"`, `limit`. `ListReadOptions` is `{ cursor?: { seq }, order?: "asc" | "desc" (default "asc"), limit? }`; the limit must be a positive safe integer, defaults to 1,000, and clamps above 10,000. + +List read semantics: ascending returns `seq > cursor.seq`, descending `seq < cursor.seq`; results are ordered before `limit`; absent and empty keys both return `[]`; callers continue with the last element's `seq`, and an empty page ends iteration. A cursor is a sequence filter, not a snapshot or key-incarnation token: concurrent later appends may appear on later ascending pages, and after a whole-key delete a read simply applies the comparison to surviving elements. There is deliberately no unbounded "read the whole list" helper. + +`scanValues(prefix)` is namespace-scoped, interprets the bound key as a prefix, and returns values in key-ascending order. Core inventory/cleanup uses only the five §1.3 prefix constructors; core call sites do not repeat raw reserved grammar. Ordinary reads use exact addresses. There is no cross-namespace value dump or durable write log. Entry inventory uses `scanEntries`, ledger reads `scanUsage`, totals the stats projection (§1.6), and test-order assertions the instrumented decorator (Part 9). + +Recovery and execution reads must be index-driven and bounded: never infer state from an absent value (no history exists to fold). Exact dereference is allowed — current typed state may name a bounded set of entries and values, and an exact list address derived from current state may be read in bounded pages and reduced by its consumer (assistant frames use `reduceAssistantMessageFrames`, §3.7). Base restore never reads lists (§4.4). Public inventory/debugging APIs expose explicit limits/pagination through Session and Branch. + +`close()` is idempotent: seal admission, reject later reads/commits on that instance, drain commits admitted before the seal, then release backend resources. Durable data reopens through the repository; writable-owner handoff belongs to the host lifecycle, not Storage. + +## 1.6 Usage ledger + +Every settled provider attempt writes one `UsageRow` — successful, failed, retried, and synthetic alike, including attempts whose operation later aborts. An orphaned structural/deferred intent that recovery discards or replaces has no settled outcome and may leave its reserved response/usage ids unused; abandonment alone writes no synthetic usage row, while any already committed usage remains. Settlement writes the response entry and its usage row together (§3.7); synthetic settlements write zero usage under the reserved usage id. Rows are append-only: terminal cleanup never deletes ledger rows, so billing survives everything that happens to orchestration state. + +- `entryId` names the entry the cost belongs to, when one exists; structural attempts that fail before producing an entry, and standalone adjustments, have none. +- `adjustment: true` marks a caller-supplied reconciliation (`recordUsage`, §5.1), not a provider report; the format-3 import writes one aggregate adjustment row (Appendix B). +- Provider-attempt usage ids are reserved in the intent commit, so settlement writes under exactly the promised id. Adjustment rows, tool-reported usage, hook-supplied compaction/navigation usage (§3.9, §3.10), and import aggregates mint ids at commit; nothing reserves them. +- `getStats()` is a maintained projection over the ledger plus the message-entry count — `messageCount` counts `message` entries only. After every commit it equals the ledger sum (asserted by conformance, Part 9). Rows reach the application through the `usage` event at commit (§5.5); `scanUsage` reads them back by seq range, so a consumer persisting the greatest applied event `seq` catches up with `scanUsage({ fromSeq })`. Recovery never reads the ledger. + +## 1.7 Backends + +Three encodings of one model ship — Memory, JSONL, SQLite — and all pass the same conformance suite (Part 9). Each records the session's `storageVersion` (Part 7): a JSONL header field, a SQLite catalog column; Memory sessions are always current. Partitioned Postgres is informative only (Part 6). + +### Memory + +Maps for entries, scalar values, list arrays, and usage rows, physically keyed by `namespace + separator + key`. One queue serializes commits. A commit checks storage invariants, assigns sequences and the transaction timestamp, then applies writes synchronously; all validation and serialization needed to admit a transaction completes before any map mutates. Value delete = map delete; list append pushes the sequenced element; whole-key list delete removes the array; list reads filter by the exclusive cursor and slice to the validated limit. Reads are map lookups; `scanBranch` walks `parentId` in RAM. Memory returns typed values without cloning and holds exactly the live state — there is no log. + +### JSONL + +The file is the **replay recipe** for the Memory maps, not the state. One physical line per `commit()`: storage assigns sequence/timestamp fields, then encodes one committed write as a JSON object line or several as one **array line**. The header line is `{"v":4,"kind":"header","id":…,"storageVersion":1,"createdAt":…,"cwd":…}` plus optional `parentSessionId`, `legacyParentSessionPath`, and the `nextSeq` high-water mark written by fork destinations and v3 normalization (and required for future J1 rewrites). + +- This is format 4. The pre-WP01 unfinished format-4 spelling was replaced in place; no migration for it exists or is required. Coding-agent format 3 remains supported (Appendix B). +- Open replays lines in order into the maps — entries/usage accumulate; a later value `set` overwrites, `delete` removes; list `append` adds `{ seq, value }`, list `delete` removes the key. That is *decoding*, not recovery logic. Open verifies persisted sequence monotonicity (strictly increasing, gaps legal) and timestamps, and never regenerates committed timestamps. All queries then run in RAM. +- **A torn final line is discarded whole**, including every element of an array line, and truncated before new writes are admitted — this makes "no crash prefix inside a transaction" true here. A malformed *interior* line or invalid framing is corruption. A future older storage version is decoded only when an explicit R11 migration defines that total mapping; post-migration compaction retires its bytes. +- Durability is process-crash level: a resolved `commit()` survives process death; no fsync promise. Optionally retain `(offset, length)` per entry and load payloads lazily — only if profiling demands it. + +**Snapshot compaction (J1 — specified, not implemented).** In SQLite a value `set` is an in-place upsert; in JSONL every `set` appends, so a 30-turn run leaves ~10 dead `pi.op.state` lines after the terminal `delete`: the file grows with write history even though logical state does not. The specified fix rewrites the file as `header + current entries + current values + surviving list elements + usage rows` via temp file + atomic rename. Surviving lines keep their original `seq` values (dropped-line gaps are legal; no renumbering). Each surviving list element is rewritten as an append record carrying its original `seq`, merged in sequence order — never collapsed into one synthetic append — so list cursors survive. Deleted lists produce no snapshot records; the `nextSeq` high-water mark is preserved so dropping a trailing delete line cannot permit sequence reuse. Compact on open when the dead-bytes ratio crosses a threshold, after a terminal or outcome-staging deletion pushes the file across it, and always after a schema migration (Part 7); between compactions, operation is append-only and O(1) per commit. + +Until J1 lands, deleted pending payloads, superseded state revisions, superseded tool checkpoints, and deleted frame lists linger as bytes indefinitely — logical deletion is immediate; physical deletion currently never happens. Tool authors therefore own bounded checkpoint values, cadence, and duplicate suppression (bash: live updates at 100 ms, checkpoints at most every two seconds, only when changed; at 50 KiB per checkpoint, continuously changing output adds ~15 MiB per ten minutes). Assistant frame lists grow linearly with model output; the [mobile assistant-output handoff](mobile-handoff/01-harness/05-assistant-output/message-update.md) replaces per-frame durable and replication writes with tracked output in scoped storage. One small immutable `pi.result` record per terminal operation is retained forever and copied into every later snapshot — result growth is linear in operation count by design. Deployments needing prompt physical removal of sensitive cancelled content compact eagerly at terminal boundaries, once J1 exists. + +### SQLite + +Backend: `packages/session-backends/sqlite-node`. **One database file per session is the default; a shared container is supported.** Without `databasePath`, safe alphanumeric/underscore/hyphen ids retain `{id}.sqlite`; every other explicit id uses a `~`-prefixed base64url encoding of its UTF-16 code units, so separators, dots, percent signs, and Unicode cannot escape `directory`. With `databasePath`, any number of Sessions share one container. Metadata reports the canonical physical container path. Every authoritative and projection row is scoped by `session_id`; shared containers are a supported deployment mode, not an implementation detail to remove. SQLite supplies atomic transactions and coherent WAL snapshots, not Session ownership. + +`001_initial.sql` (storage version 1), all scoped by `session_id`: + +```sql +entries(id, parent_id, seq, type, custom_type, timestamp, payload) WITHOUT ROWID; + -- ix_entry_parent(parent_id), ix_entry_seq(seq, type) +scalar_values(namespace, key, seq, value, PRIMARY KEY (namespace, key)) WITHOUT ROWID; +list_values(namespace, key, seq, value, PRIMARY KEY (namespace, key, seq)) WITHOUT ROWID; +usage_ledger(id, seq, entry_id, adjustment, usage, details) WITHOUT ROWID; + -- ix_usage_seq(seq) + +-- Private branch index (§2.6). Not values/lists; no equivalent in other backends. +branch_entries(branch_id, entry_id, entry_seq, entry_type, + PRIMARY KEY (branch_id, entry_id)) WITHOUT ROWID; + -- ix_be_seq(branch_id, entry_seq, entry_id, entry_type): entry_seq must directly + -- follow branch_id or ORDER BY needs a temp b-tree; trailing columns cover + -- id-only reads. ix_be_type(branch_id, entry_type, entry_seq, entry_id), + -- ix_be_entry(entry_id) +branch_meta(branch_id PRIMARY KEY, tip_entry_id, tip_seq, base_branch_id, base_seq); + -- unique ix_bm_tip(tip_entry_id) + +sessions(id, created_at, parent_session_id, storage_version, metadata, + message_count, usage_payload, next_seq); -- one row per Session +``` + +Triggers enforce the shared entry/usage id namespace and ordered parent insertion at the storage level. No pre-WP01 format-4 SQLite file is supported; migration machinery belongs to R11. + +One `commit()` is one SQL transaction: insert entries and ledger rows, replace/delete scalar values, insert/whole-list-delete list elements, maintain the branch index, bump session stats (`message_count`, aggregate `usage_payload`). Never update or delete an entry or ledger row; mutability is confined to values/lists, the branch index, stats, sequences, and the catalog row. List paging is `SELECT seq, value FROM list_values WHERE namespace = ? AND key = ? AND seq > ? ORDER BY seq ASC LIMIT ?` (descending symmetric; omit the predicate without a cursor); assert via `EXPLAIN QUERY PLAN` that it uses the primary key with no temporary sort. + +**Every transaction that may write must open with `BEGIN IMMEDIATE`.** A deferred `BEGIN` that reads before writing takes a read snapshot and must later upgrade to the write lock; if another writer committed in between, SQLite fails the upgrade — and `busy_timeout` cannot rescue it, because waiting cannot refresh a stale snapshot; the only recovery is rollback and full retry. Every commit reads the session row's `next_seq` before writing, so a read precedes a write in every writing transaction; branch creation (§2.6) also reads the newest compaction before inserting. Coherent read-only snapshot transactions — fork capture (§2.7) — may use a deferred `BEGIN` read transaction; they never upgrade to a write. The old blanket wording covered every transaction and conflicted with its own read-only fork rule; this narrows the rule to the source behavior without weakening any write path. + +**Session ownership is host-authoritative.** Exactly one worker normally owns a writable Session; create/fork administration may own a destination only until it closes that Session and hands metadata to the worker. Memory, JSONL, and SQLite do not detect a second process opening the same Session for writes; bypassing the server/worker lifecycle is a trusted-host defect. SQLite has no lease, fence, heartbeat, or replacement ownership primitive. A repository still rejects duplicate writable handles and reserves create/open/fork/delete destinations it owns in one process. The host closes a worker before deletion; shared-container deletion removes only that Session's rows in one `BEGIN IMMEDIATE` transaction, while per-file deletion removes its database and WAL/SHM sidecars. + +Database access has three explicit modes: intentional create-or-open, no-create read-write, and no-create read-only. Metadata `open` and deletion use no-create read-write access; listing and external fork sources use no-create read-only access, so missing paths never become empty databases. Writable `open`/`delete` metadata must resolve to the repository-affine physical path. A foreign fork source is instead read from its exact physical path and can never alias an active local source with the same Session ID. + +Read-only fork access may overlap the worker: WAL permits the server's repository to capture a live worker-owned source while the worker continues committing. Every source uses an independent read-only connection and one deferred read transaction that never upgrades or claims writable authority; it validates the Session row and storage version inside that transaction and sees every source transaction wholly before or wholly after its snapshot boundary. For a same-repository open source, the reader opens first and a short callback on the source commit queue begins the transaction and establishes its snapshot before releasing the queue. WAL frames become visible only when the commit record lands, so no fork sees part of a commit. Selected rows stream into a temporary on-disk staging database while the source reader remains open; after that reader closes, the stage streams into one destination `BEGIN IMMEDIATE` transaction and is removed in `finally`. Later source commits may complete while staging. Repository close seals admission, starts every open Session close, waits for all to settle, and reports one error directly or several in an `AggregateError`. + +Each physical segment of `scanBranch` uses one JOIN (§2.6 combines segment ranges): + +```sql +SELECT e.id, e.parent_id, e.seq, e.type, e.custom_type, e.timestamp, e.payload +FROM branch_entries b +CROSS JOIN entries e ON e.id = b.entry_id +WHERE b.branch_id = ? AND b.entry_seq > ? AND b.entry_seq <= ? +ORDER BY b.entry_seq; +``` + +`CROSS JOIN` is required: it forces `branch_entries` as the outer loop; left alone the planner may drive from `entries`, scan it, and sort through a temp b-tree. Assert the plan in a test (`SEARCH b USING COVERING INDEX ix_be_seq …`, then `SEARCH e USING PRIMARY KEY`); any plan with `USE TEMP B-TREE FOR ORDER BY` or an `entries` scan is a regression. `scanBranchStructure` is the same query without the payload column; `getEntries` is a primary-key `IN (...)` lookup. + +In per-session-file mode a precise rewrite (§2.9) may build a fresh database (`VACUUM INTO` or row copy over one read snapshot) and atomically swap it over the old path, like JSONL. A shared-container rewrite/fork copies only the selected Session's rows and must not rewrite unrelated Sessions. Fork staging writes to a separate temporary file in both layouts; precise-rewrite tooling remains administrative future work. + +## 1.8 Why write-once plus values and lists + +Consequences relied on throughout: attachment is bounded (fixed projection point reads per lane, §4.4; one compaction-bounded watch scan plus exact state-directed reads, §5.4; the only reducer on a durable path is pi-ai's frame reducer over one exact bounded list, §3.7); crash states are enumerable — between transactions, never inside one; cleanup is deletion, not collection — a 30-turn run replaces `operationState` ~30 times then deletes it, leaving exactly the conversation, ledger, and a few lane/session values (JSONL defers physical reclamation to J1; logical state is identical); recovery never repairs by rewrite — it appends entries and replaces only values it owns with the same transitions normal execution would commit, so interrupting and rerunning gives the same result; readers never see partial state. Staging writes are deliberate: queued content serializes into `pi.pending.entry` at enqueue and again into its entry at placement; finalized tool outcomes stage before source-ordered materialization, preventing a completed parallel effect from replaying after a crash; assistant settlements are born placed, their frames dying atomically with settlement. Staging always has one owner and dies atomically with placement or cleanup. + +--- +# Part 2 — The conversation tree + +## 2.1 Entries + +An **entry** is the complete stored row (§1.1): placement fields and payload together. `getEntries` and the scans return exactly what was committed — no materialization step, no join. + +```ts +interface MessageEntry extends EntryBase { + type: "message"; message: AgentMessage; terminate?: true; +} +interface CompactionEntry extends EntryBase { + type: "compaction"; summary: string; retainedTail: AgentMessage[]; + tokensBefore: number; details?: JsonValue; usage?: Usage; fromHook: boolean; +} +/** fromId: the summarized branch's pre-navigation tip — the producing + operation's sourceTipId (§3.10) — or null when that source is the root. */ +interface BranchSummaryEntry extends EntryBase { + type: "branch_summary"; fromId: string | null; summary: string; + details?: JsonValue; usage?: Usage; fromHook: boolean; +} +interface CustomEntry extends EntryBase { + type: "custom"; customType: string; data?: JsonValue; +} +type Entry = MessageEntry | CompactionEntry | BranchSummaryEntry | CustomEntry; +``` + +Rules: `type`/`customType` are structural fields — branch queries filter on them and the branch index denormalizes them (§2.6); `customType` is set exactly on custom entries; payload fields never drive structure. Assistant entries always contain a `SettledAssistantMessage` — reject `pending` before writing. Tool-result entries carry `terminate?: true`, orchestration state `ToolResultMessage` has no field for. Every compaction and branch summary carries `fromHook` (`true` = hook output, `false` = generated). Every compaction stores a complete `retainedTail` (`[]` when empty); **context never reads past a compaction** — a compaction is a self-contained checkpoint, not a pointer into history. Only a custom entry may lack `data`. Payloads are inline; two entries never share stored content and there is no deduplication layer. + +## 2.2 Placement + +> An **entry** is created, complete, when placement happens. Content durable *before* placement is current mutable state waiting in a `pendingEntry(id)` value; the placement transaction writes the entry and deletes the pending value. Neither is modified after that. + +**Born placed** — assistant responses and direct appends to an idle lane; content and placement arrive in one transaction (`TX[ insert entry, upsert pi.branch.tip ]`). + +**Content first — queued input.** `steer`, `followUp`, `nextRun`, and deferred tree writes mint the entry id at enqueue and construct `pendingEntry(id)`; queue state references content by that id, and the two transactions may be far apart: + +```text +t0 TX[ upsert pi.pending.entry/e_q1 = { type: "message", payload: <200KB message> }, + S(next){ ...inbox.steer += "e_q1" } ] +t1 TX[ insert e_q1 (parent e_a3), delete pi.pending.entry/e_q1, + upsert pi.branch.tip/main = "e_q1", S(next){ ...inbox.steer -= "e_q1" } ] +``` + +Crash before `t1`: still queued; after: placed, pending value gone. Until placement or cancellation exactly one of pending value and entry exists; cancellation deletes the value and the content never enters the tree (§3.11). + +**Content first — finalized parallel tool outcomes.** A tool result id begins as a plain reserved string in `pi.op.state`. When execution and `after_tool` finish, the complete final `ToolResultMessage` is staged in `pendingEntry(resultEntryId)` and the call becomes `outcome_ready`; it enters the tree only when every earlier source position is ready (`t0`: stage + `outcome_ready`; `t1`: insert after the earlier result + delete pending + `completed`). Effects settle in completion order while entries materialize in assistant source order. Crash before `t0`: uncertain effect; after `t0`: never re-executed; after `t1`: immutable entry. + +**Id reserved before content exists.** Assistant response, tool-result, and usage ids are minted as strings in operation state. Assistant settlement places its result directly; during the effect window the reserved response id also keys the auxiliary frame list, which settlement deletes (§3.7). + +Consequences: a queued or outcome-ready item is invisible to tree queries but visible through its owning state and `pendingEntry(id)`; queue placement/cancellation and outcome-ready materialization delete `pi.pending.entry` atomically with their state change; a reserved tool-result id moves through `string only → pi.pending.entry → immutable entry` with no two representations coexisting at a commit boundary; queued input pays the deliberate double-write (§1.8), and finalized tool outcomes stage once before placement when source ordering requires it — the extra write that prevents completed parallel effects from replaying after a crash. + +## 2.3 Branches and AgentLanes + +A `Branch` is data for one named path through the tree; it exists exactly when its tip value `pi.branch.tip/{name}` exists (entry id or `null`). A Branch owns only its tip, branch-relative queries, and direct append — a raw append always inserts at the current tip and moves the tip in one Session mutation. It has no model, queues, operation state, hooks, or execution policy. + +A configured `AgentLane` is a Branch plus total agent state: `pi.lane.config/{name}` (`LaneConfiguration = { model: { provider, modelId }, thinkingLevel, activeToolNames }`), `pi.lane.state/{name}` (`LaneState`, §3.3), and one `pi.result/{operationId}` per terminal operation. + +`AgentHarness.lane(name, options?, context)` is atomic get-or-create: a missing Branch writes its tip, the immutable Harness seed configuration, and idle lane state together; a data-only Branch receives configuration and idle state without moving its existing tip; a complete AgentLane returns unchanged; partial combinations fault as corruption. Concurrent acquisitions publish and return one process-local AgentLane. A fresh Session/Harness may have no Branches or AgentLanes; `main` is created only when explicitly acquired. During an active run, AgentLane append methods preserve operation-aware deferred-write semantics; a raw Branch append remains direct, and mutating that raw Branch while a Harness owns its lane is a trusted-programming defect. + +## 2.4 Session metadata and application values + +Session name and entry labels are latest-wins values outside the tree (`sessionName`, `entryLabel(entryId)`, §1.3). `getName`/`setName` and `getLabel`/`setLabel` wrap them; passing `undefined` deletes, deleting an absent value is a no-op (§1.4); these writes commit immediately and never move a tip. Applications define their own stable collision-resistant addresses (`value("my-app.state")`, `list("my-app.events")`); there is no built-in application namespace or separate application-state API. Fork behavior is defined in §2.7; applications own their migration policy. + +## 2.5 Branch queries and context + +```ts +interface BranchScan { + start?: string; // required at Storage; Branch/AgentLane default to the receiver's tip + stopAtType?: EntryType; // scan ends after the first match, inclusive + stopAtId?: string; + type?: EntryType; customType?: string; + order?: "newestFirst" | "oldestFirst"; // default newestFirst + limit?: number; + cursor?: { seq: number }; // EntryCursor +} +type StorageBranchScan = BranchScan & { start: string }; +``` + +Semantics: take the path from `start` toward the root, order it (default `newestFirst`), stop **inclusively** at the first `stopAt` match, filter by `type`/`customType`, apply the exclusive cursor (`newestFirst` retains `seq < cursor.seq`, `oldestFirst` `seq > cursor.seq`), then `limit`. A `stopAt` entry is returned only if it also passes the filter. `stopAtType` applies after ordering — `oldestFirst` with `stopAtType: "compaction"` stops at the oldest compaction segment — so canonical context reads use `newestFirst` through the newest compaction and reverse the bounded result. + +**Context projection** — how a provider request is built: + +1. `scanBranch({ start: tip, order: "newestFirst", stopAtType: "compaction" })`. +2. Reverse to oldest-first. If a compaction terminated the scan, the context is its `summary`, then its `retainedTail`, then every entry after it. **Nothing earlier is read.** +3. Drop assistant responses whose stop reason is `error`, `aborted`, or `deferred`; retain genuine output-limit `length`. +4. Run custom entries through `entryProjectors`; an unprojected custom entry never enters context. +5. Run `transform_context`, then `toProviderMessages`. + +An overflow response needs no dedicated omission rule: it is committed with stop reason `error` (§3.7) and dropped by rule 3. + +**Append-only context invariant.** Across one lane's requests, provider context must only grow at the tail: an insertion before the previous request's tail invalidates the provider's KV cache and multiplies cost. This is why mid-run writes defer to checkpoints, where they append at the tail. Compaction is the one deliberate cache invalidation, traded for a smaller context. + +## 2.6 The branch index + +Memory and JSONL walk parent pointers in RAM. SQLite maintains a private segmented branch cache so a diverging append does not copy the full root prefix. `branch_entries` stores the entries physically present in one segment; `branch_meta` stores its tip and optional `{ baseBranchId, baseSeq }`. A segment logically contains its own rows above `baseSeq` plus the referenced base prefix through `baseSeq`. + +Append: (1) if a branch tip equals the lane tip, append one row and move that tip; (2) otherwise resolve a branch that actually covers the tip, find the newest compaction at or below the tip through the complete segment chain, copy only rows after that compaction through the tip, and set the older prefix as the new segment's base; (3) append the new entry and make it the new segment tip. + +**Known contradiction (open):** the copy bound is the newest compaction, so a first divergence from a long *uncompacted* transcript copies O(history) rows — the "no unbounded copy" goal is not met in that case. The implementation follows the compaction-bounded algorithm as written. Resolving this needs a segment representation that can reference a covering segment at the parent boundary (inventoried in `post-wp05-roadmap.md`); specification and representation must change together. + +Read newest segment first; if the requested range crosses `baseSeq`, continue through the base chain with the upper bound capped at that boundary; merge segment results into the requested order before filtering/limiting. Two correctness rules are mandatory: the base branch must itself cover the tip within its logical range (containing the tip in an ancestor is insufficient), and the newest-compaction search must traverse the base chain (checking only the newest physical segment can miss it). The cache must preserve: a segment chain followed to its end yields the exact root path with no gaps or duplicates; all chains containing an entry agree below it; runtime reads never fall back to a table scan or parent walk; stale branches remain valid cache history; only an explicit repair operation rebuilds the cache from entries. Tests assert these invariants and the required query plans; no wall-clock threshold is normative. + +## 2.7 Forks + +A fork is a repository operation over one coherent source-storage boundary. Destination metadata records the source id as `parentSessionId`. + +```ts +type ForkOptions = + | { scope: "branch"; branch: string; entryId?: string; + position?: "before" | "at"; id?: string } + | { scope: "tree"; id?: string }; +``` + +**Branch scope** requires the named source Branch to be a complete configured AgentLane: tip, configuration, and lane state must all exist. A missing tip is an unknown Branch; a data-only Branch rejects; a partial configuration/state pair, or lane values without a tip, is corruption. `entryId`, when supplied, must be on the named Branch's current-tip ancestry, inclusive; omission selects the current tip. `position` defaults to `"at"`; `"before"` selects the target's parent and may produce a `null` destination tip before a root entry. A `null` source tip is legal only with no `entryId`. The destination contains exactly that one Branch under the same name, its selected path and tip, copied configuration, and fresh idle lane state. + +**Tree scope** copies every immutable entry, including entries unreachable from all current tips; every Branch tip; every configured lane's configuration plus fresh idle lane state under the same name; and every data-only Branch as data-only. A partial configuration/state pair or lane values without a tip is corruption and rejects instead of being dropped. A branchless source produces a branchless destination. + +**Both scopes** copy the session name and labels only for copied entries. They exclude the usage ledger, `pi.result`, every `pi.op.*`, and every `pi.pending.*` value/list, including pending entries, tool checkpoints, and assistant frames. Destination usage starts at zero and `messageCount` counts copied message entries. Copied entries retain ids. + +Application state follows scope rather than a historical sequence cutoff: tree scope copies every current application scalar and every surviving application list element; branch scope copies none. Current state has no replaced values or deleted list elements from which to reconstruct an earlier point, so filtering surviving rows by `seq <= selectedTipSeq` is forbidden. Applications re-derive branch-scoped state. + +One closed core policy classifies every namespace. Session name copies; labels depend on copied-entry membership; branch/lane values are reconstructed coherently; operation, pending, and result namespaces exclude; application namespaces follow scope. Exact namespace `pi` and every otherwise-undeclared `pi.*` namespace reject only when current surviving scalar or list state exists. Replaced or deleted history is absent and cannot by itself reject a fork. + +Copied entries, values, and list elements retain their source `seq`. Rewritten tips and fresh idle lane states reuse the source rows' current sequences, and destination `nextSeq` equals the source high-water mark so no sequence can be reused. Memory constructs destination state directly at its commit-queue boundary. JSONL captures a fixed read-only file prefix and uses bounded disk-backed passes without mutating the source. SQLite establishes an independent read snapshot, streams into a temporary staging database, closes the source reader, then publishes the stage in one destination transaction. Later source commits are wholly outside that fork. + +## 2.8 Session and repository boundary + +`Storage` is one-session only. `Session` owns global metadata, values/lists, entry and usage queries, Branch discovery/creation, one mutation line, and one backend lifecycle; it does not implement Branch and has no implicit main. Full declarations: `session/types.ts`. The surface, by group (every asynchronous method takes a trailing `Context`): + +- **`SessionReader`** (implemented by Session and mutation capabilities): `getEntries(ids)`, `getStats()`, `getValue(address)`, `scanValues(prefix)`, `readList(address, options?)`, `scanBranch(query: StorageBranchScan)`. +- **`SessionMutation extends SessionReader`**: `commit(writes)` — zero or one attempt, does not release — and `end()` — waits for any admitted commit, invalidates, releases. `SessionMutator = Omit`. +- **`Branch`**: `name`, `getTipId()`, `findEntries(query?: BranchScan)`, `findEntry(query?: BranchScan)`, `appendMessage(message)` and `appendCustomEntry(customType, data?)` (both return the new entry id). +- **`Session` extends SessionReader**: `metadata`, `idGenerator: { next(timestampMs?) }`, `getEntry(id)`, `findEntries`/`findEntry` (session-wide `EntryQuery`: `type?`, `customType?`, `order?: "asc"|"desc"`, `limit?`, `cursor?`), `getName`/`setName(name | undefined)`, `getLabel`/`setLabel(targetId, label | undefined)`, `branch(name)`, `createBranch(name, at)`, `beginMutation()`, `mutate(callback)`, `setValue`/`deleteValue`/`appendList`/`deleteList`, `close()`. + +All supported mutations serialize on one keyless Session line (§4.3). `beginMutation()` is the explicit scope; **every direct `beginMutation()` caller must call `end()` in `finally`**. `Session.mutate()` is the callback convenience and always ends in `finally`; normal harness/plugin code uses `mutate`. Ordinary Session and Branch reads bypass the line: each read observes the latest fully applied commit, but several reads are not a snapshot — use `mutate()` for coherent read-decide-write. + +**C1 — raw RemoteSession (contradiction, decision required).** The begin/read/commit/end lifecycle was specified as the RemoteSession transport contract: a worker runs its local callback and publication while the server holds the sole concrete Session line, then sends end; disconnect or timeout terminates that scope; no caller-selected lane key exists. No implementation, protocol schema, client facade, server-held scope, worker adapter, or conformance test exists — the shipped product deliberately deleted raw `RemoteSession` in favor of process-local Session plus routed semantic services. C1 (Part 8, roadmap) must decide whether to implement or retire this contract; if C1 commissions a remote Session, it must preserve the same read → decide → commit → process-local publication → end order (invariant 38). Until decided, treat the remote lifecycle as specification under dispute, not current behavior. + +A repository creates only metadata/header/catalog state: no Branch, configuration, or lane state. `createBranch` validates name, absence, and a non-null target atomically and writes only the tip. `SessionRepo` exposes `create`, `open`, `list`, `delete`, and `fork` with implementation-specific metadata/list-option generics. + +### Search + +**S3 — design only, not implemented.** The current `src/search/index.ts` exports a draft `SessionSearchService` skeleton (`sync()`, `notify()`, array-returning `searchEntries()`) that conflicts with this design and has no implementation; S3 must reconcile the public API before implementing. The design: + +Search is a **standalone service with its own store**; the repository knows nothing about it and exposes no search methods. A sync utility consumes `repo.list()` and read-only session opens to feed the index store; applications construct the service, run sync at startup or on a schedule, wire the notify utility to their event stream for freshness, query the service directly, and call `search.remove()` alongside `repo.delete()` (or leave stale rows to the next reconciliation). Callers join metadata and fetch entries through the repository they already hold. Draft interfaces: `SessionSearchHit { sessionId, entryId }`; `SessionSearchOptions { entryTypes?, limit?, signal? }`; `SessionSearch.search(text, options?): AsyncIterable`; `SessionSearchService { searchSessions({ text, limit? }): Promise; searchEntries?: SessionSearch; remove(sessionId); close() }` (`limit` counts sessions; `SessionSearchResult { sessionId }`; display services may extend hits/results with `timestamp`, `snippet`, `score`, `top`); catch-up targets implement `SessionSearchSyncTarget { getCursor(sessionId, storeGeneration), indexBatch(batch), remove(sessionId) }` with `SearchIndexBatch { sessionId, storeGeneration, fromSeq, toSeq, entries: { entryId, seq, text, timestamp }[] }`. + +**Indexing is pull-based; events are only hints.** The store keeps a durable cursor per session — the highest entry `seq` indexed. Sync enumerates sessions via the repository (old, new, copied files alike), reads `scanEntries({ fromSeq: cursor + 1 })`, indexes message-entry text idempotently per `(sessionId, entryId)`, and advances the cursor in the same store transaction; a crash mid-batch re-indexes into the same state, and years of existing sessions catch up with the same loop. Notify carries no content — a poke triggering a debounced pull; a lost poke is caught by the next sweep. The index is a rebuildable projection with zero authority; indexing failures never affect the harness or commits. Reading a Session its worker writes is legal through the backend's read-only path: host lifecycle prevents a second writable owner, and WAL gives cross-process snapshot reads. The precise rewrite (§2.9) may renumber seqs, so cursors key on `(sessionId, storeGeneration)`; the rewrite bumps a generation counter and a mismatch triggers full re-index. Reference implementation: one standalone SQLite database — an FTS5 table over `(session_id, entry_id, text)` plus the cursor table — working unchanged over JSONL session files; several processes may share it (WAL, `busy_timeout`, `BEGIN IMMEDIATE`, idempotent rows, monotonic cursor updates; writers serialize). + +**Open question — metadata filtering.** Coding-agent's resume flow filters by `cwd`; other repositories have no cwd concept, and search options are deliberately generic. Candidates: (a) typed filter passthrough (service generic over each repo's filter vocabulary); (b) pre-restrict via the repo's own listing, passing a possibly huge candidate id set; (c) post-filter in the app — **unsound**, filtering after ranked `limit` drops results; (d) index chosen metadata fields at sync time and filter natively, coupling the service to those fields and requiring re-sync when they change. Settled with S3. + +## 2.9 The precise rewrite + +Entries and usage rows are never deleted (§1.2); the sole sanctioned exception is the **precise rewrite**: an administrative repository operation that copies the retained set — entries, usage rows, semantic values, lane values, immutable result records — into a fresh session store over a coherent snapshot, exactly as a fork does, then atomically swaps it for the old store. Its keep-predicate can express what no runtime mechanism may: compliance-grade erasure (including content copied into `retainedTail`s and summaries), pruning abandoned branches, re-minting legacy-format ids (Appendix B). It is tooling above the harness — no harness surface exposes it, no core rule depends on it, and **no implementation exists**. + +Result records are retained even when the rewrite removes an entry named by `fromTipId`/`tipId`; those pointers then deliberately dangle — the record's identity, kind, terminal status, error, and times remain valid while transcript dereference reflects the erasure. Rewrites do not silently delete or mutate immutable operation dispositions. + +# Part 3 — The operation state machine + +## 3.1 Operations + +```ts +interface OperationMeta { + operationId: string; + lane: string; + sourceTipId: string | null; // lane tip before acceptance + startedAt: number; + intent: + | { kind: "run"; promptEntryIds: string[] } + | { kind: "compaction"; customInstructions?: string } + | { kind: "navigation"; targetId: string | null; summarize: boolean; + label?: string; customInstructions?: string }; +} +``` + +`OperationMeta` is immutable acceptance data: written once, paired with one complete `operationState(operationId)`, deleted by the terminal transaction (§3.13). For runs, `promptEntryIds` names only normalized request messages; queued items captured by acceptance and later hook messages are not prompt intent. An operation id may be supplied or minted before acceptance; it correlates a host submission with `inspectExecution`, `drive`, and the result record but is not an unbounded acceptance-idempotency index. The process-local operation `{ meta, state }` is never stored as one object. + +## 3.2 Operation state — the durable restart point + +`operationState(operationId)` holds one member of a flat 13-leaf union; every transition replaces the complete value; there is no finished state — terminal completion deletes it. Full fields: `session/types.ts`. The shared shapes: + +```ts +type Control = { status: "running" } | { status: "cancel_requested"; requestedAt: number }; + +interface OperationScope { // carried by every leaf + control: Control; + settings: { compaction: CompactionSettings; steeringMode: QueueMode; + followUpMode: QueueMode; toolExecution: "sequential" | "parallel" }; + latestAssistantEntryId: string | null; +} + +type Continuation = + | { kind: "need_assistant"; overflowRecoveryUsed: boolean } + | { kind: "may_finish"; includeFinalAssistant: boolean }; +interface CheckpointData { continuation: Continuation; triggerEntryId: string } + +type ResultBoundary = + | { kind: "resume_checkpoint"; resumeAfter: CheckpointData } + | { kind: "finish" } + | { kind: "commit_navigation"; targetId: string; label?: string }; +interface SummaryTask { + taskId: string; reason?: "manual" | "threshold" | "overflow"; + customInstructions?: string; boundary: ResultBoundary; +} + +type OperationState = // at: + | StartingOperation // "starting" + | CheckpointOperation // "checkpoint" + | AssistantReadyOperation // "assistant.ready" + | AssistantEffectPendingOperation // "assistant.effect_pending" + | AssistantRetryWaitOperation // "assistant.retry_wait" + | ToolsOperation // "tools" + | DeferredSuspendedOperation // "deferred.suspended" + | DeferredEffectPendingOperation // "deferred.effect_pending" + | SummaryDecidingOperation // "summary.deciding" + | SummaryReadyOperation // "summary.ready" + | SummaryEffectPendingOperation // "summary.effect_pending" + | SummaryRetryWaitOperation // "summary.retry_wait" + | NavigationReadyToCommitOperation; // "navigation.ready_to_commit" +``` + +The four `summary.*` leaves carry one `SummaryTask`; summary kind is derived from the closed boundary union, never duplicated. `ToolBatch`/`ToolCall` remain a nested child state machine because parallel children genuinely settle concurrently — a `ToolCall` is `{ sourceIndex, resultEntryId }` plus `planned | effect_pending{replay} | outcome_ready{terminate} | completed{terminate}`. Large content stays at referenced sibling addresses; state contains only bounded policy and the ids required to dispatch and recover. A live procedure's JavaScript continuation is finer-grained than the durable leaf: after `assistant.effect_pending` commits, a live process awaits the provider; after process loss, the same leaf means unknown-outcome recovery. + +## 3.3 Lane state and the restore projection + +```ts +interface LaneState { + currentOperationId: string | null; + lastOperationId: string | null; + inbox: Array<{ entryId: string; kind: "steer" | "followUp" | "nextRun" | "write" }>; +} +``` + +Attachment reads `branchTip`, `laneConfig`, and `laneState` per configured lane; if `currentOperationId` names O, also `operationMeta(O)` and `operationState(O)`. It validates required existence, lane/id agreement, and intent-to-leaf reachability. It never reads `operationResult`: `lastOperationId` is only an observation pointer. + +The restored process-local projection is authoritative while the Harness owns the Session; every supported mutation commits on the Session mutation line and publishes the matching projection before releasing it. Attachment does not dereference transcript, inbox payloads, deferred sources, frames, tool arguments/checkpoints/memos, preparations, or staged outcomes — `watch` and drive procedures validate those references when they consume them (§4.4). Missing optional frame lists and tool checkpoints are legal; contradictory required content faults its consumer. + +## 3.4 The atomic transition rule + +> Compute one complete next state in memory, then atomically commit every entry, usage row, value/list write, and projection change that makes it true. + +`Lane.state` supplied by the Session mutation line is the control authority. Drive procedures never reread `laneState`, `operationMeta`, `operationState`, `branchTip`, `laneConfig`, or `operationResult` to choose work; storage reads dereference ids named by current state or enumerate operation-owned cleanup addresses. §4.1's single-writer rule follows: concurrent inbox calls change only `LaneState.inbox` and `requestAbort` only `control` (draining selected inbox tags), so settlement preserves current inbox/control fields; parallel tool children retain child-status fencing. Providers, tools, hooks, timers, and event delivery run outside mutation callbacks. + +## 3.5 The graph + +```mermaid +stateDiagram-v2 + [*] --> idle + idle --> starting : accept run + idle --> summary_deciding : accept compaction / summarized navigation + idle --> navigation_ready : accept unsummarized navigation + + starting --> checkpoint : before_run consumed + checkpoint --> assistant_ready : need assistant / selected input + checkpoint --> summary_deciding : threshold preparation + checkpoint --> terminal : may finish + + assistant_ready --> assistant_pending : request intent + assistant_pending --> assistant_retry : retryable error + assistant_retry --> assistant_ready : retry elapsed + assistant_pending --> tools : tool calls + assistant_pending --> deferred_suspended : deferred handle + assistant_pending --> summary_deciding : overflow preparation + assistant_pending --> checkpoint : settled response + + tools --> tools : child intents/outcomes/placement + tools --> checkpoint : batch placed + + deferred_suspended --> deferred_pending : one poll permit + deferred_pending --> deferred_suspended : still pending + deferred_pending --> tools : ready with calls + deferred_pending --> checkpoint : ready without calls / terminal response + + summary_deciding --> summary_ready : hook selects generation + summary_ready --> summary_pending : request attempt + summary_pending --> summary_retry : retryable attempt + summary_retry --> summary_ready : retry elapsed + summary_deciding --> boundary : decline / hook result + summary_pending --> boundary : generated result / terminal failure + boundary --> assistant_ready : resume checkpoint with selected input + boundary --> checkpoint : resume may-finish resting point + boundary --> terminal : standalone finish / navigation commit / run failure + + navigation_ready --> terminal : move/label commit + terminal --> [*] +``` + +`terminal` and `boundary` are explanatory nodes, not durable leaves. Every summary result switches once on `ResultBoundary`: resume an enclosing run, finish standalone compaction, or atomically commit navigation. Cancellation is orthogonal; before ordinary dispatch it routes every one of the 13 leaves to reconciliation (§4.6). + +## 3.6 Acceptance + +`accept(request, context)` normalizes immutable input off the mutation line, then performs one acceptance command: check the lane is idle, validate durable inputs, commit metadata plus the initial leaf, publish events, return `OperationAdmission`. It installs no Drive and invokes no hook, provider, tool, timer, or process owner. Run acceptance selects eligible items from the lane's one ordered inbox: + +| Tag | Idle acceptance | +|---|---| +| `write` | all | +| `nextRun` | all | +| `steer` | all or oldest according to `steeringMode` | +| `followUp` | all or oldest according to `followUpMode` | + +Selected items place in global admission order regardless of tag; request prompt entries are newer and follow them. Selection deletes each `pendingEntry(id)` and removes only selected inbox ids in the same transaction; mode remainders and late admissions stay queued. An empty public prompt is valid only when captured queued content places at least one conversational message — the ordinary continuation-run acceptance used after structural convenience operations. + +| Request | Initial durable leaf and acceptance writes | +|---|---| +| prompt, skill, template | selected queued entries + normalized prompt entries; `OperationMeta`; payload-free `starting`; lane current id | +| compaction | durable preparation + `OperationMeta`; `summary.deciding` with boundary `finish`; lane current id | +| summarized navigation | preparation + `OperationMeta`; `summary.deciding` with boundary `commit_navigation`; lane current id | +| unsummarized navigation | `OperationMeta`; `navigation.ready_to_commit`; lane current id | + +Structural preparation may run outside the mutation line, but the acceptance command revalidates the observed source tip and idle state before committing. Pre-acceptance failures write nothing: busy lane, empty/invalid message, missing skill/template, nothing to compact, invalid navigation, unknown target; model/tool registry availability is checked only at the later effect boundary. `starting` is consumed by the Drive after the cancellation check and `before_drive`; `before_run` runs off-line, and one commit places its injected messages and enters `checkpoint` — a crash before that commit may repeat the hook, a crash after it cannot. Concurrent accepts serialize on the Session line (loser: `LaneBusy`); a crash after acceptance leaves an open initial leaf that only a later `drive` advances. + +## 3.7 Assistant generation + +Four phases: read compaction-bounded context and resolve captured model/tools → run `before_request` and commit `assistant.effect_pending` with response/usage ids → admit and consume the provider stream → commit response entry + usage + frame cleanup + one classified successor. + +The request identity is the stable lane identity `Session metadata id + ":" + lane name` (§5.7). The intent snapshots the lane configuration, stream options, retry policy, trigger, and overflow-recovery flag. An unavailable captured model or configured tool fails terminally before intent with a machine-readable configuration error and no fabricated response or usage. + +Settlement commits the complete response entry, usage row, branch tip, deletion of `pendingAssistantFrames(O, R)`, and exactly one successor: + +| Settled response | Successor | +|---|---| +| accepted tool calls | `tools` with reserved result ids | +| retryable error with attempts remaining | `assistant.retry_wait` | +| first overflow with preparation | `summary.deciding` with `resume_checkpoint` | +| valid deferred handle | `deferred.suspended` | +| stop or genuine output-limit length | `checkpoint{may_finish}` | +| terminal error, exhausted retry, invalid deferred handle, second overflow, or empty overflow preparation | terminal failed result | + +A retry timer runs off the mutation line and enters `assistant.ready` only after `notBefore`; cancellation or close wins without starting another request. Every response/usage/decision lands together or none does. + +### Streamed frame persistence + +During an admitted assistant or deferred effect, one `AssistantMessageFrameEncoder` converts provider events to compact recovery frames. A convertible event synchronously enqueues one invocation-fenced append to `pendingAssistantFrames(operationId, responseEntryId)` and emits the corresponding live message event. The provider loop never awaits storage per frame; Session FIFO preserves order, every promise carries fault observation, and settlement awaits the latest queued frame write before `after_response` and the final commit. + +Each append checks the same response id is still effect-pending: an append admitted before settlement may commit first; one reaching the line after settlement declines and cannot recreate the list. Frames are auxiliary — absence is legal, they do not prove request completion, and a complete-looking prefix still restores as unknown outcome until settlement commits. Recovery reduces the exact list with `reduceAssistantMessageFrames`, synthesizes the documented partial result, and deletes the list with its next durable decision. Structural summary streams intentionally persist no frames. JSONL records the appends physically until snapshot compaction (J1) even after logical deletion. The [mobile assistant-output handoff](mobile-handoff/01-harness/05-assistant-output/message-update.md) replaces this path with tracked pending output in ephemeral scoped storage while preserving unknown-outcome recovery. + +### Classification order + +First match wins: + +1. current durable control is `cancel_requested` → normalize to `aborted`; reconciliation terminalizes it as aborted; +2. adapter-reported or recognized context overflow → normalize to `error`; enter one overflow summary, or terminal-fail if recovery was already used; +3. valid deferred handle → suspend; invalid handle → terminal failure; +4. retryable error with attempts remaining → retry wait; otherwise terminal failure; +5. accepted tool calls → tools; +6. stop or genuine output-limit length → `checkpoint{may_finish}`. + +Overflow is checked before retryability. Error, aborted, and deferred assistant entries remain durable history but are omitted from future provider context by §2.5. A genuinely truncated response carrying calls produces synthetic error tool results rather than executing potentially corrupted arguments. + +## 3.8 Tools + +Tool execution separates effect completion from source-ordered tree placement: + +| From | Trigger | Transaction | To | +|---|---|---|---| +| call _i_ `planned` | clearance passed (`before_tool`, lookup, arg validation) | `TX[ upsert pi.op.tool_args/O:{stepId}:{i} = effective args, S(call i = effect_pending, replay) ]` | dispatch | +| call _i_ `effect_pending` | tool calls `onUpdate(partial, { checkpoint:true })` | `TX[ upsert pi.pending.tool_output/O:{resultEntryId} = partial ]` after invocation fencing; state unchanged | `effect_pending` | +| call _i_ `effect_pending` | effect settled; latest update delivery and latest checkpoint write awaited; `after_tool` applied | `TX[ upsert pi.pending.entry/{resultEntryId} = finalized result, delete pi.pending.tool_output/O:{resultEntryId}, delete pi.op.tool_memo/O:{resultEntryId}:*, S(call i = outcome_ready, terminate) ]`, with post-commit `tool_end` | `outcome_ready` | +| call _i_ `planned` | unknown tool / invalid args / `before_tool` blocks or throws / control cancelled | `TX[ upsert pi.pending.entry/{resultEntryId} = complete synthetic result, S(call i = outcome_ready, terminate) ]`, with post-commit `tool_start` followed by `tool_end`; no effect intent | `outcome_ready` | +| source-ready prefix | first non-completed calls are `outcome_ready` | `TX[ insert result entries in source order, delete their pi.pending.entry values, insert reported usage, upsert pi.branch.tip, S(calls = completed / next checkpoint) ]` | `completed` or checkpoint | + +**Updates and checkpoints.** Every `onUpdate` is a process-local `tool_update` observation: the synchronous callback emits the event and retains the latest delivery promise internally; tools neither receive nor await it. `checkpoint:true` additionally requests replacement of the invocation's bounded durable progress snapshot: each such call synchronously enqueues one invocation-fenced value replacement on the mutation line, attaches the ordinary harness-fault observer, and replaces only the process-local latest checkpoint-write promise reference. No checkpoint write is dropped or coalesced; Session FIFO preserves request order, and each mutation verifies the same call is still `effect_pending` when it executes. The tool alone controls cadence, duplicate suppression, and bounding — requesting checkpoints faster than storage commits queues memory under the trusted-tool contract, and the API imposes no generic byte cap or truncation. When the tool promise settles, the harness stops accepting updates and closes checkpoint admission; a late request returns without committing. Before `after_tool`, the procedure awaits the latest update-delivery promise **and** the latest checkpoint-write promise — each implies completion of everything earlier in its queue. Checkpoint writes order before outcome staging, and staging deletes the value; a failed checkpoint commit follows the ordinary storage-fault path and prevents staging. + +**Staging.** Outcome staging is the point after which the tool can never replay. After `after_tool`, the procedure constructs the complete canonical final result — bounded independently of progress snapshots — and stages its `ToolResultMessage`; the state carries only `terminate` and the reserved id. The staging commit publishes `tool_end` after committed state is installed, so that event is durable evidence that the call is `outcome_ready`. For a fresh synthetic call, the same staging commit publishes `tool_start` followed by `tool_end`; it never crosses the external tool-effect boundary or runs `after_tool`. Tool-reported usage stays inside the staged message until materialization, where its ledger row commits atomically with the entry; added tool names likewise become active from the materialized transcript point, never from invisible staging. + +`tool_start`/`tool_end` bracket public processing of a fresh call and finalized-result availability, not necessarily an external effect. Historical events are not replayed: an unsafe restored `effect_pending` call is represented as running by the initial snapshot and may emit only a recovery-tagged `tool_end` when its interruption result stages. A safe replay's checkpoint-clear commit publishes its recovery-tagged `tool_start`; its staging commit later publishes `tool_end`. + +After any outcome stages, the procedure materializes the contiguous `outcome_ready` prefix from the first non-completed source position; several results may enter the tree in one transaction, each parented to the previous inserted result. When the final call materializes, the same transaction deletes the addresses from `scanValues(operationToolArgsPrefix(O, stepId))` and chooses: **every** completed call set `terminate: true` → `checkpoint{may_finish, includeFinalAssistant: false}`; otherwise `checkpoint{need_assistant(overflowRecoveryUsed: false)}`. `terminate` lets a tool end the run without another provider turn (a "submit final result" tool in place of structured output); the result record still embeds no message payload. + +Modes: **sequential** — clear → intent → execute → finalize → stage → materialize, one call at a time; **parallel** — clearance and intent in source order, effects and post-effect hooks settle independently, each complete outcome stages immediately in completion order, tree materialization stays source ordered. + +Blocked and invalid calls skip intent/execution but still stage a synthetic outcome. A missing tool implementation is the ordinary unknown-tool case: stage an `isError:true` `ToolResultMessage` saying the named tool is unavailable, then continue the batch and later assistant turn; the harness constructs that message directly, omits `details`, and must not invent a value for the tool's typed details contract. A crash before staging reruns ordinary clearance, including `before_tool` under its replay contract; a crash after staging never reruns the hook or tool. + +Calls are tracked internally by `sourceIndex` (position in the assistant message's complete content array); hooks and events see provider `toolCallId` and tool name. A provider `toolCallId` is unique only within its tool-call batch and may be reused by a later assistant message. `AgentHarnessToolInvocation.invocationId` equals the reserved session-unique `resultEntryId`, is stable across safe replay, and scopes durable memos under `operationToolMemo(O, invocationId, name)`. Memo names must be non-empty with no `:`; `setMemo(name, undefined)` deletes. Memo operations synchronously enqueue on the mutation line before returning their promises, and tools must await writes; each job verifies the same effect-pending invocation when it executes, so a queued memo write cannot outlive staging. A pre-return write is FIFO-ordered before staging and then deleted by it; a post-return call rejects after capability expiry; no separate write drain exists. Flue-style named effect memoization (`step.do(name, effect)`) awaits these operations: a committed value returns on replay, while a crash before its memo commit may rerun the effect. There is no nested per-step replay state and no exactly-once external-effect promise. + +## 3.9 Summary generation — compaction and navigation summaries + +Compaction and navigation summaries share one durable quadruple, `summary.deciding → summary.ready → summary.effect_pending ↔ summary.retry_wait`. `SummaryTask.boundary` determines semantics: + +| Boundary | Use | Successful publication | +|---|---|---| +| `resume_checkpoint` | threshold/overflow inside a run | compaction entry, then one atomic boundary plan for queued input and run continuation | +| `finish` | standalone compaction | compaction entry plus terminal compaction result | +| `commit_navigation` | summarized navigation | move, summary entry, optional label, and terminal navigation result in one commit | + +Preparation is immutable content stored at `operationPreparation(operationId, taskId)` in the same transaction that enters `summary.deciding`; `before_compaction` runs off-line. A decline, hook-supplied result, generated result, model absence, or terminal generation failure all meet at one boundary switch; cancellation never takes a boundary continuation. + +If generation is selected, `summary.ready` captures configuration, stream options, retry policy, and result id. Each nested provider request has its own durable request/usage intent inside `summary.effect_pending`, and its usage commits before another nested request begins. Structural request options force `cacheRetention: "none"` and a fresh request identity; structural streams emit no assistant-message lifecycle and persist no frames. A lost effect-pending attempt is unknown and retries under the captured policy; committed attempt usage remains in the ledger. + +Threshold compaction is guarded by transcript recency: it runs only when `shouldCompact` is true and the newest compaction entry is older than the checkpoint trigger, so a successful compaction is its own durable marker; decline never commits back to the threshold-checking checkpoint, so no extra checked flag exists. + +Overflow trace: assistant settlement normalizes the response to `error` + usage + overflow preparation → `summary.deciding{boundary: resume_checkpoint{need_assistant(true)}}`; summary attempts run intent → effect → usage/result; publication commits the compaction entry + selected write/steer items + `assistant.ready` in one commit. The overflow response remains durable but is excluded from summarized context. `overflowRecoveryUsed: true` prevents a second compaction loop; a second overflow terminal-fails the run. + +## 3.10 Navigation + +Unsummarized navigation accepts directly into `navigation.ready_to_commit`; summarized navigation enters the shared quadruple with `commit_navigation`. The successful transaction is atomic: optional hook usage → move tip to target → optional summary entry parented to target (tip moves to it) → optional target label → operation cleanup + immutable navigation result + idle lane state. A summarized decline moves nothing. Abort before commit moves nothing and records `aborted`; after commit the operation is already completed. `navigation_end` tells replicas to rebase because the new tip can be outside their transcript; `WatchHandle.resnapshot()` captures the replacement snapshot (§5.4). + +## 3.11 Inbox, queues, deferred writes + +Every queued admission mints an entry id and atomically writes `pendingEntry(id)` plus one tagged item into the lane's single ordered inbox. Enqueue is accepted while idle, during any operation family, during deferred suspension, and after durable cancellation. Tags determine eligibility, not ownership: + +| Drain point | Eligible tags | +|---|---| +| idle acceptance | all `write` and `nextRun`; mode-selected `steer` and `followUp` | +| run boundary | all `write`; mode-selected `steer`; mode-selected `followUp` only at `may_finish` | +| idle direct append | all earlier `write`, then the new direct entry | +| abort | all `steer` and `followUp` removed and returned; `nextRun`/`write` remain | + +Within one drain, selected items always place in global inbox order; queue modes select per tag and leave remainders in their original relative positions. `nextRun` is never consumed mid-run and never blocks finish. A steer admitted too late for one boundary stays queued and becomes eligible at the next boundary or idle acceptance — not an error. + +`steer`, `followUp`, `nextRun`, and operation-aware append all use the same staging path and emit the authoritative full `queue_update`; there is no separate `write_pending` event. `LaneSnapshot.queues` uses the same ordered `LaneQueuedItem[]`; clients group by `kind` without reordering it. + +`cancelQueued(id)` performs one triage on the mutation line: pending item → remove it and delete its payload, `cancelled`; immutable entry exists → `already_consumed`; neither → `not_found` (a lost/retried cancellation treats `not_found` as success). Terminal cleanup never deletes lane-owned inbox payloads. Writes can remain pending for an unbounded structural operation; callers needing immediate placement use `waitForIdle()` then append, and `runWhenIdle()` provides serialized process-local callback ownership — neither creates durable scheduling state. + +## 3.12 The checkpoint and boundary procedure + +A boundary pass makes one decision and commits at most once. It may perform bounded transcript/payload reads and run `before_run_end` off the mutation line, but never commits back into `checkpoint` merely to remember a drain. For an ordinary checkpoint: + +1. select eligible `write` + `steer` in global order; +2. if none project, evaluate the transcript-derived threshold guard; +3. route `need_assistant`, or at `may_finish` select eligible `followUp`; +4. if still finishing, capture a no-write verdict and run `before_run_end` off-line; +5. re-enter the mutation line and replan; discard stale hook output if inbox/control changed; +6. commit one of: selected entries + `assistant.ready`, `summary.deciding`, hook follow-up + `assistant.ready`, or the terminal transaction. + +The shared structural `resume_checkpoint` publication uses the same planner with threshold checking disabled; its compaction entry, selected queued entries, inbox deletion, tip movement, and successor leaf commit together. A `may_finish` result with no selected input may rest in `checkpoint` so the same live Drive can run finish mediation; it cannot re-trigger threshold because the new compaction is newer than the trigger. Failures terminalize directly; they do not consume queued lane input to rescue the failed operation — that input stays available to a later ordinary run. + +## 3.13 Terminal transactions and result records + +```ts +interface OperationResultRecord { + operationId: string; + kind: "run" | "compaction" | "navigation"; + status: "completed" | "declined" | "aborted" | "failed"; + error?: OperationError; + fromTipId: string | null; + tipId: string | null; + startedAt: number; + endedAt: number; +} +``` + +Every terminal path performs one universal suffix in the same transaction as its final business writes: procedure-specific entries/usage/tip writes → delete all operation-owned `pi.op.*` and pending progress/frame/outcome addresses → set `operationResult(operationId)` exactly once → set `laneState{ currentOperationId: null, lastOperationId: operationId, inbox: preservedCurrentInbox }`. This is the implementation's normative write order. The old §3.13 prose listed result before cleanup, while its worked trace and source used cleanup first; this resolves that contradiction in favor of source and the trace. + +The record is the public settled outcome, not a pointer to a hydrated outcome object; it embeds no entries and is never read by recovery. `fromTipId`/`tipId` delimit the operation's transcript segment; a precise rewrite may make either pointer dangle (§2.9) without changing the recorded disposition. Records are immutable, lane-lived, and retained for every operation; J1 snapshot compaction must carry them forward. `getResult(id)` is one value read. `drive(id)` is total: the current id installs/joins the lane Drive, an existing record returns `{ kind: "settled", outcome: record }`, and neither returns `OperationMismatch`; `LaneState.lastOperationId` and `LaneSnapshot.lastResult` expose the newest record without limiting access to older ids. A terminal commit under `cancel_requested` always records `aborted`, so `completed`/`declined`/`failed` imply terminal control was still running. Operation cleanup never deletes the lane inbox; usage rows and immutable transcript entries survive terminal cleanup. + +# Part 4 — Execution, recovery, abort, close + +## 4.1 The live operation task + +An open operation has durable state whether or not this process executes it. A `Drive` is the lane-owned process-local continuation for one pass: it answers whether the lane already has a live continuation, supplies the effect gate, and exposes one shared completion. + +```ts +class Drive { + readonly operationId: string; + readonly completion: Promise; + readonly gate: Gate; + readonly context: Context; // installing invocation cancellation removed + readonly waitForRetry: boolean; + deferredPermits: number; // 1 when installed with pollDeferred +} +``` + +The first matching `drive` caller installs the Drive on the Session mutation line; every later matching caller observes the same `Drive.completion`. The first caller is not an owner: all callers are observation peers and the Lane owns execution. Each caller races only its own observation with `context.abortSignal` — a signal winning before installation starts nothing; after installation it rejects only that caller's invocation, never removing, replacing, or cancelling the Drive. Durable cancellation exists only through `requestAbort`. + +One Drive is the sole top-level state-advance writer. Inbox methods mutate only inbox fields, `requestAbort` only control, and close seals mutation admission — so a live procedure's operation identity and `at` leaf cannot change concurrently, and procedures do not repeatedly verify operation existence, id, kind, Drive identity, or expected `at`. After awaiting external work they re-enter the mutation line and receive the latest authoritative `Lane.state`, preserving concurrent control/inbox changes. Parallel tool children are the exception: sibling call statuses genuinely race, so call identity/status and source-ready-prefix checks remain. + +The task runs direct async procedures — no graph interpreter or action scheduler. The Lane supplies two mutation operations: `continueOperation` returns explicit `cancel_requested` without invoking the planner when control is cancelled, otherwise pairs the next state write with projection publication and returns the planner's result; `settleOperation` performs already-admitted effect settlement and tool-child transitions despite cancellation and owns the universal terminal suffix. Intent publishers use `continueOperation`, outcome publishers `settleOperation`: cancellation prevents a new durable intent but cannot erase already-admitted work. + +A pass ends at a terminal result or durable wait; that pass clears `activeDrive`, and no live pass is replaced in-process. A crash or close destroys/detaches the continuation; a later attachment rebuilds `Lane.state` from durable values before another pass starts. Normal procedures are straight-line: prepare immutable inputs → publish durable intent → perform the effect → publish one durable outcome. Recovery dispatches directly from the flat `state.at` leaf; cancellation reconciliation runs before ordinary dispatch and never starts new ordinary effects. + +## 4.2 Effect gate + +`Session.mutate` orders durable races, but ordinary hook/provider/tool/timer admission occurs outside a transaction. Each installed `Drive` owns a split gate: + +```ts +interface Gate { + readonly signal: AbortSignal; + /** Synchronously checks admission and invokes the operation with no yield between. */ + admit(invoke: () => T): T; +} +interface GateControl { + beginAbort(cancellation: Promise): void; + signalAbort(): void; + close(error: HarnessClosed | HarnessFault): void; +} +type GateState = + | { status: "open" } + | { status: "aborting"; cancellation: Promise } + | { status: "closed"; error: Error }; +``` + +Procedures receive only `drive.gate`; `Drive` privately retains `GateControl`, and there is no procedure-facing `assertOpen`. The source primitive currently types `close(error: Error)` so isolated tests can close with a generic error, but production `Drive` closure supplies only `HarnessClosed | HarnessFault`; the narrower declaration above is the production contract and the broader source type is H1 cleanup. `Gate.admit(invoke)` performs the only check and immediately returns `invoke()`: aborting → throw `AbortRequested(cancellation)`; closed → throw the closing error. The gate owns the cooperative `AbortController`, exposed as `gate.signal`. + +`requestAbort(operationId, context)` is the durable cancellation primitive. With a matching live Drive it creates the abort-mutation promise and calls `drive.beginAbort(promise)` synchronously before the lane mutation; the committed marker resolves the promise and then `drive.signalAbort()`. An id mismatch resolves it and returns `OperationMismatch`; a commit fault rejects it and closes with `HarnessFault`. With no Drive, requestAbort commits or observes the marker but starts no pass. + +**The admission boundary must be synchronous.** Preparation finishes first; then the gate check and operation invocation are one synchronous expression — wrapping preparation itself in `admit` is wrong, because abort could win while preparation awaits after admission: + +```ts +await prepareRequest(); // all preparation first +const admittedContext = withAbortSignal(drive.gate.signal, drive.context); +const stream = drive.gate.admit(() => + models.streamSimple(model, aiContext, { + ...options, + signal: admittedContext.abortSignal, + telemetryContext: admittedContext.telemetryContext, + }), +); +``` + +The admitted boundary is the public Models/tool/hook operation, not an eventual SDK syscall: a Models call synchronously returns a lazy stream, and later auth resolution, provider loading, and delegation remain part of the admitted operation and own the same signal. + +The complete admission catalog: + +- **Hook aggregates** (one `admit` wraps the complete registered pipeline, not each handler): `before_drive`, `before_run`, `before_run_end`, `transform_context`, `before_request`, `before_payload`, `after_response`, `before_tool`, `after_tool`, `before_compaction`, `before_navigation`. +- **Provider operations:** one assistant `Models.streamSimple`, each individual structural-summary request, one explicit `Models.streamDeferred` poll. Best-effort `cancelDeferred` is cancellation cleanup and uses its separate close-only signal. +- **Other:** one real `tool.execute` and creation of each assistant/structural retry timer. Unknown, invalid, blocked, and synthetic tool outcomes start no tool and use no gate. + +No other code calls `Gate.admit`. It does not wrap commits, public queue/configuration/value/tree mutations, pure classification, transaction construction, synthetic settlement, argument/system/context preparation, an already-admitted promise, cancellation reconciliation, or passive listeners. + +The two possible orders: **admission first** — `Gate.admit` checks and invokes synchronously; `requestAbort` begins durable cancellation; the marker commits; `signalAbort` pulls the already-admitted operation's signal. **Abort first** — `beginAbort` closes ordinary admission synchronously; a later `Gate.admit` throws `AbortRequested` and `invoke` never runs; the task waits for the marker and reconciles. + +The gate is not durable state, a mutex, scheduler, or mutation line. If the process dies before the cancellation commit, the closed gate disappears and no cancellation exists; recovery trusts only durable control. Every catalog item has abort-first/admission-first tests; preparation must precede `admit`, and the admitted signal must reach asynchronous Models auth/loading/provider work. + +## 4.3 The Session mutation line + +Every supported mutation uses the one keyless Session line of §2.8: reads and at most one commit happen through the capability, successful commits publish their exact process-local projection and synchronously bind event recipients, and `end()` releases. Lane commands, lane acquisition, progress writes, Branch creation/appends, metadata/value writes, and coherent restore/watch capture all use this line — deliberately sacrificing preparation overlap between lanes for a simpler ownership model. Storage retains its independent commit serializer for atomic application and session-global sequence assignment. + +`Session.mutate()` is trusted and easy to misuse: the callback must use the supplied mutator for bounded reads and its sole commit. Calling a public Session writer inside the callback queues the nested write behind the active callback; awaiting it deadlocks. Plugins must not perform nested public writes or unbounded work while holding the line. + +A Drive procedure uses the current owned Lane projection for control flow; the Lane pairs every operation-state write with publication of the matching projection, so settlement preserves newer inbox/control fields. Providers, tools, hooks, timers, event delivery, idle waits, and Drive completion stay outside the line. Raw Branch mutation while a Harness owns the corresponding AgentLane can stale the projection and is a trusted-programming defect; AgentLane methods are the operation-aware surface during ownership. + +## 4.4 Attachment and open-operation inventory + +`AgentHarness.create(options, context)` performs one bounded keyless Session mutation to inventory and restore complete AgentLanes before publishing the Harness. It starts no hook, provider, tool, timer, Drive, or application callback. + +Attachment inventories the union of Branch tips and lane configuration/state. A Branch with only a tip is data-only and not published as an AgentLane; a complete lane has tip + configuration + lane state and optional compatible current operation metadata/state; partial or orphan lane values fault attachment; zero Branches and no main are legal. Per-lane restore performs exactly the §3.3 reads and validation — nothing more. + +The returned `open` array contains one item per restored lane with a current operation and omits data-only Branches and idle lanes. It is inventory, not scheduling or ownership. Configured model identities remain unresolved strings until their actual effect boundary. + +## 4.5 Driving and crash recovery + +Recovery begins only when an open operation has no `Drive` and a matching `drive({ operationId }, context)` installs a real pass owner. `AgentHarness.create` never drives; `resume(context)` inspects and drives the current operation without exposing its id and grants the pass one deferred-poll permit; `requestAbort` with no task commits cancellation but installs nothing, and the next drive enters reconciliation directly. + +The pass first inspects the owned control projection: cancellation requested → invoke neither `before_drive` nor `before_run`, enter §4.6. Otherwise gate and invoke `before_drive`; failure rejects the pass without faulting the harness or writing durable progress. Model/tool implementations resolve only at the boundary that needs them: an unavailable provider/model or configured request tool is a non-retryable configuration failure before request intent, an unavailable requested tool a synthetic error result; neither suspends the operation. Durable phase then decides the work: `starting` runs and settles `before_run` per §3.6; a pending effect with no owner is an orphan and follows the table; all other phases continue ordinarily. + +| Orphaned restart point | Activation recovery | +|---|---| +| assistant generation `effect_pending` | Read bounded pages from `pendingAssistantFrames(O, R)`, reduce with `reduceAssistantMessageFrames`, and commit under the reserved ids a synthetic zero-usage `error` response carrying the reconstructed partial (no committed start frame → `api:"unknown"`, captured provider/model strings, empty content). Include an explicit warning: request interrupted, preceding content is the latest committed partial, newer live output may be missing, external outcome unknown. The same transaction deletes the frame list. The committed error then follows ordinary classification: attempts remaining → retry wait and a later numbered attempt under fresh ids; cap reached → terminal failure. Partial tool calls inside it never execute, and `after_response` never runs — there is no trustworthy complete provider result to transform. | +| structural generation `effect_pending` | Treat the entire attempt as uncertain, including any completed first split-turn request whose intermediate text was process-local. Advance to a later `ready` attempt under the captured policy or fail at the cap. Committed request-usage rows remain in the ledger. | +| tool call `effect_pending` | Stored and current declarations both `safe`: delete any old progress checkpoint and re-execute persisted arguments with the same invocation memos/id. Implementation absent, current declaration no longer safe, or stored declaration `never`: synthesize interruption instead of suspending — preserve checkpoint content/details/usage when present, ignore its added-tool/termination hints, append the explicit latest-durable/newer-live-may-be-missing/unknown-outcome warning, and stage a non-terminating error without `after_tool` (no checkpoint → omit `details`). | +| deferred poll `effect_pending` | No poll permit → stays suspended; may expose its durable partial in snapshots. Permit plus resolvable captured model → replace the unknown poll with fresh response/usage ids at the same poll number and fetch once; the replacement intent deletes the abandoned old frame list. Captured model unavailable → delete that old frame list and enter configuration-provenance failure without fabricating settlement. There is no cap. | + +After orphan recovery removes or takes live ownership of every pending effect, the ordinary procedures continue. Calls already `outcome_ready` need no identity or effect recovery; ordinary source-order materialization places their staged results. Recovery is not a second end-to-end driver. + +Atomic transactions have no internal prefix, so every repeat-sensitive effect has the same four durable crash positions: + +| Crash point | Durable restart point | Activation behavior | +|---|---|---| +| before intent commit | previous ordinary state | run the ordinary procedure as if nothing happened | +| after intent, before effect admission | `effect_pending` | outcome indistinguishable from a crash during the effect; apply the table above | +| during/after effect, before settlement | `effect_pending` | same unknown-outcome policy | +| after settlement commit | output + usage + next state | continue; never re-settle | + +Queue application and final structural commits remain atomic (Part 3): a crash before one sees the prior complete state, after one the next. A crash after durable abort activates reconciliation; a crash after terminal cleanup sees an idle lane and its immutable `pi.result`. + +Retry waits are ordinary restartable states with two caller policies: `waitForRetry: false` returns waiting/`notBefore` with no timer and the caller schedules a wake, later driving the same id; `waitForRetry: true` admits and starts the retry timer through `drive.gate` — the timer reaching `notBefore` verifies the same current wait and commits `ready`, `requestAbort` wakes it after durable cancellation so reconciliation runs, and close rejects the local task with no durable write. At or after `notBefore`, either policy verifies the same current durable wait in the owned projection and commits `ready` without an unnecessary timer. + +## 4.6 Abort and cancellation reconciliation + +Invocation cancellation and durable cancellation are different: aborting one caller's `Context` stops only that caller's observation and never mutates operation state. Durable cancellation exists only through `requestAbort(operationId, context)` or the `abort(context)` convenience. + +For a matching current operation, the first request, in order: (1) synchronously call `Drive.beginAbort()` when a live Drive exists, preventing new effect admission while the marker is pending; (2) on the mutation line, set `control = { status: "cancel_requested", requestedAt }`; (3) in the same commit, remove every `steer` and `followUp` item from the inbox and delete its pending payload, preserving `nextRun` and `write`; (4) after commit, publish the Lane projection, resolve the abort mutation, and signal the live gate; (5) still before releasing the mutation line, bind `operation_abort` and any `queue_update` recipients; (6) deliver those events, then return `{ operationId, newlyRequested: true, steer, followUp }`. Signal callbacks run before event recipients are bound, but no later Lane mutation can publish first because the current mutation still owns the Session line. + +The drained messages exist only in that return value and event — no durable drained-control fields. A process crash, transport loss, or lost response after the commit permanently loses those payloads: an explicit product tradeoff. A repeat request against the same still-open cancelled operation returns `newlyRequested: false` with empty drains and no duplicate event. A stale id returns `OperationMismatch` and cannot cancel another operation. `requestAbort` never installs a Drive; with none it only commits or observes the marker, and a later `drive` reconciles. `abort()` inspects the current id, requests cancellation, then ensures a same-id reconciliation pass is observed; an idle lane returns `NoActiveOperation`. + +Before every ordinary dispatch the Drive checks control; `cancel_requested` routes to one total reconciliation switch over all 13 leaves that starts no new ordinary hook/provider/tool work. It settles or reconstructs admitted assistant/deferred outcomes, preserving committed frame prefixes; interrupts unsafe orphaned tools, safely replays only where policy permits, and stages and source-orders outcomes already durable; discards process-local structural results not atomically published; best-effort cancels a deferred provider handle using the Drive's close-only signal; and deletes operation-owned values/lists, recording one terminal `aborted` result. Lane-owned `nextRun`/`write` items remain queued. Close is not abort (§4.7). + +## 4.7 Close — a controlled crash + +Close writes no cancellation or terminal state. It seals harness and Lane mutation admission, rejects caller observations through the harness-close boundary, keeps detached pass promises observed, drains Session mutations admitted before the seal, then closes storage. A provider/tool result produced after sealing cannot commit — its next Lane mutation rejects with `HarnessClosed`. The Drive is not replaced and durable operation state is unchanged, so reopening sees the same restart point as process loss. Whether a host also signals cooperative provider/tool work is local resource cleanup; it must not write cancellation, synthesize settlement, remove a durable operation, or create an ownership-loss recovery path. + +## 4.8 Faults + +A failed admitted storage commit faults the whole harness: it closes Drive gates, rejects barriers and pending/future calls with `HarnessFault`, and requires process restart — never an expected `Err` result. `faulted:true` appears in snapshots obtained before observation closes; reopen restores from the last successful transactions. + +Close rejects active drive and convenience-operation promises with `HarnessClosed`; already-resolved admissions remain durable, calls not yet accepted return `Err(Closed)`, and surfaces without a `Result` channel reject with `HarnessClosed` on and after close. Provider, tool, and isolated hook failures remain per-lane and in-band. A throw/rejection from trusted deterministic application computation (`systemPrompt`, `toolContext`, `toProviderMessages`, an `entryProjector`) faults the harness; `AgentTool.prepareArguments` is the deliberate exception, normalized to a synthetic tool error. + +# Part 5 — Public surface + +## 5.1 The lane surface + +An `AgentLane` is the execution-capable facade over one named Branch. Full declarations: `agent-harness.ts`. Every asynchronous method takes a trailing `Context`. The complete method inventory: + +- **Branch surface** (same five methods as `Branch`, §2.8, plus operation-aware append behavior): `getTipId`, `findEntries`, `findEntry`, `appendMessage`, `appendCustomEntry`. +- **Primitives:** `accept(request: OperationRequest) → OperationAdmissionResult`; `drive(options: { operationId; waitForRetry?; pollDeferred? }) → DriveResult`; `requestAbort(operationId) → AbortRequestResult`; `getResult(operationId) → OperationResultRecord | undefined`; `inspectExecution() → LaneExecutionInfo`. +- **Conveniences:** `prompt(text, images?)` and `prompt(message | message[]) → RunResult`; `skill(name, additionalInstructions?) → RunResult`; `promptFromTemplate(name, args?) → RunResult`; `compact({ customInstructions? }?) → CompactionResult`; `navigateTree(targetId, options?: { summarize?; label?; customInstructions? }) → NavigationResult`; `resume() → ResumeResult`; `abort() → AbortResult`. +- **Queues:** `steer`/`followUp`/`nextRun(message: string | AgentMessage, images?) → QueueResult`; `cancelQueued(entryId) → CancelQueuedResult`. +- **Other:** `recordUsage(usage, { entryId?; details? }?) → RecordUsageResult`; `waitForIdle()`; `runWhenIdle(callback)`; `getModel`/`setModel(identity: { provider, modelId })`; `getThinkingLevel`/`setThinkingLevel`; `getActiveTools`/`setActiveTools(names)`; `watch() → WatchHandle`. + +`OperationRequest` is the union of `prompt` (text+images or message(s)), `skill`, `prompt_template`, `compaction`, and `navigation` requests, each with an optional caller-supplied `operationId` (§1.2, §3.1). + +The four primitives are `accept`, `drive`, `requestAbort`, and `getResult`/`inspectExecution` for observation. `accept` commits no process owner; `drive` installs or joins one lane-owned pass, reports durable retry/deferred waits, and returns old result records without disturbing the current operation; each caller races only its own observation with its Context signal; `requestAbort` is expected-id fenced and the sole durable cancellation primitive. + +Conveniences add process-local waiting policy only: `prompt`/`skill`/`promptFromTemplate` compose acceptance and drive; `resume` inspects and drives any current operation, granting one deferred poll permit; `abort` requests durable cancellation and observes reconciliation; `compact`/`navigateTree` settle structural operation A, then may accept and drive an ordinary empty-prompt run B when queued conversational input remains — B has a fresh id and ordinary `run_start`, and a competing acceptance may win that idle window, in which case the convenience returns only A. Primitive and convenience histories are equivalent and externally reproducible; no scheduler, auto-start-on-reopen, or hidden continuation exists below this layer. + +### Results + +```ts +interface SuspendedRun { operationId: string; status: "suspended"; deferred: DeferredHandle } + +type RunResult = Result; +type CompactionResult = Result< + { compaction: OperationResultRecord; run?: OperationResultRecord | SuspendedRun }, + LaneBusy | NothingToCompact | Closed>; +type NavigationResult = Result< + { navigation: OperationResultRecord; run?: OperationResultRecord | SuspendedRun }, + LaneBusy | InvalidNavigation | UnknownTarget | Closed>; +type ResumeResult = Result; +type QueueResult = Result<{ entryId: string }, InvalidMessage | Closed>; +type CancelQueuedResult = Result<{ kind: "cancelled" | "already_consumed" | "not_found" }, Closed>; +type AbortResult = Result< + { operationId: string; steer: AgentMessage[]; followUp: AgentMessage[] }, + NoActiveOperation | Closed>; +type RecordUsageResult = Result<{ usageId: string }, Closed>; + +type DriveOutcome = + | { kind: "settled"; outcome: OperationResultRecord } + | { kind: "waiting"; operationId: string; reason: "retry"; notBefore: number } + | { kind: "waiting"; operationId: string; reason: "deferred"; deferred: DeferredHandle }; +type DriveResult = Result; +type AbortRequestResult = Result< + { operationId: string; newlyRequested: boolean; + steer: AgentMessage[]; followUp: AgentMessage[] }, + OperationMismatch | Closed>; +``` + +`SuspendedRun` is convenience-only and never stored. Terminal outcomes are exactly the immutable record; callers retrieve entry payloads separately through Branch/Lane queries. Queue admission returns the reserved `entryId`; `AbortResult`/`AbortRequestResult` carry the family-neutral `operationId` plus the drained steer/follow-up messages; `recordUsage` writes an adjustment row and returns its id. + +`waitForIdle` resolves after earlier admitted lane jobs settle, there is no current operation, and no idle callback owns the lane; multiple waiters may resolve together and later work may begin immediately afterward. `runWhenIdle` serializes one process-local callback owner, released on return or throw; the callback must not invoke another mutating method on the same lane (it would wait behind itself); close rejects callbacks not started and waits for one already running. `setModel` stores `ModelIdentity`, not a live registry object — an unavailable identity remains valid configuration and later fails in-band when generation resolves it. Tree browsing beyond one branch, fork administration, label inventory, and Session/repository listing are deliberately not AgentLane methods; a serving/RPC facade composes those read services beside the lane rather than widening it. + +## 5.2 The harness + +Full declarations: `agent-harness.ts`. `AgentHarness` methods (all with trailing `Context`): + +- `lane(name)` / `lane(name, { createAt?: string | null })` → `AgentLane`; `lanes() → LaneInfo[]`. +- `getName`/`setName(name | undefined)`; `getLabel`/`setLabel(targetId, label | undefined)`. +- Harness-global configuration — tool implementations are code and cannot persist, active names live in each lane's configuration, and `setTools` replaces only the registry: `getTools`/`setTools`, `getResources`/`setResources`, `getStreamOptions`/`setStreamOptions`, `getRetryPolicy`/`setRetryPolicy`, `getCompactionSettings`/`setCompactionSettings`, `getSteeringMode`/`setSteeringMode`, `getFollowUpMode`/`setFollowUpMode`. +- `watchSession() → WatchHandle`; `hooks`; `events`; `close()` (detach cleanly, §4.7 — durable open operations remain open). + +`AgentHarness.create(options, context)` returns `{ harness, open: OpenOperation[] }`, where `OpenOperation = { lane, operationId, kind, startedAt, aborting?: true }` and `LaneInfo = { name, tipId, operation: CurrentOperationInfo | null }`. + +**R12:** `watchSession` currently throws `SliceNotImplemented("watchSession")` — the sole stubbed Harness method. The current `SessionSnapshot` is `{ lanes: LaneInfo[]; faulted: boolean }`; R12 decides whether it stays that small. + +Passing an open `Session` to `create` transfers orchestration ownership to the attachment attempt and then the returned Harness until `close` resolves; if create rejects, ownership returns to the caller. During ownership, raw Branch mutation for a configured AgentLane and direct writes to reserved `pi.*` control addresses can stale the authoritative Lane projection and are trusted-programming defects; session-global application values remain available. `create` creates nothing and restores the small durable projection for every complete lane before returning (§4.4); `open` contains exactly one item per lane with a durable current operation, omits idle lanes, copies `aborting:true` only from durable cancellation control, and is inventory that may become stale — not a reservation, identity prediction, or drive claim. Detailed snapshot payloads are read only by `watch(context)`. + +### Options + +`AgentHarnessOptions`: `session`, `models`; the immutable lane seed `model`, `thinkingLevel?` (default `"off"`), `activeToolNames?` (default: initial tool names) — captured at `create`, initializing every missing AgentLane, never overriding an existing complete lane configuration; `tools?`, `toolContext?` (a `TContext` value or `(context) => TContext | Promise`), `systemPrompt?` (string or sync/async `(toolContext, context) => string`, evaluated per request), `resources?` (skills, prompt templates), `streamOptions?`, `retry?`, `compaction?`, `steeringMode?`, `followUpMode?`, `toolExecution?` (`"sequential" | "parallel"`, default parallel), `toProviderMessages?`, `entryProjectors?: Record` where `EntryProjector` is a sync/async `(entry: CustomEntry, context) => AgentMessage[] | undefined`. `Resources = AgentHarnessResources`. `AgentHarnessStreamOptions` is the curated §0.7 type; it excludes signal and provider lifecycle callbacks, which the harness owns. + +`AgentHarnessTool` replaces `AgentTool.execute` with `execute(toolCallId, params, onUpdate, toolContext, invocation, context)`; the update callback is `(partialResult, options?: { checkpoint?: true }) => void`; `AgentHarnessToolInvocation` is `{ invocationId, operationId, turnId, getMemo(name), setMemo(name, value | undefined) }` — `invocationId` is an opaque session-unique logical call id equal to the reserved result entry id, and `setMemo(name, undefined)` deletes. + +There is no harness-level telemetry default: a shared harness may serve concurrent callers, each method/callback uses only its explicit invocation Context, `context.telemetryContext` is always the telemetry parent, and runtime configuration must not reintroduce a receiver-level fallback. + +`create` copies the three seed fields into one immutable `LaneConfiguration`, storing the model as `{ provider, modelId }`; existing complete lanes use only their current config. `lane` atomically gets or creates/attaches on the Session mutation line, using the seed whenever it creates or attaches; missing lanes use `options.createAt ?? null`, existing lanes ignore it. Commit success publishes the one Lane object and synchronously binds `lane_created` recipients before line release, then awaits delivery outside. Invalid names and unknown non-null targets reject with `InvalidLane`/`UnknownTarget`; partial durable combinations fault. Lane configuration and Harness metadata setters likewise bind their events in the committing Session job. Applications opt into deferred generation through `setStreamOptions({ deferred: ... })` or initial `streamOptions`; `before_request` may patch the same curated field per attempt. Initial, replacement, and hook-patched stream options are trusted typed internal values; patch deletion semantics apply before publication, and extensions returning values outside the declared types are defective rather than runtime-validated. + +`systemPrompt`, `toolContext`, `toProviderMessages`, and `entryProjectors` are deterministic/idempotent computation callbacks: they receive the current invocation Context and may repeat after a crash; effectful interception belongs in hooks. `systemPrompt` is evaluated per provider request; `transform_context` then receives and may request-locally transform both messages and that prompt — durable run context belongs in `before_run` message injection, not request-local transformation. `toolContext` is resolved once per live batch; each bound call receives its stable invocation and a required synchronous update callback even with no live listener. A `replay:"safe"` tool may implement named durable effect memoization over `getMemo`/`setMemo`; committed values survive replay until the call reaches `outcome_ready`, and tools must await memo writes. These methods are invocation-scoped capabilities, not raw Session access. + +## 5.3 Session and Branch + +Session-global metadata, values/lists, global entry queries, Branch discovery/creation, mutation, id generation, and close live on `Session` (§2.8); Session has no tip or implicit-main methods. `Branch` is intentionally narrow (§2.8): because the receiver already names one Branch, its query methods are `findEntries`/`findEntry`, and direct appends always extend its current tip atomically. AgentLane exposes the same five methods and adds operation-aware append behavior. There is no nested tree/store/view accessor. + +## 5.4 Snapshots and subscription + +```ts +interface LaneSnapshot { + lane: string; + transcript: Entry[]; + tipId: string | null; + lastResult?: OperationResultRecord; + configuration: LaneConfiguration; + stats: SessionStats; + operation: null | { + id: string; kind: "run" | "compaction" | "navigation"; + startedAt: number; fromTipId: string | null; + status: "running" | "open" | "aborting"; + retry?: { attempt: number; maxAttempts: number; nextAttemptAt: number }; + deferred?: { handle: DeferredHandle; poll: number }; + streamingMessage?: AssistantMessage; + runningTools: Array< + | { status: "running"; toolCallId: string; toolName: string; args: unknown; + result?: AgentToolResult } + | { status: "settled"; toolCallId: string; toolName: string; args: unknown; + result: AgentToolResult; isError: boolean } + >; + }; + queues: LaneQueuedItem[]; + faulted: boolean; +} + +interface WatchHandle { + snapshot: T; + start(listener: EventListener): void; + resnapshot(context: Context): Promise; + unsubscribe(): void; +} +``` + +`OperationStatus` includes `"running" | "open" | "aborting"`, but current snapshot and reducer paths produce only `"open"` and `"aborting"`; `"running"` has no defined producer and is tracked as contract cleanup (§0.9, roadmap). + +`watch(context)` captures one coherent presentation snapshot on the Session mutation line, then exposes events serialized after it. Capture performs one compaction-bounded transcript read, the newest result lookup named by `lastOperationId`, current stats, and exact state-directed reads for inbox payloads, frames, deferred source, effect-pending tool progress, and outcome-ready staged results. A running tool's optional `result` is its latest complete progress snapshot; a settled tool's required `result` is final and remains in `runningTools` until its own `entry_added` moves presentation to the transcript. Required missing references fault capture; optional frame/checkpoint absence is legal; results remain unrelated to recovery. `queues` is the one global ordered tagged inbox including pending writes; `configuration`, `stats`, and `faulted` make the initial snapshot self-sufficient before any event arrives. First snapshot, reconnect capture, and `resnapshot()` share one path. + +`reduceLaneSnapshot(snapshot, event)` is the normative client fold: for non-navigation histories, folding a snapshot over its own events produces the next snapshot. It returns `{ rebase: true }` for `navigation_end`; the client calls `handle.resnapshot(context)` without tearing down or resubscribing. Resnapshot marks a barrier on the event-bus delivery tail while the mutation line still holds the captured boundary — queued pre-boundary watcher deliveries are invalidated and post-boundary events held until the fresh snapshot installs — so calling it from inside the listener neither deadlocks nor refolds stale queue/usage state. The reducer ignores other lanes' events, applies session-wide usage totals, and clones its input rather than mutating caller state. + +Operation-terminal events are `run_end`, `navigation_end`, and `compaction_end` only when the open snapshot operation kind is standalone compaction; in-run `compaction_start`/`compaction_end` are segment brackets inside the open run. `run_suspend` is non-terminal and leaves the operation open with a deferred descriptor; `run_resume` clears it. + +## 5.5 Events + +Events are passive committed-state/lifecycle observations: they never drive execution and are not replayed from durable history. `HarnessEvent` adds `lane` to lane-scoped payloads and may add `recovery: true` for actual orphan recovery/replay. Full payload unions: `agent-harness.ts`. The authoritative groups: + +| Group | Events and required data | +|---|---| +| operation | `run_start{runId,startedAt}`, `compaction_start{runId,reason,startedAt}`, `navigation_start{runId,targetId,startedAt}`, `operation_abort{operationId,steer,followUp}` | +| terminal/segment | `run_end{runId,status,fromTipId,tipId,endedAt,error?}`, `compaction_end{runId,reason,status,endedAt,entryId?,error?}`, `navigation_end{runId,status,fromTipId,tipId,endedAt,error?}` | +| suspended/retry | `run_suspend{runId,reason:"deferred",deferred,poll}`, `run_resume{runId}`, `retry_scheduled{step,attempt,maxAttempts,delayMs,notBefore,errorMessage}`, `retry_start`, `retry_end` | +| transcript | `message_start`, `message_update{message,event,frame?}`, `message_end{message,entryId?}`, `entry_added{entry}` | +| tools/turns | `turn_start`, `turn_end`, `tool_start`, `tool_update`, `tool_end` | +| replicated state | `queue_update{queues}`, lane/global `config_update`, `usage{row,totals}`, `lane_created{at}` | +| metadata/faults | `value_update`, `fault`, `handler_error` | + +`queue_update` carries the complete ordered `LaneQueuedItem[]` after every inbox change and is the sole authoritative queue event; there is no `write_pending`. Lane configuration updates carry `previous` and `value`; global data-bearing configuration updates do the same, while tools/resources remain notification-only because code registries are not replicated. Usage events carry authoritative committed totals from `CommitResult`/storage stats. + +Acceptance publishes after its transaction: the start event, message lifecycle plus `entry_added` for placed queued/request entries, then `queue_update` when capture changed the inbox. Standalone structural starts publish before `accept` resolves. Provider streaming and `tool_update` observations may precede the transaction persisting final content; `tool_start` is emitted from the commit that establishes fresh effect intent or synthetic outcome readiness, `tool_end` is emitted only after its finalized result stages, and `entry_added` always means the immutable entry is queryable. `tool_start` carries effective arguments for an intended effect and source arguments for an immediate synthetic result; `tool_end` carries the finalized result but does not repeat arguments. + +Clients depend on the terminal taxonomy: `run_end` closes a run; `navigation_end` closes navigation and requires snapshot rebase; standalone-compaction `compaction_end` closes compaction; in-run `compaction_start`/`compaction_end` are nested segment brackets that do not clear the run; `run_suspend` keeps the operation open. Every structural start has one matching end, including `aborted`. `compaction_end.status` is `completed | declined | failed | aborted` (success carries `entryId`); `run_end` is `completed | failed | aborted`; `navigation_end` additionally permits `declined`. + +The event bus binds recipients and Context synchronously after commit, serializes delivery in mutation order, and makes the public operation await its retained delivery promise. Listener failures emit `handler_error` and do not roll back committed state. `watch` recipients install on the mutation line so no event falls between snapshot and subscription. `reduceLaneSnapshot` (§5.4) is the supported fold; clients should not reconstruct operation terminality or queue/config/stat state with a second reducer. + +## 5.6 Hooks + +Hooks are awaited interception points. Registration is harness-global: `Hooks.on(name, handler, options?: { id? })` returns an unsubscribe function; `HookHandler` receives the event plus `{ lane, runId }` (`HookInvocation`) and the current operation Context as its final argument, and returns the result synchronously or as a promise. Registration is host-local configuration and retains no caller Context; nested handler work must derive from the invocation Context, not a harness default. A registration `id` is optional observability metadata only — not uniqueness, persisted routing, replay identity, or a durability protocol. Extension-private durable state belongs in extension-owned bound values/lists or audited custom entries keyed by lane/operation id; the extension owns replay, cleanup, and idempotency. + +The canonical hook contract (event/result field shapes as declared in `agent-harness.ts`): + +| Hook | Event | Result | Durability | +|---|---|---|---| +| `before_run` | `{ prompt: AgentMessage[], resources }` | `{ messages? }` | transition-consumed: injected messages and the checkpoint commit together | +| `before_drive` | `{ operation: "run"\|"compaction"\|"navigation" }` | `void`; failure rejects the pass with no durable progress | pass-local | +| `before_run_end` | `{ runId, messages }` | `{ followUp?: string }` | transition-consumed: a follow-up and continuation commit together, or the terminal transaction consumes the no-follow-up decision | +| `transform_context` | `{ messages, systemPrompt }` | `{ messages?, systemPrompt? }` | request-local | +| `before_request` | `{ model, step: "assistant"\|"deferred"\|"compaction"\|"branch_summary", attempt, streamOptions }` | `{ streamOptions?: AgentHarnessStreamOptionsPatch }` | request-local: the intent stores only its specified derived request metadata | +| `before_payload` | `{ model, payload: unknown }` | `{ payload }` | request-local | +| `after_response` | `{ status?, headers?, message: SettledAssistantMessage }` | `{ message? }` (must keep role) | transition-consumed: the transformed message feeds the settled response entry; cancellation or overflow may normalize it at commit | +| `before_tool` | `{ toolCallId, toolName, args }` | `{ args?, block?: { reason, terminate? } }` | transition-consumed: effective arguments commit with effect intent, or a blocked outcome is staged | +| `after_tool` | `{ toolCallId, toolName, args, content, details?, isError, usage? }` | `{ content?, details?, isError?, usage?, terminate? }` (field-by-field patch) | transition-consumed: the finalized result commits with `outcome_ready` staging | +| `before_compaction` | `{ reason: "manual"\|"threshold"\|"overflow", preparation: CompactionPreparation, customInstructions? }` | `{ decline?, compaction?: CompactResult }` | transition-consumed: decline, supplied result, or selection of generation commits as the next structural transition | +| `before_navigation` | `{ targetId, preparation: BranchPreparation, customInstructions? }` | `{ decline?, summary?: BranchSummaryResult }` | transition-consumed, as above | + +Timing and repetition: + +| Hook | When it runs / repetition | +|---|---| +| `before_drive` | once per newly installed real drive pass, after the cancellation check and before recovery or ordinary work; repeats after every wait/suspension or process loss; joiners do not rerun it | +| `before_run` | while a run is durably `starting`, after `before_drive`; may rerun until its consuming commit succeeds; never after that transition | +| `transform_context`, `before_request`, `before_payload` | once per request attempt, including retry and replay; `transform_context` at `AgentMessage` level before `toProviderMessages`; `before_payload` on the provider-specific wire payload | +| `after_response` | per settled response, after streaming settles and the latest frame write completes (§3.7), before `message_end` and the commit; unless abort wins before it starts | +| `before_tool` | after validation, before execution; per call execution; not when an orphaned unsafe call is synthesized without execution | +| `after_tool` | after execution, before outcome staging; per executed result unless abort wins before it starts; runs on safe replay | +| `before_compaction`, `before_navigation` | in `deciding`; once until a structural source commits; never once generation is durable | +| `before_run_end` | at a normal finish boundary; may repeat after a crash at that boundary; never for abort, terminal failure, or exhausted auto-compaction | + +Uniform semantics: + +- Handlers run in registration order, each seeing prior aggregate output where the hook transforms a value. A throw emits `handler_error`, skips that handler, and lets the rest continue — except **`before_drive` fails closed and rejects the pass, and `before_tool` fails closed and blocks the tool**. One accepted-operation hook invocation calls `drive.gate.admit(() => runPipeline(...))`; individual handlers are not separate gate checks. +- Aggregation: `before_run` appends messages, each later handler seeing the prompt plus prior injections, all applied once by the consuming `starting → checkpoint` transaction. `transform_context`, request/payload/response, and `after_tool` transformations chain with field-by-field patch merging. `before_tool` argument replacements chain and are revalidated; the first block is terminal and later handlers do not run. `before_compaction`/`before_navigation` stop at the first decline or supplied result; if all return neither, generation is selected; decline plus a result is a handler error, ignored like a throw. `before_run_end` uses the latest defined follow-up. +- Durability classes: **pass-local** results control only the current process-local pass — nothing records that the hook ran. **Request-local** values exist only while constructing/executing that provider request — transformed context, system prompts, stream-option patches, and provider payloads are not durable request snapshots, and a retry or rebuilt request runs fresh middleware. **Transition-consumed** output is reflected in the transaction performing the dependent durable transition: before it commits the output may be lost and the hook may run again per the recovery path; after, recovery observes the resulting state/content rather than rerunning the hook. There is no separate hook-completion record. Events expose post-hook values; passive listeners cannot transform them. +- `before_request` receives `AgentHarnessStreamOptions` and returns `AgentHarnessStreamOptionsPatch`; neither can contain a signal or provider lifecycle callback. `after_response` must preserve the assistant role and may return `aborted` only when the harness signal is already aborted. `before_navigation` runs only for summarized navigation; unsummarized navigation cannot decline. + +No external hook is globally exactly-once. Transition-consumed hooks commit their interpreted output with dependent durable progress; pass-local and request-local hooks do not. A crash before a consuming transaction may lose the output and repeat the hook when the procedure retries, while recovery paths that synthesize an unknown outcome may skip it. External side effects require extension-owned idempotency keyed by stable operation or invocation ids. + +## 5.7 Harness execution blocks + +The harness owns purpose-built execution blocks under `src/harness/execution/`; they implement provider and tool mechanics for the operation procedures and know nothing about durable operation state, lanes, retries, classification, queues, or storage. `src/agent-loop.ts` is an independent compatibility implementation, not modified or rebuilt on these blocks — its exports, injected `StreamFn`, callback shapes, mutable-context behavior, and event ordering are unchanged. + +### Assistant streaming + +`assistant.ts` owns one already-approved provider request (`streamHarnessAssistant(messages, config, context)`; shapes in source). Before the request intent commits, the assistant procedure verifies the captured durable `{ provider, modelId }` resolves in `Models` and runs `before_request`; after that commit, the request adapter resolves the same pair, derives the admitted Context, and invokes `Models` through `drive.gate.admit(...)` under its composed abort signal and telemetry parent. Block order: `transformContext` → `toProviderMessages` → construct provider `AiContext` → map curated stream options + thinking level to `SimpleStreamOptions` → install `context.abortSignal`, `context.telemetryContext`, `beforePayload`, metadata capture → `request(...)` → either `observer.start` then `observer.update`* or a pre-generation error with no start/update → settle the stream completely → `afterResponse(settled message, captured metadata)` → `observer.end` → return the settled message. + +It never mutates `messages`; every callback receives the same invocation Context unless its adapter deliberately derives a child span Context. The observer feeds actual start/update events to one per-stream `AssistantMessageFrameEncoder` and synchronously enqueues each returned invocation-fenced frame append without awaiting storage (§3.7); events already covered by a queued frame return no frame. The standalone block's source config makes `afterResponse` optional for callers that need no durable frame/hook mediation; the old inline declaration made it required. The durable Harness procedure must always install it — even with no hook listeners — because it first stops frame admission and awaits the latest frame-write promise before the optional `after_response` pipeline. A pre-generation `error` emits no synthetic start: the adapter calls only `observer.end` after the response hook. An update or successful `done` before `start`, a duplicate start, or an event after terminal is a provider protocol defect. If abort interrupts the parked `afterResponse` adapter, the block awaits the carried abort-mutation promise, skips that hook, emits `observer.end` with the raw settled message, and returns it so the caller commits it under the now-current cancellation control. `beforePayload` maps to pi-ai's payload callback; metadata capture maps to pi-ai's `onResponse`, which runs before the response body is consumed — distinct from `afterResponse`, which transforms the settled message afterward. The harness exposes neither callback through `AgentHarnessStreamOptions`. + +The request function, not the block, owns registry dispatch, auth, and admission: it resolves the captured model, derives the admitted Context, and invokes `models.streamSimple` inside `gate.admit` exactly as §4.2 shows, additionally passing ``sessionId: `${session.metadata.id}:${lane.name}` ``. No yield exists between check and invocation; asynchronous auth/lazy/provider work is part of the admitted request and owns the admitted signal. Ordinary assistant requests derive that one stable cache/affinity identity per lane; lanes in one Session never share it, and identity-prefix changes may miss old cache entries but cannot incorrectly reuse them — no durable lineage or rotation state exists. Structural summary requests use fresh identities with `cacheRetention: "none"`; deferred polling sends no cache identity. A captured identity that disappears after intent becomes an in-band provider error under the reserved ids; one unavailable before intent becomes a non-retryable configuration failure with no fabricated response or usage (§3.7, §4.5). Existing summary helpers keep their separate `Models`-based generation logic but gate their `Models` invocation the same way. + +### Tool phases + +`tools.ts` exposes phases at the exact durable boundaries of §3.8 — `prepareToolCall`, `applyBeforeToolDecision`, `executeToolCall`, `finalizeToolCall`, `createToolResultMessage` (shapes in source). Hooks remain separate gated invocations and commits remain explicit operation-procedure statements; neither hides behind a callback bag. The batch procedure composes: prepare (lookup, `prepareArguments`, initial validation) → `before_tool` → apply decision (block or validate replacement arguments) → commit `pi.op.tool_args` + effect-pending intent with post-commit `tool_start` → execute (effect + live updates + checkpoint requests) → stop updates, expire memo capability, close checkpoint admission → await latest `tool_update` delivery and latest checkpoint write → `after_tool` → finalize → commit `pi.pending.entry` + `outcome_ready` + invocation cleanup with post-commit `tool_end` → materialize source-ready outcomes as entries + usage. + +Unknown tools, `prepareArguments` failures, invalid initial/replacement arguments, and blocked calls produce an immediate raw error `AgentToolResult` with `isError: true` and no invented `details`; `createToolResultMessage` constructs the canonical synthetic message before staging `outcome_ready`. Their outcome-staging commit emits `tool_start` followed by `tool_end`; they still invoke no tool effect or `after_tool`. The old inline declarations instead put a `ToolResultMessage` directly in the immediate outcome. `AgentHarnessTool.prepareArguments` is deterministic/idempotent computation and may repeat before intent; effectful policy belongs in `before_tool`. At `tool.execute` admission, `executeToolCall(call, gate, onUpdate, toolContext, invocation, context)` derives `withAbortSignal(gate.signal, context)` and invokes `AgentHarnessTool.execute` directly through `gate.admit(...)`, with the admitted Context trailing; there is no neutral `AgentTool` adapter. The old four-argument declaration and adapter description predate this source shape. The block converts expected tool throws to an error result and stops accepting updates when the tool promise settles; the declared raw tool-effect span is not emitted until T1 (§5.8); update/checkpoint promise retention and the await-both rule follow §3.8. `finalizeToolCall` applies the field-by-field patch before outcome staging and post-commit `tool_end`. + +Before starting any call in a live batch, the procedure resolves `toolContext` once and filters the current `AgentHarnessTool` registry to the complete captured active-name set, retaining that procedure-local snapshot. The `executeToolCall` call site supplies each call's stable invocation (`invocationId: resultEntryId`, `operationId`, `turnId`, memos), update callback, tool context, and current invocation Context. An absent implementation — or a provider call outside the captured active names — becomes the §3.8 synthetic unknown-tool result and does not suspend the batch. Every call observes the same application context and its own stable invocation identity. Safe replay creates a new code/context snapshot but passes the same invocation id and memos after deleting the stale progress checkpoint. `AgentHarnessTool.replay` defaults to `"never"`. + +There is deliberately no harness `executeToolBatch`. In parallel mode the direct procedure makes one source-ordered start pass; each position either starts a real promise or retains an immediate outcome until it can be staged. Effects/finalization settle independently: each complete result commits `outcome_ready` in completion order, and a separate Session mutation job materializes the contiguous ready prefix in source order. Durably, completed calls form a prefix while the suffix may mix `planned`, `effect_pending`, and `outcome_ready` — e.g. `[effect_pending, outcome_ready, effect_pending]` after the completed prefix. A crash discards only unstaged process-local outcomes; recovery safely replays or interrupts orphaned effects, materializes already-ready outcomes without resolving tool code, and reruns ordinary clearance for planned positions. The same procedure owns cancellation and durable batch completion. Genuine-`length` calls bypass effects but stage their specified synthetic outcomes (§3.7). + +The legacy agent loop remains behavioral evidence for ordinary streaming and tool execution; harness differences are deliberate — `before_tool` returns explicit revalidated replacement arguments, hooks have explicit gate boundaries, parallel outcomes stage in completion order, entries materialize in source order. Remote protocol adapters validate untrusted wire data before returning typed provider values; the harness trusts those typed values and all in-process tool/hook/extension values, and violations are adapter or extension defects, not storage validation cases. Expected provider failures still become assistant `error` settlements, tool preparation/argument failures synthetic tool results, throwing hooks retain their documented handling, and invalid public caller operations return their declared errors before acceptance. + +## 5.8 Telemetry + +Use the existing callback-based `TelemetryContext`, no-op/reference implementations, typed schema machinery, and agent-owned schemas; do not invent a second contract. Invocation Context is passed explicitly as the trailing argument; no core `AsyncLocalStorage`, global active span, or mutable receiver default is permitted. + +Local Context propagation and request-ID RPC cancellation follow §0.2, with these additions: child work derives a new immutable Context when it starts a child span; a pre-aborted request starts no server work; one request or drive joiner cannot cancel another caller. An aborted `context.abortSignal` must not call `requestAbort()`, write `cancel_requested`, or commit a durable aborted result while control remains running — only explicit `requestAbort`/`abort` owns that transition. Context objects, signals, telemetry objects, and backend-native span objects are never stored durably or serialized as business arguments. RPC currently carries cancellation metadata and reconstructs a fresh local cancellation Context. T1 retains the old specified trace recipe: the client injects trace metadata; the server extracts the incoming trace parent into a local `TelemetryContext`; then it derives a fresh invocation Context with both `withAbortSignal` and `withTelemetryContext` before invoking core. T1 must define the trace carrier encoding and implement that reconstruction; it does not reopen the composition rule. Whether selected adapter-managed typed values may also cross remains an RPC design decision. Shared receivers retain no caller Context and expose no receiver-level telemetry default; process-local objects representing one invocation (a drive pass, an event subscription) may retain their derived Context for that invocation only. Buffered events retain `{ event, context }`; `emitBatch` binds recipients synchronously so delayed local handlers and RPC event frames preserve source lineage. + +**T1 — declared, largely unimplemented.** `src/harness/telemetry.ts` and the generated `docs/telemetry-schema.md` declare the span vocabulary below, but production starts only `pi.harness.hook`, and only for registered `before_tool`/`after_tool` handlers. AI options propagate `telemetryContext`, but no provider path starts `pi.ai.request`, and no tool-effect span is emitted anywhere. Server request ingress has request-ID cancellation signaling but no trace carrier and no client/server RPC spans. T1 must first reconcile whether every declared span is wanted, then implement or remove; RPC trace propagation and an exporter are separate follow-ups. The declared spans: + +```text +pi.harness.run | compaction | navigation +pi.harness.checkpoint | turn | step | tool | hook | sleep | event_handler +pi.session.write +pi.ai.request +``` + +Specified span semantics for the implementation T1 commissions: operation, step, tool, hook, event, and write parents follow the actual async procedure nesting; sleep spans permit run, compaction, navigation, turn, and checkpoint parents; `stepId`/`taskId` correlate retries and recovery. Every provider request/fetch/cancel uses `pi.ai.request`; each real or safely replayed phase-two tool effect uses one tool span. Every storage transaction uses one `pi.session.write` whose start attributes include `pi.session.item_count` and `pi.session.item_kinds` (`entry`, `usage`, `value`, `list`); list appends/deletes are never reported as value replacements; a calling procedure may supply its lane/operation ids and storage never infers them from payloads; end attributes include first and last committed sequence. Tool-checkpoint, invocation-memo, and assistant-frame commits are ordinary value/list writes under this span and emit no additional tool- or provider-effect span; address namespaces may be attributes, but snapshot and frame content never enters telemetry. No span is emitted when a mutation returns without committing; synthetic settlements and blocked/invalid tools emit no provider/tool-effect span. + +Telemetry attributes may contain declared ids, names, counts, durations, statuses, and usage — never prompts, completions, tool arguments/results, file contents, provider payloads, headers, handles, or credentials. Events and hooks may contain such content. The generated schema document and adapter/runtime conformance tests remain authoritative; implementation slices extend instrumentation only through those schemas. + +# Part 6 — Future: partitioned retention (Postgres) + +**Informative; no normative rule.** Memory, JSONL, and SQLite never partition and never delete entries or usage rows (§1.2); no core rule references this part. It records why §1.2's identity choices suffice for a possible Postgres deployment with TTL retention: UUIDv7 sorts bytewise in time order, so entries, the usage ledger, and `branch_entries` can use `PARTITION BY RANGE` on the id with period-boundary UUIDs as bounds and no partition column, while values, `branch_meta`, stats, and sessions stay in a hot unpartitioned catalog. Dropping a period requires an online pre-pass repairer (reparent edges crossing into the period, null dormant tips via value-seq CAS, force-expire open operations through the §3.13 terminal transaction under exclusive administrative ownership, uuid-range-delete labels), then one transactional lock barrier around delta repair plus plain `DETACH PARTITION`, so every commit sees either the fully attached period or a fully repaired store without it. A `DEFAULT` partition absorbs stray inserts whose ids predate every attached partition and is never dropped. A backend admitting an external repairer must perform value reads and CAS checks inside the commit transaction; shipping single-writer backends need no such rule. Retention policy, period granularity, and partition-count limits stay unspecified until the backend is real. + +# Part 7 — Schema evolution + +**R11 status: mechanism specified, not implemented; activation-gated.** No format-4 migration exists or is required: Memory is current-only, JSONL and SQLite reject unsupported storage versions, and SQLite runs only idempotent `001_initial.sql`. R11 becomes required immediately before the first incompatible durable change after format 4 stabilizes; format 4 is still WIP and pre-stabilization shape changes happen in place without migrations. + +**Problem and why it is small here.** Durability snapshots in-flight state shaped like *today's* state machine; ship a different machine and old durable state still exists mid-run. Migration cost is proportional to what must convert: entries and usage rows (years) cannot be rewritten and must stay read-compatible; lane/semantic values are a few per lane; `pi.op.*` exists only for open operations (usually zero); `pi.pending.entry` holds queued items plus staged tool outcomes; `pi.pending.tool_output` only optional open-call checkpoints; `pi.pending.assistant_frame` only open-response frames (usually zero). With no history retained, the entire mutable surface is a few dozen current values/lists, and the host assigns one writable owner before migration starts — migrate-on-open has no concurrent writer. + +**Mechanism: storage version plus migrate-on-open.** One session-level `storageVersion` lives in the catalog or header. A version number beats versioned namespace suffixes (`pi.lane.state.v2`): one number to check, chained `v1→v2→v3` migrations, no probing of historical namespace names, stable address components for point lookups. + +```text +open session: + version == current → proceed + version < current → run migrations in order, each one transaction: + convert lane/semantic/pending values, + handle open operations, bump the version + version > current → refuse to open (older binary, newer session) +``` + +Chained migrations run under exclusive host-assigned writable ownership before `open()` returns. Each step commits its conversions and version bump atomically, so a crash mid-chain resumes at the recorded version; conversions must be idempotent over already-converted values, which plain field mappings naturally are. + +JSONL has one wrinkle in each direction: when R11 adds migrations, replay must decode exactly the older-version value/list records the migration names, because pre-migration bytes remain in the file; a migration then triggers snapshot compaction (J1), whose temp-file-and-rename persists the new header version atomically and retires the old bytes. Between crash and compaction, version-specific decoding plus idempotent conversion keep the intermediate state harmless. None of this adds compatibility for the pre-WP01 WIP format-4 spelling. Legacy format 3 predates `storageVersion`; it normalizes through Appendix B on load and receives the current version with its first format-4 write. + +**Migrations are total.** Value conversion is a field mapping; a state-machine shape change is more — an old `pi.op.state` mid-phase may have no field-by-field equivalent in the new machine. A vN→vN+1 migration translates every stored value/list: lane/semantic values, `pi.pending.entry`, optional `pi.pending.tool_output`, invocation memos, and open operations' `pi.op.meta`/`pi.op.state` included (a migration adding `outcome_ready`, for example, must distinguish staged finalized tool results from still-uncertain effects). The author of a state-machine change writes the mapping for every reachable old state in the same change; a state with no natural successor maps to an explicit safe choice — no force-settle path or silent partial escape hatch. This is tractable because migration runs at open under exclusive host-assigned ownership over quiescent state: no task running, no effect in flight, every `pi.op.state` exactly what some transaction committed — a pure function over a small, fully enumerable, fully typed set of values. + +Address and list rules (§1.3, §1.4) extend the discipline: a bound address's namespace, key grammar, and kind are static for one storage version — changing any component or value↔list kind is an explicit migration, storage never infers or coerces kind, changing the TypeScript value shape requires a total value migration when old values are incompatible, and adding a new address with no stored value rewrites nothing. A list migration pages current elements in sequence order and either maps values preserving each element's `seq` or deletes the whole key — never loading an unbounded list at once. A migration changing `AssistantMessageFrame` shape must map every surviving element or explicitly delete the whole list, leaving `effect_pending` recovery with no partial; it must never infer completion from legacy frames. + +**Three strata as policy:** entries + usage carry the stability budget — provider-shaped messages plus three simple structural types, read-compatible forever (the precise rewrite §2.9 is administrative, not an open-time step; custom entry payloads are the application's contract). Lane/session values migrate on open, a few per lane, cheap forever. `pi.op.*`/`pi.pending.*` are ephemeral by design and few; every state-machine change ships the total mapping for its own states, and the cost is bounded by open operations — usually zero. Orchestration is ephemeral while the conversation format changes rarely, so migration cost is bounded by the small mutable surface and long-lived entries stay read-compatible. + +# Part 8 — Work packages + +A rolling plan, not a history. `harness.md` remains the normative behavior contract; a work-package handoff defines one executable implementation boundary. The evidence-backed inventory and dependency order live in [`post-wp05-roadmap.md`](post-wp05-roadmap.md); this part names packages and status only. + +Workflow: keep a future package's row here until actionable; move exact files/tests/ordering/exclusions into one handoff; move newly discovered normative behavior into Parts 0–7 or Part 9; only then reduce the row to a link. Every package implements its named concern end to end and tests its normal path, introduced states, owned crash boundaries, and both orders of owned races. Consumption-time dereference checks, implementation resolution, hooks, events, and deterministic effect controls land with the package that first needs them; earlier packages do not build generic future machinery. If implementation exposes a contradiction or a materially simpler boundary, stop for review. + +| ID | Status | Outcome | Handoff | +|---|---|---|---| +| WP00 | complete | Reconciled acceptance/hooks, harvested runtime1 scenarios, switched the public factory, deleted runtime1. | [Runtime1 removal](work-packages/00-runtime1-removal.md) | +| WP01 | complete | Bound values/lists across Session, Memory, JSONL, SQLite, instrumentation, conformance, public application access. | [Bound values and lists](work-packages/01-bound-values-lists.md) | +| WP02 | complete | Atomic prompt/skill/template acceptance, minimal open-operation attachment, Session mutation inspection, gap-free lane watch capture. | [Atomic acceptance and coherent attachment](work-packages/02-atomic-run-acceptance.md) | +| WP03 | complete | Removed the wall-clock drive deadline and non-durable yielded outcome. | [Remove drive deadlines](work-packages/03-remove-drive-deadlines.md) | +| WP04 | complete | Synchronous `emitBatch` publication; Session owns committed lane publication. | [Mutation publication and event delivery](work-packages/04-mutation-publication.md) | +| WP05 | complete | The total direct durable graph, public/replicated lane surfaces, immutable results, atomic boundaries, cancellation reconciliation, lane-safe provider identity. The [mobile assistant-output handoff](mobile-handoff/01-harness/05-assistant-output/message-update.md) is its only recorded follow-up. | [Direct durable drive](work-packages/05-direct-durable-drive.md) | +| WP06 | complete | Separated Session, Branch, AgentLane, AgentHarness; one keyless Session mutation line. | [Session, Branch, Lane separation](work-packages/06-session-branch-lane-separation.md) | +| WP07 | complete | Removed SQLite storage-layer ownership; added live read-only source forks, no-create opens, deletion reservation, physical/path safety, and all-settled close. | [SQLite host ownership and live forks](work-packages/07-sqlite-host-ownership-live-forks.md) | +| WP08 | in progress — Slice A | Replace implicit-main forks with named-branch/tree semantics and bounded-memory backend copies. | [Named-branch and tree forks with streaming copies](work-packages/08-named-branch-streaming-forks.md) | +| WP09 | complete | Project effect-pending and settled-but-unplaced tool calls continuously through snapshots and lifecycle events until transcript placement. | [LaneSnapshot settled-but-unplaced tools](work-packages/09-lane-snapshot-settled-tools.md) | + +WP05 subsumed the former R2–R12 execution rows; their implemented contract is in Parts 0–5 and the completed handoff. + +Future candidates (detail and order in the roadmap): **WP08** — complete Slice A and the JSONL/SQLite streaming slices; **H1** — resolve the `OperationStatus.running`, abort signal/event-order, and private gate-close typing contracts and audit Part 9 coverage; **C1** — resolve the §2.8 raw-RemoteSession contradiction before implementing either direction; **L1** — repository ownership of open handles and all-settled close across the three backends; **J1** — implement the §1.7 snapshot rewrite, dead-byte triggers, preserved high-water/list sequences, physical reclamation; the **[mobile assistant-output handoff](mobile-handoff/01-harness/05-assistant-output/message-update.md)** — implement tracked assistant progress, scoped durability, and delta replication without weakening unknown-outcome recovery; **R12** — implement `watchSession`; **T1** — reconcile the declared telemetry schema, then implement retained local spans (RPC trace propagation and an exporter are separate follow-ups); **S3** — reconcile the draft search API, then implement the standalone service, repository catch-up utilities, and the reference SQLite FTS5 projection (§2.8); **R11** — chained migrate-on-open under exclusive host ownership with total mappings (Part 7), activated only before the first incompatible stabilized-format change. + +Client watch/subscription incarnation fencing, SQLite branch/query performance, pending-payload measurement, and optional presentation/plugin capabilities are inventoried in the roadmap; they do not alter the Harness state machine. Protocol, client/server resnapshot, and lane reducer surfaces required by WP05 are already implemented; future protocol work extends them rather than redefining the lane contract. + +# Part 9 — Invariants and tests + +## 9.1 Invariants + +Storage: + +1. Entries and usage rows are **write-once** and share one session-wide id namespace. Writing either kind under any existing id is corruption. +2. Transactions are all-or-none, with strictly increasing `seq` in write order; gaps are legal. `seq` is monotonic session-wide. +3. Bound values and lists are the only mutable state. `setValue` replaces the current value and `deleteValue` removes it; `appendList` adds one immutable element and `deleteList` removes every element at the exact address. There are no tombstones or per-element mutations, and JSON `null` is legal only where an address's type permits it. +4. **Every payload lives in exactly one place**: an entry, a bound value/list, or the ledger. +5. No read on a hot path may fold history or infer state from an absent value — no value history exists to fold. Execution, recovery, and branch hot paths must be index-driven; inventory and debugging APIs page through indexes. Bounded paged reads of an exact list address derived from current typed state are the one sanctioned ordered read; their contents are auxiliary and never restart authority. Every bound address has one stable namespace, key, kind, and trusted value type per storage version; value helpers cannot target list addresses or vice versa. Namespace `pi` and every `pi.*` namespace are reserved by contract; every built-in namespace starts with `pi.`, and application use is a trusted-programming defect. Core and applications use the same constructors with no privilege split. Exactly five core prefix constructors encapsulate lane inventory and operation-cleanup grammar and are consumed only by `scanValues`. + +Tree: + +6. An entry's parent chain never changes. Branches share prefixes; nothing is copied. +7. Entries are trusted typed internal values. Only a custom entry may omit payload data; external shape corruption is unsupported rather than revalidated on internal reads. +8. Configuration and orchestration never enter the tree. Deleting every operation-owned value and list must leave a complete, valid conversation and ledger. +9. A lane's tip moves only by append or navigation. +10. A branch segment chain, followed to its end, yields the full root path (§2.6). +11. A missing parent is corruption — always (§1.2). + +Operations: + +12. `laneState(lane)` confers lane ownership and `operationState(operationId)` operation-state ownership. An open lane names operation O, `operationMeta(O)` holds that lane's compatible `OperationMeta`, and `operationState(O)` holds an `OperationState` compatible with O's intent kind; state values carry no duplicate owner metadata. While a harness owns the session, exactly one live `Lane` owns each lane's authoritative projection and every supported write to that lane's control addresses commits through it. +13. Operation-owned values and lists may exist only while their operation is open: the terminal transaction deletes them atomically with clearing `currentOperationId` (§3.13). The lane inbox and its `pendingEntry` payloads are lane-owned and never deleted by terminal cleanup. +14. Acceptance must observe `currentOperationId === null`, commits no `Drive`, and returns before any hook/provider/tool/timer work begins. Run acceptance commits payload-free `starting`; only its consuming command may apply `before_run` output and replace it with `checkpoint`. A supplied operation id obeys §1.2 and is the exact id written to `pi.op.meta`, events, and its eventual `pi.result` record. +15. A reserved id may exist only with the content its intent named. Queued-content ids begin in `pi.pending.entry`; settlement-family ids begin as strings in `pi.op.state`. A tool-result id may then move through `string only → outcome-ready pi.pending.entry → immutable entry`; no two representations coexist at a commit boundary (§2.2). An effect-pending response id may additionally key its auxiliary frame list (§3.7); frames are observation, not a content representation, and die with settlement. +16. Only terminal transitions construct `OperationResultRecord`. Exactly one immutable `pi.result/{operationId}` is retained per terminal operation; older records remain readable after later operations, and recovery never reads any record. +17. At most one operation is open per lane. Two is corruption. +18. `overflowRecoveryUsed` is `true` only after overflow compaction. A transition that adds projecting conversational input or tool results and requires an assistant writes `false`; an unprojected custom write preserves it. +19. A response committed with `stopReason: "aborted"` has `control.status === "cancel_requested"`; every terminal transaction under cancelled control records `status: "aborted"`. Equivalently, a terminal `completed`, `declined`, or `failed` record proves control was still running at its terminal commit. Providers must comply with the harness-owned signal contract; violation is corruption. +20. Attachment restores and validates only the small lane/operation projection (§3.3, §4.4). That owned projection is authoritative until close, fault, or process loss. Detailed presentation references are validated by `watch(context)` under the Session mutation line; drive payload references are validated by their consuming procedure. Missing or contradictory required data faults that consumer, while optional frame/checkpoint absence is legal. Top-level operation state has one live writer; only parallel tool-call status and queued progress/memo writes require child-state fencing. `pi.result` never determines an open operation's next procedure. +21. At most one terminal transaction and one immutable result-record write commit per operation. The one lane-owned Drive is the sole top-level state-advance writer, and every terminal candidate serializes on the Session mutation line. Administrative mutation of a live Lane's reserved control values is unsupported; offline administration first acquires exclusive Session ownership. +22. At most one `Drive` exists per lane. Acceptance and taskless `requestAbort` never install one. A matching `drive` installs it before releasing the Session mutation line; another matching drive joins that pass, and a stale id starts nothing. Caller cancellation ends only that caller's observation. A live Drive is never replaced in-process. Close/fault seal mutation admission and reject observations without writing operation state. Each newly installed pass invokes `before_drive` once after the cancellation check; joiners do not. `starting` under cancelled control invokes neither `before_drive` nor `before_run`. +23. The §4.2 `Gate.admit()` catalog is complete. Every listed hook/provider/tool/timer integration calls `admit(() => operation())` after preparation; no unlisted code calls it. Admitted asynchronous provider setup/delegation owns `drive.gate.signal`. +24. `drive` and `requestAbort` are fenced by expected operation id. They may affect only that current operation; `drive` may also return any matching immutable terminal result, including records older than the lane's latest. A stale wake for A cannot drive or cancel B. +25. No public drive option encodes a wall-clock budget or partial-progress return. An admitted effect settles normally or is recovered from durable state after task loss; host scheduling and process termination remain outside the harness contract. +26. Convenience operations and their explicit primitive compositions produce the same durable writes, events, results, and recovery behavior. Structural continuation is an ordinary empty-prompt acceptance with a fresh operation id; a competing acceptance may win the idle window. Convenience adds only process-local waiting/scheduling policy. +27. Each logical tool call's public `invocationId` is its reserved `resultEntryId`: unique within the session and unchanged across safe replay. Tools must await invocation-memo writes. Such writes synchronously enqueue, verify effect-pending ownership on the Session mutation line, and are deleted with outcome staging. +28. Completed tool calls form a source-ordered prefix. A sequential suffix permits at most one effect-pending or outcome-ready call before planned calls; a parallel suffix may mix `planned`, `effect_pending`, and `outcome_ready`. Completion-order outcome staging never extends the prefix; source-ordered materialization does. +29. Every outcome-ready call has exactly one matching finalized `pi.pending.entry`, no immutable result entry, no invocation memos, and no tool-output checkpoint. Outcome-ready and completed calls never execute again. +30. A tool progress checkpoint is an optional bounded complete `AgentToolResult` snapshot, selected with `checkpoint:true`. It never proves completion. Every selected checkpoint synchronously enqueues one invocation-fenced value replacement; no write is dropped or coalesced, only the latest write promise reference is retained, and awaiting it implies completion of every earlier write. Staging or terminal cleanup deletes the value and fences late recreation. +31. Assistant/deferred operation state is the sole restart authority for streamed partials. One effect-pending response id constructs exactly one `pendingAssistantFrames(operationId, responseEntryId)` address; every element is an exported pi-ai `AssistantMessageFrame`; frame order is a subsequence of provider event order because already-covered queued events produce no frame; terminal `done`/`error` events are never stored; frames never establish provider completion or suppress unknown-outcome recovery. +32. Every final or synthetic response settlement — normal, recovery, or cancellation — atomically deletes its exact frame list. Idle forks contain no frame lists. A restored partial may appear in `streamingMessage` but never in `transcript` before settlement. +33. The provider loop never awaits storage per frame; frame appends are enqueued synchronously in provider-event order, and awaiting the latest frame-write promise at stream settlement implies every accepted append completed. +34. Successful attachment publishes only complete lane projections and an open-operation inventory. It resolves no model/tool identity and starts no work. A later drive uses the authoritative owned projection; storage reads only dereference payloads named by that projection. +35. Every event-producing committing harness lane job publishes its owned projection and calls `emitBatch` with its complete event batch in the exact continuation that observes commit, as the callback's final action; this includes AgentLane appends, lane and metadata setters, acceptance, and AgentLane acquisition/attachment. The mutation never awaits delivery, but the public operation does. A lane watch registers buffering and clones live presentation synchronously, then performs bounded durable reads while holding the line. Snapshot plus buffered events has no gap or duplicate and replays no pre-registration lifecycle. `emitBatch` binds recipients and the emitting Context immediately; a delayed watcher receives the object-identical source Context, never its start Context. For non-navigation histories, `reduceLaneSnapshot` folding those events equals a later snapshot; navigation explicitly rebases through `resnapshot`. +36. Shared Harness/AgentLane/Session/Branch receivers retain no invocation Context and expose no receiver-level telemetry default. Concurrent calls preserve independent telemetry and cancellation lineage. Context and its values are neither durable operation data nor serialized business arguments. RPC cancel/disconnect reaches only the matching invocation through `context.abortSignal` and never becomes durable cancellation. +37. Process-local model/tool registry absence never becomes durable waiting state or an acceptance error. Pre-intent request-configuration absence fails in-band without fabricating a response/usage; missing requested tools stage `isError` tool-result messages with no invented details; uncertain effects settle under their existing recovery rules first. +38. `beginMutation()` acquires exactly one Session mutation line, `commit()` consumes at most one commit capability without releasing that line, and `end()` alone invalidates and releases it after any admitted commit settles. `Session.mutate()` always ends in `finally`; its callback cannot end early; direct `beginMutation()` callers end in `finally`. Local — and, if C1 commissions one, remote — implementations preserve the same read → decide → commit → process-local publication → end order (§2.8). + +## 9.2 Race catalog + +Each durable mutation race has exactly two durable histories. Matching callers install or join one lane-owned Drive; stale operation ids are rejected. Test every listed order with test-only commit gating and controlled hooks, providers, tools, and timers. + +| Race | Orders | +|---|---| +| `prompt` vs `prompt` on one lane | both compose `accept`; one accepts, one gets `LaneBusy` | +| `accept(A)` vs process loss before `drive(A)` | acceptance absent → serving layer retries; acceptance present → restored `starting` drives normally, with no unknown effect | +| `drive(A)` vs `drive(A)` | one installs the pass; the other joins exactly that pass and may drive again after its outcome | +| stale `drive(A)`/`requestAbort(A)` vs current B | expected-id mismatch; B is untouched | +| `requestAbort` vs response settlement | marker first → normalized `aborted`; terminal commit first → completed record and later abort mismatches | +| `abort` vs started tool outcome staging | abort first → real result stages under cancelled control; outcome first → finalized result is preserved and later materializes | +| checkpoint vs tool settlement | every accepted checkpoint was enqueued before settlement closed admission; settlement awaits the latest write, then staging deletes the value; a late update is fenced without committing | +| assistant frame append vs response settlement | settlement awaits the latest frame write, then its transaction deletes the list; a crash between leaves the committed frame prefix under `effect_pending` | +| live update event vs its queued frame/checkpoint commit | either finishes first; events are observation, and reconnect uses only committed frames/checkpoints | +| later tool B settles vs earlier tool A | B stages outcome-ready immediately; tree placement waits for A | +| `abort` vs `before_run_end` follow-up | marker first → stale hook output is dropped and reconciliation aborts; follow-up commit first → the run continues under the later cancellation marker | +| `cancelQueued` vs boundary consumption | cancel first → `cancelled`; consume first → `already_consumed`; abort drain first → `not_found` | +| `setModel` vs generation step start | old snapshot used; or new snapshot used | +| `abort` vs structural commit | `aborted` with no entry; or `completed` | +| `nextRun` vs acceptance | captured by this run; or stays for the next | +| structural A terminal vs convenience continuation B | B accepts queued input as an ordinary run; or a competing acceptance wins and the convenience returns A only | +| structural result boundary vs queued input | input commits first and is selected in the one publication commit; publication commits first and input remains queued for the next boundary/operation | +| abort drain response vs process/transport loss | caller receives drained steer/follow-up; or content is durably gone and the response is lost by the accepted drain-and-return tradeoff | +| manual-compaction preparation vs idle tree write | write before the final command → stale preparation is discarded/recomputed; acceptance first → the write follows active-operation rules; preparation never blocks the lane | +| deferred write vs abort | write survives abort either way | +| `requestAbort` vs `before_drive`/`before_run` admission | admission first → the complete hook pipeline runs and its consuming command observes cancellation; cancellation first → reconciliation runs and neither hook starts | +| `requestAbort` vs ordinary operation admission | admission first → operation is invoked with the signal; cancellation first → gate refuses invocation | +| attachment vs concurrent resume | attachment owns the session before publication; after return, resume uses the authoritative owned projection and stale `open` remains harmless | +| watcher registration vs state publication | watcher first → old snapshot plus the complete buffered event batch; publication/`emitBatch` first → new snapshot without that old batch | +| close vs attachment | create completes and publishes a fully open harness; or close/fault rejects attachment without a partial harness | +| snapshot capture vs resume | capture first yields pre-resume snapshot plus events; resume publication first yields post-transition snapshot | +| concurrent invocation contexts | each call/event/session write retains its own telemetry parent and abort signal; cancellation ends only that caller observation and writes no durable cancellation | +| `close` vs settlement | settlement abandoned, state stays `effect_pending`; or it committed before the flag was set | + +## 9.3 Test tiers + +**Tier A — state and drive.** For each of the 13 leaves in Part 3: construct it durably, close, reopen, drive its expected operation id, and assert the next durable transition, wait, or terminal result. Coverage includes accepted/restored `starting`; minimal projection restore; required/optional watch references; assistant unknown-outcome recovery with no/partial/authoritative-end frames; every classification and retry/deferred outcome; every tool child status and source-order placement; memo/checkpoint fencing; every summary boundary and overflow crash position; summarized/unsummarized navigation; cancellation reconciliation from every leaf; configuration failures; terminal deletion of operation-owned args, memos, checkpoints, frames, preparations, staged outcomes, and pending payloads; immutable `pi.result`; preservation of the lane inbox; representation exclusivity; and every half-completed recovery prefix. + +For each recovery prefix: close, reopen, drive, and compare against uninterrupted recovery — invoking recovery twice from the initial prefix is **not** sufficient. Every operation kind also covers accept → close before first drive → reopen → drive. At every test-controlled committed lane boundary, compare the published `Lane.state` with a fresh `restoreLaneState` result; divergence is an implementation defect, never silently healed by the next transition. One corruption assertion constructs an `aborted` response with running control directly and requires the consuming transition to reject it as an invariant defect; provider conformance separately proves implementations emit `aborted` only for the supplied signal. + +**Tier B — writer conformance.** Run the public harness against the instrumented-storage decorator (a spy wrapping `Storage.commit()` recording every transaction's writes in order); assert exact write order and content against the Part 3 transaction tables and §5.5 ordering, with faux provider/tool/hook spies interleaving starts/events with commits. It catches: effects before intent; missing awaits of latest update delivery or checkpoint write before `after_tool`; per-frame storage awaits in the provider loop; frame appends out of provider-event order or persisted for `done`/`error`; settlements missing their frame-list delete; `tool_end` before rather than after staging; missing response/usage settlement; checkpoint or frame writes after their child state settled; outcomes not staged before replay becomes impossible; out-of-order tree placement; late result-id reservation; memos or staged/checkpoint/frame values leaked by outcome/terminal cleanup. + +**Tier C — deterministic interleavings.** Every race in §9.2, in both orders, with test-only gated commits and controlled hooks, providers, tools, and timers. + +**Cross-cutting:** + +- **Backend conformance.** One suite, three backends, identical results — including explicit begin/commit/end lane exclusion, commit-without-release, end-without-commit, close waiting for end, checkpoint value set/replace/delete, list append/page/whole-key-delete with identical sequence cursors and reduced frame sequences, and torn-transaction handling exposing no list element. Memory/SQLite retain one current checkpoint; JSONL may retain superseded bytes physically, but compaction (J1, once implemented) must produce identical logical state including preserved list cursors. Internal values are not cloned or shape-validated. Write-order assertions use the instrumented decorator, never a durable log. +- **Attachment and watch.** Construct every durable phase directly and assert minimal open inventory, configured/captured identity inspection without resolution, projection corruption faulting create, presentation corruption faulting watch, exact required/optional ad-hoc reads, no attachment effects, Session mutation inspection, complete snapshots, live-over-durable partial precedence, no historical lifecycle replay, recipient binding at `emitBatch`, and both registration/publication orders without gaps or duplicates. +- **Drive equivalence.** Convenience calls and explicit `accept`/`drive`/`requestAbort` compositions produce byte-identical durable state and equivalent events/results. +- **Deterministic transition control.** Test-only storage gating parks commits without production annotations; controlled hooks, providers, tools, and timers expose effect windows. Each runtime slice tests every durable edge and both orders of each owned race. +- **Effect-start gate.** Cover every item in §4.2's catalog and assert no other path calls `Gate.admit()`. At each integration, force both orders of abort versus admission: abort-first invokes nothing; admission-first gives the complete operation `drive.gate.signal`. Provider tests assert request preparation precedes the check and the same signal reaches Models auth/lazy/provider work. Hook tests treat each aggregate pipeline as one admitted unit. A cancelled drive must enter reconciliation without invoking `before_drive` or `before_run`. +- **Invocation context.** Public operations receive trailing Context; hooks/listeners/callbacks and Session reads/writes preserve it. Cross concurrent calls on one shared receiver and assert independent telemetry/cancellation lineage. Buffered delivery retains the object-identical emitting Context. Context is never written durably. An RPC cancel/disconnect aborts only its reconstructed request signal, and invocation cancellation never writes `cancel_requested`. +- **Signal ownership.** No public surface accepts a standalone operation signal; invocation cancellation arrives through `Context.abortSignal`, operation-owned effect signals remain harness-controlled, and a `before_request` patch carrying a signal has it stripped. Assert by type and by test. +- **Ledger completeness.** Every settled attempt commits its response and its usage; failed structural attempts retain their cost; `getStats()` equals the ledger sum after every commit; a fork starts at zero. +- **Query-plan guards.** `EXPLAIN QUERY PLAN` for `scanBranch` matches §1.7 exactly — no `entries` scan or temporary ordering b-tree. Segment tests assert copied rows are bounded by the newest compaction interval. +- **Transaction discipline.** Assert every SQLite transaction that may write opens with `BEGIN IMMEDIATE`. Add a regression test that reads, lets a second connection commit, then writes — it must succeed, and would fail with `database is locked` under a deferred `BEGIN`. +- **Segment chain soundness.** Build a chain by alternating branch-and-append across several compactions, then assert a full-to-root scan through the chain returns exactly the entries a flat branch would, with no duplicates and no gaps. Both §2.6 rules — resolve-through-base coverage and the chain-searched newest compaction — fail this test when violated, and fail silently without it. + +--- + +# Appendix A — Glossary + +Shorthand vocabulary only; common terms already defined clearly in the body are omitted. + +| Term | Meaning / defined in | +|---|---| +| **Pending entry** | Complete unplaced content in `pi.pending.entry` until placement/cancellation/cleanup (§2.2). | +| **Inbox** | Lane-owned globally ordered tagged queue (§3.11). | +| **Result record** | Immutable `pi.result/{operationId}` terminal disposition (§3.13). | +| **Continuation run** | Fresh ordinary run accepted by structural convenience code when queued conversational input remains (§5.1). | +| **Operation status** | Process-relative observation: `running`, `open`, or `aborting`; idle is no current operation; never predicts registry availability. | +| **Open operation** | Attachment inventory item for a lane with durable current work; not a reservation or continuation policy (§4.4). | +| **Attachment** | Minimal lane/operation projection restore plus open inventory; starts no execution (§4.4). | +| **Drive / drive pass** | The one installed lane-owned process-local pass (§4.1). | +| **Effect** | Anything not pure computation: commit, provider request, tool, hook, timer. A **repeat-sensitive effect** is one whose repetition is observable outside the harness. | +| **Effect gate** | Process-local synchronous arbitration of effect admission against cancellation (§4.2). | +| **Reserved id** | An id minted before content exists (§2.2). | +| **Follower id** | An id minted with its leader's 48-bit timestamp so a call/result group shares one time prefix (§1.2). | +| **Session mutation line / mutation** | The Session-wide serialization point and its explicit read/one-commit capability (§2.8, §4.3). | +| **Control** | Orthogonal per-leaf cancellation flag: `running` or `cancel_requested` (§3.2). | +| **Checkpoint / boundary pass** | Durable resting leaf between turns, and the one-decision procedure that resolves it (§3.12). | +| **Continuation** | Durable answer to "does this run still owe an assistant turn?" (§3.2). | +| **Tool checkpoint** | Optional bounded complete live-update snapshot in `pi.pending.tool_output`; auxiliary, never completion authority (§3.8). | +| **Assistant frame** | Compact replayable pi-ai stream frame in `pi.pending.assistant_frame`; auxiliary, never completion authority (§3.7). | +| **Outcome ready** | Tool call whose finalized result is durable and will never execute again, awaiting source-ordered placement (§3.8). | +| **Invocation memo** | Tool-invocation-scoped durable value for replay-safe memoization (§3.8). | +| **Terminal transaction** | The commit performing the universal terminal suffix (§3.13). | +| **Segment** | A branch-index range referencing an older branch instead of copying it (§2.6). | +| **Precise rewrite** | The administrative copy-retained-and-swap rebuild of a session store (§2.9). | + +# Appendix B — Coding-agent v3-format compatibility + +"v3" here names the legacy coding-agent JSONL session format, not this document. Old v3 files must open unchanged and restore idle. Normalization on load: + +- `custom_message` becomes a custom agent message. +- `label` and `session_info` become session-name/entry-label values (latest by file position wins) and leave the tree. A label target resolves through discarded nodes to its nearest retained ancestor; if resolution produces `null`, the label is skipped. +- Legacy `model_change`, `thinking_level_change`, and `active_tools_change` nodes disappear from the tree. The importer uses the nearest change of each kind on the selected physical main path to write ordinary total main-lane configuration plus idle state before returning; an unsupported nearest value does not fall back to older history. Missing active-tools history normalizes to `[]`; missing or unsupported required model/thinking history leaves main data-only. +- Each retained child of a discarded node is reparented to its nearest retained ancestor. `main`'s tip is the final physical node resolved the same way. +- An old compaction resolves its legacy `firstKeptEntryId` field against its own branch and materializes that range as `retainedTail`. Format 4 never exposes or persists that field. +- Existing `details`, `usage`, and `fromHook` are preserved; absent `fromHook` normalizes to `false`. v3 ISO timestamps convert to Unix milliseconds. +- A v3 `parentSession` path resolves to an available parent header id; otherwise it is preserved as `legacyParentSessionPath`. +- On first format-4 write, append one aggregate adjustment usage row with `details: { source: "v3-import" }`, summing v3 node usage so ledger-derived totals remain unchanged. +- Legacy v3 ids are re-minted at import: each entry gets a UUIDv7 whose prefix is the legacy entry's own timestamp (random tail), preserving time order and §1.2's every-id-is-time-prefixed property. All references the format knows are remapped — parent chains, `main`'s tip, surviving label keys, non-null `fromId`, usage `entryId`. Ids embedded in opaque payloads are not rewritten; the opaque-payload contract (§1.2) covers them. + +Read-only open leaves the file unchanged and computes stats from normalized entry snapshots. The first format-4 write persists normalization through a temporary file and atomic rename over the original path, including the aggregate adjustment so subsequent stats are ledger-derived, and stamps the current `storageVersion` (Part 7). Forking an open legacy-v3 source rejects until a normal non-empty commit persists its normalized format-4 ids. A closed legacy-v3 source is parsed without mutation: tree forks remain available; branch forks require a reconstructable complete configured main lane and use its normalized tip when `entryId` is omitted, while data-only main rejects. + +# Appendix C — Open questions + +1. **Overflow detection remains heuristic.** The normalization specified in §3.7 is authoritative. Preserve the original reason in `errorMessage` for diagnosis. +2. **Pending-payload write amplification.** The deliberate double write (§1.8) is paid only by queued items; measure it for pathological payloads before optimizing (`INSERT … SELECT` placement exists on SQL backends, eager compaction on JSONL). diff --git a/packages/agent/docs/mobile-handoff/01-harness/01-delta/FINDINGS.md b/packages/agent/docs/mobile-handoff/01-harness/01-delta/FINDINGS.md new file mode 100644 index 000000000..396aadcc7 --- /dev/null +++ b/packages/agent/docs/mobile-handoff/01-harness/01-delta/FINDINGS.md @@ -0,0 +1,214 @@ +# 01-delta: known defects and measured findings + +Everything here was found **after** `delta.md` and the prototype implementation beside it were written. None of it is applied to the code in this directory. Production now lives in `packages/chord/src/delta/index.ts`: flush-time dirty tracking resolves D1, and production remeasurement closes D2 without a new API. See [`append-decision.md`](append-decision.md). Each item below remains the original problem, evidence, and candidate-fix record. + +Every number was measured with **`node --experimental-strip-types`**, not `tsx`. +That matters: tsx transpiles through esbuild and inflated the same benchmark by +**2.6x** (183 µs vs 63 µs per write). Do not benchmark this module through a +transpiler. + +--- + +## D1. Interleaved paths defeat coalescing (correctness-adjacent, unbounded memory) + +**Severity: high.** This is the one to fix first. + +### Problem + +`flush()` coalesces only **adjacent** ops. A producer that alternates between two +paths therefore never coalesces anything, and the ops array grows with the number +of writes rather than with the size of the value. + +This is not hypothetical. It is exactly what bash does: it updates the output text +and a byte counter on every chunk. + +```ts +const next = t.state.content[0].text + chunk; +t.state.content[0].text = next.length > CAP ? next.slice(next.length - CAP) : next; +t.state.truncation.totalBytes += chunk.length; // <- the other path +``` + +### Evidence + +50 KB window, 200-byte chunks, no flush in between: + +| held-back writes | ops | bytes | +| --- | --- | --- | +| 1 | 3 | 0.3 KB | +| 100 | 201 | 26 KB | +| **1000** | **2001** | **264 KB** | + +Without interleaving the same workload gives 1 op and 51 KB. The window is +bounded at 50 KB; the ops describing it are not. + +### Why it matters beyond memory + +Any design where the sink **holds back** — rate limiting, backpressure, a slow +consumer — depends on held-back writes costing nothing. They currently cost +everything. See `../../04-tool-output/rate-limiting.md`. + +### Candidate fix + +Fold at **record** time, keyed by path rather than by adjacency: one slot per +path holding at most two ops (`[s]`, `[d]`, `[t]`, `[a]`, or `[t, a]`), emitted in +first-touch order. + +A prototype reached **2 ops / 51 KB** on the same workload. It is not included +here because it was not verified to the standard the rest of this unit meets. + +**Four folding rules, each a specific claim about the vocabulary:** + +1. A later `s` supersedes an earlier `s` on the same path — but **not** an earlier + `d`, which must survive or the key changes position on reinsertion (§5.1). +2. Consecutive `a` merge. Once the merged append is **strictly longer** than the + value, the window has been sliding and the append is the more expensive + spelling, so collapse to `s`. Strictly: while a string is still *growing* + toward its cap the append equals the value, and collapsing there would resend + the whole string every batch. +3. `t` and `a` on one path **commute** — the truncate drops from the front, the + append adds to the end — so a rolling window's `t,a,t,a,...` folds to one of + each. Without this, pairwise merging never fires at all. +4. `t` or `a` following a pending `s` changes nothing: the `s` already means "this + path becomes its current value". + +**The trap that prototype hit.** Slots emit in first-touch order, which loses the +fact that a child write happened *before* a later parent write: + +``` +script : a={x:1}; a.b=99; a={c:2} +producer: {"a":{"c":2}} +replica : {"a":{"c":2,"b":99}} <- WRONG +``` + +A set or delete must invalidate every pending slot **below** its path, at record +time. The existing `verify-dead` style test passed 1190 sequences while this was +broken, because its generator only nested one level. Any test for this must +generate genuinely nested paths with parent overwrites interleaved between child +writes. + +--- + +## D2. `overlap()` was 94.5% of prototype tracker time — closed + +**Historical severity: high. Production decision:** do not add an explicit append/truncate API. The prototype profile was dominated by a slow `startsWith` path; production uses a flattened-slice comparison and measured 17.8–18.7 µs per 200 KB assistant append flush and 2.43–2.46 µs per 50 KB rolling-window flush locally. See [`append-decision.md`](append-decision.md). The original evidence follows. + +### Problem + +The tracker infers "you appended and evicted" by searching two 50 KB strings. +Each new value produced by `next.slice(-CAP)` is a V8 `SlicedString` — a lazy +substring — and `indexOf`/`endsWith` force it to **flatten**, copying 50 KB per +call. + +### Evidence + +CPU profile of a 20 000-write rolling-window loop, node, flush at the end: + +``` + 94.5% overlap + 2.7% (anon) + 1.2% (garbage collector) + 0.5% set + 0.3% emit + 0.0% fold +``` + +And the flattening is directly observable: + +| `overlap()` on a 50 KB window | µs | +| --- | --- | +| the same two strings reused | 37 | +| **a fresh sliced string each call** | **149** | + +An earlier benchmark reported 29 µs because it reused two already-flattened +strings 2000 times — a case that never occurs in the real loop. **Do not benchmark +string operations against a reused fixture.** + +### Candidate fix + +Do not infer what the producer already knows. `ToolOutput` performs the windowing, +so it can emit the ops directly rather than assigning a new string and having the +tracker search for the difference: + +```ts +out.append(chunk); // -> ["t", path, dropped], ["a", path, chunk] +``` + +Two integers — chars appended, chars evicted — describe everything that happened +between any two observations of a bounded window. A prototype of this shape held +**exactly the window** in memory under every load, with no ops array, no overlap +detection and no folding rules at all. + +This does **not** remove `overlap`: it stays for the general case where a producer +assigns a whole new string and the delta genuinely has to be discovered. It +removes it from the hot path. + +### Do not + +Do not "optimise" `overlap` itself before doing the above. Its algorithm is +already correct and bounded (`maxOverlapScan`); the cost is string flattening, +which no probe strategy avoids. + +--- + +## D3. Nested-path proxy overhead is next, and currently invisible + +**Severity: unknown — measure before acting.** + +`overlap` dominates so completely that nothing else shows in a profile. But two +benchmarks disagree in a way that points at path depth: + +| | µs per write | +| --- | --- | +| top-level path (`{ text }`) | 63 | +| nested path (`{ content: [{ text }] }`) | 244 | + +4x for depth alone. Each nested property access wraps a child proxy and builds a +cache key by joining the path, so a read of `content[0].text` allocates on every +access. + +**Re-profile only if production delta cost becomes material.** This may be the next bottleneck or it may be +nothing; the current numbers cannot distinguish them because `overlap` swamps both. + +--- + +## D4. Things measured and found NOT worth doing + +Recorded so nobody repeats the work. All were under **1.5% combined** in the +profile above. + +**A path trie replacing `JSON.stringify(path)` keys.** Genuinely faster in +isolation — 6x on the `LaneSnapshot` shape, 93x at 500 distinct paths — because +stringify cost scales with path depth while a trie walk is one `Map.get` per +segment. But the whole slot layer is 0.5% of runtime. Build it only if a current production profile says so. + +**Trie as storage vs trie as index.** If a trie is ever built, it should map path +to a **position in the ops array**, not hold the ops. Holding them loses first-touch +ordering and forces a sort at flush to rebuild it. Measured difference: none +(1.4x / 0.9x / 1.0x). The argument is structural, not performance. + +**A `maxDepth` heuristic to skip descendant scanning.** Rejected: the flat map was +losing on the common case too, so the heuristic would have hidden a real cost +rather than removed it. + +**Eager string concatenation in the `a`+`a` merge.** Real (it rebuilds a growing +string per write) but worth ~5%. Compare lengths and materialise only when +emitting. + +--- + +## D5. Measurement errors made while producing these numbers + +Listed because each one produced a confident wrong conclusion, and the same traps +are still there. + +- **Benchmarking through `tsx`** — inflated everything 2.6x. +- **Reusing a flattened string** in a string benchmark — hid the real cost 4x. +- **Repetitive fixtures** (`"x".repeat(n)`) — overlap detection hits its candidate + bound and gives up, so the benchmark measures the fallback path, not the real one. + Use varied text. +- **Measuring a cache hit as if it were work** — `Markdown.render()` returns + `cachedLines` when text and width are unchanged, so a repeat-the-same-call loop + measured 0.065 ms for something that costs 0.86 ms. +- **Reasoning about complexity while the constant dominated** — twice. +- **Measuring heap instead of retained size** — a fixture generating 800 x 1 MB + strings reported 40 MB of "growth" that was its own garbage. diff --git a/packages/agent/docs/mobile-handoff/01-harness/01-delta/append-decision.md b/packages/agent/docs/mobile-handoff/01-harness/01-delta/append-decision.md new file mode 100644 index 000000000..ff29bbfeb --- /dev/null +++ b/packages/agent/docs/mobile-handoff/01-harness/01-delta/append-decision.md @@ -0,0 +1,39 @@ +# Decision: no explicit text append/truncate API + +**Status: closed.** Do not add `appendText`, explicit truncate, a text-specific dirty-node kind, or proxy-to-tracker lookup machinery. + +## Why it was considered + +The tracker marks dirty paths and compares them with its accepted baseline at `flush()`. For a growing or rolling string, confirming the relationship can scan the retained string even though the producer already knows it appended or evicted text. + +The original benchmark made this look expensive because its append fast path used `after.startsWith(before)`. V8 walked the producer's cons string character by character. Production Chord now uses: + +```ts +if (after.length > before.length && after.slice(0, before.length) === before) { + // emit append +} +``` + +The slice flattens once and comparison uses the native string path. Flush-time dirty tracking also removed the prototype's retained-op amplification. + +## Local confirmation + +Measured 2026-09-01 against `origin/dev` at `1a7bc80e7`, using `packages/chord/src/delta/index.ts` directly under Node 26.0.0 on an Apple M5 Max. Each workload used 3,000 warmups followed by 11 samples of 10,000 mutation-plus-flush iterations; a second process reproduced the result. + +| Workload | Median µs/flush, run 1 | Median µs/flush, run 2 | +| --- | ---: | ---: | +| 200 KB assistant string, append 8 characters | 18.68 | 17.81 | +| 50 KB rolling window, slide 32 varied characters | 2.46 | 2.43 | +| Transcript push, one small entry | 0.74 | 0.78 | + +The assistant tracker started with 200,000 varied characters, appended `" abcdef"`, and flushed each append. The rolling tracker started with 50,000 varied characters, then assigned `text.slice(32) + chunk` using a distinct 32-character `chunk::durable-stream` value on every flush; every flush was asserted to emit `t` + `a`. The transcript tracker pushed `{ id: "e", text: "message " }` and asserted one `p` per flush. Setup and garbage collection were outside each timed loop. + +At 100 assistant updates per second, the growing-string case consumes about 1.8 ms/s, approximately 0.18% of one core. The measured rolling-window cost is itself too small to justify the abandoned explicit API. + +## Why the API is rejected + +The prototype required a text-specific dirty-node state, proxy-to-tracker lookup, repeated-append/drop folding, and interaction rules for later whole-value replacement. It produced subtle silent failures: one implementation fell through to the ordinary differ while tests still passed, and another drifted when drops reached into earlier appends. + +That complexity does not justify saving microseconds below the surrounding replication, isolation, and rendering costs. Keep ordinary string mutation and the generic Chord op vocabulary. + +Reopen only when a production profile shows delta flush time is a meaningful fraction of a real workload. Re-measure the generic fast path first; a regression there is cheaper and safer to fix than adding a producer-specific API. diff --git a/packages/agent/docs/mobile-handoff/01-harness/01-delta/delta-impl.ts b/packages/agent/docs/mobile-handoff/01-harness/01-delta/delta-impl.ts new file mode 100644 index 000000000..f47056220 --- /dev/null +++ b/packages/agent/docs/mobile-handoff/01-harness/01-delta/delta-impl.ts @@ -0,0 +1,826 @@ +// ───────────────────────────────────────────────────────────────────────────── +// harness/delta — intent-recording change tracking over plain JSON. +// +// Depends on nothing else in the harness. Session storage, the runtime and the +// facet host consume it; keep the arrows pointing that way. +// ───────────────────────────────────────────────────────────────────────────── + +export type JsonValue = null | boolean | number | string | JsonValue[] | { [k: string]: JsonValue }; + +export type Seg = string | number; +export type Path = readonly Seg[]; +export type NonEmptyPath = readonly [Seg, ...Seg[]]; + +/** A path inline, or an id assigned by the encoder on second use. */ +export type PathRef

= P | number; + +/** + * Tuples are the form — in memory, on the wire, on disk. + * + * `r` is the ONLY op that replaces a whole value. `s`/`d`/`a`/`t` cannot target + * the root: the type forbids it. `p` may, and only because a tracked value can + * itself be an array — but a `p` that replaces its entire target is normalised to + * `r`/`s` at record time, so a root `p` is always a partial modification. + * + * `Op` knows nothing about the path dictionary. Interning, id references and + * omitted paths live in `WireOp` and exist only between `encode` and `decode`. + */ +export type Op = + | readonly ["r", JsonValue] + | readonly ["s", NonEmptyPath, JsonValue] + | readonly ["d", NonEmptyPath] + | readonly ["a", NonEmptyPath, string] + | readonly ["t", NonEmptyPath, number] + | readonly ["p", Path, number, number, JsonValue[]]; + +/** + * What crosses a boundary. Adds two compressions and nothing else: + * + * ["#", id, path] defines an id, emitted on a path's SECOND use + * a numeric PathRef references a previously defined id + * a shortened tuple reuses the previous op's path; arity disambiguates + * + * ["r", value] carries no path, so it encodes to itself — which is why isBase + * works unchanged on either vocabulary. + */ +export type WireOp = + | readonly ["r", JsonValue] + | readonly ["s", PathRef, JsonValue] | readonly ["s", JsonValue] + | readonly ["d", PathRef] | readonly ["d"] + | readonly ["a", PathRef, string] | readonly ["a", string] + | readonly ["t", PathRef, number] | readonly ["t", number] + | readonly ["p", PathRef, number, number, JsonValue[]] + | readonly ["p", number, number, JsonValue[]] + | readonly ["#", number, Path]; + +// ─── Classification ────────────────────────────────────────────────────────── + +export const isReplace = (op: Op | WireOp): boolean => op[0] === "r"; + +/** + * A batch begins with a replacement. Flush guarantees `r` is at index 0 or absent, + * so this is exact rather than a heuristic. + */ +export const isBase = (ops: readonly (Op | WireOp)[]): boolean => + ops.length > 0 && ops[0]![0] === "r"; + +// ─── Overlap ───────────────────────────────────────────────────────────────── + +/** + * Longest suffix of `a` that is a prefix of `b`. Probes with indexOf and verifies + * with endsWith, so the hot loops are native. A hand-written KMP is + * asymptotically equivalent and ~47x slower in practice. + * + * Always correct: the returned n satisfies a.endsWith(b.slice(0, n)). + */ +export function overlap(a: string, b: string, scan: number, probe = 64, maxCandidates = 8): number { + if (a.length === 0 || b.length === 0 || scan === 0) return 0; + const tail = a.length > scan ? a.slice(a.length - scan) : a; + + // A probe of length h can only find overlaps of at least h — the head must + // actually occur in `a`. So try a long head first (few candidates, and it + // catches the large overlaps a rolling window produces), then fall back to one + // character, which finds any overlap at the cost of more candidates. + // + // Candidates are bounded because repetitive output — a build log, or any run of + // one character — makes a long head match at thousands of positions, each + // costing a full endsWith. Giving up returns 0, which emits a set: larger, never + // wrong. Unbounded, a 50 KB window of one repeated character took 4.7 s per + // 2000 slides against 90 ms bounded. + for (const h of [Math.min(probe, b.length), 1]) { + const head = b.slice(0, h); + let tried = 0; + for (let k = tail.indexOf(head); k !== -1; k = tail.indexOf(head, k + 1)) { + if (++tried > maxCandidates) break; + const n = tail.length - k; + if (n <= b.length && tail.endsWith(b.slice(0, n))) return n; + } + if (h === 1) break; + } + return 0; +} + +// ─── Tracker ───────────────────────────────────────────────────────────────── + +export interface TrackerOptions { + maxOverlapScan?: number; + coalesce?: boolean; +} + +export interface Tracker { + /** + * Mutate this. Plain TS: assignment, push, splice, delete, all of it. + * + * Assigning to it replaces the whole value: emits `["r", next]` and discards + * ops recorded before it, which describe a value that no longer exists. + * + * `tracker.state = tracker.target` is therefore a checkpoint — it forces a base + * batch without changing anything. Discarding the pending ops is correct + * because the proxy mutates the target directly, so the value already carries + * them. + */ + state: T; + /** The untracked backing object. Reads only — writing here emits nothing. */ + readonly target: T; + flush(): Op[]; + /** + * Clear pending ops and make the next flush a base batch, without changing + * the value. + * + * Recovery replays from the last base batch, so a producer that wants "at most + * N batches to replay" calls this every N. The pending ops are already + * reflected in the value — the proxy mutates the target directly — so + * discarding them loses nothing. + */ + rebase(): void; + /** Discard without emitting. */ + discard(): void; + readonly dirty: boolean; +} + +const isObj = (v: unknown): v is object => v !== null && typeof v === "object"; +const clone = (v: T): T => (isObj(v) ? (structuredClone(v) as T) : v); +const INDEX = /^(?:0|[1-9]\d*)$/; +const norm = (t: object, k: string | symbol): Seg | symbol => + typeof k === "symbol" ? k : Array.isArray(t) && INDEX.test(k) ? Number(k) : k; +const MUTATORS = new Set(["push", "pop", "shift", "unshift", "splice", "sort", "reverse", "fill", "copyWithin"]); + +export function track(root: T, options: TrackerOptions = {}): Tracker { + const scan = options.maxOverlapScan ?? 65_536; + const doCoalesce = options.coalesce ?? true; + + let ops: Op[] = []; + + + const emit = (op: Op) => { + ops.push(op); + }; + + /** + * Reject reserved keys at record time, not just on apply. + * + * `JSON.parse('{"__proto__":{}}')` yields an OWN property, so a producer can + * hold state containing one without doing anything unusual. The tracker would + * then emit a path the applier refuses — state that cannot be replicated. Fail + * where the mistake is, at the write. + */ + const guard = (seg: Seg | symbol): Seg => { + if (typeof seg === "symbol") throw new UnsafePathError(String(seg)); + if (typeof seg === "string" && FORBIDDEN.has(seg)) throw new UnsafePathError(seg); + return seg; + }; + + /** + * A splice covering the whole target is a replacement written the long way. + * Only the interception knows the pre-splice length, so this is a record-time + * rewrite, not something flush can recover. + */ + const spliceOrReplace = (path: Path, before: number, index: number, remove: number, items: JsonValue[]) => { + if (index === 0 && remove === before) { + if (path.length === 0) emit(["r", items]); + else emit(["s", path as NonEmptyPath, items]); + return; + } + emit(["p", [...path], index, remove, items]); + }; + + const cache = new WeakMap>(); + + const wrap = (obj: V, path: Path): V => { + let byPath = cache.get(obj); + if (byPath === undefined) cache.set(obj, (byPath = new Map())); + const cacheKey = path.join("\u0000"); + const hit = byPath.get(cacheKey); + if (hit !== undefined) return hit as V; + + const proxy = new Proxy(obj, { + get(t, k, r) { + if (Array.isArray(t) && typeof k === "string" && MUTATORS.has(k)) { + return (...args: unknown[]) => { + const before = t.length; + const result = (Array.prototype[k as "push"] as Function).apply(t, args); + const items = (xs: unknown[]) => xs.map(clone) as JsonValue[]; + switch (k) { + case "push": spliceOrReplace(path, before, before, 0, items(args)); break; + case "unshift": spliceOrReplace(path, before, 0, 0, items(args)); break; + case "pop": if (before > 0) spliceOrReplace(path, before, before - 1, 1, []); break; + case "shift": if (before > 0) spliceOrReplace(path, before, 0, 1, []); break; + case "splice": { + const raw = Number(args[0] ?? 0); + const index = raw < 0 ? Math.max(0, before + raw) : Math.min(raw, before); + const remove = args.length < 2 ? before - index : Math.max(0, Math.min(Number(args[1]), before - index)); + spliceOrReplace(path, before, index, remove, items(args.slice(2))); + break; + } + // sort/reverse/fill/copyWithin permute wholesale; intent is not a splice. + default: + if (path.length === 0) emit(["r", clone(t) as JsonValue]); + else emit(["s", path as NonEmptyPath, clone(t) as JsonValue]); + } + return result; + }; + } + const v = Reflect.get(t, k, r); + return isObj(v) ? wrap(v, [...path, guard(norm(t, k))]) : v; + }, + + set(t, k, v, r) { + // `length` is a real mutation but not a document location. + if (Array.isArray(t) && k === "length") { + const before = t.length; + const next = Number(v); + if (next < before) spliceOrReplace(path, before, next, before - next, []); + else if (next > before) emit(["p", [...path], before, 0, Array(next - before).fill(null)]); + return Reflect.set(t, k, v, r); + } + + const at = [...path, guard(norm(t, k))] as unknown as NonEmptyPath; + + if (v === undefined) { // JSON has no undefined + emit(["d", at]); + return Reflect.deleteProperty(t, k); + } + + const prev = (t as Record)[k]; + if (typeof prev === "string" && typeof v === "string") { + if (prev === v) return true; + if (v.length > prev.length && v.startsWith(prev)) { + emit(["a", at, v.slice(prev.length)]); // fast path, no probe + } else { + const ov = overlap(prev, v, scan); + if (ov === 0) emit(["s", at, v]); + else { + emit(["t", at, prev.length - ov]); + if (v.length > ov) emit(["a", at, v.slice(ov)]); + } + } + } else { + assertJsonValue(v); + emit(["s", at, clone(v) as JsonValue]); + } + return Reflect.set(t, k, v, r); + }, + + deleteProperty(t, k) { + emit(["d", [...path, guard(norm(t, k))] as unknown as NonEmptyPath]); + return Reflect.deleteProperty(t, k); + }, + }); + + byPath.set(cacheKey, proxy); + return proxy as V; + }; + + let state = wrap(root, []); + // The first flush is always a base batch. A consumer starts with nothing, so + // a stream that opens with deltas has nothing to apply them to — and that + // fails at runtime, in the consumer, rather than where the mistake is. + let forceBase = true; + + return { + get state() { return state; }, + // A setter, so the obvious thing works. Without it `tracker.state = next` + // silently swaps the proxy for a plain object and stops tracking. + set state(next: T) { + ops = []; + root = next; + state = wrap(root, []); + forceBase = true; + }, + get target() { return root; }, + + // Same effect as assigning the current value back, minus the re-wrap: the + // value has not changed, so the proxy cache is still valid. + rebase() { ops = []; forceBase = true; }, + + + get dirty() { return ops.length > 0; }, + discard() { ops = []; }, + flush() { + if (forceBase) { + forceBase = false; + ops = []; + return [["r", clone(root) as JsonValue]]; + } + if (ops.length === 0) return []; + let out = ops; + ops = []; + + // Everything before a replacement is dead. + const lastReplace = out.findLastIndex(isReplace); + if (lastReplace > 0) out = out.slice(lastReplace); + + if (doCoalesce) out = coalesce(dropDead(out)); + + return out; + }, + }; +} + +const samePath = (a: Op, b: Op): boolean => { + const pa = pathOf(a); + const pb = pathOf(b); + if (pa === undefined || pb === undefined) return false; + return JSON.stringify(pa) === JSON.stringify(pb); +}; + +function pathOf(op: Op): Path | undefined { + return op[0] === "r" ? undefined : op[1]; +} + +/** + * Drop ops that a later op makes unreachable. + * + * Ops apply in order, so an op is dead if a later one overwrites the whole + * subtree it lives in. Only `s`, `d` and `r` do that — `a`, `t` and `p` modify + * what is already there, so they never dominate. + * + * O(n * d): walk backwards keeping a map of overwritten paths, and for each op + * test its own prefixes against that map. `d` is the path depth, which is small + * and bounded. The obvious formulation — compare each op against every + * dominator — is O(n^2 * d) and degrades on exactly the wide-flush input this + * pass exists to clean up. Measured: 12-14 microseconds per op, flat from 100 to + * 5000 ops, and 10001 redundant writes collapse to 2 emitted ops. + * + * OBJECT KEY ORDER IS NOT PRESERVED. A producer that does + * `set c; set a; delete c; set c` leaves `c` last, because deleting and + * reinserting moves it. The replica receives only the surviving `set c`, applies + * it to a base where `c` never existed, and `c` lands first. Values are + * identical — verified over 1823 randomised sequences with zero value + * mismatches — but the serialised forms can differ. + * + * This is fine because nothing compares a replica against its producer: the + * consumer holds the state and nobody diffs it back. The one visible case is a + * consumer that iterates keys for display, where two views could order rows + * differently. It requires a delete-and-recreate of the same key within one + * flush. Do not hash or content-address a replicated value. + */ +function dropDead(ops: Op[]): Op[] { + const overwritten = new Map(); + const kept: Op[] = []; + + for (let i = ops.length - 1; i >= 0; i--) { + const op = ops[i]!; + if (op[0] === "r") { + kept.push(op); + overwritten.clear(); + overwritten.set("", "s"); // the root is now overwritten + continue; + } + + const path = opPath(op); + if (path === undefined) { kept.push(op); continue; } // arity form, or an interned id + + let dead = overwritten.has(""); + let key = ""; + for (let k = 0; k < path.length && !dead; k++) { + key += `\u0000${String(path[k])}`; + const kind = overwritten.get(key); + if (kind === undefined) continue; + // A `d` at exactly a later `s`'s path is NOT dead. Dropping it leaves the + // key in its original slot, while the producer's delete-then-set moved it + // to the end: same value, different serialisation. + if (k === path.length - 1 && op[0] === "d" && kind === "s") break; + dead = true; + } + if (dead) continue; + + kept.push(op); + if (op[0] === "s" || op[0] === "d") overwritten.set(key, op[0]); + } + return kept.reverse(); +} + +/** Total, because `Op` paths are always inline. */ +function opPath(op: Op): Path | undefined { + return op[0] === "r" ? undefined : op[1]; +} + +/** Merge adjacent ops describing the same path. Lossless. */ +function coalesce(ops: Op[]): Op[] { + const out: Op[] = []; + for (const op of ops) { + const last = out[out.length - 1]; + if (last !== undefined && samePath(last, op)) { + if (last[0] === "a" && op[0] === "a") { + out[out.length - 1] = ["a", last[1], `${last[2]}${op[2]}`]; + continue; + } + if (last[0] === "s" && op[0] === "s") { // later write wins + out[out.length - 1] = op; + continue; + } + if (last[0] === "s" && op[0] === "a" && typeof last[2] === "string") { + out[out.length - 1] = ["s", last[1], `${last[2]}${op[2]}`]; + continue; + } + } + out.push(op); + } + return out; +} + +// ─── Path safety ───────────────────────────────────────────────────────────── + +/** + * Segments that reach the prototype chain. + * + * `JSON.parse` is safe on its own — it makes `__proto__` an own property. What is + * not safe is `parent[key] = value`, which is exactly what an applier does, and + * paths are data: `["s", ["__proto__", "isAdmin"], true]` pollutes + * `Object.prototype` for the whole process. + * + * Ops arrive from a facet, a plugin compartment, or a tool whose details may echo + * model output, so none of it is trusted input. + */ +const FORBIDDEN = new Set(["__proto__", "constructor", "prototype"]); + +export class UnsafePathError extends Error { + // Not a parameter property: Node's --experimental-strip-types rejects those, + // and these files are meant to run under it directly. + readonly segment: Seg; + constructor(segment: Seg) { + super(`unsafe path segment: ${String(segment)}`); + this.segment = segment; + this.name = "UnsafePathError"; + } +} + +/** + * Rejects anything JSON.stringify would not round-trip. + * + * structuredClone is NOT a JSON check: it clones a Map happily, and the op then + * carries `{}` while the producer keeps a real Map — producer and replica diverge + * silently, forever. + */ +export function assertJsonValue(value: unknown, seen = new Set()): void { + if (value === null) return; + const t = typeof value; + if (t === "string" || t === "boolean") return; + if (t === "number") { + if (!Number.isFinite(value as number)) throw new TypeError("non-finite number is not JSON"); + return; + } + if (t !== "object") throw new TypeError(`${t} is not a JsonValue`); + if (seen.has(value)) throw new TypeError("cyclic value is not JSON"); + seen.add(value); + if (Array.isArray(value)) { + for (const v of value) assertJsonValue(v, seen); + } else { + const proto = Object.getPrototypeOf(value); + if (proto !== Object.prototype && proto !== null) { + throw new TypeError(`${(value as object).constructor?.name ?? "object"} is not a JsonValue`); + } + if (Object.getOwnPropertySymbols(value).length > 0) { + throw new TypeError("symbol keys are not JSON"); + } + for (const v of Object.values(value as object)) assertJsonValue(v, seen); + } + seen.delete(value); +} + +/** + * Verb, arity and payload shape for a **decoded** op: paths inline, no `#`, no + * short forms. `apply` uses this. + * + * Validating `Op` against the wire grammar would be laxer than the type: a + * two-element `["s", value]` would pass, and `apply` would then read the value as + * a path. Each vocabulary gets the validator that matches it. + */ +export function assertValidOp(op: unknown): void { + if (!Array.isArray(op) || op.length === 0) throw new TypeError("op is not a tuple"); + switch (op[0]) { + case "r": if (op.length !== 2) throw new TypeError("r arity"); assertJsonValue(op[1]); return; + case "s": if (op.length !== 3) throw new TypeError("s arity"); assertPathArg(op[1]); assertJsonValue(op[2]); return; + case "d": if (op.length !== 2) throw new TypeError("d arity"); assertPathArg(op[1]); return; + case "a": if (op.length !== 3 || typeof op[2] !== "string") throw new TypeError("a shape"); assertPathArg(op[1]); return; + case "t": if (op.length !== 3 || !Number.isInteger(op[2])) throw new TypeError("t shape"); assertPathArg(op[1]); return; + case "p": { + if (op.length !== 5) throw new TypeError("p arity"); + assertPathArg(op[1]); + if (!Number.isInteger(op[2]) || op[2] < 0) throw new TypeError("p index"); + if (!Number.isInteger(op[3]) || op[3] < 0) throw new TypeError("p remove"); + if (!Array.isArray(op[4])) throw new TypeError("p items"); + for (const v of op[4]) assertJsonValue(v); + return; + } + // Silently skipping an unknown verb is how a newer producer's op vanishes. + default: throw new TypeError(`unknown op verb: ${String(op[0])}`); + } +} + +function assertPathArg(p: unknown): void { + if (!Array.isArray(p)) throw new TypeError("path is not an array"); + assertSafePath(p as Path); +} + +/** The same, for the wire grammar: ids and short forms are legal here. */ +export function assertValidWireOp(op: unknown): void { + if (!Array.isArray(op) || op.length === 0) throw new TypeError("op is not a tuple"); + const [verb] = op as unknown[]; + const okRef = (r: unknown): void => { + if (typeof r === "number") { + if (!Number.isInteger(r) || r < 0) throw new TypeError("bad path id"); + return; + } + // A string is not a path. Unchecked, `"a".slice(0, -1)` is `""`, so it + // resolves to the ROOT and writes there — a path that is not a path, accepted. + if (!Array.isArray(r)) throw new TypeError("path is not an array"); + assertSafePath(r as Path); + }; + switch (verb) { + case "r": if (op.length !== 2) throw new TypeError("r arity"); assertJsonValue(op[1]); return; + case "s": if (op.length === 3) { okRef(op[1]); assertJsonValue(op[2]); } + else if (op.length === 2) assertJsonValue(op[1]); + else throw new TypeError("s arity"); return; + case "d": if (op.length === 2) okRef(op[1]); else if (op.length !== 1) throw new TypeError("d arity"); return; + case "a": if (op.length === 3) { okRef(op[1]); if (typeof op[2] !== "string") throw new TypeError("a value"); } + else if (op.length === 2) { if (typeof op[1] !== "string") throw new TypeError("a value"); } + else throw new TypeError("a arity"); return; + case "t": if (op.length === 3) { okRef(op[1]); if (!Number.isInteger(op[2])) throw new TypeError("t count"); } + else if (op.length === 2) { if (!Number.isInteger(op[1])) throw new TypeError("t count"); } + else throw new TypeError("t arity"); return; + case "p": { + const [i, r, items] = op.length === 5 ? [op[2], op[3], op[4]] : op.length === 4 ? [op[1], op[2], op[3]] : []; + if (items === undefined) throw new TypeError("p arity"); + if (op.length === 5) okRef(op[1]); + if (!Number.isInteger(i) || (i as number) < 0) throw new TypeError("p index"); + if (!Number.isInteger(r) || (r as number) < 0) throw new TypeError("p remove"); + if (!Array.isArray(items)) throw new TypeError("p items"); + for (const v of items) assertJsonValue(v); + return; + } + case "#": { + if (op.length !== 3 || !Number.isInteger(op[1]) || !Array.isArray(op[2])) throw new TypeError("# shape"); + assertSafePath(op[2] as Path); + return; + } + // Silently skipping an unknown verb is how a newer producer's op vanishes. + default: throw new TypeError(`unknown op verb: ${String(verb)}`); + } +} + +export function assertSafePath(path: Path): void { + for (const seg of path) { + if (typeof seg === "string") { + if (FORBIDDEN.has(seg)) throw new UnsafePathError(seg); + } else if (!Number.isInteger(seg) || seg < 0) { + throw new UnsafePathError(seg); + } + } +} + +/** + * An index may address an existing element or append exactly one past the end. + * + * This is not an arbitrary cap — it is what keeps the value a `JsonValue`. A + * sparse array does not survive a JSON round trip: holes serialise to `null` and + * return as real properties, so `arr[7] = x` on a length-3 array already produces + * state a replica cannot match. Rejecting the write is more honest than silently + * diverging. + * + * It also removes the denial of service it would otherwise permit: + * `["s", ["xs", 4294967290], 1]` allocates a 4.29-billion-entry array from one op. + * Growth stays possible and stays proportional — the tracker already emits + * `arr.length = n` as a splice of explicit nulls, whose op size grows with the + * gap, so a large growth costs a large op rather than a small one. + */ +function assertIndexInRange(parent: readonly unknown[], index: number): void { + if (index > parent.length) throw new UnsafePathError(index); +} + +// ─── Applier ───────────────────────────────────────────────────────────────── + +export class PathError extends Error { + readonly path: Path | number; + constructor(path: Path | number) { + super(`unresolvable path: ${JSON.stringify(path)}`); + this.path = path; + this.name = "PathError"; + } +} + +/** + * Apply ops to a plain mutable value. Returns the value, because `r` replaces it + * outright and cannot be done in place. + * + * Takes decoded ops. Path ids and omitted paths are a wire concern — run + * `decode` first if the ops came from a boundary. + */ +export function apply(target: T, ops: readonly Op[]): T { + let root = target as unknown as JsonValue; + + for (const op of ops) { + assertValidOp(op); + + if (op[0] === "r") { + // Adopted, not copied. The consumer owns the batch it was handed. + // + // Fanning one batch out to several consumers in-process therefore makes + // their replicas alias each other. That is an ownership rule, not a + // defect: copy the batch at the fan-out point, or let each consumer + // decode its own. A batch that crosses a real boundary is already + // distinct, because serialisation produces fresh objects. + root = op[1]; + continue; + } + + const path = op[1]; + assertSafePath(path); + + if (op[0] === "p") { + const target_ = path.length === 0 ? root : resolve(root, path); + if (!Array.isArray(target_)) throw new PathError(path); + target_.splice(op[2], op[3], ...op[4]); + continue; + } + + // s/d/a/t can never target the root — the type forbids it. + const parent = resolve(root, path.slice(0, -1)) as Record; + const key = path[path.length - 1]!; + if (Array.isArray(parent) && typeof key === "number") assertIndexInRange(parent, key); + // defineProperty rather than assignment: a setter inherited from the prototype + // chain would otherwise run on write. + const write = (value: JsonValue) => { + Object.defineProperty(parent, key, { value, writable: true, enumerable: true, configurable: true }); + }; + const read = (): unknown => (Object.hasOwn(parent, key) ? parent[key] : undefined); + switch (op[0]) { + case "s": write(op[2]); break; + case "d": Array.isArray(parent) ? (parent as unknown as JsonValue[]).splice(Number(key), 1) : delete parent[key]; break; + case "a": write(`${String(read() ?? "")}${op[2]}`); break; + case "t": write(String(read() ?? "").slice(op[2])); break; + } + } + return root as unknown as T; +} + +function resolve(root: JsonValue, path: Path): JsonValue { + let node: JsonValue = root; + for (const seg of path) { + if (!isObj(node)) throw new PathError(path); + // Own properties only: an inherited getter must not run, and a walk must not + // escape the value into the prototype chain. + if (!Object.hasOwn(node, seg as PropertyKey)) throw new PathError(path); + node = (node as Record)[seg]!; + } + if (!isObj(node)) throw new PathError(path); + return node; +} + +// ─── Codec ─────────────────────────────────────────────────────────────────── +// +// Path interning and arity omission, per `delta.md` §4. These live between the +// tracker and a boundary; `Op` and `apply` know nothing about them. +// +// ONE PAIR PER STREAM. The table spans an entire subscription or file: a path +// interned in batch 3 is referenced in batch 40. A second consumer that +// subscribes at batch 40 has never seen the definition, so it needs its own +// encoder — sharing one across consumers hands the late subscriber ids it cannot +// resolve. A base batch does not rescue it, because `["r", value]` carries no +// path refs and leaves the table empty. + +const pathKey = (path: Path): string => path.join("\u0000"); + +export interface Encoder { + encode(ops: readonly Op[]): WireOp[]; +} + +/** + * Intern on SECOND use. A definition costs more than the path it replaces, so + * interning on first use loses on the many paths written exactly once — + * measured worse than no interning at all (`delta.md` §4.1). + */ +export function encoder(): Encoder { + const seen = new Set(); + const ids = new Map(); + let nextId = 0; + let previous: string | undefined; // last path in THIS batch + + return { + encode(ops) { + // Arity omission is scoped to a batch. Letting it span batches would make + // a batch's first op depend on the previous batch's last one, so a reader + // that skips or reorders a batch decodes into the wrong path. Ids are the + // only cross-batch state, and the dictionary makes those explicit. + previous = undefined; + const out: WireOp[] = []; + for (const op of ops) { + if (op[0] === "r") { + out.push(op); + // A base batch is a RECOVERY POINT: a reader replays from the last one + // with a fresh decoder. So everything after it must be self-contained. + // Keeping ids across a replacement emits references to definitions the + // reader never saw — recovery fails with an unresolvable path id. + seen.clear(); + ids.clear(); + nextId = 0; + previous = undefined; + continue; + } + const path = op[1]; + const key = pathKey(path); + + // Same path as the previous op: drop the ref entirely. + if (key === previous) { + switch (op[0]) { + case "s": out.push(["s", op[2]]); break; + case "d": out.push(["d"]); break; + case "a": out.push(["a", op[2]]); break; + case "t": out.push(["t", op[2]]); break; + case "p": out.push(["p", op[2], op[3], op[4]]); break; + } + continue; + } + + let ref: PathRef = path; + const existing = ids.get(key); + if (existing !== undefined) { + ref = existing; + } else if (seen.has(key)) { + const id = nextId++; + ids.set(key, id); + out.push(["#", id, path]); // second use: define, then reference + ref = id; + } else { + seen.add(key); // first use: inline + } + + switch (op[0]) { + case "s": out.push(["s", ref as PathRef, op[2]]); break; + case "d": out.push(["d", ref as PathRef]); break; + case "a": out.push(["a", ref as PathRef, op[2]]); break; + case "t": out.push(["t", ref as PathRef, op[2]]); break; + case "p": out.push(["p", ref, op[2], op[3], op[4]]); break; + } + previous = key; + } + return out; + }, + }; +} + +export interface Decoder { + decode(wire: readonly WireOp[]): Op[]; +} + +export function decoder(): Decoder { + const paths = new Map(); + + return { + decode(wire) { + let previous: Path | undefined; // scoped to the batch, as in encode + const out: Op[] = []; + for (const op of wire) { + assertValidWireOp(op); + if (op[0] === "#") { + assertSafePath(op[2]); + paths.set(op[1], op[2]); + continue; + } + if (op[0] === "r") { + out.push(op); + previous = undefined; + continue; + } + + // Arity tells us whether a ref is present: the short forms omit it. + const short = + (op[0] === "d" && op.length === 1) || + (op[0] !== "d" && op[0] !== "p" && op.length === 2) || + (op[0] === "p" && op.length === 4); + + let path: Path; + if (short) { + if (previous === undefined) throw new PathError([]); + path = previous; + } else { + const ref = op[1] as PathRef; + if (typeof ref === "number") { + const resolved = paths.get(ref); + if (resolved === undefined) throw new PathError(ref); + path = resolved; + } else { + path = ref; + } + previous = path; + } + + switch (op[0]) { + case "s": out.push(["s", path as NonEmptyPath, (short ? op[1] : op[2]) as JsonValue]); break; + case "d": out.push(["d", path as NonEmptyPath]); break; + case "a": out.push(["a", path as NonEmptyPath, (short ? op[1] : op[2]) as string]); break; + case "t": out.push(["t", path as NonEmptyPath, (short ? op[1] : op[2]) as number]); break; + case "p": { + const [i, r, items] = short + ? [op[1] as number, op[2] as number, op[3] as JsonValue[]] + : [op[2] as number, op[3] as number, op[4] as JsonValue[]]; + out.push(["p", path, i, r, items]); + break; + } + } + } + return out; + }, + }; +} diff --git a/packages/agent/docs/mobile-handoff/01-harness/01-delta/delta.bench.ts b/packages/agent/docs/mobile-handoff/01-harness/01-delta/delta.bench.ts new file mode 100644 index 000000000..0c2f6b239 --- /dev/null +++ b/packages/agent/docs/mobile-handoff/01-harness/01-delta/delta.bench.ts @@ -0,0 +1,113 @@ +/** + * Not a test — a guard against the cliffs found during implementation. Reports + * units you can reason about: cost per WRITE, and bytes on the wire relative to + * bytes consumed. + * + * npx vitest bench delta.bench.ts + * + * The headline finding: cost is per write, not per byte. A 100-byte chunk costs + * about the same as a 1 KB chunk, because the fixed cost is the window slice and + * the overlap probe. So chunk size — set by the exec env's coalescing interval, + * not by this package — decides both throughput and wire size. + */ +import { bench, describe } from "vitest"; +import { decoder, encoder, track } from "./delta-impl.ts"; + +const WORDS = "the quick brown fox jumps over lazy dog while considering strategies".split(" "); +const text = (n: number): string => { + let s = ""; + while (s.length < n) s += `${WORDS[Math.floor(Math.random() * WORDS.length)]} `; + return s.slice(0, n); +}; + +const CAP = 50_000; +const WRITES = 500; +const FLUSH_EVERY = 20; + +/** One write = one chunk handed to the sink. Returns nothing; bench times it. */ +function drive(initial: string, chunkSize: number, cap = CAP): void { + const t = track({ out: initial }); + for (let i = 0; i < WRITES; i++) { + const next = t.state.out + text(chunkSize); + t.state.out = next.length > cap ? next.slice(next.length - cap) : next; + if (i % FLUSH_EVERY === FLUSH_EVERY - 1) t.flush(); + } + t.flush(); +} + +/** + * Measured on the reference machine, 500 writes each. Reproduced here so a + * regression is legible rather than just "slower". + * + * append only, 100 B 127 µs/write 8k writes/s 101% of input on wire + * rolling, 100 B 176 µs/write 6k writes/s 135% of input on wire + * rolling, 1 KB 68 µs/write 15k writes/s 104% of input on wire + * rolling, 8 KB 366 µs/write 3k writes/s 31% of input on wire + * pathological, 100 B 307 µs/write 3k writes/s 727% of input on wire + * + * Two things to notice. + * + * 135% at 100-byte chunks: past the cap every write emits `truncate` + `append`, + * and two op envelopes exceed a 100-byte payload — we put MORE on the wire than we + * consumed. At 8 KB it is 31%. Flush granularity is therefore a size lever, not + * just an update-rate lever, and the exec env's coalescing is what keeps chunks + * off the bad end of that curve. + * + * The pathological row is bounded only because the overlap candidate scan is + * capped. Unbounded it was 4.7 s for 2000 writes against 93 ms; the cap makes it + * give up and emit a set — larger, never wrong. + */ +describe("rolling window, 50 KB cap", () => { + bench("100 B chunks", () => { drive(text(CAP), 100); }); + bench("1 KB chunks", () => { drive(text(CAP), 1024); }); + bench("8 KB chunks", () => { drive(text(CAP), 8192); }); + bench("pathological: one repeated character", () => { drive("x".repeat(CAP), 100); }); +}); + +describe("below the cap", () => { + // The dominant case for most tools: startsWith fast path, no probe at all, + // and one append per flush rather than a truncate/append pair. + bench("append only, never fills", () => { drive("", 100, Number.MAX_SAFE_INTEGER); }); +}); + +// ── Codec: what path interning and arity omission actually save ──────────── +// +// Reported as wire bytes relative to inline paths, on three shapes: +// - one hot path (a rolling tool-output window): omission does the work +// - a few alternating paths (lane state): interning does the work +// - many distinct paths: neither helps, and definitions cost a little +describe("codec", () => { + const streams = { + "one hot path": () => { + const t = track({ content: [{ type: "text", text: "" }] }); + return Array.from({ length: 200 }, (_, i) => { + (t.state.content[0] as { text: string }).text += `line ${i} of build output\n`; + return t.flush(); + }); + }, + "four alternating paths": () => { + const t = track({ a: "", b: "", c: "", d: "" }); + return Array.from({ length: 200 }, (_, i) => { + for (const k of ["a", "b", "c", "d"] as const) t.state[k] += `${i}`; + return t.flush(); + }); + }, + "200 distinct paths": () => { + const root: Record = {}; + for (let i = 0; i < 200; i++) root[`f${i}`] = 0; + const t = track(root); + return Array.from({ length: 5 }, (_, r) => { + for (let i = 0; i < 200; i++) t.state[`f${i}`] = r + 1; + return t.flush(); + }); + }, + }; + + for (const [label, build] of Object.entries(streams)) { + bench(label, () => { + const enc = encoder(); + const dec = decoder(); + for (const ops of build()) dec.decode(enc.encode(ops)); + }); + } +}); diff --git a/packages/agent/docs/mobile-handoff/01-harness/01-delta/delta.examples.ts b/packages/agent/docs/mobile-handoff/01-harness/01-delta/delta.examples.ts new file mode 100644 index 000000000..b7a1bee64 --- /dev/null +++ b/packages/agent/docs/mobile-handoff/01-harness/01-delta/delta.examples.ts @@ -0,0 +1,133 @@ +/** + * Runnable illustrations of the two things that are easy to get wrong. + * + * node --experimental-strip-types delta.examples.ts + * + * Nothing here is a test — `delta.test.ts` covers the same ground with + * assertions. This exists to be read. + */ +import { apply, decoder, encoder, isBase, track, type Op } from "./delta-impl.ts"; + +const show = (label: string, ops: Op[]) => { + console.log(` ${label.padEnd(22)} ${ops.length} op(s), base=${isBase(ops)}`); + for (const op of ops) console.log(` ${trunc(JSON.stringify(op), 66)}`); +}; +const trunc = (s: string, n: number) => (s.length <= n ? s : `${s.slice(0, n)}…`); + +// ───────────────────────────────────────────────────────────────────────── +// 1. Replacing the whole value +// ───────────────────────────────────────────────────────────────────────── +console.log("\n1. REPLACING THE WHOLE VALUE\n"); + +const before = { user: { id: "u1", name: "ada" }, items: ["a", "b"], note: "n".repeat(120) }; +const after = { user: { id: "u2", name: "bob" }, items: ["x"], note: "m".repeat(120) }; + +{ + // The wrong way. `state` is a property of the TRACKER, not of the tracked + // object. Without a setter this swaps the proxy for a plain object and every + // later mutation is silently untracked — no error, no ops. + // + // The implementation defines `state` as a setter precisely so this works. + const t = track>(structuredClone(before)); + t.flush(); // drain the opening base batch + t.state.items = ["changed first"]; // recorded… + t.state = structuredClone(after); // …then discarded: it described a dead value + show("assign to .state", t.flush()); + + t.state.user = { id: "u3" }; // still tracked afterwards + show(" and still tracked", t.flush()); +} + +{ + // Dead-op elimination: a field written three times emits one op, and a parent + // replaced after its child emits one. Object key order is not preserved by + // this pass — see delta.md §5.1. + const t = track>(structuredClone(before)); + t.flush(); // drain the opening base batch + t.state.items = ["first"]; + t.state.items = ["second"]; + t.state.user.name = "dropped"; + t.state.user = { id: "u9" }; + show("4 writes, 2 survive", t.flush()); +} + +// ───────────────────────────────────────────────────────────────────────── +// 2. Why checkpointing exists +// ───────────────────────────────────────────────────────────────────────── +console.log("\n2. RECOVERY LENGTH\n"); + +/** + * A bash-shaped workload: output arrives continuously and the sink writes + * durably on an interval. Each flush is one batch appended to a list. + * On resume the reader folds back to the last BASE batch, so that distance is + * the cost of recovery. + */ +function simulate(writes: number, checkpointEvery: number | null) { + const CAP = 50_000; + const t = track({ out: "" }); + const isBaseBatch: boolean[] = []; + + for (let i = 0; i < writes; i++) { + const next = `${t.state.out}[${i}] cc -c src/file${i}.c -o build/file${i}.o\n`; + t.state.out = next.length > CAP ? next.slice(next.length - CAP) : next; + + if (checkpointEvery !== null && i % checkpointEvery === checkpointEvery - 1) t.rebase(); + + const ops = t.flush(); + if (ops.length > 0) isBaseBatch.push(isBase(ops)); + } + + const lastBase = isBaseBatch.lastIndexOf(true); + return { written: isBaseBatch.length, replay: isBaseBatch.length - lastBase - 1 }; +} + +for (const [label, every] of [["never", null], ["rebase() every 50", 50]] as const) { + const r = simulate(500, every); + console.log(` ${label.padEnd(18)} ${r.written} batches written → ${r.replay} to replay on recovery`); +} + +console.log(` + Nothing produces a base batch on its own: flush() emits ops, and a replacement + happens only when the producer asks for one. So a stream of appends stays a + stream of deltas indefinitely, and recovery grows without bound. + + Two callers need rebase(): + - pendingToolOutput, during a long-running command. Without it a ten-minute + build leaves hundreds of batches to fold on resume. + - a facet host on resubscribe. facets.md §9.2 requires the first batch of a + subscription to be a base batch, and the host must be able to produce one + on demand. +`); + +// ───────────────────────────────────────────────────────────────────────── +// 3. A replica always agrees +// ───────────────────────────────────────────────────────────────────────── +console.log("3. PRODUCER AND REPLICA AGREE\n"); +{ + // The whole loop: tracker -> encode -> [boundary] -> decode -> apply. + const t = track>(structuredClone(before)); + const enc = encoder(); + const dec = decoder(); + let replica: unknown; + + const step = (label: string, mutate: () => void) => { + mutate(); + const ops = dec.decode(enc.encode(t.flush())); + if (ops.length === 0) return; + // No base-batch branch: apply handles `r` by replacing, and tolerates an + // undefined target because a root op never reads it. + replica = apply(replica, ops); + const same = JSON.stringify(replica) === JSON.stringify(t.target); + console.log(` ${label.padEnd(22)} ${ops.length} op(s) ${isBase(ops) ? "BASE " : "delta"} matches: ${same}`); + }; + + step("opening base batch", () => {}); + + step("append to a string", () => { (t.state.note as string); t.state.note = `${t.state.note}!`; }); + step("push", () => { (t.state.items as string[]).push("c"); }); + step("nested set", () => { (t.state.user as Record).name = "cy"; }); + step("delete", () => { delete t.state.note; }); + step("whole replacement", () => { t.state = structuredClone(after); }); + step("rebase", () => { t.rebase(); }); +} +console.log(); diff --git a/packages/agent/docs/mobile-handoff/01-harness/01-delta/delta.md b/packages/agent/docs/mobile-handoff/01-harness/01-delta/delta.md new file mode 100644 index 000000000..506f7e4c9 --- /dev/null +++ b/packages/agent/docs/mobile-handoff/01-harness/01-delta/delta.md @@ -0,0 +1,617 @@ +# Delta Tracking and the Op Vocabulary + +> **Production status:** landed in `packages/chord/src/delta/index.ts`, with tests in `packages/chord/test/delta.test.ts`. Chord owns the dependency-free `Op`/`WireOp`, tracker, applier, codec, and validation boundary; Session storage, the Harness, and facets consume it. The implementation and tests beside this document are historical prototype and benchmark evidence, not production source. +> +> The landed tracker computes deltas at `flush()` from a dirty tree and baseline rather than retaining one op per mutation. That resolves FINDINGS D1. Production remeasurement also closes FINDINGS D2: the generic string path is below surrounding replication/rendering costs, so an explicit append/truncate API is rejected ([decision](append-decision.md)). + +One mechanism covers assistant partials, tool output, tool details, lane state, and arbitrary facet state — on the wire and in durable storage. + +```bash +node "$(git rev-parse --show-toplevel)/node_modules/vitest/dist/cli.js" --run test/delta.test.ts +# run from packages/chord +``` + +## 1. Why not an existing library + +Immer, Valtio, Mutative and Colyseus all **record effects, not intent**. A string +is a leaf, so `text += chunk` is one write of the whole new string; there is no +representation in which the library could know you appended. `unshift` is the +same story — the engine shifts every index, so each is a write. + +Measured, all four: + +| operation | what they emit | +| --- | --- | +| `text += "x"` on 40 KB | `replace` carrying 40 KB | +| `arr.unshift(x)` on 100 items | ~100 index writes | +| `arr.pop()` | a write to a `length` path, which is not a document location | + +Immer's own pitfalls page states the guarantee: patches are correct and +**explicitly not minimal**. Colyseus documents the same array weakness — removing +the first of 20 items costs 38 extra bytes. + +Diffing base against result ourselves does not fix it either. A differ sees two +values, not intent. Prefix comparison catches pure growth, but a rolling window +that drops from the front *and* appends is neither a prefix nor a suffix of the +old value; measured on 5000 elements the fallback was +`{index: 0, remove: 5000, items: 5000}` — a full replace, in exactly the case the +optimisation existed for. + +So the win is not a blind whole-value differ. The tracker records which paths and array operations became dirty, then compares only those subtrees against its accepted baseline at `flush()`. + +## 2. Ops + +**Tuples are the form. Everywhere. There is no keyed variant.** + +```ts +export type Seg = string | number; +export type Path = readonly Seg[]; +/** Non-empty: `s`/`d`/`a`/`t` cannot target the root. Enforced by the type. */ +export type NonEmptyPath = readonly [Seg, ...Seg[]]; + +export type Op = + | readonly ["r", JsonValue] // replace the value + | readonly ["s", NonEmptyPath, JsonValue] // set + | readonly ["d", NonEmptyPath] // delete + | readonly ["a", NonEmptyPath, string] // append + | readonly ["t", NonEmptyPath, number] // truncate, in chars + | readonly ["p", Path, number, number, JsonValue[]] // splice: index, remove, items +``` + +Interning, id references and omitted paths are **not** here — they live in +`WireOp` and exist only between `encode` and `decode` (§4). + +**`r` is the only op that replaces a whole value.** It is an op rather than a +frame kind for the same reason `a` and `t` exist: they are specialisations of `s` +that are smaller and state intent. + +Do not encode a replacement as a set at the root path. Base-batch detection is a +**correctness boundary** — recovery stops there (§9) — so it must be a token +comparison, `ops[0]?.[0] === "r"`, and not a path inspection. A path-inspecting +predicate has to handle the two-element arity form, where `op[1]` is a value +rather than a path, and misclassifies `["s", []]`. + +**Only `p` may target the root**, and only because a tracked value can itself be +an array — `entries.push(x)` on a root array is `["p", [], 3, 0, [x]]`. A `p` +covering its entire target is normalised at flush time to `r` (root) or `s` +(nested), so a root `p` is always a partial modification. The other four verbs +take a `NonEmptyPath`: `["s", [], v]` and `["d", []]` do not typecheck. + +**Do not add a keyed in-memory shape with a codec at the serialization boundary.** +Two representations of one thing means two appliers, two size estimates, and a +conversion nobody needs. Ops are tuples in memory, on the wire, and on disk; +readability is what a debug formatter is for. + +Six verbs. No `move`, `copy` or `test`: the first two are size optimisations for +a case we do not have, and `test` belongs to a conflict model we do not have, +since there is exactly one authoritative writer. + +`chars` counts UTF-16 code units, matching `String.prototype.slice`. **Not bytes.** +Byte caps are a producer concern and never cross a boundary. + +The format is JSON-Patch-*shaped*, not RFC 6902 conformant. That costs nothing: +RFC 6902 has had no successor since 2013, still has the same six ops, and nothing +anywhere adds a string splice. Immer was never conformant either. + +## 3. The tracker + +State is **plain TypeScript**. No containers, no handles, no schema, no +decorators. You mutate normally. + +```ts +const t = track(laneView); +t.state.operation.streamingMessage.content[0].text += delta; // -> append +t.state.transcript.push(entry); // -> splice +t.state.tools[0].details.failures.push({ name, msg }); // -> splice +delete t.state.config.model; // -> delete +const ops = t.flush(); +``` + +Three mechanisms, one per case: + +**Objects** — ordinary `set` / `deleteProperty` traps. + +**Arrays — mutator methods intercepted in the `get` trap.** `arr.push(x)` passes +through `get(arr, "push")` first, so the tracker returns its own function that +records `splice(len, 0, [x])` and then delegates. Intent is captured *before* the +engine performs its index writes, which is why `unshift` is one op rather than +O(n). This is the thing no other library does. + +**Strings — dirty marking on the `set` trap, overlap detection at `flush()`.** Strings are primitives, so there is nothing to intercept. Given the accepted `prev` and final `next`, find the longest suffix of `prev` that is a prefix of `next`: + +``` +overlap === prev.length -> append(next.slice(overlap)) +overlap > 0 -> truncate(prev.length - overlap) + append(rest) +overlap === 0 -> set(next) +``` + +Always correct, because the overlap is **verified**, not guessed. The rolling +window works: + +```ts +s.out = s.out.slice(4) + "!!!"; +// ["truncate", ["out"], 4] +// ["append", ["out"], "!!!"] +``` + +### 3.1 Overlap must not use a hand-written KMP + +A KMP failure function is asymptotically right and **47x slower in practice**, +because it runs character by character in JS. Measured on 2000 window slides over +a 50 KB string: 2870 ms for KMP, 61 ms for a native probe. + +The probe: `indexOf` a short head of `next` within `prev`, verify each candidate +with `endsWith`. Both are native. Verified identical to KMP across 200 window +slides. + +A `startsWith` fast path runs first, so a pure append — the dominant case — skips +the probe entirely. + +### 3.2 Rules the implementation must hold + +**Inserted values are adopted; emitted payloads are cloned at flush.** Callers may retain read-only references but must not mutate adopted objects outside the tracker. `flush()` clones payloads and advances a cloned accepted baseline, so emitted ops never alias producer state. + +**`x = undefined` normalises to `delete`.** JSON has no `undefined`, and a `set` +with a missing value is indistinguishable from a lost value after a +`JSON.stringify` round trip. + +**`arr.length = n` is translated, never emitted as a path.** Shrink becomes a +splice, grow becomes a splice of nulls. `length` is a real mutation but not a +document location — this is the bug Immer ships (issue 208). Dropping it silently, +as an early prototype did, means `arr.length = 0` never reaches the replica. + +**Numeric array keys are normalised to numbers.** Proxy traps deliver `"0"`; +paths carry `0`. Valtio ships the string form and it is worse for both size and +comparison. + +**Proxies are cached per object and path**, so `s.a === s.a` and proxies are not +rebuilt on every access. + +**Scope is `JsonValue` only, and it must be CHECKED, not assumed.** +`structuredClone` is not a JSON check — it clones `Map`, `Set`, `Date` and +`RegExp` happily, and the result then differs from what `JSON.stringify` produces, +so producer and replica diverge silently. Assert the value at record time and +reject symbol keys. See §7.4 for what leaks without it. + +**Exactly two ops reach the root**, and the applier's root branch must handle +both: `r` always, and `p` when the tracked value is itself an array. +`s`/`d`/`a`/`t` cannot, by type, which makes the branch's fallthrough provably +unreachable. Handling only `r` there passes every hand-written test and fails on +the first randomised sequence that produces a root splice. + +**Two writes to one address inside a transaction chain.** When several writes to +the same address are emitted in one commit, each must be recorded against the +previous one's result, not against pre-transaction state. The applier replays them +in order; anything else applies the second against a stale base. This did not fire +in the workload that surfaced it — it would have corrupted silently when it did. + +### 3.2.1 Replacing the whole value + +Assigning to `state` replaces it: + +```ts +tracker.state = next; // emits ["r", next]; discards ops recorded before it +``` + +`state` **must be a setter on the tracker**, not a plain property. Without one, +`tracker.state = next` swaps the proxy for a plain object and every later +mutation is silently untracked — no error, no ops, `target` unchanged. It is also +the first thing anyone tries. + +Prior ops are discarded because they describe a value that no longer exists. + +Only a *full* replacement collapses to `r`. Rewrite part of the value and the ops +survive, because they are genuinely cheaper: + +``` +partial rewrite 2 op(s), base=false + ["s",["user"],{"id":"u2","name":"bob"}] + ["s",["items"],["x"]] +``` + +### 3.2.2 The first flush is a base batch + +`track()` opens a stream, and a consumer starts with nothing, so the first flush +always emits `["r", value]` — carrying any mutations made before it: + +```ts +const t = track({ x: 0, l: [] }); +t.state.x = 100; +t.state.l.push("xyz"); +t.flush(); // [["r", { x: 100, l: ["xyz"] }]] +t.state.x = 101; +t.flush(); // [["s", ["x"], 101]] +``` + +Requiring the producer to remember a `rebase()` first would fail at runtime, in +the consumer, far from the mistake. + +### 3.2.3 Forcing a base batch later + +```ts +tracker.rebase(); // next flush is ["r", value]; value unchanged +``` + +Discarding the pending ops is correct: the proxy mutates the target directly, so +the value already carries them. `tracker.state = tracker.state` has the same effect in the landed implementation, but `rebase()` states the intent directly. + +**Nothing produces a base batch on its own.** `flush()` emits ops; a replacement +happens only when the producer asks for one. So a stream of appends stays a +stream of deltas indefinitely. + +Recovery replays from the last base batch (§9). From `delta.examples.ts`, a +bash-shaped workload of 500 durable writes into a 50 KB rolling window: + +| | batches written | to replay on recovery | +| --- | --- | --- | +| never | 500 | **499** | +| `rebase()` every 50 | 500 | 0 | + +Two callers need this: + +- **A durable sink.** `pendingToolOutput` accumulates one batch per checkpoint for + as long as a command runs. A `make -j8` running for ten minutes otherwise leaves + hundreds of batches to fold on resume. Checkpointing every N makes "at most N + batches to replay" a policy rather than an accident. +- **A facet host on resubscribe.** `facets.md` §9.2 requires the first batch of + every subscription to be a base batch, and a consumer that sees a gap + resubscribes to get one. The host has to be able to produce one on demand rather + than wait for one to happen by chance. + +### 3.3 Known gaps + +- `sort` / `reverse` / `fill` / `copyWithin` mark the array dirty and emit the resulting structural/index changes rather than preserving the producer's method intent. Add a dedicated op only if a measured workload needs one. +- A manual index-shift loop (`for (…) a[i] = a[i+1]`) can still cost O(n) sets. Correct, not minimal, and unavoidable — the producer genuinely wrote every element. +- Rolling-window string assignment still runs overlap discovery at flush, deliberately: production measurements do not justify text-specific tracker state or API surface ([decision](append-decision.md)). + +### 3.4 Constraints the string algorithm must hold + +Each of these is cheap to get wrong in a way that passes hand-written tests. + +**A fixed-length probe cannot find an overlap shorter than the probe.** The head +must actually occur in `a`: `"abcdefgh"` -> `"defghxyz"` overlaps by 5, and a +64-character head cannot occur in an 8-character string. Try a long head first, +then fall back to a one-character head. A test using only 50 KB strings, where +overlaps are huge, will pass against a broken implementation. + +**Bound the candidate scan.** Repetitive output — a build log, or a run of one +character — makes a long head match at thousands of positions, each costing a full +`endsWith`. Unbounded, 2000 slides over a 50 KB window take 4.7 s against 93 ms +bounded. Exceeding the bound returns 0, which emits a set: larger, never wrong. + +**Property tests before anything depends on this.** 3000 random mutation +round-trips find the root-splice hole in §3.2 on the first run; the hand-written +cases do not. `delta.test.ts` is the suite to port. + +## 4. The codec + +Two vocabularies, not one. + +`Op` is what the tracker produces and `apply` consumes. **Paths are always +inline.** It knows nothing about a dictionary. + +`WireOp` is what crosses a boundary. It adds exactly two compressions: + +``` +["#", id, path] defines an id, emitted on a path's SECOND use +a numeric PathRef references a previously defined id +a shortened tuple reuses the previous op's path; arity disambiguates +``` + +`encoder().encode(ops): WireOp[]` and `decoder().decode(wire): Op[]` are the only places either exists. Keeping them out of `Op` means `apply` has no id resolution, no `#` +case, and no previous-path state — three branches removed from the hot path — +and dirty-tree generation always works with inline paths before encoding. + +`["r", value]` carries no path, so it encodes to itself. That is why `isBase` +works unchanged on either vocabulary. + +### 4.1 Two rules that are easy to get wrong + +**One encoder/decoder pair per stream.** The id table spans a whole subscription +or file: a path interned in batch 3 is referenced in batch 40. A second consumer +that subscribes at batch 40 has never seen the definition, so it needs its own +encoder. Sharing one across consumers hands the late subscriber ids it cannot +resolve, and a base batch does not rescue it — `["r", value]` carries no refs and +leaves the table empty. + +**Reset the table on a base batch.** A reader replays from the *last* base batch +with a fresh decoder, so everything after one must be self-contained. Carrying +ids across a replacement emits references to definitions the reader never saw: + +``` + 4 : [["a",0,"x4"],["a",1,"y4"]] + 5 BASE: [["r",{…}]] + 6 : [["a",0,"x6"]] <- id 0 was defined in batch 1 + -> RECOVERY FAILED: PathError - unresolvable path: 0 +``` + +**Arity omission is scoped to a batch.** Letting it span batches makes a batch's +first op depend on the previous batch's last one, so a reader that skips or +reorders a batch decodes into the wrong path. Ids are the only cross-batch state, +and the dictionary makes those explicit. + +### 4.2 Intern on second use, not first + +A definition costs more than the path it replaces, so interning on first use +loses on every path written exactly once — and most are. Measured on a bulk +workload: **255.5 KB first-use versus 179.6 KB with no interning at all.** + +### 4.3 Measured + +Wire bytes against inline paths, round-trip verified: + +| stream | inline | wire | saved | +| --- | --- | --- | --- | +| one hot path (a rolling tool-output window) | 11,290 B | 7,538 B | **33.2%** | +| four alternating paths (lane state) | 14,160 B | 11,032 B | **22.1%** | +| 200 distinct paths | 16,455 B | 16,035 B | 2.6% | + +Omission does the work in the first case, interning in the second. In the third +neither helps and the definitions cost a little — which is the case second-use +interning exists to bound. + +## 5. Flush emits ops, and drops the dead ones + +`flush()` computes ops for the dirty paths against the last accepted baseline. It does not retain one op per mutation, so repeated and interleaved writes stay bounded by changed state rather than write count. There is no size comparison or replacement heuristic: a replacement is something the producer asks for by assigning `state` or calling `rebase()`. + +An earlier design compared op bytes against the value's serialised size and +replaced when ops were larger. It was removed. Measured against emitting ops +unconditionally, on six workloads, it changed the output in two — both requiring +hundreds of distinct paths to change in a single flush. Neither the harness nor +the replication path does that: `LaneSnapshot` is folded one event at a time, and +the widest case (`run_end`) touches four fields. The rule cost a per-flush +comparison, a running size estimate, and an invalidation rule for the ops that +could not maintain it. + +### 5.1 Dirty-tree collapse + +The landed tracker records only which subtrees are dirty. At flush it compares each dirty subtree's accepted baseline with its final value and emits the surviving structural change. Repeated writes to one field collapse naturally; alternating writes to two fields retain two dirty paths rather than one op per mutation; a parent replacement subsumes dirty descendants where the final comparison permits it. `packages/chord/test/delta.test.ts` pins the interleaved rolling-window case at at most three ops after 1,000 alternating writes. + +This replaces the prototype's backwards dead-op pass and adjacent-only coalescer. Do not port those algorithms into Chord: flush-time generation is the D1 fix. + +> **Object key order is not a replicated invariant.** Values round-trip, but delete-and-reinsert activity within one flush can produce a different insertion order on a replica. Do not hash or content-address a replicated value, and sort explicitly where display order matters. + +## 6. There is no frame type + +The logical batch is `Op[]`; transport and durable storage carry the statefully encoded `WireOp[]`. Do not wrap either batch. + +**`seq` does not belong in the payload.** It would defend against a lossy +transport we do not have: a durable list element already carries `seq` from +storage, an in-process callback cannot skip, and SSE either delivers in order or +breaks — and a break means resubscribe, which means a base batch. A second copy of +storage's `seq` could only disagree with the first. The SSE binding stamps `id:`, +which is where transport metadata belongs. + +**Nothing else needs wrapping either.** A `kind` discriminator is redundant once a +replacement is an op (§2), and the address already says which value a batch +belongs to. What would remain is a struct around an array. + +A producer batch is just `Op[]`, and its encoded boundary form is `WireOp[]`. A **base batch** is one whose first op is `r`; +`isBase(ops)` is `ops[0]?.[0] === "r"`, exact rather than heuristic because flush +guarantees `r` appears at index 0 or not at all (§5). + +Everything a frame used to do is now done by something that already existed: + +| was on the frame | now | +| --- | --- | +| `seq` | the list element's `seq`, or the SSE `id:` | +| `kind: "replace"` | the `r` op | +| which value it belongs to | the address | +| "this is a snapshot" | the `"base"` storage tag | + +**Resubscription is a base batch plus buffered batches** — the same thing the lane +adapter already does for the harness: snapshot first, then whatever accumulated +while the client caught up. There is no `Last-Event-ID` and no cross-subscription +resume; that would need a retained op log, which §5 deliberately does not keep. + +A consumer that sees a gap, fails to resolve a path, or connects cold takes one +path: ask for a base batch. That is why there is no `Rebase` type. + +The cost, stated plainly: **a consumer cannot detect staleness from the payload +alone.** It relies on the transport reporting a break. For SSE and in-process that +is sound. If a lossy or multiplexed transport ever appears, sequencing goes on that +transport's envelope — not back into the ops. + +## 7. Safety + +Ops arrive from a facet, a plugin compartment, or a tool whose details may echo +model output. **None of it is trusted input**, and the boundary where untrusted +data meets trusted machinery is where this design has to hold. + +The reference point is CVE-2025-55182 — RCE in React Server Components, CVSS 10.0, +exploited in the wild. Their Flight protocol is a compact tagged wire format that +reconstructs structure, so the resemblance is real. The failure was *"fails to +validate the structure correctly... treats the fake object as genuine"*: a forged +Chunk resolved as a Promise and exposed internal state containing gadgets to reach +`Function`. + +**We are structurally safer, and not because of diligence:** + +| | Flight | ops | +| --- | --- | --- | +| can describe runtime objects | yes — Chunks resolve as Promises | no | +| can reference code or modules | yes — client components | no | +| values | arbitrary object graphs | `JsonValue` | +| worst case from a forged payload | RCE | corrupted replica state | + +Flight *must* reference code; that is its job. An op can only put a `JsonValue` at +a path, so there is no first link in a gadget chain. This is why an attacker cannot +plant a non-resolving `then` on `Object.prototype`: a data-only `then` is not +callable, and `await` resolves normally when `then` is not callable. + +> **Rule: no op may name something the host resolves.** Not a mutation name, not a +> component id, not a module reference. Mutation names were removed for an +> unrelated reason (§8); this is the second and better reason never to reintroduce +> them, because it is the ingredient that made Flight exploitable. + +### 7.1 Paths are the dangerous part + +`JSON.parse` is safe on its own — `{"__proto__":{}}` becomes an *own* property. +The hazard is `parent[key]`, which is exactly what applying a path does. + +A single key is not enough. `x["__proto__"] = v` swaps *x's own* parent, which is +local. It takes a **walk** — `x["__proto__"]["polluted"] = v` — to reach the shared +prototype, and a path is precisely a walk. + +`constructor` is worse than `__proto__`, because `({}).constructor.constructor` is +`Function`. That ladder is closed here only because op values cannot be functions; +it is closed properly by rejecting the segment. + +**Reserved path segments: `__proto__`, `constructor`, `prototype`.** Rejected at +record time *and* at apply time, including through an interned path id. + +They are reserved as *segments*, not as values. An object with a literal +`"__proto__"` key replicates fine as a whole value; only a path walking *through* +it is refused. This is a genuine restriction on what is mutable — document it, do +not pretend it away. + +Two further measures on the same hazard: + +- **Write with `Object.defineProperty`**, never assignment, so an inherited setter + cannot run. +- **Resolve own properties only** (`Object.hasOwn`), so a walk cannot escape into + the prototype chain and an inherited getter cannot fire. This blocks + `constructor` a second time, since it is inherited. + +What prototype pollution actually buys an attacker here is narrower than it first +appears: **it flips defaults, it does not overwrite explicit values.** An own +property shadows. So `{name:"bob", isAdmin:false}` is unaffected; an option bag read +as `opts.skipSandbox` is not. Our own `ShellExecOptions`, `ShellOutputCaptureOptions`, +`ListReadOptions` and `TrackerOptions` are exactly that shape. Note also that +`"x" in {}` and `const {x = false} = {}` both lie under pollution — which is why +the applier uses `Object.hasOwn`. + +### 7.2 Array indices + +An index may address an existing element or append exactly one past the end. + +This is not an arbitrary cap; it is what keeps the value a `JsonValue`. A sparse +array does not survive a JSON round trip — holes serialise to `null` and return as +real properties — so `arr[7] = x` on a length-3 array already produces state a +replica cannot match. Rejecting the write is more honest than diverging. + +It removes a denial of service as a side effect rather than as its purpose: +`["s",["xs",4294967290],1]` would otherwise allocate a 4.29-billion-entry array +from one op. Growth stays available and stays proportional, because +`arr.length = n` is emitted as a splice of explicit nulls whose op size grows with +the gap. + +### 7.3 Validate op structure on decode + +This is the RSC lesson applied to us. A decoder must not trust tuple shape. +Measured against an unvalidated applier: + +| malformed op | result | +| --- | --- | +| `["p",["xs"],0,0,"not-an-array"]` | string spread into the array: `["n","o","t",…]` | +| `["s","a",9]` — path is a string | accepted; `"a".slice(0,-1)` is `""`, so it wrote at the root | +| `["ZZZ",["a"],9]` | silently ignored, replica diverges with no error | + +Validate: verb is known, arity matches the verb, a path is an array of strings and +non-negative integers, `p` carries integer index and count plus an array of items, +`#` defines an array path. An unknown verb is an **error**, not a no-op — silently +skipping it is how a newer producer's op disappears and a replica drifts. + +**One validator per vocabulary.** `assertValidOp` guards `apply` and rejects ids, +short forms and `#`; `assertValidWireOp` guards `decode` and permits them. +Validating a decoded op against the wire grammar is laxer than its own type: a +two-element `["s", value]` would pass, and `apply` would then read the value as a +path. + +### 7.4 The tracker's cage leaks on types, not shape + +A facet cannot forge an op: it mutates plain objects and the tracker builds the +tuples, so shape is well-formed by construction. Values and keys are another +matter, and `structuredClone` is **not** a JSON check. + +| what a facet writes | what happens | +| --- | --- | +| a function | throws (`DataCloneError`) | +| a BigInt, a cycle | throws | +| `new Map([[1,2]])` | op carries `{}`, producer keeps a real Map — **silent divergence** | +| `new Date(0)` | op carries an ISO string, producer keeps a Date | +| `state[Symbol("s")] = 1` | emits `["s",[null],1]` — **a malformed op from our own tracker** | + +So the tracker must check what it is given, at record time, where the mistake is: + +- **Reject symbol keys.** They are not path segments. +- **Assert `JsonValue` on every recorded value.** Reject `Map`, `Set`, `Date`, + `RegExp`, typed arrays and class instances explicitly. The scope restriction in + §3.2 was a comment; it has to be a check. + +A getter on state is safe — the trap records the computed result — and a facet +value that *looks* like an op is nested inside `["s", path, value]` and can never +be read as a top-level op, because nothing flattens. + +### 7.5 Applier + +```ts +export function apply(target: T | undefined, ops: readonly Op[]): T; +``` + +Six verbs, no domain knowledge, no library, no tool code, no registry lookup, and +**no path table** — ids and omitted paths are resolved by `decode` before `apply` +sees anything (§4). + +It returns the value rather than mutating in place, because `r` replaces it +outright. +Runs against a **plain mutable object the consumer owns**. It must not be pointed +at a value produced by Immer, which deep-freezes and would throw. + +A page of code in any language, which is the property that keeps non-JS consumers +viable. Colyseus ships decoders for C#, Lua and Haxe on the same basis. + +## 8. What this removes from the codebase + +Concrete deletions, not simplifications in principle: + +- **Immer**, from the harness and from the facet layer. See §1 for why a + patch-producing library cannot serve this purpose. +- **Per-type reducers.** There is one applier, `apply(target, ops)`, with no + domain knowledge and no registry. Nobody writes a fold for `ToolOutputState` or + for a plugin's shape. +- **`detailMutations` and `initialDetails`**, and with them the argument that + `details: unknown` forces special handling. A structural tracker never needs + the type. +- **`Rebase` as a reducer return value.** A fold that cannot apply leaves state + alone and the host sends a base batch. +- **Mutation names on the wire**, and therefore mutation-name version skew. Names + never cross a boundary, so adding or renaming one is not a breaking change. + +And three things not to build, each of which looks reasonable until §1: + +- **A blind whole-value differ.** The landed tracker compares only dirty subtrees against its accepted baseline; it does not scan unrelated state. +- **A keyed op shape plus a codec.** Tuples are the form everywhere (§2). +- **A frame wrapper.** What travels is `Op[]` (§6). + +## 9. Durable form + +A tracked value is stored as a **list of encoded `WireOp[]` batches**, one appended per flush. Base batches carry the storage tag `"base"`; one stateful decoder per value decodes them before `apply`. + +Recovery reads backwards to the last base batch and applies forward: + +```ts +readList(address, { order: "desc", stopAtTag: "base", limit: 100 }) +``` + +`stopAtTag` is a stop condition *within a page*: if no base batch is in the page, +the consumer pages again with the cursor. The tag lives on the storage record +beside `seq`, never inside the value, so storage never parses ops. See +[scopes.md](../02-scopes/scopes.md) §11. + +This is what makes "a replacement truncates recovery" (§5) real rather than +aspirational — a reader stops at the last base batch instead of replaying from +the beginning. + +## 10. Open questions + +- Prefix interning (a trie over path heads) if a workload emerges with many + distinct paths sharing long prefixes. Second-use interning (§4) removes the + pathological case; this would go further. +- Whether array reordering needs a dedicated op; the landed tracker currently marks the array dirty and emits the resulting structural/index changes. +- Cross-language replicas: the applier is a page of code in any language, but the + interning tables and the wire framing are not specified for a non-JS consumer. + +Address interning in the JSONL log is the same trick as path interning, applied +one layer up over `namespace` + `key`. It is **not** an open question — it is +specified in [scopes.md](../02-scopes/scopes.md) §12 and is a separate dictionary from this one. diff --git a/packages/agent/docs/mobile-handoff/01-harness/01-delta/delta.test.ts b/packages/agent/docs/mobile-handoff/01-harness/01-delta/delta.test.ts new file mode 100644 index 000000000..a0e09c1cb --- /dev/null +++ b/packages/agent/docs/mobile-handoff/01-harness/01-delta/delta.test.ts @@ -0,0 +1,563 @@ +import { describe, expect, it } from "vitest"; +import { apply, assertJsonValue, assertValidOp, assertValidWireOp, decoder, encoder, isBase, overlap, track, type Op, type WireOp } from "./delta-impl.ts"; + +const PAD = "p".repeat(400); // see "adaptive emission": a delta must be able to win + +describe("tracker: intent", () => { + it("records an append, not a replacement", () => { + const t = track({ s: "", pad: PAD }); + t.flush(); // drain the base batch + t.state.s += "ab"; + t.state.s += "cd"; + const ops = t.flush(); + expect(ops).toEqual([["a", ["s"], "abcd"]]); + expect(apply({ s: "", pad: PAD }, ops)).toEqual({ s: "abcd", pad: PAD }); + }); + + it("recovers truncate+append from a rolling window", () => { + // The case every effect-recording library degrades to a whole-value set. + const t = track({ s: "abcdefgh", pad: PAD }); + t.flush(); // drain the base batch + t.state.s = t.state.s.slice(3) + "xyz"; + const ops = t.flush(); + expect(ops).toEqual([["t", ["s"], 3], ["a", ["s"], "xyz"]]); + expect(apply({ s: "abcdefgh", pad: PAD }, ops)).toEqual({ s: "defghxyz", pad: PAD }); + }); + + it("records array intent as one splice", () => { + const t = track({ xs: [1, 2], pad: PAD }); + t.flush(); // drain the base batch + t.state.xs.push(3); + expect(t.flush()).toEqual([["p", ["xs"], 2, 0, [3]]]); + }); + + it("normalises undefined to a delete", () => { + const t = track({ a: 1, pad: PAD }); + t.flush(); // drain the base batch + (t.state as Record).a = undefined; + expect(t.flush()).toEqual([["d", ["a"]]]); + }); +}); + +describe("tracker: root ops", () => { + it("splices a value that is itself an array", () => { + const t = track([1, 2, 3, PAD]); + t.flush(); // drain the base batch + t.state.push(4); + const ops = t.flush(); + expect(ops).toEqual([["p", [], 4, 0, [4]]]); + expect(apply([1, 2, 3, PAD], ops)).toEqual([1, 2, 3, PAD, 4]); + }); + + it("normalises a splice covering the whole root to a replacement", () => { + const t = track([1, 2, 3]); + t.flush(); // drain the base batch + t.state.splice(0, 3, 9); + const ops = t.flush(); + expect(ops).toEqual([["r", [9]]]); + expect(isBase(ops)).toBe(true); + }); + + it("normalises a nested splice-all to a set, not a replacement", () => { + const t = track({ xs: [1, 2, 3], pad: PAD }); + t.flush(); // drain the base batch + t.state.xs.splice(0, 3, 9); + expect(t.flush()).toEqual([["s", ["xs"], [9]]]); + }); + + it("normalises length = 0 on the root", () => { + const t = track([1, 2, 3]); + t.flush(); // drain the base batch + t.state.length = 0; + const ops = t.flush(); + expect(ops).toEqual([["r", []]]); + expect(apply([1, 2, 3], ops)).toEqual([]); + }); +}); + +describe("replacing the whole value", () => { + // The proxy cannot observe `tracker.state = next`: `state` belongs to the + // tracker, not to the tracked object, so assigning to it swaps the proxy for a + // plain object and silently stops tracking. Replacement must be a method. + it("assigning to state emits a base batch and keeps tracking", () => { + const t = track>({ p: 1, q: PAD }); + t.flush(); // drain the base batch + t.state.p = 2; + t.state = { r: 9, s: "new" }; + const ops = t.flush(); + expect(ops).toEqual([["r", { r: 9, s: "new" }]]); + expect(isBase(ops)).toBe(true); + expect(apply({}, ops)).toEqual(t.target); + + t.state.r = 10; // the new value is tracked + expect(t.flush()).toEqual([["s", ["r"], 10]]); + }); + + it("assigning to state discards ops recorded before it", () => { + // They describe a value that no longer exists. + const t = track>({ p: 1, q: PAD }); + t.flush(); // drain the base batch + t.state.p = 2; + t.state = { z: 1 }; + expect(t.flush()).toEqual([["r", { z: 1 }]]); + }); + + it("a partial rewrite keeps its ops", () => { + const t = track>({ a: "x".repeat(200), b: "y".repeat(200) }); + t.flush(); // drain the base batch + t.state.a = "p"; + const ops = t.flush(); + expect(isBase(ops)).toBe(false); + expect(ops).toEqual([["s", ["a"], "p"]]); + }); + + it("rebase() forces a base batch without changing the value", () => { + // This is the checkpoint. Recovery replays from the last base batch, so a + // producer must be able to bound that rather than depend on the adaptive + // size rule, which compares each flush independently. + const t = track>({ p: 1, pad: PAD }); + t.flush(); // drain the base batch + t.rebase(); + const ops = t.flush(); + expect(isBase(ops)).toBe(true); + expect(apply({}, ops)).toEqual(t.target); + }); + + it("rebase() applies once, not to every later flush", () => { + const t = track>({ p: 1, pad: PAD }); + t.flush(); // drain the base batch + t.rebase(); + t.flush(); + t.state.p = 2; + expect(t.flush()).toEqual([["s", ["p"], 2]]); + }); +}); + +describe("the first flush", () => { + it("is always a base batch", () => { + // A consumer starts with nothing, so a stream opening with deltas has + // nothing to apply them to. Requiring the producer to remember would fail + // at runtime, in the consumer, rather than where the mistake is. + const t = track>({ x: 0 }); + t.state.x = 100; + const ops = t.flush(); + expect(isBase(ops)).toBe(true); + expect(ops).toEqual([["r", { x: 100 }]]); // carries mutations made before it + }); + + it("is followed by deltas", () => { + const t = track>({ x: 0, pad: PAD }); + t.flush(); + t.state.x = 1; + expect(t.flush()).toEqual([["s", ["x"], 1]]); + }); + + it("emits nothing when untouched", () => { + // An unmutated tracker has nothing to say; the base batch is owed to a + // consumer, and there is no consumer until someone flushes. + const t = track>({ x: 0 }); + expect(t.flush()).toEqual([["r", { x: 0 }]]); + expect(t.flush()).toEqual([]); + }); +}); + +describe("apply and fan-out", () => { + it("adopts an `r` payload rather than copying it", () => { + // The consumer owns the batch it was handed. Copying every base batch + // defensively doubles the memory of the one op carrying the whole value. + // + // The rule that follows: do not hand one batch to two in-process + // consumers. This test pins the aliasing so nobody "fixes" it by adding a + // clone back. + const t = track>({ n: 0 }); + const batch = t.flush(); + const a = apply(undefined, batch) as Record; + const b = apply(undefined, batch) as Record; + a.n = 1; + expect(b.n).toBe(1); + }); + + it("does not alias the producer", () => { + const t = track>({ x: 0 }); + const replica = apply(undefined, t.flush()) as Record; + t.state.x = 999; + t.flush(); + expect(replica.x).toBe(0); + }); + + it("folds a whole stream without a base-batch branch", () => { + // `apply` handles `r` by replacing, and tolerates an undefined target, + // so a consumer needs no isBase check and no clone of its own. + const t = track>({ x: 0, l: [] as string[] }); + const enc = encoder(); + const dec = decoder(); + let replica: unknown; + const send = () => { replica = apply(replica, dec.decode(enc.encode(t.flush()))); }; + + t.state.x = 100; + t.state.l.push("xyz"); + send(); + t.state.x = 101; + send(); + expect(replica).toEqual(t.target); + }); +}); + +describe("dead-op elimination", () => { + // Ops apply in order, so an op is dead if a later one overwrites the whole + // subtree it lives in. Only s/d/r dominate; a/t/p modify what is there. + const run = (mutate: (s: any) => void, init: any = { a: { b: 1 }, x: 1, y: 2 }) => { + const t = track(structuredClone(init)); + t.flush(); // drain the base batch + mutate(t.state); + return t.flush(); + }; + + it("collapses repeated writes to one field", () => { + expect(run((s) => { s.x = 1; s.x = 2; s.x = 3; })).toEqual([["s", ["x"], 3]]); + }); + + it("drops a write superseded by a later, non-adjacent one", () => { + expect(run((s) => { s.x = 10; s.y = 20; s.x = 30; })) + .toEqual([["s", ["y"], 20], ["s", ["x"], 30]]); + }); + + it("drops a child write when the parent is replaced after it", () => { + expect(run((s) => { s.a.b = 99; s.a = { c: 5 }; })).toEqual([["s", ["a"], { c: 5 }]]); + }); + + it("keeps a child write that follows the parent replacement", () => { + expect(run((s) => { s.a = { b: 1 }; s.a.b = 7; })) + .toEqual([["s", ["a"], { b: 1 }], ["s", ["a", "b"], 7]]); + }); + + it("collapses set-then-delete to the delete", () => { + expect(run((s) => { s.x = 5; delete s.x; })).toEqual([["d", ["x"]]]); + }); + + it("keeps delete-then-set, to preserve key position", () => { + // Dropping the delete would leave the key in its original slot while the + // producer moved it to the end. Same value, different serialisation. + expect(run((s) => { delete s.x; s.x = 5; })).toEqual([["d", ["x"]], ["s", ["x"], 5]]); + }); + + it("is linear in the number of ops", () => { + // The naive formulation compares every op against every dominator, which + // is quadratic and degrades on exactly the wide flush this pass cleans up. + const wide = (n: number) => { + const root: Record = {}; + for (let i = 0; i < n; i++) root[`f${i}`] = i; + const t = track(root); + t.flush(); // drain the base batch + const started = performance.now(); + for (let i = 0; i < n; i++) t.state[`f${i}`] = i + 1; + t.flush(); + return performance.now() - started; + }; + wide(200); // warm + const small = Math.max(wide(250), 0.1); + const large = wide(2500); + expect(large / small).toBeLessThan(40); // linear would be ~10x + }); + + it("collapses a pathological redundant producer", () => { + const t = track({ a: { b: 1 }, x: 1 }); + t.flush(); // drain the base batch + for (let i = 0; i < 2000; i++) { t.state.x = i; t.state.a.b = i; } + t.state.a = { done: true }; + expect(t.flush()).toHaveLength(2); + }); +}); + +describe("flush", () => { + it("drops everything before a replacement", () => { + const t = track([1, 2]); + t.flush(); // drain the base batch + t.state.push(3); + t.state.splice(0, 3, 7); + expect(t.flush()).toEqual([["r", [7]]]); + }); + + it("keeps the prefix when the replacement is nested", () => { + // `s` on a subtree is not a root replacement, so earlier ops stay live. + const t = track({ a: 1, xs: [1, 2, 3], pad: PAD }); + t.flush(); // drain the base batch + t.state.a = 2; + t.state.xs.splice(0, 3, 9); + expect(t.flush()).toEqual([["s", ["a"], 2], ["s", ["xs"], [9]]]); + }); + + it("rejects a constructor walk", () => { + // ({}).constructor.constructor is Function — the classic escape ladder. + expect(() => apply({}, JSON.parse('[["s",["constructor","prototype","gadget"],true]]'))).toThrow(); + expect(({} as Record).gadget).toBeUndefined(); + }); + + it("rejects a forbidden path reached through an interned id", () => { + const dict = new Map([[0, ["__proto__", "w"]]]); + expect(() => apply({}, [["s", 0, true]], (id) => dict.get(id)!)).toThrow(); + }); + + it("does not run an inherited setter", () => { + Object.defineProperty(Object.prototype, "trap", { + get() { throw new Error("inherited getter ran"); }, + configurable: true, + }); + try { + expect(() => apply({}, [["s", ["trap"], 1]])).not.toThrow(); + } finally { + delete (Object.prototype as Record).trap; + } + }); + + it("allows a reserved name as a VALUE key", () => { + // Reserved as segments, not as values. + const out = apply({}, [["s", ["a"], JSON.parse('{"__proto__":{"z":1}}')]]); + expect(({} as Record).z).toBeUndefined(); + expect(out).toBeDefined(); + }); +}); + +describe("safety: array indices", () => { + // An index may address an existing element or append exactly one past the end. + // Not an arbitrary cap: a sparse array does not survive a JSON round trip, so + // arr[7] = x on a length-3 array already produces unreplicable state. + it("writes an existing index", () => { + expect(apply({ xs: [1, 2, 3] }, [["s", ["xs", 1], 9]])).toEqual({ xs: [1, 9, 3] }); + }); + it("appends one past the end", () => { + expect(apply({ xs: [1, 2, 3] }, [["s", ["xs", 3], 9]])).toEqual({ xs: [1, 2, 3, 9] }); + }); + it("rejects a gap", () => { + expect(() => apply({ xs: [1, 2, 3] }, [["s", ["xs", 5], 9]])).toThrow(); + }); + it("rejects a huge index", () => { + // Would otherwise allocate 4.29 billion entries from one op. + expect(() => apply({ xs: [] }, [["s", ["xs", 4_294_967_290], 1]])).toThrow(); + }); + it("allows explicit growth with nulls", () => { + expect(apply({ xs: [1] }, [["p", ["xs"], 1, 0, [null, null, 9]]])).toEqual({ xs: [1, null, null, 9] }); + }); +}); + +describe("safety: op structure", () => { + // The CVE-2025-55182 lesson: a decoder must not trust tuple shape. + it("rejects an unknown verb rather than skipping it", () => { + // Silently skipping is how a newer producer's op vanishes and a replica drifts. + expect(() => apply({ a: 1 }, [["ZZZ", ["a"], 9] as never])).toThrow(); + }); + it("rejects non-array splice items", () => { + // Unvalidated, this spreads a string: ["n","o","t","-","a",…] + expect(() => apply({ xs: [1] }, [["p", ["xs"], 0, 0, "not-an-array"] as never])).toThrow(); + }); + it("rejects a string path", () => { + // Unvalidated, "a".slice(0,-1) is "", so it resolved to the ROOT and wrote. + expect(() => apply({ a: 1 }, [["s", "a", 9] as never])).toThrow(); + }); + it("rejects a non-tuple op", () => { + expect(() => apply({ a: 1 }, [{ op: "s" } as never])).toThrow(); + expect(() => apply({ a: 1 }, [null as never])).toThrow(); + }); + it("clamps a splice remove past the end, as Array.prototype.splice does", () => { + // Deterministic and identical on both sides — not a hole. + expect(apply({ xs: [1, 2] }, [["p", ["xs"], 0, 1e9, []]])).toEqual({ xs: [] }); + }); +}); + +describe("safety: what a facet can put on the wire", () => { + // A facet cannot forge an op — it mutates plain objects and the tracker builds + // the tuples. Values and keys are the leak, and structuredClone is NOT a JSON + // check: it clones a Map happily, and the op then carries {} while the producer + // keeps a real Map. + const rejects = (name: string, mutate: (s: Record) => void) => + it(`rejects ${name}`, () => { + const t = track>({ pad: PAD }); + t.flush(); // drain the base batch + expect(() => mutate(t.state)).toThrow(); + }); + + rejects("a function", (s) => { s.f = () => {}; }); + rejects("a Map", (s) => { s.m = new Map([[1, 2]]); }); + rejects("a Date", (s) => { s.d = new Date(0); }); + rejects("a BigInt", (s) => { s.b = 1n; }); + rejects("a cycle", (s) => { const a: Record = {}; a.self = a; s.c = a; }); + rejects("a symbol key", (s) => { s[Symbol("k") as unknown as string] = 1; }); + + it("allows a getter on state, recording the computed result", () => { + const t = track>({ pad: PAD }); + t.flush(); // drain the base batch + Object.defineProperty(t.target, "g", { get: () => 42, enumerable: true }); + t.state.x = 1; + expect(t.flush()).toEqual([["s", ["x"], 1]]); + }); + + it("accepts a value that merely looks like an op", () => { + // Nested inside ["s", path, value]; nothing flattens, so it cannot be read + // as a top-level op. + const t = track>({ pad: PAD }); + t.flush(); // drain the base batch + t.state.x = ["r", { evil: true }]; + expect(t.flush()).toEqual([["s", ["x"], ["r", { evil: true }]]]); + }); +}); + +describe("assertions", () => { + it("assertJsonValue rejects non-JSON", () => { + for (const bad of [new Map(), new Set(), new Date(), /re/, () => {}, Number.NaN, Infinity]) { + expect(() => assertJsonValue(bad)).toThrow(); + } + for (const good of [null, true, 1, "s", [1, "a"], { a: { b: [null] } }]) { + expect(() => assertJsonValue(good)).not.toThrow(); + } + }); + it("assertValidOp accepts decoded ops and rejects wire forms", () => { + // Validating Op against the wire grammar would be laxer than the type: a + // two-element ["s", value] would pass, and apply would read the value as a + // path. Each vocabulary gets the validator that matches it. + for (const op of [ + ["r", { a: 1 }], ["s", ["a"], 1], ["d", ["a"]], + ["a", ["a"], "x"], ["t", ["a"], 2], ["p", ["a"], 0, 0, []], + ] as never[]) { + expect(() => assertValidOp(op)).not.toThrow(); + } + for (const wireOnly of [["s", 1], ["d"], ["a", "x"], ["t", 2], ["p", 0, 0, []], ["#", 0, ["a"]], ["s", 0, 1]] as never[]) { + expect(() => assertValidOp(wireOnly)).toThrow(); + expect(() => assertValidWireOp(wireOnly)).not.toThrow(); + } + }); +}); + +describe("codec: path interning and arity omission", () => { + // One pair per stream. The table spans a whole subscription or file, so a + // second consumer joining later needs its own encoder. + const roundTrip = (batches: Op[][]) => { + const enc = encoder(); + const dec = decoder(); + return batches.map((ops) => dec.decode(enc.encode(ops))); + }; + + it("round-trips a stream exactly", () => { + const t = track>({ a: { deep: "" }, b: { deep: "" } }); + t.flush(); // drain the base batch + const batches: Op[][] = []; + for (let i = 0; i < 6; i++) { + t.state.a.deep += `x${i}`; + t.state.b.deep += `y${i}`; + batches.push(t.flush()); + } + expect(roundTrip(batches)).toEqual(batches); + }); + + it("interns on second use, not first", () => { + const enc = encoder(); + const p = ["a", "deep"] as const; + const first = enc.encode([["a", p, "1"]]); + const second = enc.encode([["a", p, "2"]]); + expect(first).toEqual([["a", p, "1"]]); // inline + expect(second).toEqual([["#", 0, p], ["a", 0, "2"]]); // define, then use + }); + + it("omits the path when it repeats", () => { + const enc = encoder(); + const p = ["a"] as const; + expect(enc.encode([["s", p, 1], ["s", p, 2]])).toEqual([["s", p, 1], ["s", 2]]); + }); + + it("resets the table on a base batch, so recovery is self-contained", () => { + // A reader replays from the LAST base batch with a fresh decoder. Ids + // defined before it were never seen, so keeping them breaks recovery with + // an unresolvable path id. + const enc = encoder(); + const p = ["a", "deep"] as const; + enc.encode([["a", p, "1"]]); + enc.encode([["a", p, "2"]]); // id 0 assigned + const base = enc.encode([["r", { a: { deep: "x" } }]]); + const after = enc.encode([["a", p, "3"]]); + expect(base).toEqual([["r", { a: { deep: "x" } }]]); + expect(after).toEqual([["a", p, "3"]]); // inline again + + const dec = decoder(); + expect(() => dec.decode(base)).not.toThrow(); + expect(dec.decode(after)).toEqual([["a", p, "3"]]); + }); + + it("survives recovery from the last base batch", () => { + const enc = encoder(); + const t = track>({ a: { deep: "" }, b: { deep: "" } }); + t.flush(); // drain the base batch + const wire: WireOp[][] = []; + for (let i = 0; i < 8; i++) { + t.state.a.deep += `x${i}`; + t.state.b.deep += `y${i}`; + if (i === 5) t.rebase(); + wire.push(enc.encode(t.flush())); + } + const lastBase = wire.map(isBase).lastIndexOf(true); + const dec = decoder(); + let replica: unknown; + for (const w of wire.slice(lastBase)) { + const ops = dec.decode(w); + replica = replica === undefined + ? structuredClone((ops[0] as ["r", unknown])[1]) + : apply(replica, ops); + } + expect(replica).toEqual(t.target); + }); + + it("round-trips random streams", () => { + for (let round = 0; round < 300; round++) { + const t = track>({ a: { p: "", q: "" }, b: [] as unknown[], c: 0 }); + t.flush(); // drain the base batch + const batches: Op[][] = []; + for (let i = 0; i < 8; i++) { + const r = Math.random(); + if (r < 0.3) t.state.a.p += "x"; + else if (r < 0.5) t.state.a.q += "y"; + else if (r < 0.65) t.state.b.push(i); + else if (r < 0.8) t.state.c = i; + else if (r < 0.9) delete t.state.c; + else t.rebase(); + const ops = t.flush(); + if (ops.length > 0) batches.push(ops); + } + expect(roundTrip(batches)).toEqual(batches); + } + }); +}); + +describe("property: random round-trip", () => { + // 3000 random sequences found the root-splice hole on the first run, before any + // hand-written case did. + const rnd = (d = 0): unknown => { + const r = Math.random(); + if (d > 2 || r < 0.3) return Math.floor(Math.random() * 5); + if (r < 0.45) return ["x", "y", null, true][Math.floor(Math.random() * 4)]; + if (r < 0.7) return Array.from({ length: Math.floor(Math.random() * 4) }, () => rnd(d + 1)); + const o: Record = {}; + for (const k of ["a", "b", "c"]) if (Math.random() < 0.6) o[k] = rnd(d + 1); + return o; + }; + + it("a replica always matches the producer", () => { + let checked = 0; + for (let i = 0; i < 3000; i++) { + const base = rnd(); + if (typeof base !== "object" || base === null) continue; + const t = track(structuredClone(base) as object); + t.flush(); // drain the base batch + const next = rnd(); + if (Array.isArray(t.state) && Array.isArray(next)) { + (t.state as unknown[]).splice(0, (t.state as unknown[]).length, ...next); + } else if (!Array.isArray(t.state) && typeof next === "object" && next !== null && !Array.isArray(next)) { + const s = t.state as Record; + for (const k of Object.keys(s)) if (!(k in next)) delete s[k]; + for (const [k, v] of Object.entries(next)) s[k] = v; + } else continue; + expect(apply(structuredClone(base), t.flush())).toEqual(t.target); + checked++; + } + // Most random pairs are shape-incompatible and skipped; this only guards + // against the loop silently checking nothing. + expect(checked).toBeGreaterThan(200); + }); +}); diff --git a/packages/agent/docs/mobile-handoff/01-harness/01-delta/delta.ts b/packages/agent/docs/mobile-handoff/01-harness/01-delta/delta.ts new file mode 100644 index 000000000..80c0b5b96 --- /dev/null +++ b/packages/agent/docs/mobile-handoff/01-harness/01-delta/delta.ts @@ -0,0 +1,285 @@ +// ───────────────────────────────────────────────────────────────────────────── +// harness/delta — intent-recording change tracking over plain JSON. +// +// Depends on nothing else in the harness. Session storage, the runtime and the +// facet host consume it; keep the arrows pointing that way. +// +// State is plain TS. You mutate it normally. Ops come out carrying what you +// *did*, not what changed — `text += chunk` yields an append, not a 50 KB +// replacement; `arr.unshift(x)` yields one splice, not O(n) index writes. +// ───────────────────────────────────────────────────────────────────────────── + +export type JsonValue = null | boolean | number | string | JsonValue[] | { [k: string]: JsonValue }; + +export type Seg = string | number; +export type Path = readonly Seg[]; + +/** Non-empty: `s`/`d`/`a`/`t` cannot target the root. Enforced by the type. */ +export type NonEmptyPath = readonly [Seg, ...Seg[]]; + +/** A path inline on first use, an interned id thereafter. */ +export type PathRef

= P | number; + +/** + * Tuples are the form — in memory, on the wire, on disk. There is no keyed + * variant and no codec between them. + * + * `r` is the ONLY op that replaces a whole value. It is an op rather than a + * frame kind for the same reason `a` and `t` exist: they are specialisations of + * `s` that are smaller and state intent. It also makes base detection a token + * comparison rather than a path inspection, which matters because recovery stops + * at the last base batch — a correctness boundary. + * + * Only `p` may target the root, and only because a tracked value can itself be + * an array. A `p` covering its entire target is normalised at RECORD time to `r` + * or `s`, so a root `p` is always a partial modification. + * + * The two-element forms reuse the previous op's path; arity disambiguates. + */ +export type Op = + | readonly ["r", JsonValue] + | readonly ["s", NonEmptyPath, JsonValue] + | readonly ["d", NonEmptyPath] + | readonly ["a", NonEmptyPath, string] + | readonly ["t", NonEmptyPath, number] + | readonly ["p", Path, number, number, JsonValue[]]; + +/** + * What crosses a boundary. Adds path interning and arity omission, and nothing + * else. `["r", value]` carries no path, so it encodes to itself — which is why + * `isBase` works on either vocabulary. + */ +export type WireOp = + | readonly ["r", JsonValue] + | readonly ["s", PathRef, JsonValue] | readonly ["s", JsonValue] + | readonly ["d", PathRef] | readonly ["d"] + | readonly ["a", PathRef, string] | readonly ["a", string] + | readonly ["t", PathRef, number] | readonly ["t", number] + | readonly ["p", PathRef, number, number, JsonValue[]] + | readonly ["p", number, number, JsonValue[]] + | readonly ["#", number, Path]; + +/** + * There is no frame type. What travels is `Op[]`. + * + * A **base batch** is one whose first op is `r`; the first batch of a stream is + * always one, and storage tags it "base" so recovery can stop there. Sequencing + * lives on the transport — a list element's `seq`, or an SSE `id:` — never in the + * payload. See `delta.md` §6. + */ + +// ─── Safety ────────────────────────────────────────────────────────────────── + +/** + * Path segments that reach the prototype chain. Rejected at record time AND at + * apply time, including through an interned id. + * + * Reserved as SEGMENTS, not as values: an object with a literal `__proto__` key + * replicates fine as a whole value; only a path walking through it is refused. + * That is a real restriction on what is mutable — see `delta.md` §7.1. + */ +export declare const RESERVED_SEGMENTS: ReadonlySet; + +export declare class UnsafePathError extends Error { + readonly segment: Seg; +} + +/** Rejects reserved segments, symbol keys, and negative or non-integer indices. */ +export declare function assertSafePath(path: Path): void; + +/** + * Rejects anything `JSON.stringify` would not round-trip: Map, Set, Date, RegExp, + * typed arrays, class instances, functions, BigInt, cycles. + * + * `structuredClone` is NOT a JSON check — it clones a Map happily, and the op then + * carries `{}` while the producer keeps a real Map. Silent divergence. + */ +export declare function assertJsonValue(value: unknown): asserts value is JsonValue; + +/** + * Validate an op's structure before applying it. A decoder must not trust tuple + * shape: an unvalidated applier spreads a string into an array given + * `["p",["xs"],0,0,"not-an-array"]`, and writes at the root given a string path. + * + * An unknown verb is an error, not a no-op — skipping it is how a newer producer's + * op vanishes and a replica drifts. This is the CVE-2025-55182 lesson + * (`delta.md` §7.3). + */ +export declare function assertValidOp(op: unknown): asserts op is Op; + +/** + * The same, for the wire grammar, where ids and short forms are legal. + * `decode` uses this; `apply` uses `assertValidOp`. Validating a decoded op + * against the wire grammar is laxer than its type — a two-element `["s", value]` + * would pass and then be read as a path. + */ +export declare function assertValidWireOp(op: unknown): asserts op is WireOp; + +// ─── Classification ───────────────────────────────────────────────────────── +// +// Ops are tuples, so classification is easy to hand-roll and easy to get wrong. +// `op[0] === "s"` is not enough — a set on any path is "s"; what makes a base is +// that it targets the ROOT. And because a tuple may omit the PathRef (arity +// disambiguates), `op[1]` is a PathRef or a value depending on length. These +// exist so nobody writes that comparison twice. + +/** True for the `r` op. */ +export declare function isReplace(op: Op): boolean; + +/** + * True when this batch begins with `r` — a **base batch**. Exact rather than a + * heuristic, because flush guarantees `r` appears at index 0 or not at all. + * Storage tags such elements `"base"` so recovery can stop at the last one. + */ +export declare function isBase(ops: readonly (Op | WireOp)[]): boolean; + +// ─── Codec ─────────────────────────────────────────────────────────────────── + +/** + * Path interning and arity omission. ONE PAIR PER STREAM — the id table spans a + * whole subscription or file, so a consumer joining later needs its own. + * + * The table resets on a base batch, because a reader replays from the last one + * with a fresh decoder and everything after it must be self-contained. + */ +export declare function encoder(): { encode(ops: readonly Op[]): WireOp[] }; +export declare function decoder(): { decode(wire: readonly WireOp[]): Op[] }; + +// ─── Tracker ───────────────────────────────────────────────────────────────── + +export interface TrackerOptions { + /** + * Cap on characters compared when deriving append/truncate from a string + * write. A window slide's overlap lies near the end, so a cap bounds cost + * without changing the result. Default 65536; 0 disables detection. + */ + maxOverlapScan?: number; + /** Merge compatible adjacent ops on flush. Default true. */ + coalesce?: boolean; +} + +export interface Tracker { + /** + * Mutate this. Plain TS: assignment, push, splice, delete, all of it. + * + * Assigning to it replaces the whole value: emits `["r", next]` and discards + * ops recorded before it. It must be a SETTER on the tracker — a plain + * property would be overwritten, swapping the proxy for a plain object and + * silently ending tracking. + * + * To force a base batch without changing the value, call `rebase()`. + */ + state: T; + /** The untracked backing object. Reads only — writing here emits nothing. */ + readonly target: T; + /** + * Drain. Empty when nothing changed. + * + * The FIRST flush is always a base batch, carrying the value including any + * mutations made before it — a consumer starts with nothing, so a stream must + * open with something to apply deltas to. + * + * Ops that a later op makes unreachable are dropped first — a field written + * three times emits one op, and a parent replaced after its child emits one. + * Object key order is not preserved by that pass; see `delta.md` §5. + */ + flush(): Op[]; + /** Discard without emitting. For a producer that just sent a base batch. */ + discard(): void; + get dirty(): boolean; +} + +export declare function track(root: T, options?: TrackerOptions): Tracker; + +// ─── Applier ───────────────────────────────────────────────────────────────── + +/** + * Apply ops to a plain mutable object. Five verbs plus the dictionary entry, no + * domain knowledge, no library, no registry lookup. + * + * Returns the value, because a root set replaces it outright and cannot be done + * in place. Throws `PathError` when a path does not resolve, which a consumer + * treats exactly like a sequence gap: request a fresh base frame. + * + * Must not be pointed at a value produced by Immer, which deep-freezes. + */ +/** + * Fold ops into a value and return it — returned rather than mutated in place, + * because `r` replaces the value outright. Tolerates an `undefined` target, + * since a root op never reads it, so a consumer needs no base-batch branch. + * + * An `r` payload is **adopted, not copied**: the consumer owns the batch it was + * handed. Do not hand one batch to two in-process consumers, or their replicas + * will alias. + */ +export declare function apply(target: T, ops: readonly Op[]): T; + +export declare class PathError extends Error { + readonly path: Path; +} + +// ─── Replica ───────────────────────────────────────────────────────────────── + +export interface Replica { + /** Current value. `undefined` until the first base frame. */ + readonly value: T | undefined; + /** + * Feed one batch, with the sequence number the transport supplied. Returns + * "gap" when `seq` is not contiguous or a path failed to resolve; the caller + * then requests a fresh base batch. + */ + receive(ops: readonly Op[], seq: number): "ok" | "gap"; +} + +export declare function replica(): Replica; + +// ─── Semantics ─────────────────────────────────────────────────────────────── +// +// ROOT OPS +// `["s", [], v]` replaces the value; `["p", [], …]` splices it when the value +// is an array. The applier must special-case `path.length === 0` before +// resolving a parent — a prototype handling only a root set threw on the first +// randomised sequence that produced a root splice. +// +// STRINGS +// A write of `next` over `prev` derives ops from the longest suffix of `prev` +// that is a prefix of `next`: +// overlap === prev.length -> append(next.slice(overlap)) +// overlap > 0 -> truncate(prev.length - overlap) + append(rest) +// overlap === 0 -> set(next) +// Always correct: the overlap is verified, not guessed. This is what makes a +// rolling window cheap — the case every effect-recording library degrades on. +// +// Use a native probe, not a hand-written KMP: `indexOf` a short head of `next` +// in `prev`, verify with `endsWith`. Measured 47x faster over 2000 slides on a +// 50 KB string, verified identical across 200. +// +// `t` counts UTF-16 code units, matching String.prototype.slice. Not bytes. +// +// ARRAYS +// Mutator methods are intercepted in the `get` trap, so intent is recorded +// before the engine performs index writes: +// push/unshift/pop/shift/splice -> one `p` +// sort/reverse/fill/copyWithin -> `s` of the whole array (rare) +// Direct index writes are honest `s` ops. `a.length = n` is translated — +// shrink to a splice, grow to a splice of nulls — never emitted as a path, +// because `length` is not a document location. +// +// TRANSACTIONS +// Several writes to one address in one commit must each be recorded against +// the previous one's result, not against pre-transaction state. The applier +// replays in order; anything else applies the second against a stale base. +// +// ALIASING +// Op payloads are structured-cloned at record time. Without this a later +// mutation retroactively changes an already-emitted op and a local replay +// aliases the source — an observed bug, not a hypothetical. +// +// UNDEFINED +// `x = undefined` normalises to `d`. JSON has no undefined, and a set with a +// missing value is indistinguishable from a lost value after a round trip. +// +// SCOPE +// JsonValue only. No Map, Set, Date, RegExp, class instances, getters, symbol +// keys, or cycles. That keeps this to five traps and one string algorithm, and +// makes structuredClone on payloads always safe. diff --git a/packages/agent/docs/mobile-handoff/01-harness/02-scopes/implementation-handoff.md b/packages/agent/docs/mobile-handoff/01-harness/02-scopes/implementation-handoff.md new file mode 100644 index 000000000..67e3703b5 --- /dev/null +++ b/packages/agent/docs/mobile-handoff/01-harness/02-scopes/implementation-handoff.md @@ -0,0 +1,604 @@ +# Scoped storage Step 1 — actionable implementation handoff + +**Status: actionable, not implemented.** + +**Prepared against:** `c4b0e35ab` (`dev`). Re-audit named source files if the implementation baseline changes materially. + +This handoff implements storage scopes only. **Read this file before [`scopes.md`](scopes.md).** It supersedes conflicting details there, especially per-scope sequence spaces, operation-derived orphan detection, one-use scope IDs, raw-string scope arguments, sidecars ending in `.jsonl`, the proposed interned retirement tuple, and bundling JSONL delta/address encoding into the same change. The broader motivation and measurements in `scopes.md` remain design evidence; its signatures and implementation order are not current where this file differs. + +Step 2—Chord-encoded JSONL values, address interning, compact tuple records, and their measurements—is a separate package. Do not begin Step 2 until Step 1 has been tested, reviewed, explicitly approved by the user, committed, and pushed. + +## 0. Delivery protocol + +This package has a mandatory user checkpoint: + +1. Implement Step 1 only. +2. Run every required focused test, `npm run check`, and `./test.sh`. +3. Review the complete diff and report results. Delegated reviews, if used, run with provider `anthropic` and model `claude-fable-5`. +4. **Stop and wait for explicit user approval.** +5. After approval, commit only this package's files and push that commit. +6. Confirm the pushed commit before designing or implementing Step 2. + +Do not combine the Step 1 and Step 2 commits. Do not commit or push before approval. + +## 1. Mandatory reading + +Read completely before editing: + +1. `packages/agent/docs/harness.md` §§0.2–0.9, 1.1–1.7, 2.7–2.8, 3.7–3.8, 3.13, 4.4–4.8, 5.4, Part 7, Part 9. +2. `packages/agent/docs/values.md`. +3. `packages/agent/docs/mobile-handoff/README.md`. +4. [`scopes.md`](scopes.md), using this handoff where they conflict. +5. `../01-delta/delta.md` for vocabulary ownership only; Step 1 does not store Chord ops. +6. `../04-tool-output/harness-tools.md` §§7.1–7.7 for later-consumer context; do not implement its output redesign here. +7. `../05-assistant-output/message-update.md` §7 for later-consumer context; do not implement it here. +8. `packages/agent/src/harness/session/{types,values,commit,in-memory-storage-state,memory,session,index}.ts`. +9. Every file under `packages/agent/src/harness/session/jsonl/` and the `FileSystem` declaration/implementation in `packages/agent/src/harness/{types,env/nodejs}.ts`. +10. `packages/agent/src/harness/runtime/{progress,lane}.ts`, `runtime/drive/{terminal,response,deferred,tools,reconcile,structural,boundary}.ts`, and every caller of `operationCleanupWrites`. Inspect `tool-placement.ts` through the grep audit, but current source has no scoped delete there. +11. `packages/session-backends/sqlite-node/src/{index,sqlite/index}.ts`, `sqlite/{storage,repo}.ts`, `migrations/001_initial.sql`, and `session/{values,session-sequences}.ts`. +12. Shared storage/repository conformance and every focused test named in §9. +13. `scopes.variance.ts` for the covariant-address/invariant-write phantom mechanism only. Its current no-ID `EphemeralScope` and `retireScope(id: string)` signatures are superseded by §2 and must be updated in Slice A. + +Do not use `dist/` as implementation input. Format 4 remains WIP; no R11 migration is part of this package. + +## 2. Fixed scope contract + +### 2.1 Runtime scope identity + +A scope ID is a reusable logical lifetime name, not a globally one-use token: + +```ts +export type SessionScope = { readonly kind: "session" }; + +export interface EphemeralScope { + readonly kind: "ephemeral"; + readonly id: string; +} + +export type Scope = SessionScope | EphemeralScope; + +export function ephemeralScope(id: string): EphemeralScope; +``` + +`ephemeralScope(id)` requires a non-empty well-formed Unicode string and returns a frozen value. Equal IDs identify the same physical scope; object identity has no meaning. JSONL encodes IDs with `encodeURIComponent`. The encoded component must be at most 180 ASCII characters; reject longer IDs and lone-surrogate input. The physical filename always has a fixed `scope-` prefix, so encoded `.`/`..` values cannot become path segments. Do not directly interpolate unencoded caller input into a path. + +No create/open-scope transaction exists. The first scoped value/list write creates physical state. A later `retireScope(scope)` ends everything in that scope through its assigned global sequence. A write under the same ID after retirement begins a new logical lifetime. + +Harness operation scopes use `ephemeralScope(operationId)`. Generic application scopes may use stable names. Storage never parses a scope ID as an operation ID and never reads Harness operation state to decide lifetime. + +### 2.2 Reuse and retirement boundary + +Global sequence order defines reusable generations: + +```text +seq 10 scoped write S +seq 20 scoped write S +seq 30 retireScope(S) +seq 40 scoped write S # new lifetime +``` + +For a given scope ID, only scoped records with `seq` greater than its latest committed retirement sequence are live. Memory and SQLite realize this by deleting current rows/maps at retirement and allowing later writes to recreate them. JSONL uses the retirement sequence as its replay boundary (§6). + +Retirement is idempotent in the storage sense: retiring a scope with no current state is legal and advances the boundary. A later write still begins a new lifetime. Storage does not maintain an all-time used-ID registry or reject reuse. + +A scoped write reaching the commit line after retirement is therefore a new-lifetime write. Harness invocation fencing and settlement drains must continue to ensure a late assistant/tool progress job cannot recreate operation output after terminal retirement. Add explicit tests for this boundary. + +### 2.3 Explicit retirement is the sole authority + +```ts +export function retireScope(scope: EphemeralScope): Write; +``` + +Storage never infers orphanage from operation absence, namespace, current state, or a missing owner. The only valid lifetime boundary is a committed `retireScope` write. If an owner never retires its scope, that is an owner defect and the scope remains live. + +Crash behavior: + +- crash before the retirement transaction commits: the scope remains live and reopens; +- crash after retirement commits but before JSONL unlink: replay sees the retirement boundary, ignores pre-boundary sidecar records, and retries physical deletion; +- crash during a later reused lifetime: records after the latest retirement reopen normally. + +Repository deletion separately removes every physical sidecar belonging to the deleted Session. It does not need operation semantics. + +### 2.4 Address and write typing + +Addresses carry a covariant scope tag; writes carry an invariant scope tag. Preserve value-type invariance independently: + +```ts +interface Value { /* existing fields + scope */ } +interface ValueList { /* existing fields + scope */ } + +declare function value(namespace: string, key?: string): Value; +declare function value(namespace: string, key: string, scope: EphemeralScope): Value; +declare function list(namespace: string, key?: string): ValueList; +declare function list(namespace: string, key: string, scope: EphemeralScope): ValueList; +``` + +A session-scoped address has no runtime scope ID. An ephemeral address carries its scope ID. Scope is part of physical identity: equal namespace/key addresses in session scope and ephemeral scope, or in two different ephemeral IDs, are distinct. + +`setValue`, `deleteValue`, `appendList`, and `deleteList` preserve the address scope in `Write`. Entry and usage writes are session-scoped. `retireScope` is session-scoped even though applying it deletes ephemeral state. + +All writes in an ordinary transaction have one static scope. A session transaction may include session values, entries, usage, and one or more `retireScope` writes. It may not include a direct ephemeral set/delete/append. An ephemeral transaction may include value/list writes only. At runtime, every ephemeral write in one transaction must carry the same scope ID; two different IDs have the same TypeScript tag and require this assertion. + +Make the minimum generic changes necessary to preserve this through `Write`, `CommittedWrite`, `Storage.commit`, Session mutation capabilities, `CommitDecision`, and lane commands. Follow the variance proof in `scopes.variance.ts`; do not make readers invariant or propagate unnecessary scope type parameters through read-only APIs. + +### 2.5 One global sequence space + +Memory, JSONL main records, JSONL sidecar records, and SQLite rows share one Session-global sequence space. Every admitted commit remains serialized and receives increasing sequences in admission order. Gaps remain legal. + +- Memory continues using one `nextSeq`. +- SQLite continues allocating from `sessions.next_seq` inside the write transaction. +- JSONL main and sidecar appends share one commit queue and one resident `nextSeq`. + +JSONL open computes the high-water mark from the main header and every complete record in every matching sidecar **before** deleting or ignoring retired physical files. A retired sidecar's sequences may then become gaps but can never be reused. Torn uncommitted final transactions do not advance the durable high-water mark. + +No reservation record, per-scope sequence counter, or range allocator is permitted. + +## 3. List tags + +Step 1 adds only the mechanism required by later tracked-output recovery: + +```ts +export interface ListElement { + seq: number; + value: T; + tag?: string; +} + +export interface ListReadOptions { + cursor?: ListCursor; + order?: "asc" | "desc"; + limit?: number; + stopAtTag?: string; +} + +export function appendList( + address: ValueList, + element: NoInfer, + tag?: string, +): ListAppendWrite; +``` + +Tags are non-empty strings when present. Storage stores and returns the tag without interpreting the element. + +Read semantics: + +1. Apply address scope, exclusive cursor, and order. +2. Inspect at most the normalized `limit` elements. +3. Stop at and include the first element carrying `stopAtTag`. +4. Return that possibly shortened page. + +`stopAtTag` never searches beyond the page limit. If absent from the page, the caller pages again from the last returned sequence. Do **not** add the draft `tag` filter; no current consumer needs it and combining filtering with the existing cursor shape is ambiguous. + +SQLite may fetch the indexed `limit` rows and truncate at the first returned tag in TypeScript; storage still never parses payloads. Preserve the existing primary-key query plan with no temporary sort. + +## 4. Step 1 durable payloads remain unchanged + +This package changes lifetime and routing, not output representation: + +- `pendingToolOutput` remains `Value, EphemeralScope>`; +- `pendingAssistantFrames` remains `ValueList`; +- `operationToolMemo` remains a `JsonValue` scalar; +- JSONL transaction records remain readable keyed objects; +- assistant progress still appends one frame per accepted frame; +- tool checkpoints still replace whole snapshots; +- `message_update` and `tool_update` event shapes do not change; +- safe replay retains its current checkpoint behavior, including the known delete-before-replay bug owned by the later tool-output package. + +Do not rename `pendingAssistantFrames`, introduce `pendingAssistantOutput`, store `WireOp[]`, add Chord trackers/codecs, redesign `AgentHarnessTool`, or change output cadence here. + +## 5. Runtime ownership and cleanup + +### 5.1 Addresses moved to the operation scope + +Construct these addresses with `ephemeralScope(operationId)`: + +- `operationToolMemo(operationId, invocationId, name)`; +- `pendingToolOutput(operationId, invocationId)`; +- `pendingAssistantFrames(operationId, responseEntryId)`. + +All other current built-ins remain session-scoped. In particular, `pi.op.state`, `pi.op.meta`, `pi.op.tool_args`, `pi.op.preparation`, and `pi.pending.entry` remain in the main scope because their writes coordinate atomically with lane/operation state. + +### 5.2 No cross-file cleanup transaction + +Remove direct ephemeral deletes from transactions that also write session state: + +- assistant response settlement; +- deferred-response supersession; +- tool outcome staging in `drive/tools.ts`; +- cancellation/recovery decisions; +- terminal per-address cleanup. + +Current `drive/tool-placement.ts` has no ephemeral delete; do not invent a change there. + +`operationCleanupWrites` stops scanning tool memos and tool outputs and stops constructing a state-directed assistant-frame delete. It keeps session-scoped operation/meta/args/preparation/pending-entry cleanup and adds exactly one `retireScope(ephemeralScope(operationId))` in the universal terminal suffix. + +After removing the current staging/terminal scans, remove `operationToolMemoPrefix` and `pendingToolOutputPrefix`; current source has no other production consumers. Update the normative exported prefix-constructor count from five to three and update exact constructor tests rather than retaining dead inventory APIs. + +An ephemeral-only delete remains legal. For example, the current safe-replay checkpoint delete remains because its transaction contains no session write; fixing its behavior belongs to the later tool-output package, not scoped storage or JSONL optimization. + +### 5.3 Unreachable scoped residue + +After a response settles, a deferred response is superseded, or a tool reaches `outcome_ready`, its former frames/checkpoints/memos may remain physically and logically addressable inside the still-active operation scope. Current scalar operation state no longer references them; no recovery or snapshot path scans the scope to infer authority. The terminal retirement deletes the whole scope. + +Call this **unreachable scoped residue**, not orphanage. Update `harness.md` invariants that currently require an `outcome_ready` call to have no physically present memo/checkpoint. The replacement invariant is that settled child state never consumes or exposes residue and terminal retirement removes all scoped state atomically with operation completion. + +Close and fault are controlled crashes and never retire the scope. Reopen restores scoped progress only through addresses derived from current authoritative operation state. + +## 6. JSONL sidecars + +### 6.1 Physical layout + +The main session file remains unchanged except for committed scope-retirement records. Every main file owns one known sidecar directory at `${mainPath}.scopes`; this requires no dirname API or `FileSystem` extension. Create it lazily with `createDir(..., { recursive: true })`. Discover scopes with `listDir` on that exact path. A sidecar filename is exactly `scope-${encodeURIComponent(scope.id)}.scope`, subject to §2.1's 180-character encoded-component limit. The suffix does not end in `.jsonl`, and the sidecar directory itself is a directory, so repository listing never mistakes either for a Session. + +A sidecar begins with this exact header: + +```ts +interface JsonlScopeHeader { + v: 4; + kind: "scope_header"; + sessionId: string; + scopeId: string; + storageVersion: 1; +} +``` + +Validate the exact session/scope identity before replay. Create the first header through a temp file inside `${mainPath}.scopes` plus atomic rename. Temporary filenames end in `.tmp` and are never discovered as sidecars. Use the existing `FileSystem` capability; do not add a dirname method, extend `JsonlStorageOptions`, or add Node-only filesystem access to agent core. + +Step 1 sidecar writes retain the current object record spelling and carry/validate their scope ID in committed shapes. Each physical transaction remains one complete line or one array line. Sidecar transactions may contain only value/list writes for that exact scope. + +### 6.2 Retirement record + +The main log records an explicit committed write: + +```ts +interface CommittedScopeRetireWrite { + kind: "scope"; + op: "retire"; + seq: number; + scopeId: string; +} +``` + +Its Step 1 JSON representation is the corresponding readable object. Do not use the draft `['!', addrId, seq]`: a scope is not an interned value/list address. Compact tuple spelling belongs to Step 2 and must be designed there. + +Applying a committed retirement removes resident values/lists in that scope. After the complete main transaction is durable, JSONL synchronously serializes lifecycle cleanup on the same commit queue: close any owned sidecar resource, attempt unlink, then release the queue. Failure to unlink is non-fatal to the already-committed business transaction; the sidecar remains logically bounded by the retirement sequence and deletion is retried on reopen. + +### 6.3 Replay and reusable IDs + +Open performs these logical stages before admitting writes: + +1. Read/repair the main file and parse complete main transactions. +2. Discover and validate matching sidecars; ignore unrelated files. +3. Parse complete sidecar transactions and repair torn final transactions. +4. Compute the global sequence high-water mark from the header and **all** complete main/sidecar records, including records later excluded by retirement. +5. Determine the latest retirement sequence per scope from main transactions. +6. Keep sidecar writes whose sequence is greater than that scope's latest retirement. +7. Merge retained main and sidecar transactions by global sequence and replay them in global order, preserving transaction boundaries and rejecting duplicate/non-monotonic sequences. +8. Advance to the precomputed high-water mark. +9. Remove sidecars with no records after their latest retirement; cleanup failure does not make retired contents live. + +Main transactions have sequence gaps where sidecar transactions occurred. Sidecars have gaps where other scopes/main commits occurred. Both remain valid. + +A sidecar containing records after a retirement is an active reused lifetime and must not be deleted. Serialize retirement cleanup against later writes so an old unlink cannot race and remove a newly reused sidecar. + +Malformed interior records or identity mismatch in an active sidecar are storage corruption and fail open. A torn final transaction is discarded wholly. Temporary files left by interrupted atomic creation are cleanup artifacts and never become sidecars. + +### 6.4 Legacy v3 + +Legacy v3 has no scopes. An ordinary Harness operation commits session-scoped acceptance before any progress write, so it upgrades first in the normal path. Still define and test the generic storage behavior when the first caller write is ephemeral: complete the existing v3-to-v4 main rewrite/usage adjustment under the global commit queue, then commit the scoped transaction without placing it in the main file. A crash before the sidecar append leaves no committed caller transaction and may reuse its uncommitted sequence. + +### 6.5 Repository and fork behavior + +- Repository `list()` ignores `${mainPath}.scopes` directories and their files through its existing non-file/`.jsonl` filtering. +- Repository `delete()` removes the main file through its current path, then removes `${mainPath}.scopes` recursively with `force: true`; a scope-directory cleanup failure rejects deletion after the main-file result is known, without inferring Harness state. Missing scope directories are legal. +- `JsonlStorage.close()` drains its admitted main/sidecar commits as today; `JsonlSessionRepo.close()` retains its current lifecycle behavior because repository ownership is a separate package. No progress scope is retired merely because a handle closes. +- Preserve the current fork allowlist exactly. Do not add generic application values or lists to either fork scope. Ensure source snapshots and backend fork readers ignore all ephemeral values/lists regardless of namespace; current destinations remain logically unchanged except that ephemeral state cannot leak. +- Fork destinations contain no sidecars or retirement state. +- WP08 may later replace fork materialization but must preserve this scope policy. + +J1 compaction is excluded. When J1 eventually lands, it may omit a historical retirement record only after no physical sidecar can be made live by losing that boundary. + +## 7. Memory backend + +Extend `InMemoryStorageState` physical value/list identity with scope. Session scope and each ephemeral ID are distinct. Preserve one global `nextSeq` and current stats behavior. + +Applying `CommittedScopeRetireWrite` deletes every scalar/list in that scope in the same synchronous application of the surrounding transaction. It does not delete entries or usage, which cannot be ephemeral. A later scoped write recreates state under the same ID. + +Snapshots expose scope so fork code can reject ephemeral state, but `createForkSnapshot` keeps its current built-in allowlist and must not start copying generic session-scoped application values or lists. Instrumentation must still report exact committed write order, including retirement. + +## 8. SQLite backend + +Change WIP schema in place with no migration: + +```sql +scalar_values( + session_id TEXT NOT NULL, + scope_id TEXT NOT NULL, + namespace TEXT NOT NULL, + key TEXT NOT NULL, + seq INTEGER NOT NULL, + value TEXT NOT NULL, + PRIMARY KEY(session_id, scope_id, namespace, key) +) WITHOUT ROWID; + +list_values( + session_id TEXT NOT NULL, + scope_id TEXT NOT NULL, + namespace TEXT NOT NULL, + key TEXT NOT NULL, + seq INTEGER NOT NULL, + value TEXT NOT NULL, + tag TEXT, + PRIMARY KEY(session_id, scope_id, namespace, key, seq) +) WITHOUT ROWID; +``` + +Use `scope_id = ''` for session scope and the exact scope ID otherwise. Every point read, prefix scan, set/delete, append/delete, snapshot, and query-plan assertion includes scope identity. + +Inside the same `BEGIN IMMEDIATE` transaction, a retirement write executes: + +```sql +DELETE FROM scalar_values WHERE session_id = ? AND scope_id = ?; +DELETE FROM list_values WHERE session_id = ? AND scope_id = ?; +``` + +It still consumes its assigned global sequence through `sessions.next_seq`, even though no retired-scope tombstone row remains. A later write under that ID recreates rows naturally. Retirement rollback must restore both scoped rows and all sibling main writes. + +Fork paths copy only `scope_id = ''`. Shared-container deletion remains Session-scoped and removes all values/lists regardless of scope along with the other Session rows. + +## 9. Required tests + +Port tests before or with each implementation slice. Use the repository Vitest binary from the owning package. Do not run the full Vitest suite directly. + +### 9.1 Type and constructor tests + +- Session addresses default to `SessionScope`. +- Ephemeral overloads retain `EphemeralScope` while preserving invariant `T`. +- Address scope is covariant for reads; write scope is invariant. +- Same-scope session transactions typecheck. +- Same-scope ephemeral transactions typecheck. +- Session writes plus `retireScope` typecheck. +- Direct session + ephemeral writes in one transaction fail with `@ts-expect-error`. +- Two ephemeral IDs evade static distinction but fail the runtime same-ID assertion. +- Scope object identity is irrelevant; equal IDs access the same state. +- Empty and overlong encoded IDs reject; separators/Unicode encode without path escape. +- Session and ephemeral addresses with equal namespace/key do not alias. + +### 9.2 Shared backend conformance + +Run identically on Memory, JSONL, and SQLite: + +- ephemeral scalar set/get/replace/delete; +- ephemeral list append/page/whole-list-delete; +- independent scopes with equal namespace/key; +- session scope independent from equal ephemeral address; +- mixed session/ephemeral transaction rejection before mutation; +- two ephemeral IDs in one transaction rejection before mutation; +- retirement deletes every value/list in the exact scope and no other scope; +- retirement atomic with session sibling writes; +- retirement of absent state is legal; +- write after retirement recreates a new lifetime; +- repeated retirement advances the boundary without making later writes disappear; +- main, scope A, scope B, retirement, and reused-A commits receive globally increasing sequences in admission order; +- retired/deleted physical elements leave legal gaps and first post-reopen commit allocates above every complete prior record; +- list tags round-trip, page limits remain bounded, and ascending/descending `stopAtTag` includes the marker; +- stats ignore scoped values/lists and retirement; +- close drains admitted scoped commits and rejects later reads/writes. + +### 9.3 JSONL focused tests + +- first scoped write creates exactly one valid sidecar and no main value/list record; +- several writes in one scoped transaction remain one physical sidecar line; +- main and multiple sidecars replay in global sequence order; +- header `nextSeq` and all complete sidecar records contribute to reopen high-water; +- torn sidecar final transaction is discarded wholly and repaired; +- malformed interior active sidecar and header identity mismatch fail open; +- crash before retirement leaves scoped data live; +- crash after main retirement but before unlink leaves data logically absent and cleanup retries; +- unlink failure does not reject the already-committed terminal transaction; +- reuse after retirement replays only records after the latest retirement sequence; +- multiple retirement/reuse cycles choose the latest boundary; +- retirement cleanup is serialized so a later reuse cannot be deleted by an earlier unlink; +- ignored pre-boundary records still prevent sequence reuse; +- sidecars and temp files never appear in repository listing; +- repository deletion removes all sidecars; +- forks exclude generic application-owned ephemeral values/lists; +- legacy-v3 first session write and first scoped-write paths both produce valid v4 state; +- close/reopen preserves active operation progress and does not retire it. + +### 9.4 SQLite focused tests + +- schema and query helpers include `scope_id`; list rows preserve nullable tags; +- one transaction atomically deletes scoped scalar/list rows and writes terminal main state; +- forced failure after scope deletion rolls back deletion and sibling writes; +- reuse after retirement recreates rows under the same scope ID; +- `sessions.next_seq` advances for retirement; +- list paging plan still uses the scoped primary key with no temp b-tree; +- both per-file and shared-container layouts isolate Session and scope IDs; +- fork and Session deletion exclude/remove scoped rows correctly. + +### 9.5 Harness integration + +Update existing exact-write tests rather than weakening them: + +- assistant/deferred settlement drops mixed-scope frame deletes; +- tool outcome staging drops mixed-scope checkpoint/memo deletes; +- unreachable residue is never consumed by snapshot/recovery after its owning child settles; +- safe-replay's current ephemeral-only delete remains ordered and fenced; +- every terminal leaf emits one retirement in the documented cleanup position; +- close/fault emits no retirement and reopen recovers current frames/checkpoint/memos; +- cancellation and unknown-outcome recovery retire only at terminal completion; +- late queued assistant/tool progress cannot commit after terminal retirement or reopen the scope; +- terminal cleanup no longer scans memo/output prefixes; +- every focused test and fixture found by the §12 old-delete/prefix greps is updated deliberately; do not rely on a historical numeric test count; +- every terminal path reached through `operationCleanupWrites` callers in `drive/{response,reconcile,structural,boundary}.ts` is covered, including each operation family and cancellation. + +Likely focused files include: + +```text +packages/agent/test/harness/values.test.ts +packages/agent/test/harness/{memory,jsonl}-storage*.test.ts +packages/agent/test/harness/{memory,jsonl}-session-repo*.test.ts +packages/agent/test/harness/runtime/drive-{terminal,retry-deferred,tools,reconcile,generation}.test.ts +packages/session-backends/sqlite-node/test/{storage,storage-conformance,repo,repo-conformance}.test.ts +``` + +Use compiler and grep guards to find additional affected tests rather than assuming this list is exhaustive. + +## 10. Implementation slices + +### Slice A — contract, types, Memory reference, list tags + +Primary files: + +```text +CREATE packages/agent/src/harness/session/scope.ts for `SessionScope`, `EphemeralScope`, `Scope`, `ephemeralScope`, scope phantoms/helpers, and runtime scope-ID validation +MODIFY packages/agent/src/harness/session/{types,values,commit,in-memory-storage-state,memory,session,index}.ts +MODIFY packages/agent/src/harness/session/testing/{conformance/storage,instrumented-storage,storage-decorator,gating-storage}.ts +MODIFY packages/agent/test/harness/{values,memory-conformance,memory-storage,storage-backed-session}.test.ts +MODIFY packages/agent/docs/mobile-handoff/01-harness/02-scopes/scopes.variance.ts +``` + +1. Add scope values/address/write variance and reusable-ID semantics. +2. Add retirement committed write and one-scope transaction validation. +3. Add tag/`stopAtTag` behavior. +4. Implement Memory physical identity, retirement, global sequences, snapshots. +5. Pass type tests and Memory conformance before continuing. + +### Slice B — SQLite + +Primary files: + +```text +MODIFY packages/session-backends/sqlite-node/src/sqlite/migrations/001_initial.sql +MODIFY packages/session-backends/sqlite-node/src/sqlite/{storage,repo}.ts +MODIFY packages/session-backends/sqlite-node/src/sqlite/session/{values,session-sequences}.ts +MODIFY packages/session-backends/sqlite-node/test/{storage,storage-conformance,repo,repo-conformance}.test.ts +``` + +1. Add scoped schema/query identity and tags. +2. Implement atomic `DELETE ... WHERE scope_id` retirement. +3. Preserve global sequence allocation, stats, branch indexes, shared-container behavior. +4. Exclude ephemeral rows from forks. +5. Pass SQLite focused/conformance tests before continuing. + +### Slice C — JSONL sidecars + +Primary files: + +```text +CREATE packages/agent/src/harness/session/jsonl/scope-files.ts for fixed directory/file naming, exact header parsing/serialization, discovery, atomic first creation, torn-tail parsing/repair, and recursive cleanup helpers +MODIFY packages/agent/src/harness/session/jsonl/{types,codec,storage,repo,legacy-v3,index}.ts +MODIFY packages/agent/test/harness/jsonl-{storage,storage-conformance,session-repo,session-repo-conformance,v3-migration}.test.ts +``` + +1. Add sidecar naming/header/create/discovery. +2. Route scoped commits to sidecars while allocating globally. +3. Persist main retirement records and serialized unlink. +4. Implement multi-file high-water calculation and globally ordered replay with reusable boundaries. +5. Handle torn tails, corruption, v3, listing, delete, close, and forks. +6. Pass all JSONL tests before runtime migration. + +### Slice D — Harness migration + +Primary files: + +```text +MODIFY packages/agent/src/harness/session/values.ts +MODIFY packages/agent/src/harness/runtime/{progress,lane}.ts to propagate scoped write generics through progress channels and lane commands +MODIFY packages/agent/src/harness/runtime/drive/{terminal,response,deferred,tools,reconcile,structural,boundary}.ts +INSPECT packages/agent/src/harness/runtime/drive/tool-placement.ts; current source needs no scoped cleanup change +MODIFY all focused runtime tests found by the §12 greps +``` + +1. Scope memo/tool/frame addresses by operation ID. +2. Remove mixed-scope deletes and cleanup scans. +3. Add terminal retirement in the universal suffix. +4. Preserve current output/replay/event behavior. +5. Prove late-write fencing and every terminal/close/recovery path. + +### Slice E — documentation and final validation + +Update current normative/reference docs: + +```text +packages/agent/docs/harness.md +packages/agent/docs/values.md +packages/agent/docs/post-wp05-roadmap.md +packages/agent/docs/mobile-handoff/README.md +packages/agent/docs/mobile-handoff/01-harness/02-scopes/scopes.md +packages/session-backends/sqlite-node/README.md +packages/agent/src/harness/telemetry.ts and generated `packages/agent/docs/telemetry-schema.md`: add the `scope` session-write item kind without implementing spans +``` + +Do not rewrite historical WP00–WP07 handoffs or released changelog sections. On `dev`, do not add changelog entries under the repository's main-only rule. + +Run: + +```bash +npm run check +./test.sh +``` + +Also run every modified focused test after each slice. Record full command results for the approval checkpoint. + +## 11. Exclusions + +Do not include: + +- Chord `Op`/`WireOp` storage integration; +- JSONL address/path interning or compact tuple records; +- pending-output address renames; +- `ToolOutput`, tool API changes, replay seeding, memo/checkpoint atomicity, progress cadence, rate limiting, exec-env changes; +- compact `message_update`, assistant incremental reducer changes, or protocol replication changes; +- J1 main-log snapshot compaction; +- WP08 fork redesign beyond excluding ephemeral state from current forks; +- repository lifecycle contract changes; +- telemetry implementation beyond keeping the declared schema accurate for the new write kind; +- migrations or compatibility for older WIP format-4 SQLite schemas; +- application-level automatic scope ownership, operation-derived orphan inference, leases, TTLs, or background scope collection. + +If implementation requires an excluded item, stop and revise this handoff before expanding scope. + +## 12. Grep guards + +Before final review, inspect every remaining match: + +```text +operationToolMemoPrefix +pendingToolOutputPrefix +deleteList(pendingAssistantFrames +deleteValue(pendingToolOutput +scopeId +retireScope +stopAtTag +``` + +Expected outcomes: + +- no session-scoped transaction directly deletes an ephemeral address; +- no terminal path omits retirement; +- no backend point read/write omits scope identity; +- no fork copies ephemeral state; +- no JSONL sidecar is treated as a Session file; +- no per-scope sequence counter exists; +- no operation-state/orphan inference exists in storage. + +## 13. Exit condition + +Step 1 is implementation-complete when: + +- scope/address/write typing rejects mixed-scope transactions and runtime validation rejects two ephemeral IDs in one transaction; +- IDs are reusable through latest-retirement global sequence boundaries with no historical ID registry; +- Memory, JSONL, and SQLite share one global sequence space and pass common conformance; +- SQLite retirement deletes exact scoped rows atomically with terminal state; +- JSONL writes ephemeral values/lists only to sidecars, records retirement only in the main log, replays all files in global order, preserves high-water marks, and deletes retired sidecars without operation inference; +- list tags and bounded `stopAtTag` work identically across backends; +- current tool/assistant payload and event formats are unchanged; +- every terminal path retires once, while close/fault preserves active scope recovery; +- forks and repository listing/deletion handle sidecars correctly; +- focused tests, `npm run check`, and `./test.sh` pass; +- final review has no blocker; +- the complete diff and results have been presented to the user and implementation has stopped for approval. + +Only after explicit approval may Step 1 be committed and pushed. Step 2 starts only after that push is confirmed. diff --git a/packages/agent/docs/mobile-handoff/01-harness/02-scopes/scopes.md b/packages/agent/docs/mobile-handoff/01-harness/02-scopes/scopes.md new file mode 100644 index 000000000..19aa2de6b --- /dev/null +++ b/packages/agent/docs/mobile-handoff/01-harness/02-scopes/scopes.md @@ -0,0 +1,574 @@ +# Session Storage: Scopes + +> **Status:** Design evidence. The actionable Step 1 contract is [`implementation-handoff.md`](implementation-handoff.md), which supersedes this document on sequence allocation, reusable scope IDs, retirement authority/records, sidecar layout, implementation order, and the separation of JSONL encoding into Step 2. +> +> **Scope:** `packages/agent/src/harness/session/`, plus the call sites in +> `runtime/` listed in §10. +> +> **Depends on [01-delta](../01-delta/delta.md)** for the landed Chord op vocabulary that pending state is stored in (§2, §11) and for the `WireOp` form the record encoding carries (§12). The two +> are otherwise independent and compound: encoding shrinks every write, scopes +> move one class of write out of the main log entirely. +> +> Every claim here was checked against `runtime/lane.ts`, `runtime/progress.ts` +> and `runtime/drive/*.ts`. The type-level enforcement in §6 was verified with +> `tsc --strict`, including that mixed-scope commits are the *only* thing it +> rejects. + +## 1. The problem + +Operation state — pending tool output, pending assistant output — matters only +while the operation runs. It is crash-recovery scaffolding, not history. + +The JSONL log is append-only, so it persists anyway. Worse, deleting it *adds* +records: `deleteValue` and `deleteList` write a line each, and those lines also +persist. Today only a full snapshot rewrite reclaims the space, and no in-place +compaction exists (`jsonl/storage.ts` has `createFromSnapshot`, used for fork). + +Measured on 20 operations, each 400 assistant frames plus 2 tools at 60 +checkpoints of a 50 KB window: + +| | size | +| --- | --- | +| total JSONL | 93.89 MB | +| settled transcript entries — the part that is actually history | **0.06 MB** | + +Three orders of magnitude of the file is scaffolding for finished operations. + +**Two independent fixes, and they are not alternatives:** + +- **Encoding** ([delta.md](../01-delta/delta.md)): value writes carry Chord ops rather than whole values, and addresses are interned. Takes the same workload to **5.32 MB in a single file**, with no change to atomicity. The op vocabulary and flush-time tracker have landed; storage integration has not. +- **Scopes** (this document): pending state leaves the main log entirely, so it is + never history to begin with. + +Do the encoding first. Scopes are worth it because 5.32 MB of dead weight per 20 +operations still accumulates, and there is no compaction pass to reclaim it. + +## 2. Scope on the address + +Scope has two halves that must not be confused: a **type-level tag** carrying no +data, and a **runtime scope id** that names the file. + +```ts +export type SessionScope = { readonly kind: "session" }; +export type EphemeralScope = { readonly kind: "ephemeral" }; +export type Scope = SessionScope | EphemeralScope; + +// Passing a scopeId is what makes an address ephemeral. +export function value(namespace: string, key: string): Value; +export function value(namespace: string, key: string, scopeId: string): Value; + +export function list(namespace: string, key: string): ValueList; +export function list(namespace: string, key: string, scopeId: string): ValueList; + +/** A main-log record retiring an ephemeral scope. See §5. */ +export function retireScope(id: string): Write; +``` + +The suffix is not decoration: `Session` already names the session interface in +`harness/session/types.ts`, and reusing it produces `TS2440`/`TS2484` plus an +ambiguous re-export from `session/index.ts`. + +```ts +interface Value extends ScopedAddress { + readonly namespace: string; + readonly key: string; + /** Present iff Sc is Ephemeral. Routes the write and names the sidecar. */ + readonly scopeId?: string; +} +``` + +The id is a plain runtime string, because it is an operation id — generated at +runtime and unavailable to the type system. That is precisely why §6 can +distinguish *session from ephemeral* statically but cannot distinguish *two +different ephemeral scopes*: the tag is in the type, the id is not. + +```ts +// Durable lists carry encoded batches; one encoder/decoder pair belongs to each value stream. +export const pendingToolOutput = (operationId: string, invocationId: string) => + list("pi.pending.tool_output", `${operationId}:${invocationId}`, operationId); + +export const pendingAssistantOutput = (operationId: string, entryId: string) => + list("pi.pending.assistant_output", `${operationId}:${entryId}`, operationId); + +export const operationToolMemo = (operationId: string, invocationId: string, name: string) => + value("pi.op.tool_memo", `${operationId}:${invocationId}:${name}`, operationId); + +// unchanged — no scopeId, so Session by inference +export const laneStateValue = (lane: string) => value("pi.lane.state", lane); +``` + +**Scope is the file.** A session-scoped write goes to the main log; an +ephemeral-scoped write goes to `..jsonl`. + +Other backends need no sidecars at all: in-memory drops a `Map` keyed by scope id, +SQLite issues one `DELETE ... WHERE scope = ?`. Scope is meaningful to them only as +a lifetime hint. It is JSONL that needs the file split, and that is where the +constraints in §4 and §5 come from. + +## 3. Which addresses are scoped, and why + +The boundary is drawn by **write isolation**, not by lifetime. + +Scoping everything under `pi.op.*` on the grounds that it dies with the operation +looks right and is wrong. `lane.ts` shows why: **every** operation commit bundles +`operationState` with `laneState`. + +```ts +// lane.ts:405/431, 659/660, 749/750, 1071/1072 — the same shape each time +writes: [ + ...decision.writes, + setValue(operationStateValue(operationId), decision.operationState), + setValue(laneStateValue(this.name), durableLaneState(...)), +] +``` + +That pair is coordination state: lane state says the operation is at step N, +operation state holds the data for step N. Splitting them across files makes a +crash produce a lane that believes it is somewhere the data does not support. + +The **bulk** values behave differently. `openProgress` commits exactly one write +(`runtime/progress.ts:51`), and `pendingToolOutput` / `pendingAssistantFrames` +appear in multi-write transactions only as *deletes*. + +| | addresses | +| --- | --- | +| **ephemeral (sidecar)** | `pi.pending.tool_output`, `pi.pending.assistant_output`, `pi.op.tool_memo` | +| **session (main log)** | `pi.lane.state`, `pi.op.state`, `pi.op.meta`, `pi.result`, `pi.pending.entry`, `pi.branch.tip`, `pi.op.tool_args`, `pi.op.preparation` | + +`operationToolMemo` is ephemeral so that the memo-plus-checkpoint bundling in +`harness-tools.md` §7.5 is a single-file transaction. `terminal.ts:34` already +treats `operationToolMemoPrefix` and `pendingToolOutputPrefix` as the same cleanup +class, so this matches how the code already thinks about them. + +The size win is unaffected by leaving `operationState` behind: effectively all of +the 93.89 MB in §1 is pending tool and assistant output. + +## 4. Two files are not atomic + +Serialising on the Session line gives **ordering, not atomicity**. Two `write()` +calls to two descriptors, two fsyncs, no journal spanning them. A crash between +leaves the files disagreeing, and JSONL has nothing to repair it with. + +So the design does not try to make cross-file transactions work. It ensures none +exist: + +> **All writes in a transaction share one scope.** + +That is achievable because it is already true of the code — see §5 for the audit. + +Two consequences of *not* needing cross-file commits: + +- **No write ordering between files is required.** The commit-record pattern + (sidecar first, fsync, then a main-log record acknowledging it) is unnecessary if + no transaction spans files. There is nothing to acknowledge. +- **No commit marker for sidecar-only writes.** One tempting design is a small + main-log record for `openProgress`, since nothing acknowledges its write. It is + not needed: that transaction is one write in one file and already atomic. Torn + tails are handled the same way as in the main log — every line is a complete op + or a whole batch, so a reader stops at the last intact one. + +What remains is a **durability** choice, not a consistency one: if the sidecar is +fsynced less aggressively, recent checkpoints can be lost. That is the same bounded +loss the checkpoint interval already accepts, but it should be a deliberate policy +rather than an accident. + +## 5. Audit: does the existing code satisfy the rule? + +Every commit in `lane.ts` (12 sites) and every `writes:` producer in +`runtime/drive/*.ts` (32 sites) was read. + +**Result: yes, with one adjustment.** No transaction writes to two files. The +spanning cases are all *deletes* of ephemeral state bundled with main-log writes: + +- `response.ts:340` — the settle commit: `insertEntry`, `insertUsage`, + `setValue(branchTip)`, plus `deleteList(pendingAssistantFrames(...))`. +- `terminal.ts:50` `operationCleanupWrites` — bundles main-log deletes with + `toolMemos` and `toolOutputs` deletes, returned into the same array as + `operationResultValue` and `laneStateValue`. +- `tools.ts:230, 257` — `deleteValue(pendingToolOutput)` with main-log writes. +- `deferred.ts:157` — `deleteList(pendingAssistantFrames)`. + +**The adjustment: retirement becomes a main-log record.** + +```ts +retireScope(operationId): Write +``` + +The log is the truth; the sidecar file is a cache. Recovery reads the retire record +and ignores — then removes — the sidecar. Unlinking eagerly after commit is a pure +optimisation; losing the unlink costs disk, never correctness. + +This makes all four sites above single-scope, because the individual ephemeral +deletes disappear. It also *simplifies* `operationCleanupWrites`: the `scanValues` +calls for `operationToolMemoPrefix` and `pendingToolOutputPrefix` are replaced by +one `retireScope`. + +And it makes the open-time sweep principled rather than heuristic. A sidecar is +dead iff the main log retired it or its operation is absent — no inference from +operation status. + +### 5.1 Two ephemeral scopes in one transaction cannot arise + +A lane holds `operation: { meta, state } | null` — **singular**. Every write set +uses one `operationId`, either `drive.operationId` or the newly created one in +`startOperation`, which asserts no operation is active. Retire and start are always +separate transactions: the settle commits at `lane.ts:427` and `response.ts:339` +set lane state to `null`. + +Concurrency is across **lanes**, and each lane has at most one active operation, so +two ephemeral scopes never meet in a commit. The runtime assertion in §6 is defence +against a future change, not a live gap. + +## 6. Static enforcement + +> **`scopes.variance.ts` ships alongside this doc** and is the executable form of +> this section: `npx tsc --noEmit --strict --lib es2023 scopes.variance.ts`. +> Silence means the rule holds. Its three `@ts-expect-error` lines fail the +> compile if they ever start type-checking, so it catches the enforcement being +> weakened as well as broken. + + +Two phantom types, differing only in where `Sc` appears. This distinction is +load-bearing and is the single easiest thing to get wrong here. + +```ts +declare const storedScopeType: unique symbol; + +/** Addresses: COVARIANT — Sc only in return position. */ +export interface ScopedAddress { + readonly [storedScopeType]?: () => Sc; +} + +/** Writes: INVARIANT — Sc in both parameter and return position. */ +export interface Scoped { + readonly [storedScopeType]?: (scope: Sc) => Sc; +} + +export interface Value + extends StoredAddressBase, ScopedAddress { … } + +export interface ValueSetWrite extends Scoped { … } +``` + +**Why they differ.** Reading an address is safe at any scope — `getValue` does not +care which file a value lives in, so `Value` must be usable where +`Value` is expected. Forming a transaction is *not* safe at any scope, +because two files are not atomic, so a commit must pin down exactly one. + +Enforcement therefore belongs where transactions are formed, not where addresses +are read. `setValue(address: Value, …): Write` is the bridge: it +infers the scope from a covariant address and stamps it onto an invariant write. + +> **Getting this wrong is expensive.** Making addresses invariant too breaks every +> reader — `getValue`, `scanValues`, `readList` and everything downstream — and +> produced a 36-error cascade across storage backends that had done nothing wrong. +> The tempting workaround, making each reader generic over `Sc` to mean "any +> scope", propagates a type parameter through the whole storage stack to express +> something variance gives for free. + +Verified with `tsc --noEmit --strict`. Reads pass at both scopes: + +```ts +getValue(laneState); // Value +getValue(pendingOutput); // Value +``` + +Single-scope commits pass: + +```ts +commit([setValue(laneState, a), setValue(operationState, b)]); +commit([setValue(toolMemo, m), setValue(pendingOutput, o)]); +commit([setValue(laneState, a), retireScope("op_1")]); +``` + +Mixed-scope commits fail, and are the *only* things that fail: + +```ts +commit([setValue(laneState, a), setValue(pendingOutput, o)]); // ERROR +commit([setValue(pendingOutput, o), retireScope("op_1")]); // ERROR +``` + +### 6.1 What the type system cannot catch + +Two different ephemeral scopes both type as `Write`, because the id +is a runtime string (§2). A runtime assertion at commit covers it — compare +`scopeId` across all writes — and per §5.1 it should never fire. + +## 7. Recovery and sweeping + +On open, enumerate sidecars alongside the main log and load them. Their records +participate in replay exactly as main-log records do. + +A sidecar is removed when the main log holds a `retireScope` record for it, or when +its operation is absent — a crash between commit and unlink. Torn-tail semantics +are unchanged: each line is self-contained, so a crash mid-write loses exactly the +trailing line. + +## 8. Sequence numbers + +Scoped writes get their own sequence space per scope. Nothing outside a scope +orders against its contents, and keeping them separate means `nextSeq` in the main +header does not account for numbers consumed by files that will be deleted. + +Shared numbering would leave large gaps after several operations settled, and the +high-water mark would need separate persistence. + +## 9. Measured effect + +Same workload as §1: + +| | size | of today | +| --- | --- | --- | +| today, single JSONL | 93.89 MB | — | +| ops + interned addresses, single JSONL | 5.32 MB | 5.7% | +| **plus scopes — main log** | **0.06 MB** | **0.06%** | +| — sidecars, retired on settle | 5.26 MB | peak 0.26 MB per operation | + +The 5.7% figure is **rate-dependent and must not be quoted alone**: it holds at +2 KB of tool output per checkpoint and inverts above the cap, where a whole-value +replacement wins. + +The scope result is **not** rate-dependent. The main log holds settled entries +regardless of encoding, and that is the number that governs session file growth. + +## 10. What changes in the existing code + +The compiler finds these for you: once addresses carry a scope and `Write` is +invariant, every site that mixes scopes in one transaction fails to typecheck. +This list is what a full pass turned up, so you know when you are done. + +**New, in `session/values.ts` and `session/types.ts`** + +`SessionScope` / `EphemeralScope` / `Scope`; the two phantoms from §6; +`Value` and `ValueList`; `scopeId` on the address; the `value` / +`list` overloads; `retireScope`; scope-preserving `setValue` / `deleteValue` / +`appendList` / `deleteList`; `Write` with entries, usage and retirement +session-only by construction. + +**Generified** + +`CommitDecision`, `lane.command`, `Storage.commit`. +Give `Sc` a default of `SessionScope`: conditional types are not inference sites, +so without a default a mixed array falls back to the constraint `Scope` and every +call site fails. `settleOperation` and `continueOperation` stay **non**-generic — +see §5. + +**Committed writes** + +`CommittedScopeRetireWrite`, and `scopeId` carried through the committed shapes so +`JsonlStorage` can route on it. + +**Call sites that must change** — each currently mixes scopes in one transaction: + +| site | what it does today | what it becomes | +| --- | --- | --- | +| `drive/terminal.ts` `operationCleanupWrites` | enumerates tool memos and tool outputs via `scanValues`, deletes each alongside session deletes | one `retireScope(operationId)`; two of its four scans disappear | +| `drive/response.ts` (settle) | `deleteList(pendingAssistantOutput)` bundled with operation state | drop it — the sidecar is discarded at retire | +| `drive/deferred.ts` (superseded response) | same | same | +| `drive/tools.ts` (tool settle) | `deleteValue(pendingToolOutput)` plus memo deletes in a session write array | drop them | +| `drive/tool-placement.ts` | `deleteValue(pendingToolOutput)` | drop it | + +The pattern is the same everywhere: **intra-operation cleanup of ephemeral state +is impossible**, because those commits also write operation and lane state. The +sidecar is discarded wholesale at retire. The cost is that a multi-turn operation +holds superseded pending output until settle — bounded by turns, and never +reaching the main log. + +**Storage** + +Sidecar routing in `JsonlStorage.commit` (assert one scope id per transaction, +route the append), unlink on a `scope` record **after** the commit, and an +open-time sweep that loads sidecars and removes any the main log already retired. +In-memory drops a `Map` keyed by scope id; SQLite issues one +`DELETE ... WHERE scope = ?` and treats a retire record as a no-op, because it has +no sidecar and the terminal transaction deletes the scoped values explicitly. + +**Expected test churn** + +Nine tests assert the old cleanup write set and will fail — `drive-terminal` (3), +`drive-retry-deferred` (2), `drive-tools`, `drive-reconcile`, `drive-generation`, +and one pinning reserved namespaces. They expect six or seven individual deletes +where the new code emits fewer plus one `retireScope`. That is the change landing, +not a regression. + +## 11. List tags and stop conditions + +Tracked state is stored as a list of op batches (`delta.md` §9). Recovery needs "the +batches since the last base batch" without unpacking every row, which lists cannot +express today: `ListReadOptions` has `cursor`, `order` and `limit` only. + +`BranchScan` already solves the same problem for entries, and its vocabulary is +the one to copy — `stopAtType` / `stopAtId` for a terminator, plain field names +for a filter. + +```ts +appendList(address: ValueList, element: T, tag?: string): ListAppendWrite; + +export interface ListElement { + seq: number; + value: T; + tag?: string; +} + +export interface ListReadOptions { + cursor?: ListCursor; + order?: "asc" | "desc"; + limit?: number; + /** Include elements up to and including the first carrying this tag, then stop. */ + stopAtTag?: string; + /** Return only elements carrying this tag. */ + tag?: string; +} +``` + +**`stopAtTag` is a stop condition within a page, not a guarantee.** It can only end +a page earlier than `limit` would; it never overrides it. If the tagged element is +not in the page, the consumer sees no tagged element and pages again with the +cursor — the same loop `readAssistantFrames` already runs. There is no ordering +question between the two bounds and no "not found" error. + +For that loop to work, `readList` must return the tag on each element. Otherwise a +consumer cannot tell "page ended at the tag" from "page ended at the limit" +without parsing the value, which is the thing the tag exists to avoid. + +**The producer sets the tag, not storage.** For a tracked value the caller +already knows, because `isBase(ops)` is a token comparison on the first op +(`delta.md` §2): + +```ts +const ops = tracker.flush(); +if (ops.length === 0) return; +writes: [appendList(address, enc.encode(ops), isBase(ops) ? "base" : undefined)]; +``` + +Storage must never inspect an element to derive a tag. That is what keeps op +batches opaque to the storage layer, and it is the same property that makes +`EntryScan.type` work: the discriminant is a stored column, not something derived +from the payload. + +**The tag lives on the storage record**, beside `seq` and `value`: + +```jsonc +["l",7,9,[["r",{…}]],"base"] // per §12.1 +``` + +Storage must never parse an element to evaluate a predicate. That is what keeps +ops opaque to storage and the durable path free of domain knowledge +(`harness-tools.md` §7.7). It is also why `EntryScan.type` works: the discriminant +is a stored column, not something derived from the payload. + +`order` stays `"asc" | "desc"` rather than `BranchScan`'s +`"newestFirst" | "oldestFirst"`. Branch order is semantic — a walk from a tip +through a tree. List order is over `seq`. Borrowing the branch words would imply a +traversal that is not happening. + +Backends: SQLite gets a `tag` column and a real predicate. JSONL holds the list in +memory already, so it scans backwards and checks a field — no worse than today. +In-memory likewise. + +The primitive generalises past frames: any list wanting "the tail since the last +checkpoint" gets it. + +## 12. JSONL record encoding + +Two dictionaries at two layers. They are the same trick and are independent: the +**address** dictionary is over `namespace` + `key` on the record; the **path** +dictionary is over paths *inside* a value's ops (`delta.md` §4). + +### 12.1 Records + +``` +["@", addrId, namespace, key] address definition +["v", addrId, seq, wireOps] value write — WireOp[], delta.md §4 +["l", addrId, seq, element, tag?] list append — element is WireOp[] + for a tracked value +["x", addrId, seq] delete (value or list) +["!", addrId, seq] retire an ephemeral scope +``` + +**Record verbs and op verbs are separate vocabularies read at different levels**, +but they must not look alike. `!` rather than `r` for retire: `r` is delta's +replace op, and a reader scanning a file should not have to remember which nesting +level they are at to know what a tuple means. + +Retire takes an address id like everything else, defined by an `["@", id, …]` +whose namespace is the scope. That keeps one interning rule for the whole file +rather than a special case for one record. + +Entries and usage rows keep their current keyed form: they are written once, never +repeat an address, and are read by machinery that has nothing to do with ops. + +An address is defined on its **second** use, matching path interning — a +definition is pure overhead for an address written once, and a session has many of +those. + +### 12.2 Worked example + +Four writes across two addresses, before and after. + +```jsonc +// today — 1250 bytes +{"kind":"value","op":"set","seq":7,"namespace":"pi.op.state","key":"01a04cf6-…","value":{"at":"starting","control":{…},"settings":{…},"latestAssistantEntryId":null}} +{"kind":"value","op":"set","seq":8,"namespace":"pi.lane.state","key":"main","value":{"currentOperationId":"01a04cf6-…",…}} +{"kind":"value","op":"set","seq":9,"namespace":"pi.op.state","key":"01a04cf6-…","value":{"at":"checkpoint",…}} +{"kind":"value","op":"set","seq":12,"namespace":"pi.op.state","key":"01a04cf6-…","value":{"at":"assistant.ready",…}} +``` + +```jsonc +// with both dictionaries — 547 bytes +["@",0,"pi.op.state","01a04cf6-…"] +["v",0,7,[["r",{"at":"starting","control":{…},"settings":{…},"latestAssistantEntryId":null}]]] +["@",1,"pi.lane.state","main"] +["v",1,8,[["r",{"currentOperationId":"01a04cf6-…",…}]]] +["v",0,9,[["#",0,["at"]],["s",0,"checkpoint"]]] +["v",0,12,[["s","assistant.ready"]]] +``` + +| line | before | after | +| --- | --- | --- | +| address definition | – | 61 | +| op.state -> `starting` (base batch) | 353 | 256 | +| address definition | – | 31 | +| lane.state (base batch) | 181 | 114 | +| op.state -> `checkpoint` | 355 | **48** | +| op.state -> `assistant.ready` | 361 | **37** | +| **total** | **1250** | **547** (44%) | + +The shape matters more than the total. Base batches barely shrink — 353 to 256 is +envelope only, since the value ships whole either way. The transitions collapse +~8x because they carry one changed field instead of the whole state including the +never-changing `settings` block. + +Both dictionaries appear on line 5: `0` in `["v",0,9,…]` is an **address** id, +while `["#",0,["at"]]` defines a **path** id inside the value's ops. Same trick, +different layers, separate tables. Line 6 then drops the path entirely — arity +omission, because it targets the same path as the previous op in that batch. + +**Do not quote 44% as the file-level saving.** Dictionary entries are one-time, so +a real operation's ~11 op.state writes amortise them and the ratio improves; but +transcript entries are untouched, and they were 18 KB of a 67 KB tool-heavy run. +Applied there this hits the ~27 KB of value writes and would take the file down by +roughly a fifth, not by half. + +### 12.3 Reading + +A reader builds both tables while replaying, so a torn tail costs exactly the +trailing lines and no header needs rewriting. A snapshot rewrite starts fresh +tables and re-emits definitions naturally. + +A numeric `addrId` for which no definition has been seen is a corrupt file, not a +recoverable state — unlike a missing path id, which cannot occur because path +definitions are inside the same record as their first use. + +## 13. Open questions + +- File-handle pressure across concurrent lanes, each with a live sidecar. Probably + fine, unmeasured. +- fsync policy for sidecars (§4) — matched to the main log, or relaxed given the + contents are bounded-loss scaffolding. +- Whether a long-running operation should rotate its sidecar. `rebase()` + (`delta.md` §3.2.3) bounds *recovery* length but not file size, so a command + running for hours still grows its sidecar without bound. +- Whether an in-place compaction pass for the main log is worth building anyway. + `createFromSnapshot` plus atomic replace is the mechanism; scopes reduce the need + but do not remove it, since superseded session-scoped values still accumulate. diff --git a/packages/agent/docs/mobile-handoff/01-harness/02-scopes/scopes.variance.ts b/packages/agent/docs/mobile-handoff/01-harness/02-scopes/scopes.variance.ts new file mode 100644 index 000000000..364ff423a --- /dev/null +++ b/packages/agent/docs/mobile-handoff/01-harness/02-scopes/scopes.variance.ts @@ -0,0 +1,65 @@ +/** + * Executable form of scopes.md §6. + * + * npx tsc --noEmit --strict --lib es2023 scopes.variance.ts + * + * Silence means the rule holds: reads accept either scope, single-scope commits + * pass, and mixed-scope commits are the ONLY thing rejected. The three + * `@ts-expect-error` lines fail the compile if they ever start type-checking, + * so this catches the enforcement being weakened as well as broken. + * + * The variance split is the part that is easy to get backwards. Making addresses + * invariant too breaks every reader — getValue, scanValues, readList and + * everything downstream — and the tempting fix, making each reader generic over + * Sc, propagates a type parameter through the whole storage stack to express + * what variance gives for free. + */ +declare const tag: unique symbol; +type SessionScope = { readonly kind: "session" }; +type EphemeralScope = { readonly kind: "ephemeral" }; +type Scope = SessionScope | EphemeralScope; + +/** Addresses: COVARIANT — Sc only in return position. Readers take any scope. */ +interface ScopedAddress { readonly [tag]?: () => Sc } +/** Writes: INVARIANT — Sc in both positions. Mixed commits cannot unify. */ +interface Scoped { readonly [tag]?: (s: Sc) => Sc } + +interface Value extends ScopedAddress { k: "v" } +interface ValueList extends ScopedAddress { k: "l" } +interface VSet extends Scoped { kind: "value"; op: "set" } +interface VDel extends Scoped { kind: "value"; op: "delete" } +interface LApp extends Scoped { kind: "list"; op: "append" } +interface EntryWrite { kind: "entry" } +interface RetireWrite extends Scoped { kind: "scope" } + +type Write = + | VSet | VDel | LApp + | (Sc extends SessionScope ? EntryWrite | RetireWrite : never); + +declare function setValue(a: Value, v: T): VSet; +declare function deleteValue(a: Value): VDel; +declare function appendList(a: ValueList, v: T): LApp; +declare function retireScope(id: string): RetireWrite; +declare function insertEntry(e: unknown): EntryWrite; +declare function commit(w: readonly Write[]): void; +declare function getValue(a: Value): T; // reader: any scope + +const laneState = null as unknown as Value; // session +const pendingOut = null as unknown as ValueList; // ephemeral +const toolMemo = null as unknown as Value; + +// READS accept either scope — that is what covariance buys. +getValue(laneState); +getValue(toolMemo); + +// Single-scope commits pass. +commit([setValue(laneState, "a")]); +commit([setValue(laneState, "a"), insertEntry({}), retireScope("op_1")]); +commit([appendList(pendingOut, "x"), setValue(toolMemo, "m")]); + +// @ts-expect-error mixed scopes in one transaction +commit([setValue(laneState, "a"), appendList(pendingOut, "x")]); +// @ts-expect-error a retire is session-only, so it cannot join an ephemeral commit +commit([appendList(pendingOut, "x"), retireScope("op_1")]); +// @ts-expect-error entries are session-only +commit([insertEntry({})]); diff --git a/packages/agent/docs/mobile-handoff/01-harness/03-execenv/bash.after.ts b/packages/agent/docs/mobile-handoff/01-harness/03-execenv/bash.after.ts new file mode 100644 index 000000000..95a6e3e74 --- /dev/null +++ b/packages/agent/docs/mobile-handoff/01-harness/03-execenv/bash.after.ts @@ -0,0 +1,172 @@ +import { type Static, Type } from "typebox"; +import type { Context } from "../context.ts"; +import type { AgentHarnessTool } from "../types.ts"; +import { getOrThrow } from "../types.ts"; +import { DEFAULT_MAX_BYTES, DEFAULT_MAX_LINES, formatSize, type TruncationResult } from "../utils/truncate.ts"; +import type { ExecutionToolContext } from "./tool-context.ts"; + +const MAX_TIMEOUT_SECONDS = 2_147_483_647 / 1000; +/** + * Coalescing cadence for live updates. The env owns capping and coalescing now + * (`executionenv.md` §3), so bash no longer has an update throttle or a + * checkpoint interval of its own. + */ +const BASH_UPDATE_INTERVAL_MS = 100; + +const bashSchema = Type.Object({ + command: Type.String({ description: "Bash command to execute" }), + timeout: Type.Optional(Type.Number({ description: "Timeout in seconds (optional, no default timeout)" })), +}); + +export type BashToolInput = Static; + +export interface BashToolDetails { + truncation?: TruncationResult; + /** Set by the exec env, on the machine the command ran on. */ + spillPath?: string; +} + +export interface BashExecution { + command: string; + cwd: string; + env: Record; + inheritEnv: boolean; +} + +export type BashPrepare = ( + execution: BashExecution, + toolContext: TContext, + context: Context, +) => void | Promise; + +export interface BashToolOptions { + commandPrefix?: string; + prepare?: BashPrepare; +} + +function validateTimeout(timeout: number | undefined): void { + if (timeout === undefined) return; + if (!Number.isFinite(timeout) || timeout <= 0) { + throw new Error("Invalid timeout: must be a finite number of seconds"); + } + if (timeout > MAX_TIMEOUT_SECONDS) { + throw new Error(`Invalid timeout: maximum is ${MAX_TIMEOUT_SECONDS} seconds`); + } +} + +export function createBashTool( + options?: BashToolOptions, +): AgentHarnessTool { + return { + name: "bash", + label: "bash", + description: `Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last ${DEFAULT_MAX_LINES} lines or ${DEFAULT_MAX_BYTES / 1024}KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.`, + parameters: bashSchema, + async execute(_toolCallId, { command, timeout }, onUpdate, toolContext, _invocation, context) { + validateTimeout(timeout); + const { env } = toolContext; + const execution: BashExecution = { + command: options?.commandPrefix ? `${options.commandPrefix}\n${command}` : command, + cwd: env.cwd, + env: {}, + inheritEnv: true, + }; + await options?.prepare?.(execution, toolContext, context); + + // The env caps, coalesces and spills at the source. bash keeps a mirror + // only to build the settled result; it never truncates and never chooses + // a spill path. + let view = ""; + let truncation: TruncationResult | undefined; + let spillPath: string | undefined; + + const publish = (checkpoint: boolean): void => { + onUpdate( + { + content: [{ type: "text" as const, text: view }], + details: { truncation, spillPath }, + }, + checkpoint ? { checkpoint: true } : undefined, + ); + }; + + onUpdate({ content: [], details: undefined }); + + const result = await env.exec( + execution.command, + { + cwd: execution.cwd, + env: execution.env, + inheritEnv: execution.inheritEnv, + timeout, + capture: { + limits: { maxBytes: DEFAULT_MAX_BYTES, maxLines: DEFAULT_MAX_LINES, retain: "tail" }, + intervalMs: BASH_UPDATE_INTERVAL_MS, + spill: true, + }, + onUpdate: (update) => { + switch (update.kind) { + case "chunk": + view += update.text; + break; + case "snapshot": + view = update.output.text; + truncation = update.output.truncation; + spillPath = update.output.spillPath; + break; + case "counters": + truncation = update.truncation; + break; + } + // A snapshot is the durable-worthy state: it is the only kind that + // can express eviction, so it is where a crash loses least. + publish(update.kind === "snapshot"); + }, + }, + context, + ); + + if (!result.ok) { + const error = result.error; + const status = + error.code === "timeout" + ? `Command timed out after ${timeout} seconds` + : error.code === "aborted" + ? "Command aborted" + : error.message; + throw new Error(view ? `${view}\n\n${status}` : status, { cause: error }); + } + + const captured = result.value.output; + view = captured.text; + truncation = captured.truncation; + spillPath = captured.spillPath; + + let outputText = captured.text; + let details: BashToolDetails | undefined; + if (captured.truncation.truncated) { + details = { truncation: captured.truncation, spillPath }; + const startLine = captured.truncation.totalLines - captured.truncation.outputLines + 1; + const endLine = captured.truncation.totalLines; + if (captured.truncation.lastLinePartial) { + // One line alone exceeded the byte cap. Line numbers would be + // misleading here, so report the line's own size instead — and only + // the env can measure it, since it holds the raw buffer. + const lastLineSize = formatSize(captured.lastLineBytes ?? captured.truncation.outputBytes); + outputText += `\n\n[Showing last ${formatSize(captured.truncation.outputBytes)} of line ${endLine} (line is ${lastLineSize}). Full output: ${spillPath}]`; + } else if (captured.truncation.truncatedBy === "lines") { + outputText += `\n\n[Showing lines ${startLine}-${endLine} of ${captured.truncation.totalLines}. Full output: ${spillPath}]`; + } else { + outputText += `\n\n[Showing lines ${startLine}-${endLine} of ${captured.truncation.totalLines} (${formatSize(DEFAULT_MAX_BYTES)} limit). Full output: ${spillPath}]`; + } + } + + if (result.value.exitCode !== 0) { + throw new Error( + `${outputText ? `${outputText}\n\n` : ""}Command exited with code ${result.value.exitCode}`, + ); + } + return { content: [{ type: "text", text: outputText || "(no output)" }], details }; + }, + }; +} diff --git a/packages/agent/docs/mobile-handoff/01-harness/03-execenv/execenv.md b/packages/agent/docs/mobile-handoff/01-harness/03-execenv/execenv.md new file mode 100644 index 000000000..ec4d05a57 --- /dev/null +++ b/packages/agent/docs/mobile-handoff/01-harness/03-execenv/execenv.md @@ -0,0 +1,148 @@ +# ExecutionEnv: bounded shell output + +**Status:** implemented in production source. The prototype files beside this document are historical evidence; production lives in: + +- `packages/agent/src/harness/utils/adaptive-publisher.ts` +- `packages/agent/src/harness/utils/output-capture.ts` +- `packages/agent/src/harness/env/nodejs.ts` +- `packages/agent/src/harness/tools/bash.ts` + +The generic `ToolOutput` use of the same publisher remains in `04-tool-output`. + +## 1. Problem + +The old `Shell.exec()` accumulated complete strings inside `NodeExecutionEnv`: + +```ts +stdout += chunk; +stderr += chunk; +``` + +Bash truncated only after those strings had already been built. `cat 1gb.txt` therefore materialized a gigabyte in the worker before any tool-level bound could help. + +Spilling also belongs where bytes originate. If execution is remote, a spill on the worker is inaccessible to the model's `read` and `grep` tools, and creating it after transport would first send the complete gigabyte over the connection. + +## 2. Boundary + +`ExecutionEnv` now owns: + +- a bounded head or tail view; +- complete byte and line totals; +- lazy source-local spill after the view first crosses its limits; +- persistent source-local spill with bounded write-stream backpressure; +- adaptive publication of the latest bounded state; +- a forced final publication before settlement. + +It does not return or retain separate `stdout` and `stderr` values. Both pipes feed one arrival-ordered model-visible text view, matching bash and `ToolOutput`; preserving stream styling through tail eviction would require a segmented retained state that the Harness does not expose. Text is folded from updates, and `ShellExecResult` contains only exit and truncation/spill metadata. + +Bash owns only command semantics and its model-visible footer. Its old rolling buffer, spill creation, 100 ms throttle, and full-output accumulation are gone. The existing two-second durable checkpoint request remains temporarily until `ToolOutput` owns durable cadence. + +## 3. Contract + +```ts +interface ShellOutputLimits { + maxBytes: number; + maxLines: number; + retain?: "head" | "tail"; +} + +interface ShellOutputCaptureOptions { + limits: ShellOutputLimits; + spill?: boolean; +} + +type ShellOutputTruncation = Omit; + +interface ShellOutputMetadata { + truncation: ShellOutputTruncation; + spillPath?: string; + lastLineBytes?: number; +} + +interface ShellOutputView extends ShellOutputMetadata { + text: string; +} + +type ShellOutputUpdate = + | { kind: "replace"; output: ShellOutputView } + | { kind: "append"; text: string; metadata: ShellOutputMetadata } + | { kind: "slide"; drop: number; text: string; metadata: ShellOutputMetadata } + | { kind: "metadata"; metadata: ShellOutputMetadata }; + +interface ShellExecResult extends ShellOutputMetadata { + exitCode: number; +} +``` + +`drop` counts JavaScript string code units, matching `slice()`. Updates are ordered. A consumer applies them with `applyShellOutputUpdate()`. + +A complete replacement establishes the initial state or recovers when no verified overlap exists. An append carries only a growing suffix. A slide drops a prefix and appends the new suffix. Metadata updates move totals or the spill path without resending text. + +The compatibility `executeShellWithCapture()` helper still returns one bounded final view. Its `onChunk` callback receives initial, append, and slide text only; metadata and post-turnover replacements are not mislabeled as new bytes. + +## 4. Adaptive publication + +`AdaptivePublisher` keeps only the latest dirty state. Intermediate process writes never become a queue of output updates. + +Policy is harness-global rather than per tool: + +```ts +minIntervalMs = 100; +targetBytesPerSecond = 100 * 1024; +nextDelayMs = max(minIntervalMs, encodedUpdateBytes * 1000 / targetBytesPerSecond); +``` + +The first dirty state after idle is immediate. Writes received before the deadline collapse into the latest bounded state. One trailing timer guarantees eventual publication. Finalization bypasses the deadline once, still bounded by the retained cap. + +The publisher commits its baseline before invoking the consumer. If a consumer applies an update and then throws, finalization cannot emit the same delta twice. The command fails with `callback_error`. + +### Workload behavior + +| workload | result | +| --- | --- | +| Finishes below cap | immediate initial state, small appends, forced final state | +| Below-cap trickle | isolated writes are immediate; sustained writes are at most 100 ms apart | +| Full-force output after cap | complete turnovers are cap-sized replacements spaced by their encoded size | +| Post-cap trickle | small verified `slide` updates remain responsive; the complete window is not resent | +| Burst then silence | one leading update and one trailing update | +| Error, timeout, or abort | latest bounded state is forced before the error settles | + +At a 50 KB cap and 100 KB/s target, repeated complete turnovers settle near two updates per second. Small post-cap slides still use the 100 ms floor. + +The rate bound is amortized. An immediate-after-idle update and forced terminal update may each create one cap-bounded burst. + +## 5. Capture and spill + +`OutputCapture` trims its decoded line-aware working buffer back to twice the byte cap when it exceeds four times the cap. Tail mode drops old text; head mode preserves the original prefix. This amortizes UTF-8 trimming instead of rescanning the retained window for every process chunk. + +Spill creation is lazy. Before crossing, the source keeps at most the bounded prefix needed to create a complete archive. On crossing it: + +1. pauses stdout and stderr while it creates the file inside the execution environment; +2. opens one persistent append stream with a bounded 8 MB high-water mark; +3. writes the preserved raw prefix and subsequent raw chunks in arrival order; +4. resumes immediately while the writer accepts data; +5. pauses again only when `write()` reports backpressure, then resumes on `drain`. + +This avoids both an unbounded promise chain and one async file-open/append cycle per process chunk. A spill create or stream-write failure kills the child and fails execution rather than publishing lossy success. + +The spill path is force-published as metadata when it becomes available. Spill writes are awaited before the final output flush. + +Node streams remain raw for spill throughput and exact archival bytes. `OutputCapture` uses one streaming `TextDecoder`, so a read boundary cannot split a code point; invalid display control characters are removed only from bounded snapshots, not by scanning the complete raw stream. Line totals count a final unterminated line, and `lastLineBytes` remains exact even when one line exceeds the working buffer. + +## 6. Remote execution + +When worker and execution environment are colocated, updates are in-process and this publisher primarily bounds capture work. `ToolOutput` remains the downstream event/durability limiter. + +When an execution environment is physically remote from its worker, the same bounded updates cross that transport. Full-force output cannot transfer the complete stream: intermediate writes collapse at the source, while the complete stream remains in the source-local spill. + +Each real costly boundary gets its own publisher instance. A colocated deployment may bypass transport serialization; a custom tool that bypasses `ExecutionEnv` still passes through the future `ToolOutput` publisher. + +## 7. Remaining work + +- Move generic custom-tool composition, text retention, event publication, and durable cadence into `ToolOutput`. +- Replace whole `AgentToolResult` progress with Chord operations at that downstream boundary. +- Make memo and output checkpoint persistence one atomic transaction. +- Seed replay from durable output rather than deleting it. +- Put spills in an environment-owned session directory and sweep them after session lifetime plus a crash-retention floor. +- Decide explicit limits for images and structured details; text is bounded, those values are not yet. +- Define raw binary-output behavior. Current shell output remains lossy UTF-8 text. diff --git a/packages/agent/docs/mobile-handoff/01-harness/03-execenv/nodejs-env.diff b/packages/agent/docs/mobile-handoff/01-harness/03-execenv/nodejs-env.diff new file mode 100644 index 000000000..2e3719970 --- /dev/null +++ b/packages/agent/docs/mobile-handoff/01-harness/03-execenv/nodejs-env.diff @@ -0,0 +1,137 @@ +diff --git a/packages/agent/src/harness/env/nodejs.ts b/packages/agent/src/harness/env/nodejs.ts +index 732dc84db..db9eb9a57 100644 +--- a/packages/agent/src/harness/env/nodejs.ts ++++ b/packages/agent/src/harness/env/nodejs.ts +@@ -19,6 +19,7 @@ import { basename, isAbsolute, join, resolve } from "node:path"; + import { createInterface } from "node:readline"; + import { fileURLToPath } from "node:url"; + import type { Context } from "../context.ts"; ++import { OutputCapture } from "../utils/output-capture.ts"; + import { + type ExecutionEnv, + ExecutionError, +@@ -28,7 +29,10 @@ import { + type FileKind, + ok, + type Result, ++ type ShellCapabilities, + type ShellExecOptions, ++ type ShellExecResult, ++ type ShellStream, + toError, + } from "../types.ts"; + +@@ -365,11 +369,15 @@ export class NodeExecutionEnv implements ExecutionEnv { + return ok(join(...parts)); + } + ++ capabilities(): ShellCapabilities { ++ return { capture: true, spill: true, liveUpdates: true }; ++ } ++ + async exec( + command: string, + options: ShellExecOptions | undefined, + context: Context, +- ): Promise> { ++ ): Promise> { + const signal = context.abortSignal; + if (signal?.aborted) return err(new ExecutionError("aborted", "aborted")); + const timeoutMsResult = resolveTimeoutMs(options?.timeout); +@@ -392,9 +400,28 @@ export class NodeExecutionEnv implements ExecutionEnv { + ); + } + ++ // Spill lives in the exec env, on the machine where the model's own `read` ++ // and `grep` run — the whole reason capping happens here rather than on the ++ // agent side (`executionenv.md` §1). ++ let spillHandle: string | undefined; ++ let spillChain: Promise = Promise.resolve(); ++ const wantSpill = options?.capture?.spill === true; ++ + return await new Promise((resolvePromise) => { +- let stdout = ""; +- let stderr = ""; ++ const capture = new OutputCapture(options?.capture, options?.onUpdate); ++ const spillChunk = (chunk: string): Promise => { ++ if (!wantSpill) return Promise.resolve(); ++ spillChain = spillChain.then(async () => { ++ if (spillHandle === undefined) { ++ const created = await this.createTempFile({ prefix: "pi-output-", suffix: ".log" }, context); ++ if (!created.ok) return; ++ spillHandle = created.value; ++ capture.setSpillPath(spillHandle); ++ } ++ await this.appendFile(spillHandle, chunk, context); ++ }); ++ return spillChain; ++ }; + let settled = false; + let timedOut = false; + let callbackError: ExecutionError | undefined; +@@ -407,7 +434,7 @@ export class NodeExecutionEnv implements ExecutionEnv { + } + }; + +- const settle = (result: Result<{ stdout: string; stderr: string; exitCode: number }, ExecutionError>) => { ++ const settle = (result: Result) => { + if (timeoutId) clearTimeout(timeoutId); + if (signal) signal.removeEventListener("abort", onAbort); + if (child?.pid) this.activeChildPids.delete(child.pid); +@@ -458,31 +485,32 @@ export class NodeExecutionEnv implements ExecutionEnv { + } + } + ++ // setEncoding("utf8") is what keeps a chunk boundary from splitting a ++ // codepoint; without it a multi-byte sequence straddling a read bakes in a ++ // replacement character (`executionenv.md` §8). + child.stdout?.setEncoding("utf8"); + child.stderr?.setEncoding("utf8"); +- child.stdout?.on("data", (chunk: string) => { +- stdout += chunk; +- try { +- options?.onStdout?.(chunk, context); +- } catch (error) { +- const cause = toError(error); +- callbackError = new ExecutionError("callback_error", cause.message, cause); +- onAbort(); +- } +- }); +- child.stderr?.on("data", (chunk: string) => { +- stderr += chunk; ++ const feed = (stream: ShellStream, chunk: string) => { + try { +- options?.onStderr?.(chunk, context); ++ capture.push(stream, chunk, context); ++ void spillChunk(chunk); + } catch (error) { + const cause = toError(error); + callbackError = new ExecutionError("callback_error", cause.message, cause); + onAbort(); + } +- }); ++ }; ++ child.stdout?.on("data", (chunk: string) => feed("stdout", chunk)); ++ child.stderr?.on("data", (chunk: string) => feed("stderr", chunk)); + + void waitForChildProcess(child).then( +- (code) => { ++ async (code) => { ++ // The spill chain is async and lazy: the file is created on the first ++ // chunk. Await it before snapshotting, or the returned CapturedOutput ++ // reports no spillPath for output that was in fact archived. ++ await spillChain; ++ capture.flush(context); ++ capture.dispose(); + if (callbackError) { + settle(err(callbackError)); + return; +@@ -495,7 +523,7 @@ export class NodeExecutionEnv implements ExecutionEnv { + settle(err(new ExecutionError("aborted", "aborted"))); + return; + } +- settle(ok({ stdout, stderr, exitCode: code ?? 0 })); ++ settle(ok({ output: capture.snapshot(), exitCode: code ?? 0 })); + }, + (error: Error) => settle(err(new ExecutionError("spawn_error", error.message, error))), + ); diff --git a/packages/agent/docs/mobile-handoff/01-harness/03-execenv/nodejs-env.test.diff b/packages/agent/docs/mobile-handoff/01-harness/03-execenv/nodejs-env.test.diff new file mode 100644 index 000000000..c27be5981 --- /dev/null +++ b/packages/agent/docs/mobile-handoff/01-harness/03-execenv/nodejs-env.test.diff @@ -0,0 +1,149 @@ +diff --git a/packages/agent/test/harness/nodejs-env.test.ts b/packages/agent/test/harness/nodejs-env.test.ts +index 361040495..ee28ef506 100644 +--- a/packages/agent/test/harness/nodejs-env.test.ts ++++ b/packages/agent/test/harness/nodejs-env.test.ts +@@ -8,7 +8,6 @@ import { afterEach, describe, expect, it } from "vitest"; + import { BACKGROUND_CONTEXT, withAbortSignal } from "../../src/harness/context.ts"; + import { NodeExecutionEnv } from "../../src/harness/env/nodejs.ts"; + import { FileError, getOrThrow } from "../../src/harness/types.ts"; +-import { executeShellWithCapture } from "../../src/harness/utils/shell-output.ts"; + import { createTempDir } from "./session-test-utils.ts"; + + const chmodRestorePaths: string[] = []; +@@ -303,7 +302,8 @@ describe("NodeExecutionEnv", () => { + BACKGROUND_CONTEXT, + ), + ); +- expect(result).toEqual({ stdout: `${await realpath(root)}:ok`, stderr: "", exitCode: 0 }); ++ expect(result.output.text).toContain(`${await realpath(root)}:ok`.trim()); ++ expect(result.exitCode).toBe(0); + }); + + it.each([ +@@ -334,7 +334,7 @@ describe("NodeExecutionEnv", () => { + ), + ); + +- expect(result.stdout).toBe(`${expectedSessionFile}|true|preserved`); ++ expect(result.output.text).toBe(`${expectedSessionFile}|true|preserved`); + }, + ); + +@@ -358,7 +358,7 @@ describe("NodeExecutionEnv", () => { + ), + ); + +- expect(result.stdout).toBe("::explicit"); ++ expect(result.output.text).toBe("::explicit"); + } finally { + if (previousInherited === undefined) delete process.env[inheritedKey]; + else process.env[inheritedKey] = previousInherited; +@@ -396,7 +396,9 @@ describe("NodeExecutionEnv", () => { + await wslEnv.exec(`name='World'; echo "Hello, ${nameExpansion}!"`, undefined, BACKGROUND_CONTEXT), + ); + +- expect(result).toEqual({ stdout: "Hello, World!\n", stderr: "args:-s\n", exitCode: 0 }); ++ expect(result.output.text).toContain("Hello, World!"); ++ expect(result.output.text).toContain("args:-s"); ++ expect(result.exitCode).toBe(0); + } finally { + process.chdir(originalCwd); + process.env.PATH = originalPath; +@@ -425,7 +427,7 @@ describe("NodeExecutionEnv", () => { + () => controller.abort(), + ), + ); +- expect(result.stdout).toContain("child-exiting"); ++ expect(result.output.text).toContain("child-exiting"); + } finally { + controller.abort(); + cleanupDetachedChild(pidFile); +@@ -448,25 +450,30 @@ describe("NodeExecutionEnv", () => { + it("streams stdout and stderr chunks", async () => { + const root = createTempDir(); + const env = new NodeExecutionEnv({ cwd: root }); +- let stdout = ""; +- let stderr = ""; ++ // Streams are aggregated in arrival order now; `chunk` carries `stream` so a ++ // renderer can still colour stderr (`executionenv.md` §6). ++ let seen = ""; ++ const streams: string[] = []; + const result = getOrThrow( + await env.exec( + "printf out; printf err >&2", + { +- onStdout: (chunk) => { +- stdout += chunk; +- }, +- onStderr: (chunk) => { +- stderr += chunk; ++ onUpdate: (update) => { ++ if (update.kind !== "chunk") return; ++ seen += update.text; ++ streams.push(update.stream); + }, + }, + BACKGROUND_CONTEXT, + ), + ); +- expect(result).toEqual({ stdout: "out", stderr: "err", exitCode: 0 }); +- expect(stdout).toBe("out"); +- expect(stderr).toBe("err"); ++ expect(result.exitCode).toBe(0); ++ expect(result.output.text).toContain("out"); ++ expect(result.output.text).toContain("err"); ++ expect(seen).toContain("out"); ++ expect(seen).toContain("err"); ++ expect(streams).toContain("stdout"); ++ expect(streams).toContain("stderr"); + }); + + it("reports a missing working directory before spawning", async () => { +@@ -484,7 +491,8 @@ describe("NodeExecutionEnv", () => { + const root = createTempDir(); + const env = new NodeExecutionEnv({ cwd: root }); + const result = getOrThrow(await env.exec("exit 7", undefined, BACKGROUND_CONTEXT)); +- expect(result).toEqual({ stdout: "", stderr: "", exitCode: 7 }); ++ expect(result.output.text).toContain("".trim()); ++ expect(result.exitCode).toBe(7); + }); + + it("returns timeout errors for commands exceeding the timeout", async () => { +@@ -501,7 +509,7 @@ describe("NodeExecutionEnv", () => { + const result = await env.exec( + "printf out", + { +- onStdout: () => { ++ onUpdate: () => { + throw new Error("callback failed"); + }, + }, +@@ -542,12 +550,23 @@ describe("NodeExecutionEnv", () => { + const root = createTempDir(); + const env = new NodeExecutionEnv({ cwd: root }); + const result = getOrThrow( +- await executeShellWithCapture(env, "yes line | head -n 15000", undefined, BACKGROUND_CONTEXT), ++ await env.exec( ++ "yes line | head -n 15000", ++ { ++ capture: { ++ limits: { maxBytes: 50 * 1024, maxLines: 2000, retain: "tail" }, ++ spill: true, ++ }, ++ }, ++ BACKGROUND_CONTEXT, ++ ), + ); +- expect(result.truncated).toBe(true); +- expect(result.fullOutputPath).toBeDefined(); +- const fullOutput = getOrThrow(await env.readTextFile(result.fullOutputPath!, BACKGROUND_CONTEXT)); ++ expect(result.output.truncation.truncated).toBe(true); ++ // The spill lives inside the exec env, on the machine the command ran on — ++ // which is where the model's own read and grep run (`executionenv.md` §1). ++ expect(result.output.spillPath).toBeDefined(); ++ const fullOutput = getOrThrow(await env.readTextFile(result.output.spillPath!, BACKGROUND_CONTEXT)); + expect(fullOutput.split("\n").length).toBeGreaterThan(10000); +- expect(result.output.length).toBeLessThan(fullOutput.length); ++ expect(result.output.text.length).toBeLessThan(fullOutput.length); + }); + }); diff --git a/packages/agent/docs/mobile-handoff/01-harness/03-execenv/output-capture.ts b/packages/agent/docs/mobile-handoff/01-harness/03-execenv/output-capture.ts new file mode 100644 index 000000000..3d10fea34 --- /dev/null +++ b/packages/agent/docs/mobile-handoff/01-harness/03-execenv/output-capture.ts @@ -0,0 +1,194 @@ +import type { Context } from "../context.ts"; +import type { CaptureOptions, CapturedOutput, ShellOutputUpdate, ShellStream } from "../types.ts"; +import { + DEFAULT_MAX_BYTES, + DEFAULT_MAX_LINES, + type TruncationResult, + truncateHead, + truncateTail, + utf8ByteLength, +} from "./truncate.ts"; + +/** + * Bounded capture for a Shell implementation. + * + * Owns the two-stage structure that `shell-output.ts` had inside `bash.ts`: a raw + * buffer trimmed to `maxBytes * 2` as a memory guard, and a retained view + * computed from it per snapshot. `truncateTail` is line-aware — it walks + * backwards line by line and takes a partial only when one line alone exceeds the + * cap — so eviction is not byte arithmetic and cannot be done incrementally. + * + * Emits the three update kinds from `executionenv.md` §4 and enforces the + * ordering invariant: once the cap is crossed, a `chunk` never follows without an + * intervening `snapshot`. + */ +export class OutputCapture { + readonly #maxBytes: number; + readonly #maxLines: number; + readonly #retain: "head" | "tail"; + readonly #intervalMs: number; + readonly #onUpdate?: (update: ShellOutputUpdate, context: Context) => void; + + /** Raw rolling buffer, held at twice the cap. */ + #buffer = ""; + /** Everything ever produced. */ + #totalBytes = 0; + /** Newlines seen. A trailing partial line is counted separately — a command + * emitting one 60 KB line with no newline has produced one line, not zero. */ + #newlines = 0; + #endsWithNewline = true; + /** True once the retained view has begun evicting (tail) or filled (head). */ + #crossed = false; + /** Pending chunks for the current coalescing window. */ + #pending: { stream: ShellStream; text: string }[] = []; + #lastFlush = 0; + #timer: ReturnType | undefined; + #spillPath: string | undefined; + + constructor( + options: CaptureOptions | undefined, + onUpdate: ((update: ShellOutputUpdate, context: Context) => void) | undefined, + ) { + this.#maxBytes = options?.limits.maxBytes ?? DEFAULT_MAX_BYTES; + this.#maxLines = options?.limits.maxLines ?? DEFAULT_MAX_LINES; + this.#retain = options?.limits.retain ?? "tail"; + this.#intervalMs = options?.intervalMs ?? 0; + this.#onUpdate = onUpdate; + } + + push(stream: ShellStream, text: string, context: Context): void { + if (text === "") return; + this.#totalBytes += utf8ByteLength(text); + this.#newlines += countNewlines(text); + this.#endsWithNewline = text.endsWith("\n"); + this.#buffer += text; + + // Memory guard. The retained view is computed from this, never equal to it. + const guard = this.#maxBytes * 2; + if (utf8ByteLength(this.#buffer) > guard) { + this.#buffer = trimToLastBytes(this.#buffer, guard); + } + + this.#pending.push({ stream, text }); + this.#schedule(context); + } + + setSpillPath(path: string | undefined): void { + this.#spillPath = path; + } + + /** The bounded view, recomputed from the raw buffer. */ + snapshot(): CapturedOutput { + const truncation = + this.#retain === "head" + ? truncateHead(this.#buffer, { maxBytes: this.#maxBytes, maxLines: this.#maxLines }) + : truncateTail(this.#buffer, { maxBytes: this.#maxBytes, maxLines: this.#maxLines }); + // The buffer already lost its front, so totals come from counters rather + // than from what survives. + const totalLines = this.#totalLines(); + const truncated = this.#totalBytes > this.#maxBytes || totalLines > this.#maxLines; + return { + text: truncation.content, + truncation: { + ...truncation, + lastLinePartial: truncation.lastLinePartial, + truncated, + truncatedBy: truncated ? (totalLines > this.#maxLines ? "lines" : "bytes") : null, + totalBytes: this.#totalBytes, + totalLines, + }, + ...(this.#spillPath === undefined ? {} : { spillPath: this.#spillPath }), + ...(truncation.lastLinePartial ? { lastLineBytes: this.#lastLineBytes() } : {}), + }; + } + + #totalLines(): number { + return this.#newlines + (this.#endsWithNewline || this.#totalBytes === 0 ? 0 : 1); + } + + /** Bytes in the final line of the raw buffer, for the single-oversized-line case. */ + #lastLineBytes(): number { + const start = this.#buffer.lastIndexOf("\n"); + return utf8ByteLength(start === -1 ? this.#buffer : this.#buffer.slice(start + 1)); + } + + #schedule(context: Context): void { + if (this.#onUpdate === undefined) return; + if (this.#intervalMs === 0) { + this.flush(context); + return; + } + if (this.#timer !== undefined) return; + const wait = Math.max(0, this.#lastFlush + this.#intervalMs - Date.now()); + this.#timer = setTimeout(() => { + this.#timer = undefined; + this.flush(context); + }, wait); + } + + /** Emit one update for everything buffered since the last flush. */ + flush(context: Context): void { + if (this.#onUpdate === undefined) return; + this.#lastFlush = Date.now(); + const pending = this.#pending; + this.#pending = []; + if (pending.length === 0) return; + + const nowCrossed = this.#totalBytes > this.#maxBytes || this.#totalLines() > this.#maxLines; + + if (this.#retain === "head") { + // Head never evicts. Below the cap the appended text is complete; above + // it, only totals move — resending maxBytes to change a number is the + // thing `counters` exists to avoid. + if (!nowCrossed) { + this.#onUpdate({ kind: "chunk", stream: pending[0]!.stream, text: joined(pending) }, context); + } else { + this.#onUpdate({ kind: "counters", truncation: this.snapshot().truncation }, context); + } + this.#crossed = nowCrossed; + return; + } + + // Tail. Once the buffer is full every byte in evicts a byte out, so there is + // no cheap-trickle regime past the cap: chunks until full, snapshots after. + if (nowCrossed) { + this.#crossed = true; + this.#onUpdate({ kind: "snapshot", output: this.snapshot() }, context); + return; + } + if (this.#crossed) { + // Defensive: never let a chunk follow the crossing. + this.#onUpdate({ kind: "snapshot", output: this.snapshot() }, context); + return; + } + this.#onUpdate({ kind: "chunk", stream: pending[0]!.stream, text: joined(pending) }, context); + } + + dispose(): void { + if (this.#timer !== undefined) clearTimeout(this.#timer); + this.#timer = undefined; + } +} + +const joined = (pending: { text: string }[]): string => pending.map((p) => p.text).join(""); + +function countNewlines(text: string): number { + let count = 0; + for (let index = text.indexOf("\n"); index !== -1; index = text.indexOf("\n", index + 1)) count++; + return count; +} + +/** Keep the last `bytes` UTF-8 bytes without splitting a codepoint. */ +function trimToLastBytes(text: string, bytes: number): string { + if (utf8ByteLength(text) <= bytes) return text; + let low = 0; + let high = text.length; + while (low < high) { + const mid = Math.floor((low + high) / 2); + if (utf8ByteLength(text.slice(mid)) > bytes) low = mid + 1; + else high = mid; + } + return text.slice(low); +} + +export type { TruncationResult }; diff --git a/packages/agent/docs/mobile-handoff/01-harness/04-tool-output/bash-worked-example.md b/packages/agent/docs/mobile-handoff/01-harness/04-tool-output/bash-worked-example.md new file mode 100644 index 000000000..dc81efc99 --- /dev/null +++ b/packages/agent/docs/mobile-handoff/01-harness/04-tool-output/bash-worked-example.md @@ -0,0 +1,211 @@ +# Worked Example: `bash` End to End + +One tool through every layer. `execute` is simplified — timeout validation, +cancellation and exit-code branches are elided. Everything on the output path is +shown. + +Layers: exec env → `ToolOutput` sink → harness events → durable storage → lane +state → facet → wire → consumer. + +--- + +## 1. The tool + +```ts +// packages/agent/src/harness/tools/bash.ts +export interface BashToolDetails { spillPath?: string; truncation?: ShellOutputTruncation } + +export function createBashTool(): AgentHarnessTool { + return { + name: "bash", + parameters: bashSchema, + output: { retain: "tail", maxBytes: DEFAULT_MAX_BYTES, maxLines: DEFAULT_MAX_LINES }, + + async execute(_id, { command, timeout }, signal, out, context) { + const env = context.env; + let view: ShellOutputView | undefined; + + const result = getOrThrow(await env.exec(command, { + cwd: env.cwd, + inheritEnv: true, + timeout, + capture: { limits: this.output, spill: true }, + onUpdate: (u) => { + view = applyShellOutputUpdate(view, u); + if (u.kind === "append") out.write(u.text); + else out.replace(view.text); + out.details.truncation = view.truncation; + if (view.spillPath) out.details.spillPath = view.spillPath; + }, + }, context)); + + if (result.spillPath) out.details.spillPath = result.spillPath; + if (result.truncation.truncated) out.write(`\n\n[${describe(result.truncation)}]`); + if (result.exitCode) throw new Error(`Command exited with code ${result.exitCode}`); + }, + }; +} +``` + +What is gone from today's implementation: the rolling `tailOutput` buffer, +`truncateTail`, `ensureFullOutputFile`, `createTempFile`, +`BASH_UPDATE_THROTTLE_MS`, `BASH_CHECKPOINT_INTERVAL_MS`, `updateDirty`, +`lastCheckpoint`, `scheduleOutputUpdate`, `emitOutputUpdate`, +`clearUpdateTimer`. About 40 lines of tool-local machinery, replaced by a capture +policy every tool gets. + +`execute` returns `void`. `details` is one field, set once — and note the tool no +longer computes `truncation`, because the env owns the window and the tool no +longer knows what was dropped. + +## 2. The exec env caps at the source + +`env.exec` applies `ShellOutputLimits` where the bytes originate. For a sandbox host +that means `cat 1gb.txt` never ships 1 GB to the agent machine, and the spill +lands where the model's own `read` and `grep` run. See [`execenv.md`](../03-execenv/execenv.md). + +The initial state is a bounded `replace`. Growth emits `append`; a moving tail emits `slide { drop, text }`; complete turnover falls back to a bounded `replace`; metadata moves totals and spill paths without resending text. The adaptive publisher keeps small slides responsive and spaces cap-sized turnovers according to their encoded size. + +## 3. The sink + +`ToolOutput` folds those into `ToolOutputState`: + +```ts +{ + content: [{ type: "text", text: "…the retained window…" }], + details: { spillPath: "/tmp/pi-session-x/bash-8f2.log" }, + usage: undefined, + addedTools: undefined, + terminate: false, + truncation: { truncated: true, truncatedBy: "bytes", totalLines: 8123, totalBytes: 262144 }, +} +``` + +`out.write(text)` appends to `content[0].text`; `out.replace(text)` assigns +the whole retained view when the env sends a `snapshot`, because an append cannot +express eviction. The tracker (`delta.md`) records intent either way: an append is +an `a`, and a window slide assigned wholesale becomes `t` + `a` via verified +overlap detection. + +## 4. Ops + +A 256 KB build over ~20 flushes, 50 KB window: + +The first batch is a base batch — it begins with `r`, carrying the initial state. Everything +after is deltas. The repeated content path is interned on its second use, and +consecutive ops on it omit the id entirely: + +```jsonc +[["r",{"content":[{"type":"text","text":""}],"details":{},"terminate":false,"truncation":{…}}]] + +[["a",["content",0,"text"],"make: Entering directory …\n"]] +[["#",0,["content",0,"text"]],["a",0,"cc -c src/a.c …\n"]] +[["a",0,"cc -c src/b.c …\n"]] +… +[["s",["details","spillPath"],"/tmp/pi-session-x/bash-8f2.log"]] +… +[["t",0,4096],["a","cc -c src/z.c\n"],["s",["truncation","totalBytes"],262144]] +``` + +Details are one `s`, in one flush out of twenty. The `t` + `a` pair is the window slide. The current tracker recovers it by verified overlap; production remeasurement shows the generic path is already negligible, so no explicit append/truncate producer API is added ([decision](../01-delta/append-decision.md)). + +## 5. Harness events + +```ts +{ type: "tool_start", toolCallId: "call_7", toolName: "bash", + args: { command: "make -j8" } } + +{ type: "tool_update", toolCallId: "call_7", ops: [ … ] } + +{ type: "tool_end", toolCallId: "call_7", isError: false } +``` + +`tool_start` is identity only — no `caps`, no `initial`. The first batch is a base +batch, so the initial state arrives on the update channel, and the caps are +already inside it as `truncation.maxBytes` / `maxLines`. + +`tool_end` carries no content and no details. Every byte already went out; +re-sending would duplicate any images. + +## 6. Durable storage + +`pendingToolOutput(operationId, "call_7")`, **ephemeral-scoped** to the operation, +so it lives in `.op_….jsonl` and is retired on settle rather than +persisting in the main log forever. Retirement is a main-log `retireScope` record, +so it commits atomically with the settle writes; the unlink is a consequence of +replaying that record, not part of the transaction ([scopes.md §5](../02-scopes/scopes.md)). + +A `list`, one encoded batch appended per flush, base batches tagged `"base"` so recovery reads backwards with `stopAtTag` and stops there ([scopes.md §11](../02-scopes/scopes.md#11-list-tags-and-stop-conditions)). The tracker emits structural ops; the producer calls `rebase()` periodically to write a capped root replacement and bound recovery replay. The durable interval is a sink policy. Shell capture controls only source-state and transport publication; memo, terminal, and recovery-base flushes are forced by `ToolOutput`. + +Recovery seeds a fresh `ToolOutput` from that state — it does **not** delete it, +which is what `clearReplayCheckpoint` does today and is a bug +(`harness-tools.md` §7.4). If bash memoised "spilled to /tmp/…" and crashed, a +replay that discarded the state would create a second spill file and lose the +first. + +The memo write and this checkpoint commit in the same transaction. Both are +ephemeral-scoped, so both land in the same sidecar — and the type system rejects a +commit that mixes scopes, so this cannot regress silently. + +## 7. Lane state and the facet + +```ts +export function reduceLaneSnapshot(view: LaneView, event: HarnessEvent): void { + switch (event.type) { + case "tool_start": + view.operation.tools.push({ id: event.toolCallId, name: event.toolName, + args: event.args, output: undefined }); + return; + case "tool_update": { + const tool = view.operation.tools.find((t) => t.id === event.toolCallId); + if (tool === undefined) return; // host will send a base batch + tool.output = apply(tool.output, event.ops); + return; + } + case "tool_end": { + const i = view.operation.tools.findIndex(t => t.id === event.toolCallId); + if (i >= 0) view.operation.tools.splice(i, 1); + return; + } + } +} +``` + +Plain mutation, no Immer, no return value. An event for a tool the view has never +seen simply does nothing; the host sends a `replace`. + +A lane facet runs this same function under the tracker, so the facet's own ops +fall out against **its** shape — which need not be `LaneView` and generally is +not. The facet never writes an op. + +## 8. Wire and consumer + +```jsonc +{ "seq": 0, "ops": [["r",{"transcript":[],"operation":null}]] } +{ "seq": 1, "ops": [ … ] } +``` + +One shape: a batch of ops. The first is a base batch — it begins with `r` — +and everything after is deltas. A gap, a reconnect, a provider +reload, or a fold that could not apply all take one path: send a fresh `replace`. + +The consumer is `apply` — six verbs, no domain knowledge, no library, no tool +code, against a plain mutable object it owns. + +## 9. What this run costs + +256 KB of output, ~20 flushes, 50 KB window: + +| | today | this design | +|---|---|---| +| durable writes | full `AgentToolResult` per checkpoint | structural ops plus explicit periodic capped base batches | +| durable location | main log, forever | sidecar, unlinked on settle | +| wire per flush | whole snapshot | one `truncate` + one `append` | +| details written | rebuilt whole, every flush | one `set`, once | +| truncation logic | in `bash.ts` | in the exec env, shared by all tools | +| spill location | `/tmp`, OS-cleaned | exec env, session-scoped | + +The details row is the one worth dwelling on. Nothing about bash's details +changed — they were always small. What changed is that they stopped riding inside +a container replaced wholesale on every update, which is why no tool anywhere +currently bothers to mutate them incrementally. diff --git a/packages/agent/docs/mobile-handoff/01-harness/04-tool-output/harness-tools.md b/packages/agent/docs/mobile-handoff/01-harness/04-tool-output/harness-tools.md new file mode 100644 index 000000000..3d52141b6 --- /dev/null +++ b/packages/agent/docs/mobile-handoff/01-harness/04-tool-output/harness-tools.md @@ -0,0 +1,462 @@ +# Tool Output and Progress + +> **Scope:** harness-local. Nothing here depends on the facet system, RPC, or any +> presentation. Depends on [delta tracking](../01-delta/delta.md) (the landed Chord op vocabulary and tracker), [execution environments](../03-execenv/execenv.md) (where truncation happens), and [scoped storage](../02-scopes/scopes.md) (durability). + +## 1. The problem + +Three faults, one cause. + +**Tools each implement their own truncation.** `bash.ts` owns a rolling buffer, +`truncateTail`, a spill file, an update throttle and a checkpoint interval. +`read.ts` has its own truncation. Every future tool producing bulk output +reimplements this, differently. + +**Progress is a whole value.** `onUpdate(partialResult)` hands over a complete +`AgentToolResult` on every update, `tool_update` carries that whole result, and +`openToolProgress` persists it with `setValue`. + +**`details: unknown` was believed to force this** — you cannot append to a value +whose shape you do not know. That premise is now false. A structural tracker +(`delta.md`) records ops against JSON without knowing the type, so details need +no special handling at all. + +## 2. `ToolOutput` + +The harness constructs a sink per invocation and passes it to `execute`. **Tools +return nothing**; the sink holds the result. + +```ts +interface ToolOutput { + /** Append to the text block. */ + write(text: string): void; + /** Append an image block. Images are never windowed. */ + image(image: ImageContent): void; + /** Replace the retained text wholesale. Chord still recovers a verified `t` + `a` slide when possible. */ + replace(text: string): void; + /** Apply source-owned truncation totals and spill metadata without resending text. */ + capture(metadata: ShellOutputMetadata): void; + + /** The tool's own details object. Mutate it directly. */ + readonly details: TDetails; + + /** Accumulates. A subagent making several model calls adds to it. */ + usage(usage: Usage): void; + /** Replaces. */ + addTools(names: string[]): void; + /** Replaces — a tool that decides to terminate and then recovers can say so. */ + terminate(value: boolean): void; +} +``` + +```ts +execute( + toolCallId: string, + params: Static, + signal: AbortSignal, + out: ToolOutput, + context: Context, +): Promise; +``` + +`execute` returns `void`. Everything the tool produces flows through the sink — +including `usage` and `addTools`, which cannot be settle-time return values +because a replayed tool must be able to seed from durable state (§7.4). + +**Failure is a thrown error.** `isError` is set by the harness when `execute` +rejects. A tool may throw anything, including errors from libraries it did not +write. + +**`terminate` is orthogonal to how execution ended**, so a failing tool can ask +the loop to stop: + +```ts +try { await thing(); } catch (error) { out.terminate(true); throw error; } +``` + +This closes a gap in the current implementation, where `executeToolCall`'s catch +hardcodes `isError: true` with no terminate, and `immediateError`'s terminate +parameter is only ever passed `true` by `applyBeforeToolDecision` for a hook +block. + +### 2.1 Details are just an object + +```ts +async execute(id, params, signal, out, context) { + out.details.total = 42; + out.details.passed += 1; + out.details.failures.push({ name, message }); + out.details.current = undefined; // -> delete +} +``` + +Fully granular ops fall out, verified against the prototype: + +```jsonc +["#",0,["details","passed"]] +["s",0,1] +["p",["details","failures"],0,0,[{…}]] +["d",["details","current"]] +``` + +No recipes, no mutation map, no `initialDetails` on `tool_start`, no Immer, and +no consumer ever runs tool code. `TDetails` remains the tool's own exported type +so a renderer can cast `call.details` to it — that is all it is for. + +A survey of `packages/agent/src/harness/tools/` found **no tool mutating details +incrementally today**; only `bash.ts` writes them mid-stream and it rebuilds them +whole, because the container was replaced wholesale anyway. This design removes +that cause, and costs nothing if nobody takes it up. + +**Caveat:** details are now unbounded. A tool pushing to `failures` in a loop +grows without limit, and unlike text there is no cap. Nothing shipped does this; +the door is open in a way it was not when details were a replace-only value. + +### 2.2 Partial output survives failure + +Today `executeToolCall` catches and returns `createErrorToolResult(message)`, +building a fresh result from the error string alone — a tool that streamed 8 KB +and then threw reports only the error. + +With the sink, what was written is already the result. The harness appends the +error text as content and keeps the rest. The error text is model-visible +content, not presentation: the model needs to read why the call failed. + +`abortedMessage` and `interruptedMessage` should be made consistent with this; +today they build replacement results via `syntheticMessage`, so a cancelled +long-running command loses its partial output too. + +## 3. Content + +Content is exactly **one text block followed by zero or more image blocks**. A +tool cannot interleave text between images — text written before and after an +image lands in the same block. Deliberate: truncation only ever touches a string, +and an image is never partially anything. + +Images are **never windowed**. A byte or line cap over base64 payloads is +meaningless and `truncateTail` operates on text. `maxBytes` / `maxLines` govern +text only. + +### 3.1 Retention mode + +Declared on the tool definition: + +```ts +output?: { + retain?: "head" | "tail"; // default "tail" + maxBytes?: number; + maxLines?: number; +} +``` + +**`head`** — append until the cap, then stop. Nothing is ever removed. Right for +output meaningful from the start: file reads, listings, greps. + +**`tail`** — a rolling window. Right for anything whose interesting part is at +the end: builds, test runs. + +Head+tail is **not** offered. `truncate.ts` exports `truncateHead` and +`truncateTail`; neither combines them and none is being added. + +### 3.2 The tool does not truncate — the exec env does + +For output originating in the execution environment, capping, coalescing and +spilling happen **at the source**. See [`execenv.md`](../03-execenv/execenv.md). The tool passes its +`ShellOutputLimits` into `env.exec` and pipes the resulting updates into the sink. + +This is not a convenience. Reading a 1 GB file on a sandbox host must not ship +1 GB to the agent machine to be capped there, and the spill file must land where +the model's own `read` and `grep` run. + +For output originating on the agent machine (subagents, in-process work), +`ToolOutput` applies the same logic locally. Same code, different location. + +Note this reverses an earlier decision that there is no spill in the sink and a +temp-file path is tool-specific. The exec-env argument — model reachability +across a machine boundary — is what changed it. + +## 4. Tool output state + +`tool_update` carries `Op[]` (`delta.md` §6) targeting the invocation's +`ToolOutputState`: + +```ts +interface ToolOutputState { + content: (TextContent | ImageContent)[]; + details: JsonValue; + usage?: Usage; + addedTools?: string[]; + terminate: boolean; + truncation: ShellOutputTruncation; // totals over everything ever written, without duplicate text +} +``` + +Ops rather than a typed variant union, for one decisive reason: **only ops give +details granularity without the harness knowing `TDetails`**. A typed union would +need per-tool recipes, which is the machinery §2.1 deletes. + +Text still gets delta treatment, because the sink applies the cap *before* +mutating, so a rolling window produces `truncate` + `append` on +`content[0].text` rather than a whole-value set. + +Interned ops also measure **smaller than a typed frame vocabulary** on every +workload, and 10x smaller on details, because a frame repeats `toolCallId` where +an interned op carries one integer. See `delta.md` §4.1. + +There is no `drop` event. The sink knows what it evicted and expresses it as +`truncate`; a consumer needs no separate signal and derives nothing. + +## 5. Harness events + +```ts +| { type: "tool_start"; + runId; turnId; toolCallId; toolName; + args: unknown } + +| { type: "tool_update"; + runId; turnId; toolCallId; + ops: Op[] } // a base batch begins with `r` + +| { type: "tool_end"; + runId; turnId; toolCallId; + isError: boolean } +``` + +**`tool_start` carries identity only.** An earlier draft added `caps` and +`initial`; both were redundant and are gone. The first batch is always a base batch +(`delta.md` §6), so the initial state arrives on the update channel — carrying it +twice means two ways to establish the base, which can disagree. And `caps` are +already inside the state: `ToolOutputState.truncation` carries `maxBytes` and +`maxLines`, which is what a renderer needs to say "50 KB limit". The draft itself +conceded a consumer "no longer has to" fold identically, since the producer's ops +encode eviction — that was the last reason for `caps` to travel, and it does not +hold. + +`tool_update` carries ops. A base batch travels the same channel as a delta, +because a replacement is itself an op (`delta.md` §2) — there is no second shape. `message_update` has the identical shape +(`message-update.md` §5.1); a consumer folds tool output and assistant output with +one code path. + +**`tool_end` carries no content, no details, no usage, no terminate.** Every byte +already went out. Re-sending would duplicate each base64 image at the moment +nothing new has happened. + +> **The fold is the result.** Nothing a consumer folded is ever re-sent to +> confirm it. + +This dissolves an earlier open question about settle-time truncation disagreeing +with the running fold. There is no separate settle-time truncation: the sink's +window *is* the truncation. Anything a tool wants to add at the end — bash's +`[Showing lines 8000-8123 of 8123]` footer — is `out.write(footer)`, one more +append. + +The harness still assembles `AgentToolResult` in process for the model, and +`createToolResultMessage` still writes the settled `ToolResultMessage` to the +transcript with `content`, `details`, `usage`, `isError`. +Neither is a wire event. + +## 6. Lane reduction + +```ts +export function reduceLaneSnapshot(view: LaneView, event: HarnessEvent): void; +``` + +Plain mutation on a plain object. No `Draft`, no `produce`, no Immer, and **no +`Rebase` return value** — a fold that cannot apply an event leaves state alone, +and the host sends a `replace`. (An earlier draft had `void | Rebase`; under +Immer that throws, and without Immer there is nothing to return to. Note +`reducer.ts:4` currently defines `LaneSnapshotReduction = LaneSnapshot | { rebase: true }`, +so this is a real change to existing code, and `navigation_end` is the event that +drives it.) + +The event is the only input besides the view. No registry, no `resolve`, no tool +code — so a tool rewritten between crash and resume cannot make a persisted +stream unreadable. + +When the harness is wrapped by a facet, the same mutation runs under the tracker +from `delta.md` and ops fall out. The harness itself does not know this. + +## 7. Durability + +### 7.1 What exists today + +`pendingToolOutput(operationId, invocationId)` is a +`value>`, and — importantly — `progress.write(partial)` +fires **only** when `options?.checkpoint === true` (`drive/tools.ts:325`). Not on +every update. bash checkpoints every 2 s with a `JSON.stringify` dedupe; every +other tool never checkpoints at all. + +It is also **not a progress buffer**. It is the interruption checkpoint. On +resume, if the tool is not replay-safe, `readCheckpoint` turns it into a real +`ToolResultMessage` for the model: `[...checkpoint.content, INTERRUPTION_MARKER]` +plus `details` and `usage`. + +Note the implication that misled an earlier draft: because the checkpoint must +*be* the current state, it was read as needing `value` semantics. It does not. It +needs to be *derivable*, and folding encoded batches from the last base batch derives it — which is the point of tagging base batches (§7.3). `checkpoint: true` +requests a durable write, not a replacement. + +`pendingAssistantFrames` is a `list` appended per frame, +`deleteList` on settle in `response.ts:345`, `deferred.ts:157`, `terminal.ts:47`. + +Note `operationCleanupWrites` (`terminal.ts:26`) does four `scanValues` calls to +enumerate what to delete at settle. Under scopes, the two covering +`operationToolMemoPrefix` and `pendingToolOutputPrefix` are replaced by a single +`retireScope(operationId)`. + +### 7.2 Renaming + +`pendingAssistantFrames` → **`pendingAssistantOutput`**, matching +`pendingToolOutput`. Frames stop being the durable unit; both addresses now hold +tracked state written as ops or snapshots. + +### 7.3 What to write, and when + +Both addresses are **ephemeral-scoped** ([scopes.md](../02-scopes/scopes.md)), so they live in a +sidecar retired on settle rather than in the main log forever. + +Two independent wins, in order of importance: + +- **Encoding.** Ops instead of whole values, plus address interning: 93.89 MB to + 5.32 MB in a single file, no change to atomicity. Do this first. +- **Scopes.** Pending state leaves the main log entirely: 5.32 MB to 0.06 MB + surviving. + +**Both are `list`**, not values. The sink owns one stateful Chord encoder/decoder pair per durable value stream. One encoded batch is appended per flush, and a logical batch whose first op is `r` is tagged `"base"` on the storage record. +Recovery reads backwards with `stopAtTag: "base"` and applies forward +([delta.md §9](../01-delta/delta.md#9-durable-form), [scopes.md §11](../02-scopes/scopes.md#11-list-tags-and-stop-conditions)). + +Writing one flush: + +```ts +const ops = out.flush(); +if (ops.length === 0) return; +const wire = enc.encode(ops); +writes: [appendList(address, wire, isBase(ops) ? "base" : undefined)]; +``` + +`isBase` comes from Chord. It checks for the root replacement op `r`; ordinary nested sets use `s`. Classification lives beside the vocabulary so the comparison is written once. + +The landed tracker emits structural ops unconditionally. There is no serialized-size comparison or adaptive replacement heuristic. A producer requests a base batch explicitly with `rebase()`; the output sink's cap bounds that replacement, while periodic rebasing bounds recovery work. Production remeasurement rejects a text-specific append/truncate API: the generic path measured 2.43–2.46 µs per 50 KB rolling-window flush locally, below surrounding costs. Keep ordinary tracked string mutation; see the [decision record](../01-delta/append-decision.md). + +**Checkpointing is not deleted.** `BASH_CHECKPOINT_INTERVAL_MS` remains only as an interim compatibility mechanism. The generic sink owns durable frequency because Shell cannot price a storage write or enforce memo/output atomicity. Forced memo, terminal, and recovery-base writes bypass ordinary pacing while remaining cap-bounded. + +### 7.4 Replay must seed, not discard + +`clearReplayCheckpoint` currently writes `deleteValue(pendingToolOutput(...))` +before re-executing a replay-safe tool (`drive/tools.ts:257`). **This is a bug.** + +Replay-safe means the tool is re-executed, but memos exist precisely so it does +*not* redo work it already did — and skipped work emits nothing. Any output for +memoised work is lost today. + +The fix: seed a fresh `ToolOutput` from the durable state, then re-execute. The +tool appends to a sink that already holds what it produced before the crash. + +This is also why nothing may be settle-only. `usage` and `addTools` must survive +a seed, so they flow through the sink like everything else. + +### 7.5 The memo invariant + +> A tool's memo write and its output checkpoint must commit in the same +> transaction. + +Otherwise a tool does work, sets a memo, crashes before the next checkpoint, and +on replay skips the work while the seeded output has no record of it. + +**This does not hold today.** `setMemo` (`drive/tools.ts:112`) and `openProgress` +(`runtime/progress.ts:44`) are two separate `lane.command` calls, hence two +transactions. Making it hold means bundling the checkpoint into the memo's commit: + +```ts +setMemo(name, value) { + validateMemoName(name); + if (!active) return Promise.reject(ended()); + return lane.command((state) => { + if (!ownsEffect(state)) return { kind: "reject", error: ended() }; + const memo = operationToolMemo(drive.operationId, call.resultEntryId, name); + return { + kind: "commit", + writes: [ + value === undefined ? deleteValue(memo) : setValue(memo, value), + setValue(pendingToolOutput(drive.operationId, call.resultEntryId), out.snapshot()), + ], + next: state, + materialize: () => undefined, + }; + }, drive.context); +} +``` + +Both addresses are **ephemeral-scoped**, so this is a single-file transaction and +statically enforced as one ([scopes.md §3 and §6](../02-scopes/scopes.md)). `operationToolMemo` is +placed in that scope precisely for this. Ordering on the Session line would not be +enough — two file writes are not atomic. The periodic checkpoint stays as it +is — best-effort, for the interruption path; this forces one where correctness +requires it. + +The invariant holds because the tool does X, writes X's output to the sink, *then* +calls `setMemo("did X")` — so the sink's state at commit time already contains X's +output. + +**Ordering alone does not work**, in case it looks tempting: + +- memo first, checkpoint second → crash between → replay skips X and the seeded + output lacks it → silent loss; +- checkpoint first, memo second → crash between → replay redoes X and appends + again → duplicated output. + +Duplication is the less bad failure, so ordered writes are a tolerable fallback, +but neither is correct. + +**Cost:** `setMemo` now writes a full capped output state rather than a small +value. Memos are rare — a handful per invocation — so this is bounded by +`memo count x cap`, not by output volume. + +### 7.6 `openProgress` has a write-ordering race + +`commitWrite(item)` captures `item` when `write()` is called, and the write is +fire-and-forget with only `latest` tracked. A checkpoint captured at T1 can +therefore commit *after* a memo-bundled checkpoint at T2, overwriting newer state +with older — reintroducing exactly the loss §7.5 prevents. + +Fix: resolve the sink's state inside the command planner rather than at call time. + +```ts +commitWrite: () => setValue(address, out.snapshot()) // evaluated under the Session line +``` + +`lane.command` serializes on the Session line, so checkpoint writes become +monotonic by construction. This removes the race generally, not only against +memos. + +### 7.7 Nothing on the durable path runs tool code + +Ops are interpreted by a six-verb applier with no domain knowledge, so a tool +rewritten between crash and resume cannot make a persisted stream unreadable. + +> **The durable path uses only harness-owned reducers.** + +This also rules out persisting *facet* ops. The harness has no facet state, +facets come and go, and a recovering harness must rebuild its working values with +no facet present. + +## 8. Open questions + +- **Property tests for the tracker** (`delta.md` §3.3). Everything here rests on + producer and replica agreeing; nothing currently proves they do. +- Coalescing window: per tick, or a byte/time threshold. +- Whether image count needs a bound. Images are unwindowed, so a tool pushing + them in a loop grows `content` without limit. Currently treated as a tool bug. +- Whether details need a bound for the same reason (§2.1). +- Whether `retain: "head"` should keep emitting counter-only updates once capped + so a renderer can report how much was suppressed. [`execenv.md`](../03-execenv/execenv.md) answers this + for exec-originated output; agent-side output needs the same answer. +- Whether a failing tool *should* be able to terminate, or whether the current + inability is deliberate — there is a reasonable argument the model should + receive the error and decide. +- **Derived values.** `arguments` parsed from accumulated JSON should not be + replicated; derive it on demand. Safe because `parseStreamingJson` is total — + four fallbacks ending in `{}`, it cannot throw — so a replica derives with no + error path and no agreement protocol. Generalises to: no derived fields in + replicated state. diff --git a/packages/agent/docs/mobile-handoff/01-harness/04-tool-output/rate-limiting.md b/packages/agent/docs/mobile-handoff/01-harness/04-tool-output/rate-limiting.md new file mode 100644 index 000000000..eef04ebd5 --- /dev/null +++ b/packages/agent/docs/mobile-handoff/01-harness/04-tool-output/rate-limiting.md @@ -0,0 +1,125 @@ +# Bounded output publication + +**Status:** the shared adaptive publisher and its `ExecutionEnv` use are implemented. Generic `ToolOutput` integration remains design work. + +Depends on landed Chord delta tracking, source-bounded execution output, and scoped storage for durable batches. + +## 1. Invariant + +> The durable record, what the model sees, and what the UI shows are the same bounded view. + +A spill file is not a second view. It is a file inside the execution environment that the model reaches through ordinary file tools. + +Every uncontrolled producer boundary needs two independent bounds: + +- **state size:** the latest retained text is capped; +- **publication:** both encoded bytes and event count are paced. + +Delta encoding is complementary. It compresses a published change; it does not cap state or decide when publication occurs. + +## 2. Boundaries + +```text +remote process + -> optional ExecutionEnv publisher + -> worker ToolOutput publisher + -> events + durability + replication +``` + +A publisher instance is required only before a real costly boundary: + +- a physically remote execution environment limits output before transport; +- `ToolOutput` limits custom tools and downstream event/storage traffic; +- a colocated environment can feed its bounded updates in process without another serialized transport; +- custom tools bypass `ExecutionEnv` but cannot bypass `ToolOutput`. + +The control algorithm is shared. Payload vocabularies differ: Shell uses replace/append/slide/metadata; `ToolOutput` uses Chord operations. + +## 3. Why size or cadence alone is insufficient + +A fixed 50 KB snapshot is safe per event but not over time. At 100 ms it permits ten complete snapshots per second, approximately 500 KB/s plus envelopes. + +A byte budget alone also permits excessive tiny events and durable transactions. A minimum interval bounds count; encoded-size debt bounds bandwidth. + +The original handoff incorrectly treated `intervalMs = 100` as 100 emits/s. It is ten emits/s. The fixed interval was still non-adaptive: it delayed small trickles while allowing complete windows at the same frequency. + +## 4. Landed adaptive algorithm + +`packages/agent/src/harness/utils/adaptive-publisher.ts` implements: + +```ts +nextDelayMs = max(globalMinEmitInterval, encodedUpdateBytes * 1000 / globalTargetBytesPerSecond); +``` + +Current harness-global policy: + +```ts +minEmitInterval = 100 ms; +targetBytesPerSecond = 100 KB/s; +``` + +Behavior: + +1. The first dirty state after idle publishes immediately. +2. Writes before the next deadline collapse into the latest state. +3. One trailing timer publishes held state after the deadline. +4. Completion and correctness boundaries force one bounded publication. +5. The publisher commits its baseline before consumer delivery, preventing duplicate deltas if a consumer applies and then throws. + +This is an amortized token bucket with a cap-sized burst. A leading or forced terminal update may exceed the target over a short interval, but sustained encoded bytes converge to the target and sustained event count cannot exceed the minimum-interval floor except for explicit forced correctness writes. + +## 5. Scenario traces + +Assume a 50 KB cap, 100 KB/s target, and 100 ms floor. + +### Below cap, completes + +The initial state publishes immediately. Small appends publish no faster than the floor, and final dirty state is forced. Total encoded text is approximately the produced text. + +### Below cap, trickling + +Writes arriving more than 100 ms apart publish immediately because the preceding deadline has already passed. Faster writes collapse into one append per floor interval. + +### Above cap, full force + +The retained state never exceeds 50 KB. Complete window turnovers encode as bounded replacements. A roughly 50 KB update buys about 500 ms of silence, yielding approximately two updates and 100 KB per second regardless of raw producer throughput. + +For shell execution, the complete stream goes to a source-local spill under backpressure rather than over the output-update channel. + +### Above cap, trickling + +A small tail movement encodes as truncate plus append (Chord) or `slide` (Shell). Its encoded size is small, so the 100 ms floor dominates and output remains responsive. Resending a complete 50 KB snapshot for each tiny slide would be both slower and larger. + +### Burst then silence + +The leading state is immediate. Held writes collapse, and one trailing timer publishes the latest residue. There is no polling timer. + +### Huge single write + +The producer may already have allocated its input, but the boundary retains and publishes only the configured cap. Shell spill keeps the complete source stream. Arbitrary custom-tool images and structured details still require separate limits. + +## 6. Forced writes + +These bypass pacing once, while remaining state-size bounded: + +- command/tool completion; +- error or abort; +- a memo and output checkpoint committed atomically; +- an explicit recovery base/rebase. + +A terminal flush cancels the trailing timer before `tool_end`, preventing a late update for a settled invocation. + +## 7. `ToolOutput` application + +The sink will own one Chord tracker and encoder per invocation. Mutations remain local while publication is blocked; flush-time dirty tracking means held writes collapse without retaining one op per write. + +One sink flush feeds the live event and current model-visible state. Durable batches use the same logical flush, encoded as per-stream `WireOp[]`. Periodic `rebase()` bounds recovery replay; a memo correctness flush writes a tagged base batch in the same transaction as the memo. + +Durable cadence belongs to the sink, not to Shell or bash. Bash's existing two-second checkpoint request remains only as an interim compatibility mechanism until the sink migration. + +## 8. Remaining decisions + +- Explicit byte/count rejection policy for images. +- Bounds for structured details; arbitrary JSON cannot be meaningfully tail-windowed. +- Whether ordinary durable writes ride every live sink publication initially or use a slower measured cadence. Memo and terminal correctness flushes are not optional. +- Exact global production values after profiling real remote transport and storage. diff --git a/packages/agent/docs/mobile-handoff/01-harness/05-assistant-output/message-update.md b/packages/agent/docs/mobile-handoff/01-harness/05-assistant-output/message-update.md new file mode 100644 index 000000000..f96f15c61 --- /dev/null +++ b/packages/agent/docs/mobile-handoff/01-harness/05-assistant-output/message-update.md @@ -0,0 +1,227 @@ +# `message_update` Write Amplification + +> **Scope:** harness-local. Depends on [delta tracking](../01-delta/delta.md) for the landed Chord `Op`/`WireOp` vocabulary and [scoped storage](../02-scopes/scopes.md) for durability. Chord delta tracking has landed; scoped storage and this Harness integration have not. + +## 1. The problem + +```ts +{ type: "message_update", runId, message, event, frame? } +``` + +Three representations of the same thing travel together: + +- `message` — the full `AssistantMessage`; +- `event` — an `AssistantMessageEvent`, which itself carries `partial: AssistantMessage`, + a second full copy; +- `frame` — the actual delta, optional. + +Per streamed token this is roughly two complete snapshots plus a delta, so bytes grow +quadratically over a response. + +The reducer does not even use the delta: + +```ts +case "message_update": + if (next.operation?.id === event.runId && event.message.role === "assistant") { + next.operation.streamingMessage = event.message; + } +``` + +A straight assignment. So on any wire, sending the event is *worse* than sending a +snapshot — it ships two where `replace` ships one. + +The wire adapter is already halfway to the fix: it drops `event` and sends `message` +plus `frame`. That is the full snapshot together with the delta that produced it. + +## 2. The compact form already exists + +```ts +/** + * Compact, replayable assistant-message progress. Terminal settlement is + * intentionally excluded and must be persisted separately. + */ +export type AssistantMessageFrame = + | { type: "start"; partial: AssistantMessage } + | { type: "text_start" | "text_delta" | "text_end"; contentIndex: number; ... } + | { type: "thinking_start" | "thinking_delta" | "thinking_end"; ... } + | { type: "toolcall_start" | "toolcall_checkpoint" | "toolcall_delta" | "toolcall_end"; ... }; +``` + +`AssistantMessageFrameEncoder` produces it, `reduceAssistantMessageFrames` folds it, +and `openFrameProgress` already persists it with `appendList`. The delta format is +built, used, and durable. It simply is not what `message_update` carries. + +## 3. Precedent + +pi already does wire-is-delta one layer down. `PiMessagesEvent` — the serialized form +a pi-messages backend sends — has no `partial`: + +```ts +| { type: "text_delta"; contentIndex: number; delta: string } +| { type: "text_end"; contentIndex: number; content: string; contentSignature?: string } +``` + +`pi-messages.ts` then rehydrates: it holds a local `partial`, mutates it per event +(`partial.content[i].text += event.delta`), and returns the in-process +`AssistantMessageEvent` with `partial` attached. + +So the convention is established. It just stops at the provider edge instead of +continuing through the harness. + +## 4. The change + +```ts +| { type: "message_update"; runId: string; entryId: string; frame: AssistantMessageFrame } +``` + +`message` and `event` are removed; `frame` stops being optional. + +The blast radius is small because `HarnessEvent` is not what most streaming consumers +read. `AgentEvent` (`agent-loop.ts`) and `AgentSessionEvent` (`agent-session.ts`) are +separate unions that happen to share the tag name and build their own +`message_update` from the pi-ai event directly. They are out of scope here. + +Real consumers and producers of `HarnessEvent.message_update`: + +| site | change | +| --- | --- | +| `runtime/drive/response.ts` | emit the required semantic frame; stop attaching full snapshots | +| `runtime/reducer.ts` | fold the frame instead of assigning `event.message` | +| lane/facet state adapter | run that fold under the Chord tracker and emit `Op[]`/encoded `WireOp[]` | +| `experimental/harness-wire-adapter.ts` | stop treating raw `HarnessEvent` as the final replication format | +| `harness/telemetry.ts` | name list only | +| `protocol/harness.ts` | replication carries encoded `WireOp[]`, not the raw event | + +`message_end` continues to carry the settled message, because frames deliberately +exclude terminal settlement. That is once per message, not once per token. + +## 5. An incremental applier is needed + +`reduceAssistantMessageFrames` is a whole-stream fold over an `Iterable`. The +reducer needs a step function: + +```ts +export function applyAssistantMessageFrame( + state: AssistantFrameState, + frame: AssistantMessageFrame, +): void; +``` + +Plain mutation on a plain object. **No `Draft`, no Immer.** An earlier draft +argued for a draft-mutating signature so it would compose inside a `produce` +recipe; that motivation is gone ([delta.md §8](../01-delta/delta.md#8-what-this-removes-from-the-codebase)). The step function is still +needed — the whole-stream version becomes a loop over it — just for the simpler +reason that the reducer folds one frame at a time. + +### 5.1 pi-ai frames stay at the pi-ai boundary; Chord ops cross replication boundaries + +`AssistantMessageFrame` is pi-ai's semantic delta vocabulary (`text_delta`, `text_end`, …). [Delta tracking §6](../01-delta/delta.md#6-there-is-no-frame-type) defines no second frame wrapper: in-process replication carries `Op[]`, and a wire adapter carries encoded `WireOp[]`. Pi-ai frames stop at the fold; Chord ops cross the replication boundary. + +`AssistantMessageFrame` is pi-ai's own delta vocabulary and stays. What changes is +that it is no longer the durable unit or the replication unit. + +The harness folds frames into `LaneView` by plain mutation. Under the Chord tracker that yields: + +```json +["a",["operation","streamingMessage","content",0,"text"],"Let me "] +``` + +Measured, interned ops are **smaller than frames** on this workload — 13.6 KB +against 21.5 KB raw for 200 deltas — because a frame carries three keys of its own +where an interned op carries one integer. So the size argument for putting frames +on the wire is dead. + +What frames keep is semantic: `text_end` carries authoritative content plus a +signature in one atomic unit, where ops would need two with a weaker contract +about their relationship. That is why they remain the *input* vocabulary and are +folded before anything crosses a boundary. + +### 5.2 Reducer state must live in the reduced value + +Text and thinking fold purely: `block.text += frame.delta`, and `*_end` overwrites +with authoritative content plus signature. The `ended` flag in `ReducerBlockState` +is used only for validation and can be dropped. + +Tool calls cannot. `toolcall_delta` does `state.json += frame.delta`, accumulating +a **raw JSON string that is never stored on the message** — the block holds +`arguments`, the parsed value. You cannot recover the accumulator from the +snapshot, and you cannot append a delta to a parsed object. + +So the accumulator has to become part of the value being folded — e.g. +`operation.frameState[contentIndex].json` on `LaneView` — leaving +`AssistantMessage` clean. Generally: + +> **A replicated reducer's state must be part of the replicated value.** Anything +> held beside it diverges on any consumer that did not run the producer's fold. + +The corollary is that `arguments` itself should **not** be replicated. It is +derived from `json`, and a parsed object is a fresh reference on every parse, so +storing both ships two copies of the same information. Derive it on demand. + +This is safe precisely because `parseStreamingJson` is **total** — four fallbacks +ending in `{}`, it cannot throw — so a replica derives unconditionally with no +error path and no agreement protocol. There is no parse-failure state to +represent, and no block needs an error slot. + +### 5.3 Parse cost + +`parseStreamingJson` on a growing string once per delta is quadratic per message. +Since `arguments` is now derived rather than replicated, this cost falls on whoever reads it rather than on every consumer. Presentation can refresh derived arguments at semantic checkpoints and `toolcall_end` instead of parsing on every delta; that policy is separate from why the encoder currently emits a checkpoint (§6). + +## 6. What `toolcall_checkpoint` is for + +`EncoderBlockState` carries `caughtUp` and `catchupJson` because a queued provider event's shared `partial` may already be ahead of that event's delta. The checkpoint catches the semantic frame stream up to the authoritative tool-call arguments visible at block start; it is not currently a general late-subscriber protocol. + +After frames fold into tracked state, a Chord root replacement (`r`) is the replication and durable-recovery resync point. `toolcall_checkpoint` remains semantic input to that fold rather than carrying transport responsibility. + +## 7. Write volume for pending output + +`openFrameProgress` calls `appendList(pendingAssistantFrames(...))`, so every +frame is a line. A long response is thousands of writes. + +**The address is renamed `pendingAssistantOutput`** and becomes a `list`, matching `pendingToolOutput` ([tool-output handoff §7.2](../04-tool-output/harness-tools.md#72-renaming)), and it stops being a list of frames. The progress sink encodes tracked `Op[]` with one stateful encoder per response before appending each durable `WireOp[]` batch; explicit `rebase()` calls produce bounded root-replacement batches for recovery. The list lives in an **ephemeral scope** so it is unlinked on settle rather than persisting in the main log ([scoped storage](../02-scopes/scopes.md)). + +The important property is that this list is **not history**: `deleteList` runs on +settle in `response.ts`, `deferred.ts`, and `terminal.ts`. It exists so a crash +mid-response can recover a partial assistant message. The settled message is persisted +separately. + +That means per-frame durability buys almost nothing, and the write rate can be traded +away directly: + +- **Coalesce with a bounded, non-resetting window.** The first pending frame opens the + window; later frames join it *without* extending the deadline, so a continuously + streaming response cannot postpone the first write indefinitely — the failure mode a + naive debounce has. Frames admitted during an active write form the next batch. +- **Concatenate on flush.** A run of `text_delta` frames for one `contentIndex` + collapses to a single frame with the joined text. The fold result is identical, so + this is lossless for our purposes. + +A crash then loses at most one window of in-flight streaming. + +### 7.1 Contrast with DeepSeek Harness + +DSH cannot make this trade. Its `assistant/chunk` events are canonical log entries, so +per-token durability is mandatory and it attacks size instead: + +- the same bounded non-resetting write-behind window; +- **packed rows** — runs of consecutive chunk deltas stored as + `text-chunks` / `reasoning-chunks` / `tool-call-chunks`, lossless and roughly 60% + smaller on a real session, with reading unconditional so layout never depends on the + write switch; +- checksummed zstd frames by default, with recovery from a torn final frame. + +Their packing must reconstruct exact event boundaries, sequence numbers and timestamps, +because `seq = log.length` and validation requires a contiguous logical log. Ours does +not, because the frames are discarded at settle — so plain concatenation is available +to us and packing machinery is not needed. + +## 8. Consequence for ad-hoc listeners + +`HarnessEvent.message_update` stops being self-describing. A listener that attaches +mid-stream sees a delta against a partial it does not hold. + +Any correct listener already has the base, because `watch()` installs the subscription +and captures the snapshot inside one `readLane` critical section, buffering until +`start()`. But a listener that calls `on("message_update")` directly, without a watch, +is no longer viable — worth knowing before committing. diff --git a/packages/agent/docs/mobile-handoff/02-plugins/01-facets/facets.md b/packages/agent/docs/mobile-handoff/02-plugins/01-facets/facets.md new file mode 100644 index 000000000..e576fabc8 --- /dev/null +++ b/packages/agent/docs/mobile-handoff/02-plugins/01-facets/facets.md @@ -0,0 +1,1855 @@ +# Plugin and Facet Architecture + +> **Status:** Design specification. Supersedes the facet/service model in `plugins.md` +> where the two disagree. Read `rpc.md` for transport framing. + +## 1. Shape of the system + +A **plugin** is a package with up to three entry points, one per host kind: + +```text +my-plugin/ + contract.ts service tokens + JSON DTOs (shared, no host imports) + server.ts server facet (optional) + worker.ts session-worker facet (optional) + tui.ts presentation facet (optional) +``` + +Nothing links the entry points at runtime. They share only the tokens in +`contract.ts`. Each is built into a single JavaScript file by esbuild, pointed at +that entry. + +A **facet** is the in-process unit: one object with a static manifest and one +construct function. A **host** is a process that assembles a facet graph — server, +session worker, or presentation. + +The topology is a tree of connections: + +```text +server +├─ TUI A +├─ web B +├─ session worker S0 +└─ session worker S1 +``` + +## 2. Tokens + +A token is the identity of a service contract. It carries the service type as a +phantom, its stable ID, whether it is RPC-capable, and — critically — its **mode**. + +```ts +type ServiceMode = "singleton" | "keyed" | "peer"; + +interface Service { + readonly id: string; + readonly mode: M; + readonly rpc: boolean; + readonly __type?: T; // phantom +} + +function defineService( + id: string, + options?: { rpc?: boolean }, +): Service; + +function defineKeyedService( + id: string, + options?: { rpc?: boolean }, +): Service; + +/** One instance per connected peer; each peer sees exactly its own. */ +function definePeerService( + id: string, + options?: { rpc?: boolean }, +): Service; +``` + +The three modes differ only in how many instances exist and who sees them: + +| mode | instances | consumer declares | consumer gets | +| --- | --- | --- | --- | +| `singleton` | one, shared | `uses` | the service | +| `keyed` | many, all visible to everyone | `observes` | one task per instance | +| `peer` | one per connected peer, visible only to that peer | `uses` | the service | + +`peer` is the answer to per-client state (§10.2). It is not a fourth mechanism — +the host instantiates lazily per peer and announces to that peer alone — but the +consumer ergonomics matter: from a client's side there is exactly one, so it says +`uses` and calls it, exactly like a singleton. Watching a *set* of instances is +what `observes` is for, and per-client state is not that. + +Mode lives on the token because it is a property of the contract, not a choice the +provider makes. This deletes an entire class of assembly error: a token cannot be +provided as a singleton and consumed as keyed, because there is nothing to declare. + +Tokens are RPC-capable by default. `{ rpc: false }` confines a token to its +providing process; such tokens are never announced and never resolvable remotely. + +## 3. Facets + +```ts +const modelSelectionTui = defineFacet({ + id: "@pi/model-selection:tui", + + uses: [Models, Tui], + provides: [], + observes: [], + + construct(ctx) { + const models = ctx.use(Models); + const tui = ctx.use(Tui); + + tui.commands.register("models.select", async (context) => { /* ... */ }); + + return []; + }, +}); +``` + +Three declaration fields, all pure token data, all readable without executing +anything: + +- **`uses`** — singleton tokens this facet requires. +- **`provides`** — tokens this facet implements, singleton or keyed. +- **`observes`** — keyed tokens this facet watches instances of. + +`construct` runs only after the kernel has validated the whole graph and +constructed every dependency. Everything `ctx.use()` returns is a **real object**, +never a proxy that becomes valid later. There is no phase during which a facet +holds something unusable. + +**`construct` is synchronous.** It wires objects and returns provisions; it does no +I/O. Anything asynchronous registers through `ctx.onActivate`, which runs after the +entire graph is constructed, in dependency order. `ctx.onDeactivate` runs before +disposal in reverse order. Three phases, each with a single job: + +| phase | sync? | may call dependencies? | purpose | +| --- | --- | --- | --- | +| `construct` | yes | no | wire objects, return provisions | +| activate | no | yes | I/O, subscriptions, initial fetches | +| deactivate | no | yes | orderly shutdown before disposal | + +Keeping `construct` synchronous is what makes the ordering guarantee simple: a facet +cannot observe a half-built graph, because nothing runs until all of it exists. + +### 3.1 Why not declaration-by-side-effect + +`plugins.md` derives the dependency ledger from `env.use()`/`env.provide()` calls +made during a synchronous `setup()`. That avoids writing the manifest twice, but it +forces `use()` to return a disconnected lazy proxy — an object whose type is a lie +until assembly finishes. Every facet author then has to remember a rule the language +cannot enforce. + +Separating declaration from construction costs one extra mention of each token and +buys: + +- graph validation with **zero facet code executed** — decisive for third-party + plugins, where a bad set should be rejected before it runs; +- honest types throughout construct; +- a facet whose entire surface is readable in ten seconds. + +The duplication is eliminated by type checking (§4), so it cannot drift. + +### 3.2 Construct context + +```ts +interface ConstructContext { + /** Resolved singleton dependency. Only accepts tokens declared in `uses`. */ + use(token: S): ServiceType; + + /** Owner handle for a keyed token declared in `provides`. */ + owner>(token: S): ServiceInstances>; + + /** Host-built replicated state; see §9.2. */ + state(definition: StateDefinition, initial: T): MutableState; + + /** Runs after the whole graph has constructed, in dependency order. */ + onActivate(fn: (context: Context) => void | Promise): void; + /** Runs before disposal, in reverse dependency order. */ + onDeactivate(fn: (context: Context) => void | Promise): void; +} + +interface ServiceInstances { + /** Invokes this facet's factory for `key`, registers it, announces it. */ + add(key: string): () => void; +} +``` + +`ctx.use()` is a typed lookup rather than a positional tuple or a named bag: no +invented labels, no positional matching, and passing an undeclared token is a +compile error. + +Owner handles arrive through `ctx` rather than being returned, because they are +kernel-owned machinery that exists before the factory does. Construct receives +dependencies and its own handles; it returns implementations. + +## 4. Return type and completeness + +`construct` returns an array of entries built by two helpers: + +```ts +function provide(token: Service, impl: T): ProvideEntry; +function provide(token: Service, factory: (key: string, scope: InstanceScope) => T): ProvideEntry; +function provide(token: Service, factory: (principal: Principal, scope: InstanceScope) => T): ProvideEntry; + +/** Instance-lifetime equivalent of the construct context's state(). */ +interface InstanceScope { + state(definition: StateDefinition, initial: T): MutableState; +} + +function watch( + token: Service, + handler: (instance: Instance, context: Context) => void | Promise, +): WatchEntry; + +interface Instance { + readonly key: string; + readonly service: T; +} +``` + +The union of tokens in the returned array must equal the union of `provides` plus +`observes` — mutual assignability in both directions, which gives both +**completeness** (nothing forgotten) and **no extras** (nothing undeclared): + +```ts +type TokensIn = R[number]["token"]; + +type CheckComplete = + [TokensIn] extends [Declared] + ? [Declared] extends [TokensIn] + ? unknown + : { __error: "missing implementation for declared token" } + : { __error: "returned an undeclared token" }; + +function defineFacet< + const U extends readonly AnySingleton[], + const P extends readonly AnyService[], + const O extends readonly AnyKeyed[], + const R extends readonly Entry[], +>(facet: { + id: string; + uses: U; + provides: P; + observes: O; + construct: (ctx: ConstructContext) => R & CheckComplete; +}): Facet; +``` + +Note this is an array of token/value pairs, not an object map. Tokens are objects, +and objects cannot be keys. String-ID keys reintroduce a seam between the token in +the manifest and a string literal in construct; unique symbols do not survive +`defineService`'s return type, collapsing to plain `symbol` and merging every key. +Pairing tokens with values in an array keeps each entry individually typed against +its own token and keeps completeness checkable. + +> **Open:** error message ergonomics for `CheckComplete` need experimentation. +> The failure points at construct's return type, which is correct but not pretty. + +### 4.1 Worked example + +```ts +export const questionSession = defineFacet({ + id: "@pi/question:session", + + uses: [Tools], + provides: [QuestionDialogs], // keyed token + observes: [], + + construct(ctx) { + const tools = ctx.use(Tools); + const dialogs = ctx.owner(QuestionDialogs); + const pending = new Map(); + + tools.add((draft) => { + draft.set("question", { + /* ... */ + async execute(_id, params, _u, _tc, invocation, context) { + const completion = Promise.withResolvers(); + pending.set(invocation.invocationId, { params, completion }); + const close = dialogs.add(invocation.invocationId); + try { + return toResult(await awaitAbortable(completion.promise, context.abortSignal)); + } finally { + close(); + pending.delete(invocation.invocationId); + } + }, + }); + }); + + return [ + provide(QuestionDialogs, (key) => { + const entry = pending.get(key)!; + return { + request: entry.state, + async submitAnswer(candidate, _context) { /* memoOnce, resolve */ }, + }; + }), + ]; + }, +}); +``` + +The factory receives the key and closes over facet-private state. `dialogs.add()` +triggers it. Construct still runs exactly once. + +## 5. Ordering and cycles + +`uses` and `provides` are static, so the kernel computes a topological order over +facets **before running any of them**. Every facet is constructed after all of its +dependencies. Failures — missing provider, duplicate singleton owner, cycle — are +reported against the manifest with no facet code executed. + +**Cycles between facets are rejected.** No ordering satisfies them, and the survey +of coding-agent features found no genuine construction-time cycle: tools and +providers are fan-in through contribution registries; hooks are the host calling +you; telemetry is a leaf; wrappers are ordered composition. + +The cases that look circular are late *call-time* references, not construction +dependencies. Those get an explicit, visible escape hatch: + +```ts +uses: [Tools, deferred(QuestionDialogs)] +// ctx.use(deferred(X)) returns () => X, resolved on first call, after assembly +``` + +Deferral is opt-in and rare, so the cost of a knot is local and legible rather than +being paid by every facet in the form of universally unusable proxies. + +### 5.1 Same-facet composition + +A facet's services share module and closure scope. Two services that need each +other are ordinary JavaScript: build both inside construct, wire the references by +hand, return both. Shared private state is a variable both close over. The kernel +is not involved and there is no intra-facet graph. + +If two services in one facet need each other symmetrically, that is usually one +service with two faces, or one private object behind two facades. + +## 6. Hosts, connections, and direction of authority + +**A host may depend only on what it connects to, and connections form a tree.** +That single rule replaces any global sort and keeps the cross-process graph acyclic +without special-casing the server as "built-ins that are just there." + +Each host sorts locally. Remote provisions are **leaves** — the upstream process +constructed them before announcing, so they are already satisfied by definition. + +Concretely: + +- A **session worker** connects to the server, so worker facets may `use` server + tokens. This is required: a `spawn_subagent` tool needs the server's session + management. +- A **presentation** connects to the server, so TUI facets may `use` server tokens. +- A presentation also consumes **session** services — but the provider from its + point of view is the server, which routes. The server's catalogue simply grows + when a session is attached. + +There is no direct presentation↔worker connection. + +### 6.1 The server never depends on a worker + +The server must construct before any worker exists, so it cannot `use` a worker +service. Inverted flows use a **reporting registry**: the server provides a token +that workers push into. + +```ts +// contract.ts +export interface SessionStatusReporting { + report(status: SessionStatus, context: Context): Promise; +} +export const SessionStatusReporting = + defineService("pi.session-status-reporting"); + +export interface SessionStatusView { + readonly state: State>; +} +export const SessionStatusView = + defineService("pi.session-status"); +``` + +The worker `uses` the reporting token and pushes; the server aggregates into +replicated state; presentations read the aggregate. Dependencies still point +upward, data flows down. + +This gets the lifecycle right for free: the aggregate exists immediately and is +empty; workers appear, register, and vanish; a crashed worker is an entry removal, +which the server already knows about because it owns the connection. + +## 7. Facet delivery and generations + +**A presentation ships with no plugin facets.** It has host services (`Tui`) and +nothing else. All plugin facets arrive over the wire as built bundles. + +This dissolves the problem of a detached presentation holding unresolved +requirements: while detached there are no requirements, because there are no +facets. + +Two generations, with different lifetimes: + +| generation | source | lifetime | example | +| --- | --- | --- | --- | +| **connection** | server | the server connection | session picker | +| **attachment** | session worker | one attachment | question dialog, chat | + +Switching sessions tears down and rebuilds only the attachment generation. The +picker keeps running throughout — which it must, since it is what triggers the +switch. + +The attachment generation resolves against the connection generation (upstream +first, per §6). The reverse is forbidden: a connection-generation facet cannot +depend on an attachment-generation token, because the attachment can go away. + +### 7.1 Why the worker chooses the presentation facets + +Plugins may be **global** or live in a session's **working directory**. So the set +of TUI facets depends on where the TUI was started — same binary, different +capabilities per project. + +The worker therefore knows which TUI bundles belong with it, and ships them through +the server on attach. The working directory is not merely a spawn parameter; it is +an input to the graph. + +> **Security:** attachment-generation bundles are third-party code arriving from a +> project directory and executing in the user's presentation process. Trust policy +> for directory-local plugins is an open decision and must be settled before this +> ships. + +## 8. Startup and handshakes + +### 8.1 Presentation connect + +```text +TUI → server: connect +server → TUI: catalogue of server RPC provisions + connection-generation bundles +TUI: assemble, validate, construct connection generation +``` + +### 8.2 Reaching a session + +Three routes, converging on one attach: + +| invocation | route | +| --- | --- | +| `pi` (bare, in a directory) | ask server to create a session for cwd | +| `pi --resume` | invoke the picker command at startup | +| `pi --session ` | attach directly | + +`--resume` needs no special machinery: the picker is an ordinary command registered +by the server-sourced picker facet, and resume invokes it at startup instead of +waiting for a keystroke. Same code path either way. + +Create carries the working directory, since the server needs it to spawn and the +worker needs it to resolve local plugins. + +### 8.3 Attach + +```text +TUI → server: attach(sessionId) +server: authorize; tear down previous attachment generation; + bind routing to worker +server → worker: client attached (identity) +worker → server: catalogue of session RPC provisions + TUI bundles +server → TUI: catalogue + bundles +TUI: assemble, validate, construct attachment generation +worker/server: hydrate state state values +``` + +The TUI's kernel validates once, with the worker's catalogue already in hand. There +is no provisional or degraded resolution state. + +### 8.4 Worker startup + +```text +worker → server: connect +server → worker: catalogue of server RPC provisions +worker: load global + cwd-local plugins; assemble; construct +``` + +The worker's own dependencies on server tokens resolve here, before any attachment. + +## 9. Replication primitives + +Exactly three things cross a connection. + +### 9.1 Service calls + +Ordinary request/response. For actions (`select`, `submitAnswer`) and one-shot +reads of things that do not change (scrolling back into chat history). Nothing is +replicated. Arguments and results are strict JSON; `Context` is stripped by the +proxy and reconstructed at the endpoint. + +### 9.2 Replicated state + +One authoritative writer, many readers. The stream carries one base op batch followed by delta batches; the vocabulary is in [delta.md](../../01-harness/01-delta/delta.md). + +Full-value replication is the degenerate configuration where the producer explicitly replaces or rebases on every update. It is not a separate primitive: Chord emits the root replacement op `r`. + +#### State is plain TypeScript + +Facets never write ops. A provider mutates its state object normally; the +framework's tracker (`delta.md`) records intent and emits ops. + +```ts +// contract.ts +export const TranscriptState = defineState("pi.transcript.tail"); +``` + +```ts +const tail = scope.state(TranscriptState, initial); + +tail.mutate((s) => { + s.entries.push(entry); // -> splice + s.entries[0].text += chunk; // -> append + delete s.pending; // -> delete +}); +``` + +No mutation map, no recipes, no declared operations, no Immer. There is no +`defineValueState` / `defineReducedState` split either — an earlier draft needed +one because reduced states carried typed deltas a differ could not recover, and +the tracker recovers them. + +`mutate` is synchronous and takes no `Context`. A mutation is a pure state +transition: it calls nothing, cannot be cancelled, and has no caller identity to +consult. Authority is checked in the method that decides to mutate. + +#### Ops never carry provider code + +Consumers apply ops. They never see mutation names and never run provider code. + +This matters for a specific reason: a consumer running provider code would have +to resolve a definition from a registry, making the fold depend on ambient +registration — a reload changes the answer for the same input. Ops remove that, +and make non-JS consumers trivial, since the applier is six verbs. + +Consequences: + +- mutation names are not part of the wire contract and appear in no member table, + so there is no mutation-name version skew; +- a consumer needs no schema for arguments, only for the value in the root `r` op; +- `x = undefined` normalises to `delete`, since JSON has no `undefined`. + +#### Wire protocol + +The producer and applier use `Op[]`; a subscription carries encoded `WireOp[]`. There is no frame type and no `seq` in the payload: the SSE binding stamps `id:`, which is where transport +metadata belongs (`delta.md` §6). + +`Op` and its encoding — six verbs, array paths, tuple form, second-use path interning — are specified in [delta.md](../../01-harness/01-delta/delta.md) §2 and §4. They are not restated here. + +**A replacement is an op**, `["r", value]`. There is no frame discriminator. A batch whose first op is `r` is a **base batch**, and batch zero is always one. This collapses several things `plugins.md` treats separately: + +- there is no distinct wire hydration shape — the snapshot is simply the first batch; +- snapshot buffering remains a local adapter detail rather than a second wire protocol; +- cold start, reconnect, provider reload, session switch, a sequence gap, and a + fold that could not apply are **one code path**: send a fresh base batch. + +That last item is why there is no `Rebase` type anywhere in this design. + +The SSE `id:` is stamped by the host binding, starts at zero on every +subscription, and is contiguous. A consumer seeing a gap discards its replica and +resubscribes. + +#### Resubscription is a base batch plus buffered batches + +There is no cross-subscription resume and no `Last-Event-ID`. Resubscribing works +the way the harness already hands out lane state: snapshot the current value as +base batch zero, buffer whatever arrives while the client is catching up, then deliver the buffer as ordinary batches once it is ready. + +That is the same two-phase shape as `AgentHarness.watch` — `snapshot` then +`start` — and it is a purely local detail of the provider binding, invisible on +the wire. A consumer sees a base batch followed by contiguous batches, exactly as on cold start. + +Real resume would need a retained op log so a client could ask for "everything after seq N". We deliberately do not keep one: the durable form is a list of batches per *value* ([delta.md §9](../../01-harness/01-delta/delta.md#9-durable-form)), retired with its scope, not a per-subscription history. Buffering costs a bounded amount of memory for the duration of a handshake; an op log would cost unbounded disk forever. + +The landed tracker emits structural ops without a serialized-size heuristic. Provider replacement, reconnect, and policy-driven recovery bounds call `replace()`/`rebase()` explicitly; those are the only sources of a root `r` batch. + +#### The harness does not emit ops + +The harness emits typed `HarnessEvent`s and always will. Ops appear one layer up. + +A lane facet calls the harness in process, subscribes to typed events, and folds +them into **its own** state shape by plain mutation. The tracker turns that into +ops. The facet never writes an op and never thinks about paths. + +This fold has to exist regardless, because **the replicated shape is a facet's +choice, not the harness's**. Emitting ops from the harness would not remove the +fold; it would only replace "events into state" with "ops into state, with worse +types", and it would make `LaneSnapshot`'s field layout the wire contract. + +```ts +// packages/agent/src/harness/runtime/reducer.ts +export function reduceLaneSnapshot(view: LaneView, event: HarnessEvent): void; +``` + +Plain mutation, no Immer, no return value. See `harness-tools.md` §6. + +#### Shaping state to avoid write amplification + +Two rules, both consequences of the tracker recording intent: + +- **Keep growing strings at stable paths** — one per content block, one per tool + operation's output. A delta then touches exactly one path, which is also what + makes path interning collapse it to a single integer. +- **No derived fields in replicated state.** A parsed value recomputed from a raw + one is a fresh reference on every parse, so it ships as a whole-value `set` and + duplicates information already present. Derive on demand instead + (`message-update.md` §5.2). + +An array of content blocks is fine: it takes one `splice` when a block appears and +then nothing. What changes afterwards is a string *inside* a block. + +#### Requesting a state value + +A provider asks the host to build the state value during `construct`. Facets never +construct one themselves, because the state value is a binding: the host owns its +subscriber table, revision stamping, and disposal. + +```ts +interface State { + readonly value: T | undefined; // undefined until base batch zero + subscribe(listener: (value: T) => void): void; // returns nothing (§13) +} + +// on ConstructContext and InstanceScope +state(definition: StateDefinition, initial: T): MutableState; +``` + +Provider side: + +```ts +export const transcriptSession = defineFacet({ + id: "@pi/transcript:session", + uses: [Agent], + provides: [Transcript], + observes: [], + + construct(ctx) { + const agent = ctx.use(Agent); + const tail = ctx.state(TranscriptState, { entries: [] }); + + agent.onEntry((entry, context) => { + tail.append([entry]); + if (tail.value.entries.length > TAIL_LIMIT) { + tail.evict(tail.value.entries.at(-TAIL_LIMIT)!.id); + } + }); + + return [ + provide(Transcript, { + tail, + page: (params, context) => archive.read(params.before, params.limit, context), + }), + ]; + }, +}); +``` + +A `mutate()` call advances the authoritative value **and** publishes the resulting ops — one statement, so the value and the wire cannot disagree. `replace()` publishes an explicit root `r` batch and is what reconnect, provider reload, resnapshot, and policy-driven recovery bounds use. + +Consumer side: + +```ts +export const transcriptTui = defineFacet({ + id: "@pi/transcript:tui", + uses: [Transcript, Tui], + provides: [], + observes: [], + + construct(ctx) { + const transcript = ctx.use(Transcript); + const tui = ctx.use(Tui); + + transcript.tail.subscribe((tail) => render(tail.entries)); // base batch zero arrives here too + tui.commands.register("transcript.older", async (context) => + render(await transcript.page({ before: oldestId(), limit: 100 }, context)), + ); + + return []; + }, +}); +``` + +The consumer never sees opcodes. It receives values, and base batch zero is delivered as an ordinary update — so there is no separate "ready" callback and no hydration +branch in facet code. + +#### Worked: adapting `AgentHarness.watch` + +The real API is asynchronous throughout: + +```ts +interface AgentHarness { + lane(name: string, context: Context): Promise; + lanes(context: Context): Promise; +} +interface AgentLane { + watch(context: Context): Promise>; +} +interface WatchHandle { + snapshot: T; + start(listener: EventListener): void; + resnapshot(context: Context): Promise; + unsubscribe(): void; +} +``` + +`watch()` installs the subscription **and** captures the snapshot inside a single +`readLane` critical section, so both are taken under the same lock. Events arriving +before `start()` are buffered, then flushed. `resnapshot()` does the same with a +`markBoundary()` callback re-establishing where the stream resumes. That is exactly +the guarantee base batch zero needs, so the adapter is thin. + +`LaneSnapshot` is per lane, so `Lane` is keyed — one instance per lane, each taking +its own `watch()`. The harness already filters by lane +(`event.type === "usage" || !("lane" in event) || event.lane === this.name`), so the +adapter does no routing. + +Because `watch()` is async, **instance factories may be async**. The instance is +announced once the factory resolves; `add(key)` returns its closer immediately. + +```ts +export const Lane = defineKeyedService("pi.lane"); // key = lane name + +export const laneSession = defineFacet({ + id: "@pi/lane:session", + uses: [Harness], + provides: [Lane], + observes: [], + + construct(ctx) { + const harness = ctx.use(Harness); + const lanes = ctx.owner(Lane); + + ctx.onActivate(async (context) => { + for (const info of await harness.lanes(context)) lanes.add(info.name); + }); + + return [ + provide(Lane, async (laneName, scope, context) => { + const lane = await harness.lane(laneName, context); + const handle = await lane.watch(context); // phase 1: snapshot + subscribe + // The facet's own shape, not LaneSnapshot. reduceLaneView is its code. + const state = scope.state(LaneState, toLaneView(handle.snapshot)); + + handle.start((event) => { // phase 2: buffered, then live + if (event.type === "navigation_end") { + void handle.resnapshot(context).then((fresh) => state.replace(toLaneView(fresh))); + return; + } + state.mutate((v) => reduceLaneView(v, event)); // plain mutation; ops fall out + }); + + return { snapshot: state, setModel: (ref, ctx2) => lane.setModel(ref, ctx2) }; + }), + ]; + }, +}); +``` + +Consumers declare `observes: [Lane]` and get one task per lane, each with its own +replica. + +Four things to notice. + +- **The `watch`/`start` handshake does not cross the wire.** It is a local detail of + one adapter. Remote consumers see base batch zero, then ops. Mini's buffering dance + disappears, because the harness already buffers between the two phases and the flush + lands as ordinary mutations after the initial value. +- **The facet owns the replicated shape.** `LaneView` is the facet's, not the + harness's — hence `toLaneView`. This is the point made above: the fold exists + regardless, so the harness gains nothing by emitting ops itself. +- **Rebase never reaches consumers, and is not a type.** `navigation_end` is an + ordinary event the adapter answers with `resnapshot()` plus `replace()` — the + same base batch a reconnecting client receives. `markBoundary()` supplies the + ordering. No reducer signals anything by returning a value. +- **`HarnessEvent` is not a wire format.** `message_update` today carries the full + message *and* an `AssistantMessageEvent` holding a second copy *and* the delta, + so shipping it would be worse than shipping a snapshot. See `message-update.md`. + It never ships here, because ops travel. +- **Most of the union is already lane state.** `usage` folds into `stats.usage`; config + `value_update` into `configuration.*`; `message_update` into + `operation.streamingMessage`; `tool_start`/`tool_update` into + `operation.runningTools`; retries into `operation.retry`. The state value is very + nearly everything a presentation needs. + +The three that do **not** belong in it are a useful test of §9.4: + +| event | why not a mutation | where it goes | +| --- | --- | --- | +| `lane_created` | creates a lane, does not change one | `lanes.add(name)` — a new instance | +| `handler_error` | diagnostic; no late joiner needs it | events primitive | +| global `value_update` | not lane-scoped | state on whichever service owns that value | + +Note `lane_created` is delivered on the lane event stream, so discovering new lanes +after startup requires a session-level watch rather than the per-lane ones above — +an open question in §16. + +#### Host responsibilities + +Everything below is host machinery, not facet API: + +- stamping `seq` per batch within the provider binding, so business snapshots and ops carry no transport metadata; +- delivering a root `r` base batch before deltas on a fresh subscription; +- resetting each subscriber's encoder dictionary on that base batch; +- applying only consecutive batches, and requesting a fresh base batch after a gap, reconnect, or provider replacement; +- hardening values as they cross the boundary (§14.3); +- announcing the state value definition ID in the member table. Mutation names are + **not** announced, because they never travel — ops are structural, so a consumer + needs no knowledge of how a value was produced. + +Two rules follow from the provider and every replica running the same fold: + +> **A reducer's state must be part of the replicated value.** Anything accumulated +> beside it — the classic case being a raw JSON string being parsed incrementally — +> diverges on any consumer that did not run the producer's fold. + +> **The durable path uses only host-owned reducers.** Plugin-interpreted data must be +> skippable, so an unreadable stream degrades that one value instead of failing the +> record that contains it. + +Mutation-name version skew does not exist: mutations never cross a boundary, so +adding, renaming or removing one is not a breaking change. The version-skew surface +is the *value shape* alone, which the root `r` batch's schema describes. + +Prefer several coarse state values to one large value. A cold replica has +`value === undefined`; that is local readiness and never crosses the wire. + +### 9.3 Events + +Fire-and-forget broadcast. No history, no durability, dropped on disconnect. For +presence, cursors, in-progress drags, transient notifications. + +### 9.4 Which to use + +> **Does a late joiner need it?** +> Yes → replicated state. No → event. + +State: transcript, model catalogue, session status, chat tail, canvas document. +Events: who is typing, where someone's cursor is, an uncommitted stroke. + +### 9.5 Worked: chat, and the tail/archive split + +A chat room is not one state. It splits along the same line: + +- **live tail** — last N messages as a state value. The first batch replaces with + the window; appends and evictions are ops. +- **archive** — a plain service call returning a page of older messages. Immutable + history, no liveness requirement, not replicated onto anything. + +There is no `subscribe` method. Attaching to the state value *is* the subscription, and it +produces the snapshot, so the server registers the reader before it reads. That +closes the window between a query and a later subscribe, and means the server never +has to answer "what happened after an arbitrary client-supplied cursor" — the +class of race that client-chosen timestamps create. + +The lane transcript is the same pattern: recent entries in the state value, older ones +fetched by query when the user scrolls up. + +### 9.6 Worked: shared canvas + +Document state is a state value; strokes are ops. Multi-writer is handled by +**optimistic apply with acknowledgement**, not CRDT: + +1. client generates an op ID, applies locally as unacknowledged, sends it; +2. server accepts first-come-first-served and publishes the resulting ops; +3. client sees its own op ID return and promotes it to acknowledged. + +Note this is the one place the single-writer assumption behind `delta.md` is +relaxed, and it is relaxed by serialising at the server rather than by merging. +The op vocabulary has no `test` verb precisely because there is no conflict model +to support. + +In-progress drags never touch the document. They are events, keyed by peer, +latest-wins, dropped on disconnect — which is exactly why the events primitive +exists. + +## 10. Peers, principals, and authority + +### 10.1 Principals on the context + +Every service method already takes a `Context`. The proxy strips it on the way out +and **reconstructs it at the endpoint**, so the kernel fills in the caller's +principal from the authenticated connection. It is control-plane data: never an +argument, never forgeable. + +```ts +type Principal = + | { kind: "local" } // same process, full authority + | { kind: "user"; peer: PeerId; userId: string; role: Role } + | { kind: "process"; peer: PeerId; host: "worker" | "server"; sessionId?: string }; + +interface Context { + readonly abortSignal: AbortSignal; + readonly principal: Principal; + // ... +} +``` + +There is exactly one `Context` type, and `principal` is always populated — the +same interface locally and across a connection, so a method body reads +`context.principal` without knowing which it was. That is the point: a service must +not have an unchecked path that only appears in-process. + +Peers are not only humans — a worker calling the server through a reporting +registry is a peer too, and server facets often want to distinguish *my own worker* +from *some TUI*. In-process calls carry an explicit `local` principal rather than an +absent one, so a missing check cannot masquerade as a missing peer. + +### 10.2 Per-peer services + +**Authority-bearing state belongs on a `peer` service, not a singleton.** A +singleton has one value by definition, so it cannot show an owner and a guest +different roots — there is nowhere for the difference to live. + +The provider declares the token `peer` and returns a factory. The host calls it +once per connected peer, passing that peer's principal, and announces the result to +that peer only. Each instance gets its own state values from its own `scope`. + +The consumer declares it in `uses` and calls it. No `observes`, no reconciliation, +no set to watch — from a client's side exactly one instance exists, because exactly +one was ever announced to it. + +### 10.3 Worked example: file browsing + +```ts +// contract.ts +export interface BrowseRoot { + readonly handle: string; // opaque, per-principal, revocable + readonly label: string; +} + +export interface BrowseView { + readonly roots: BrowseRoot[]; + readonly note?: string; // e.g. "Ask the owner for wider access" +} + +export const BrowseState = defineState("pi.browse.view"); + +export interface FileBrowsing { + readonly view: State; + list(handle: string, path: string, context: Context): Promise; +} + +export const FileBrowsing = definePeerService("pi.file-browsing"); +``` + +Server facet: + +```ts +export const fileBrowsingServer = defineFacet({ + id: "@pi/file-browsing:server", + + uses: [Fleet], + provides: [FileBrowsing], + observes: [], + + construct(ctx) { + const fleet = ctx.use(Fleet); + + return [ + provide(FileBrowsing, (principal, scope) => { + // One table per instance. Handles are minted here, so a handle a peer + // was never given is not merely rejected — it does not exist for them. + const table = new Map(); + + function publish() { + const roots = rootsForPrincipal(principal, fleet).map((path) => { + const handle = newHandle(); + table.set(handle, path); + return { handle, label: labelFor(path) }; + }); + return roots.length > 0 + ? { roots } + : { roots: [], note: "No browsable locations for this account" }; + } + + const view = scope.state(BrowseState, publish()); + + return { + view, + + async list(handle, path, context) { + const root = table.get(handle); + if (root === undefined) throw new ServiceError("forbidden", "unknown root"); + return readDirectory(root, path, context); + }, + }; + }), + ]; + }, +}); + +function rootsForPrincipal(principal: Principal, fleet: Fleet): string[] { + if (principal.kind === "local") return [ROOT]; + if (principal.kind === "user" && principal.role === "owner") return [ROOT, ...fleet.workspaces()]; + if (principal.kind === "user") return fleet.workspacesFor(principal.userId); + return []; +} +``` + +TUI facet — note that it contains no permission logic at all: + +```ts +export const fileBrowsingTui = defineFacet({ + id: "@pi/file-browsing:tui", + + uses: [FileBrowsing, Tui], + provides: [], + observes: [], + + construct(ctx) { + const browsing = ctx.use(FileBrowsing); + const tui = ctx.use(Tui); + + tui.commands.register("files.browse", async (context) => { + const view = browsing.view.value; + if (view === undefined) return; + + if (view.roots.length === 0) { + await tui.notify(view.note ?? "Browsing unavailable"); + return; + } + + const root = await tui.select( + "Browse", + view.roots.map((r) => ({ label: r.label, value: r })), + { signal: context.abortSignal }, + ); + if (root === undefined) return; + + let path = ""; + for (;;) { + const entries = await browsing.list(root.handle, path, context); + const chosen = await tui.select( + root.label, + entries.map((e) => ({ label: e.name, value: e })), + { signal: context.abortSignal }, + ); + if (chosen === undefined) return; + if (!chosen.isDirectory) return void open(root.handle, join(path, chosen.name)); + path = join(path, chosen.name); + } + }); + + return []; + }, +}); +``` + +An owner's `view` state carries both a machine root and workspace roots; a guest's +carries one or none. Same bundle, same code path, different data. The TUI never +branches on a role, never greys anything out, and cannot construct a handle it was +not given. + +### 10.4 Server-side validation is still mandatory + +The state governs rendering. It does not constrain the wire: a hostile client can +call any method with any argument, including one it never received. + +`list()` therefore checks its handle against the instance's table. Prefer +**unforgeable handles over parsed paths** — a table lookup fails by construction, +where path validation invites `..`, symlink, normalisation and case-folding bugs. +Revocation is free: drop the entry and outstanding handles stop working. + +Where an argument cannot be a handle, the method reads `context.principal` and +decides. The rule is that authority is checked at the provider, every time, no +matter what the client was previously shown. + +### 10.5 The peers cell + +Presence is replicated state the host provides: + +```ts +export const ConnectedPeers = + defineService<{ readonly state: State> }>("pi.peers"); +``` + +Server and worker facets read it to enumerate who is attached. A facet needs it +only for genuinely multi-peer features — a presence roster, a "3 viewers" badge. +Per-peer state does **not** need it, because `peer` mode handles instantiation and +teardown for you. + +### 10.6 Keyed instances + +`keyed` remains what it always was: many instances, all visible to every consumer, +each producing one abortable task under `observes`. Question dialogs are the +canonical case — three concurrent invocations produce three instances, and every +attached presentation sees all three. + +An instance is addressed by `(service, key, generation, member)`, so its state values are +naturally per-instance. Reusing a closed key creates a new generation, so stale +proxies cannot address the replacement. + +The distinction to keep straight: + +- **Do I want all of them?** → `keyed` + `observes`. +- **Do I want mine?** → `peer` + `uses`. + + +## 11. The presentation surface + +`Tui` is an ordinary singleton token, declared in `uses` and resolved through +`ctx.use` like any other. It is provided by the presentation kernel rather than a +plugin, so it sits at the root of the local sort, but facets cannot tell the +difference. + +```ts +interface TuiHost { + readonly slots: SlotContributions; + readonly commands: CommandContributions; // handlers take an AbortSignal; see §11.3 + readonly keybindings: KeybindingContributions; + readonly toolRenderers: ToolRendererContributions; + notify(message: string): Promise; + acquireModal(signal: AbortSignal): Promise; + select(title: string, items: SelectItem[], options: { signal: AbortSignal }): Promise; +} + +const Tui = defineService("pi.local.tui", { rpc: false }); +``` + +### 11.1 Slots + +A slot is a named region of host chrome a facet may fill. + +```ts +interface SlotContributions { + /** Exclusive. A second claim on the same slot is an assembly error. */ + claim

(slot: string, factory: (props: P) => Component): void; + /** Additive. Ordered by contribution order. */ + add

(slot: string, factory: (props: P) => Component): void; +} +``` + +```ts +ctx.use(Tui).slots.claim("footer", (props) => new FooterComponent(props, models)); +``` + +`claim` is exclusive and a duplicate is an **assembly error, not last-write-wins** — +two plugins silently fighting over the footer is a bug that must surface at +validation, alongside missing providers and duplicate singletons. + +Props are DTOs. A factory never receives the `TUI` or `Theme` instance, so a +component cannot reach back into host internals through its own arguments. + +### 11.2 Mounting is host-owned + +Facets do not mount or unmount. They contribute a factory; the host instantiates +it, tracks the mount against the contributing facet, and **unmounts it on +disposal** — restoring the built-in chrome for a claimed slot, or dropping the +entry for an additive one. + +There is therefore no `dispose?()` on a contribution, and no teardown for a facet +author to forget. This is the same rule as §13: registration is ownership. + +Two consequences fall out of the generation model in §7: + +- **Attachment drop** — presentation facets consuming session services are + disposed, so their mounts vanish and host chrome renders in their slots. Only + connection-generation chrome reads attachment state. +- **Session switch** — attachment-generation facets are disposed and their mounts + go with them; connection-generation mounts survive, so there is no flicker and + no lost scroll state. + +### 11.3 Contributions with in-flight work + +§11.2 covers *removal*: the host tracks each mount against its contributing facet +and unmounts on disposal, so there is no teardown to forget. That is complete for +anything whose disposal is synchronous. + +It is not complete for contributions that can be **executing** when their +contributor is disposed — the presentation analogue of an in-flight tool call +(§13.2). Three cases, and only one of them is hard. + +**Synchronous callbacks are free.** A keypress handler, a render call, a click +listener. JavaScript is single-threaded, so a handler runs to completion before +disposal can be scheduled — disposal cannot interleave with a synchronous frame. +Deregistration is a list removal. Nothing further is needed, and this is most of +the surface: slots, keybindings, tool renderers, theme tokens. + +**Modals and pickers already carry a signal.** `acquireModal(signal)` and +`select(title, items, { signal })` take an `AbortSignal` because the host may need +to dismiss them. Disposal aborts it, the promise rejects, and the facet's +`await` unwinds. The mechanism exists; disposal just has to use it. + +**Async command handlers are the hard case**, and they are exactly a tool call in +different clothing: + +```ts +ctx.use(Tui).commands.add("deploy", async (args, signal) => { + await longRunningThing(signal); +}); +``` + +The user runs `/deploy`, the facet is disposed ten seconds later, and the handler +is still awaiting. So `CommandContributions` is a registry with in-flight work and +takes the same four phases as the tool registry: deregister, signal, race the +kernel deadline, and settle by outcome — here dismissing any progress UI and +restoring host chrome rather than producing a `ToolResultMessage`. + +Two consequences for the API: + +- **A command handler receives an `AbortSignal`.** Not optional. Without it the + registry can deregister but cannot signal, and phase 2 is unavailable. +- **The host renders command progress, not the facet.** If a facet paints its own + progress and is disposed mid-command, the paint outlives the painter. Host-owned + progress is removed with the mount, which is the §11.2 rule applied to a case + §11.2 did not name. + +The rule this generalises to: + +> A contribution registry needs settlement machinery **iff** a contribution can be +> mid-execution across disposal. Synchronous contributions never can; anything +> handed an `AbortSignal` always can. + +Which makes the signal the marker. A registry whose contributions take a signal +owns invocation tracking and a deadline; one whose contributions do not is a plain +list and disposal is a removal. + +### 11.4 Reload + +Reload is unload plus load, so mounts are torn down and rebuilt by the same path. +Cross-host reload is two-phase, so no facet is ever alive with a dead provider: + +```text +1. server → presentations with dependent facets: dispose, ack + (the provider is still ALIVE, so in-flight calls abort against a live peer) +2. server → session worker: dispose + reload +3. server → presentations: load +race(acks, 2s); a presentation that misses the window goes degraded, +which disposes those facets anyway. +``` + +Calls carry a caller generation and the provider drops retired ones. Because a +reload is a round trip to every connected presentation, it stays user-initiated — +it must not be wired to a file watcher. + +## 12. Contribution registries + +Services are one owner, many consumers. Providers and tools invert that: many +contributors, one host-owned result. A registry replays ordered contributions over +a fresh working copy, so removal is a rebuild rather than an inverse mutation. +(Unrelated to the tracker in `delta.md`; nothing here is replicated.) + +```text +fresh working copy +→ built-in providers (@pi/providers-builtin) +→ remote catalogue (@pi/providers-catalog) +→ models.json transform (@pi/providers-models-json) +→ auth/availability mark (@pi/auth) +→ validated state +``` + +Tools additionally support ordered wrapping — `telemetry(permission(sandbox(bash)))` +— which recomposes deterministically when a contributor disappears. Only the host +finalizes a draft; facets never call `setTools()`. + +Slots, commands, keybindings, and tool renderers (§11) are all instances of this +same shape on the presentation host. + +**Registries split by whether a contribution can be in flight.** A registry whose +contributions are values — slots, keybindings, theme tokens, tool renderers — is a +list, and disposal is a removal (§11.2). A registry whose contributions are +*invoked and take an `AbortSignal`* — tools, commands — owns invocation tracking, a +signal chained from the caller's, and settlement against the kernel's disposal +deadline (§13.2). The signal in the contribution's signature is the marker for +which kind it is. + +Contributions configure rebuilt behaviour; hooks intercept live operations. They +remain separate mechanisms. + +## 13. Lifecycle, failure, disposal + +- Construct in dependency order; dispose in reverse. +- A facet owns its provisions, keyed instances, and observations. Ownership is + **implicit**: every handle a facet receives is a host-built binding that + registers its own disposer, so `subscribe`, `on`, timers, and instance handles + are released with the facet. There is no `own()` and nothing to remember. + This covers handles the host hands over; it does not cover ambient authority a + facet can reach without being handed anything (§14.3). + Consequently `subscribe()` returns nothing — there is no unsubscribe handle to + hold or leak. +- Admitted inbound calls may finish while their provider deactivates; withdrawal + rejects new calls. +- Disconnect aborts in-flight requests and closes that peer's subscriptions and + instance tasks. It must **not** perform service-owned or durable cancellation. +- Three cancellation domains stay distinct: per-invocation, service-owned + (`job.cancel()`), and durable Harness (`requestAbort()`). +- Never blindly replay a mutation after an uncertain disconnect. Reconnect, + hydrate, reconcile — or design around a stable operation ID. +- Errors cross as `{ code, message }` with stable codes; unexpected exceptions + become `internal_error` with no stack. + +Reload: shape-preserving replacement by `Facet.id` is permitted when the +replacement declares an identical manifest. Because the manifest is static, this +check happens **before** constructing the candidate — a real improvement over +validating shape by running setup. Structural changes require graph reassembly or a +process restart. + +### 13.1 Teardown, not swapping + +The alternative is a proxy per dependent, swapping the implementation behind +consumers so they are never torn down. It is rejected, and the prior art is +unusually clear. + +**OSGi ships both.** Bundle refresh computes the transitive closure of everything +wired to the old exports and restarts it; Declarative Services also offers +`ReferencePolicy.DYNAMIC`, where a consumer keeps running across a swap. Their own +guidance makes `STATIC` the default, because dynamic requires every consumer to be +defensive about the service vanishing mid-call. + +**Cordis is the proxy model, and the ergonomics show.** `ctx.get(name)` returns +`undefined` when absent and the guidance is to *"handle their absence"* — the +defensive check is the recommended path. `inject` opts *into* teardown: the plugin +enters waiting and is reactivated when the service returns. + +**JS HMR is this design with one addition.** Updates propagate up the import graph +until a module calls `import.meta.hot.accept()`; if nothing accepts, full reload. +React Fast Refresh bails to full reload when a hook signature changes — the same +conclusion, reached from a different direction: implementation swaps survive, shape +changes do not. + +**Erlang is the counterexample and it does not transfer.** Two module versions +coexist and `code_change/3` migrates state at the transition. That works because +contracts are messages rather than types, so a process can handle both shapes +during changeover, and because isolation means there is no shared state to +reconcile. Neither holds here. + +Three specific objections, in increasing order of severity: + +**Contract changes.** JVM HotSwap permits method bodies only; DCEVM and JRebel go +further and still break on shape change. We are better placed than any of them, +because a token's `protocol` block carries a TypeBox schema and `CheckComplete` +already does mutual assignability — so a swap *could* be gated on the new +provider's schema being assignable both ways. This is the one objection we can +actually answer. + +**In-flight registrations.** A tool contributed by the old provider may be +mid-execution. It cannot be cancelled (side effects have happened), handed over +(different closure), or allowed to settle into a registry that no longer contains +it. A proxy does not help: the invocation is bound to the implementation that +started it. §13.2 is required either way, which is most of why the proxy earns +nothing. + +**State.** This is the one with no answer. A facet holds a replica derived from the +old provider's stream. After a swap it is either stale — silently wrong — or the +new provider sends a base batch, which *is* a resubscribe. The state was torn down; +only the facet was not. Erlang solves this with an explicit migration hook, which +is Erlang's design without Erlang's isolation. + +**Cached references make the proxy worse, not better.** Cordis's guard lives on the +access path, not on the value: `ctx.foo` throws if the service is gone, but a +reference you stored keeps working and calls into the dead plugin's closure. Our +`ctx.use(Token)` returns a value at construct time by design, so we have the same +hazard — and teardown-of-dependents is what makes it safe, because the holder dies +with the provider. A proxy does not fix cached references; it makes them silently +wrong instead of impossible. + +> **Deferred, not adopted.** If teardown ever proves too coarse, the addition is +> HMR's `accept()` rather than OSGi's dynamic policy: a per-dependency opt-in, +> `uses: [Harness, accepts(Models)]`, meaning *my acquisition can be re-pointed, I +> derive no state from this provider, and I hold no in-flight registrations against +> it*. The kernel would permit the swap only if the schema check passes and +> silently fall back to teardown otherwise. **Teardown must remain the path that +> always works**; the moment `accepts` is load-bearing for correctness, every +> consumer is writing defensive code again. +> +> Worth measuring the cost of teardown before building it. If disposing a +> presentation facet is a repaint, the mechanism buys nothing. + +### 13.2 Disposal with in-flight work + +Disposal removes a registration. That is sufficient for a service, and **not** for +a registry whose contributions can be executing when their contributor goes away. + +The forcing constraint is settlement, not cleanup: the harness owes the model a +`ToolResultMessage` for every call it started. "The facet went away" is not a +result, and an operation waiting on one hangs. + +Neither reference system solves this. Cordis's `_unload` is +`await Promise.all(disposers)` with a try/catch and no deadline — a disposer that +hangs hangs the reload. DSH goes further and places the obligation on the tool +author: async work must *"observe or forward `exec.signal` and settle only after"* +reaching *"quiescence"*, with the registry rechecking cancellation afterwards. That +is the settlement concept Cordis lacks, but it is stated in prose and enforced by +nothing, so a tool that ignores its signal still wedges the unload. + +**Four phases, and the last two are what neither system has:** + +1. **Deregister.** The contribution leaves the registry immediately. Synchronous, + cheap, and it stops the problem growing. +2. **Signal.** Abort each in-flight invocation. +3. **Race a deadline.** Kernel-imposed, on every disposer. +4. **Settle by outcome.** At expiry the registry resolves *its own* promise with an + aborted outcome and abandons the contributor's, attaching a catch and dropping + the reference. The harness sees an ordinary aborted invocation and its existing + `abortedMessage` path produces the result. No new settlement machinery. + +Phase 4 is what makes the deadline safe rather than a leak with extra steps: **the +invocation settles even when the contribution does not.** + +**Layering.** The kernel knows facets and contributions, not invocations, so it +bounds disposal generically: every disposer gets a deadline, and one that overruns +is logged and dropped. A registry with in-flight work is what makes that deadline +meaningful, because only it knows a call is outstanding. + +The signal is **registry-owned and chained** from the harness signal, never the +harness signal itself — so deregistering one tool aborts that tool's invocations +without cancelling the operation. DSH's warning that *"replacement cannot detach +caller cancellation"* is about exactly this; the chain must be one-way. + +**Invocation tracking belongs to the registry, not the contributor.** A facet could +track its own outstanding calls and signal them, but then a buggy or hostile facet +has no bound at all. Every invocation already routes through the registry, so it +can hold the signal and the count without trusting anyone. + +**What the deadline actually guarantees, stated honestly:** disposal completes in +bounded time and the kernel holds no reference afterwards. **Not** that the bundle +is collected. If the abandoned contribution has a live interval, an open socket, or +a pending child process, its own async work still references the closure and the +compartment stays alive. We can bound our retention; we cannot bound the runtime's. +The only real fix is process isolation, where teardown is a kill — which is why +Erlang supervisors terminate rather than negotiate, and why OSGi `refresh` can hang +on a bundle that will not stop. + +## 14. Isolation and trust + +Condensed; a full treatment lives in the isolation spec. Recorded here because it +constrains the API surface above. + +### 14.1 Threat model + +**A malicious server ships a facet to an unsuspecting client.** + +§7 states that a presentation ships with no plugin facets and that all of them +arrive over the wire as built bundles. So third-party code executes in the user's +process by design, and connecting to a server is not consent to run its code. The +victim is the *user*, with their filesystem, their credentials, their SSH keys — +not the operator. + +An earlier draft scoped this to "a careless plugin author, not a hostile one", +with a determined attacker explicitly out of scope. **That was wrong** and it +selected the wrong mechanism. The attacker is hostile, the delivery is remote, and +the target is a workstation. + +**Availability is not in scope.** A malicious server can already refuse to serve, +hang, or send garbage; crashing the client is a subset of that. What must be +prevented is **disk access and exfiltration**. + +That asymmetry decides several things below, and it is the reason this section is +shorter than a general sandboxing treatment would be. + +### 14.2 Mechanism: a V8 isolate behind a string-only membrane + +Facet code runs in an `isolated-vm` isolate. Measured behaviour, not inference — +see the sandbox PoC: + +``` +require / process / fetch undefined +globals visible 64 +Function("return process")() undefined +spinning guest interrupted at its timeout, isolate reusable after +``` + +**What was rejected, and why.** + +*SES / `lockdown()`* — an earlier draft chose this. It hardens intrinsics but +leaves guest and host **in the same VM**, which is precisely the class Figma's +Realms shim failed on: "confusing an object from outside the sandbox with an +object from inside... possible because the shim uses the same JavaScript VM for +all code both inside and outside". Figma shipped Realms, was breached within two +months, and moved to a different VM. Choosing SES repeats their first attempt. + +*Node `worker_threads`* — `terminate()` is a genuine kill (3 ms on a tight loop), +but a worker has full `fs`, `env` and `child_process`. Node's `--permission` model +does apply inside workers, but Node documents it as a **"seat belt"** that +"malicious code can bypass", and it does not inherit per-worker. Availability +without authority is the wrong half for this threat model. + +*Deno workers* — `permissions: "none"` gives real per-worker authority reduction, +verified. But `terminate()` does **not** stop a spinning worker: 2990 ms of CPU +over 3 s wall, measured after terminate. And it is a runtime switch. + +*QuickJS-WASM* — a genuinely different VM, so object confusion is impossible, and +it is what Figma shipped. Rejected on two measurements: **8–17× slower** (300 +markdown components repaint in 1118 ms versus 108 ms), and **no `Intl`**, which +`packages/tui` needs for grapheme segmentation in every width calculation. It +remains the fallback if a native addon is unacceptable. + +*ShadowRealm* — same thread, same VM, and its own explainer disclaims being "a +full spectrum mechanism against security issues". Not a successor for this +purpose. + +**The membrane is what makes `isolated-vm` safe, and it is structural.** + +`isolated-vm` can be used unsafely, and an earlier draft rejected it for exactly +that: handing the guest a live handle via `derefInto()`, or returning a +`Reference` from a host call, lets the guest walk that object's prototype chain +into the host realm. (The draft also called the project "maintenance mode" — that +is out of date; releases run 6.0.1 in Jul 2025 through 7.0.1 in Aug 2026.) + +The membrane forecloses it by construction rather than by care: + +1. `encode()` is the only path from host to guest, and it is `JSON.stringify` + with a replacer. **Its output is a string.** A string cannot carry a reference. +2. Host callables never cross. They become an integer id into a host-side table. + The guest receives a **number**. +3. Exactly **one** `Reference` reaches the guest — the single call-in point — and + its `deref()` throws cross-isolate. +4. `derefInto()` is used once, on the guest's **own** global. No host object is + ever its argument. + +The guest's entire view of the host is `{ number, string }`. There is no object +graph, so there is nothing to walk. Audited: `deref`, `copySync` and `getSync` all +blocked; `derefInto()` yields an inert marker that is not callable and exposes no +host globals. + +**One isolate, N contexts** — not one isolate per facet. Contexts give separate +globals, and a timeout on one context leaves the others running. They share a +`memoryLimit`, so one facet's allocation bomb disposes the isolate and every +context in it — which does not matter here, because availability is out of scope +(§14.1). The cost difference is real: **164 KB per context versus 1124 KB per +isolate**. + +A facet that must survive its neighbours — host chrome running as a facet — takes +its own isolate. That is a per-facet decision, not a global one. + +**Limits worth knowing.** + +- **Budgets do not nest.** A timeout bounds one evaluation. A guest function + re-entered *by the host* — a contributed callback, a component method — gets its + own budget, not the outer one. Bounding total facet time needs separate + accounting. +- **References release, but lazily.** The membrane interns by id with `WeakRef` + plus a `FinalizationRegistry`, which both gives identity across crossings and + releases on collection. Finalization is not prompt, so **do not create + references in a hot path**: a factory registered at construct is one reference + forever; a component method returning a fresh closure each frame is one per + frame. +- **Native addon.** Prebuilds cover linux-x64/arm64, darwin-arm64 and win32-x64 + for Node 22 and 24 only; anything else falls back to a source build needing + Python and a toolchain. Shipping SEA builds moves that from every user's install + to CI. +- **Still one engine.** A separate isolate is a far stronger boundary than SES, + but it is not Figma's "different VM" claim. The guarantee here is V8's isolate + boundary plus the membrane's discipline. + +### 14.3 Ambient authority defeats registration-is-ownership + +§13 states that every handle a facet receives is a host-built binding that +registers its own disposer, so there is nothing to remember. That holds for +handles the host *hands over*. It says nothing about authority a facet can reach +**without** being handed anything: `setInterval`, `process.on`, +`document.addEventListener`, a WebSocket. + +Registration-is-ownership is a property of the API surface, and it is only as +complete as the surface is exclusive. Where a facet has another way to reach the +outside world, disposal is back to author discipline — exactly the position §13.1 +criticises Cordis for (*"do not assume unload automatically removes arbitrary +third-party callbacks"*). + +How complete we can make it is **not uniform**, and pretending otherwise would be +the wrong kind of tidy. + +**Session and server facets: closable, and this argues for compartments.** They run +in Node with full ambient authority today — a facet can `import` timers, `fs`, +`net`. Inside a compartment with no module access, the only globals are the ones we +endow, so an endowed `setTimeout` that registers its own disposer is the *only* +`setTimeout`. That is a real argument for resolving the open question above toward +compartmenting session facets, on ergonomics rather than on trust. + +**TUI: closable for the same reason.** A presentation facet already gets a +compartment (§14.2) and reaches the terminal only through the `TuiHost` facade. +There is no ambient terminal to grab. Endow timers the same way and the surface is +exclusive. + +**Web: not closable, and it is worth being explicit about why.** The DOM is an +ambient mutable graph reachable from *any node in it*. Hand a component one +element and it has `ownerDocument`, `parentNode`, `window` — and from there +`addEventListener`, `MutationObserver`, a detached subtree that outlives its +mount. `lockdown()` does not help: the escape is not a prototype, it is a live +object graph we deliberately gave away. + +Two mechanisms would actually close it, and both cost more than the problem is +currently worth: + +- **An iframe per facet.** Real isolation, `postMessage` boundary, teardown is + removing the frame. VS Code does this for webviews. The cost is no shared DOM: + styling, layout, and focus all become protocol. + + Figma goes further and is worth studying, because it is the strongest form of + "make the surface exclusive". They split by *capability*: plugin logic runs in + QuickJS-on-WASM with the scene graph and **no browser APIs at all**, while UI + runs in an iframe with browser APIs and **no scene access**, connected by + `postMessage`. Plugin code never touches the DOM — not "sandboxed DOM access", + no `document` in the environment. That is what lets them guarantee cleanup. + + Two things to take from their history. They shipped **Realms** first and it was + found insecure within two months, which is the same escape class §14.2 cites for + rejecting `isolated-vm` — production evidence rather than inference. And the + QuickJS cost is real: practitioners report *"truly impenetrable errors"* and a + badly degraded debugging story, which §14.2 should weigh when it names QuickJS as + the upgrade path. As one put it, sandboxing is a thing *"everyone wants, but + there are so few examples of it actually ever working."* +- **Declarative-only components.** The facet never receives a node — it returns a + description and the host reconciles. §11.1 already points this way (*"props are + DTOs; a factory never receives the `TUI` or `Theme` instance"*), and extending it + to *never receives a DOM node* would hold. But it rules out refs, and therefore + most component frameworks worth using. + +**So the rule is: make the safe path the easy path, and do not pretend the escape +is closed.** + +Provide `ctx.dom.on(el, event, fn)` and `ctx.timer.every(ms, fn)` that register +their own disposers, and make them the obvious way to do the thing. §14.1's threat +model is a *careless* author, not a hostile one, and a careless author uses the +ergonomic path. The escape remains reachable; it just is not the one you fall into. + +Where the surface can be made exclusive — session, server, TUI — enforce it with +endowments rather than relying on the ergonomics. Where it cannot — web — document +that facet authors own their own DOM cleanup, and treat the iframe as the upgrade +path if the threat model ever moves from careless to hostile. + +This is the same shape as §14.2's isolate decision: take the mechanism that fits +the current threat model, name the stronger one, and record what would trigger +moving to it. + +### 14.4 Consequences for the API + +Isolation does not change the shapes in §3–§4, but it fixes three things that would +otherwise be conventions: + +- **Ownership is implicit.** Every handle a facet receives is a host-built binding + that registers its own disposer. Hence no `own()`, and `subscribe()`/`on()` + return nothing — there is no unsubscribe handle to hold, and therefore none to + leak. Local `{ rpc: false }` services are the exception: they hand back the raw + implementation with no interception and no auto-disposal. +- **Every object a binding returns is itself a binding** — host-built, plain + prototype, hardened, registering its own disposer. This is the invariant that + makes the previous point hold transitively. +- **Borrowed immutable JSON is enforced, not advised.** Values are hardened as they + cross the boundary and rejected if unfrozen, so §9's no-clone rule is safe rather + than a documented hope. + + This does **not** extend to a replica's internal buffer. `apply` from `delta.md` + mutates a plain object in place and would throw against a frozen one, so the host + keeps the replica mutable and hardens only the value it hands to a facet through + `State.value`. Freezing happens at the binding, not at the buffer. The same rule + governs the producer side: the tracker proxies a mutable object and hardens what + it exposes. + +Facets are built with `await` compiled to generators, so the host owns resumption. +Disposal then stops three distinct things, none of which subsumes the others: +cancel the scope's root context (stops the **operation**), unwind the driver +(stops the **continuation**, running every `finally` and no `catch`), then release +registrations in reverse (stops the **effects**). + +Endowments are minimal: `Date` and `Math` are replaced with clock-backed versions, +timers return opaque integers and are scope-owned, `process` and `console` are +shape-only stubs, and `fetch`, `Buffer`, `require`, and `process.exit` are never +endowed. **Network access is a binding, never an endowment.** + +## 15. Protocol: exposing services to foreign clients + +Everything above assumes both ends are TypeScript compiling against the same tokens. +A browser, a script, or another language cannot do that — it needs a description it +can fetch. + +**Schemas are opt-in and per service.** Most plugins never expose anything, and pay +nothing. A service with no `protocol` block is in-process and native-RPC only: it is +absent from the catalogue and the HTTP router returns `no_such_member`. + +### 15.1 Declaring a protocol + +```ts +import { object, array, string, int, oneOf, literal } from "@pi/schema"; + +export const TranscriptEntry = object({ + id: string(), + role: oneOf([literal("user"), literal("assistant")]), + text: string(), +}); +export type TranscriptEntry = Static; + +export const TranscriptState = defineState("pi.transcript.tail", { + schema: object({ entries: array(TranscriptEntry) }), +}); + +export interface Transcript { + readonly tail: State; + page(params: { before: string; limit: number }, context: Context): Promise; + subscribeRaw(sink: (e: TranscriptEntry) => void): Unsubscribe; +} + +export const Transcript = defineService("pi.transcript", { + protocol: { + methods: { page: { params: object({ before: string(), limit: int() }), + result: array(TranscriptEntry) } }, + state: { tail: TranscriptState }, + }, +}); +``` + +`page` and `tail` are published. `subscribeRaw` is absent from `protocol`, so it is +local — nothing marks it, and publishing it later means adding one entry. + +Only the state **value** needs a schema, for the root `r` batch. Mutations do not, +because recipes never leave the provider (§9.2) and ops are structural. + +Method parameters are a single object, not positional. Names then survive into the +TypeScript signature, into the JSON Schema as `properties`, and into the request body, +and adding an optional field does not change arity. + +`@pi/schema` re-exports TypeBox constructors as free functions so declarations stay +readable. They are functions rather than constants because TypeBox schemas are mutable +objects and a shared constant would alias into every schema referencing it. + +### 15.2 The protocol must match the type + +A schema that drifts from the interface is the failure this design exists to prevent, +so it is checked, with the same mutual-assignability technique as `CheckComplete` (§4): + +- every key in `protocol.methods` must exist on the service interface — a typo is a + compile error, not a silently unpublished member; +- for each, `(params: Static

, context: Context) => Promise>` must be + assignable to the interface member and vice versa; +- every key in `protocol.state` must be a `State` member whose `T` matches the + definition's value schema. + +Omission is always legal. Omission is the feature. + +### 15.3 Routes + +``` +GET /v1/catalogue +POST /v1/call/{service}/{member} +POST /v1/call/{service}/{key}/{member} keyed instance +GET /v1/state/{service}/{member} SSE +GET /v1/state/{service}/{key}/{member} SSE +``` + +`peer` services take no key: the server resolves the instance from the authenticated +session, so a client cannot address another peer's (§10.2). + +**Catalogue** + +```json +{ "version": "1", + "services": { + "pi.transcript": { + "mode": "singleton", + "methods": { "page": { "params": { "$ref": "#/definitions/PageParams" }, + "result": { "type": "array", + "items": { "$ref": "#/definitions/TranscriptEntry" } } } }, + "state": { "tail": { "value": { "$ref": "#/definitions/TranscriptTail" } } } + } + }, + "definitions": { "TranscriptEntry": { "type": "object", "properties": { "...": {} } } } } +``` + +**Call** + +``` +POST /v1/call/pi.transcript/page +{ "before": "entry_88", "limit": 50 } + +200 { "result": [ { "id": "entry_87", "role": "assistant", "text": "..." } ] } +403 { "code": "forbidden", "message": "unknown root" } +404 { "code": "no_such_member" } ← unpublished members land here +422 { "code": "invalid_params", "message": "limit: expected integer" } +``` + +**State over SSE** + +``` +GET /v1/state/pi.lane/main/snapshot +Accept: text/event-stream +id: 0 +event: ops +data: [["r",{"lane":"main","transcript":[],"operation":null}]] + +id: 1 +event: ops +data: [["p",["transcript"],12,0,[{"id":"e12"}]]] + +id: 2 +event: ops +data: [["a",["operation","streamingMessage","content",0,"text"],"Sure, I"]] +``` + +`id` is stamped by the binding, not carried in the payload. The first batch is +always a base batch, so hydration is not +a separate route. `Last-Event-ID` is **not** honoured: `seq` restarts per +subscription, and resubscription is a base batch plus buffered batches (§9.2); a gap, an unknown id, and a provider reload all take that same path. A +closed keyed instance ends the stream with `event: closed` and the client does not +retry. + +A foreign client needs only the six ops of [delta.md §2](../../01-harness/01-delta/delta.md#2-ops) — `replace`, `set`, `delete`, `append`, `truncate`, `splice`. No mutation names, no recipes, no provider code. The applier is +a page of code in any language. + +**This is where non-conformance becomes visible**, and it is the one place worth +being deliberate about it. The format is JSON-Patch-*shaped*, not RFC 6902: paths are +arrays rather than string pointers, and `append`, `truncate` and `splice` have no +RFC equivalent. A client reaching for an off-the-shelf `jsonpatch` library will not +work. + +Two mitigations, neither of which is "conform": + +- The catalogue advertises the op vocabulary explicitly, so a client discovers the + six verbs rather than assuming a different vocabulary. +- If a client genuinely needs RFC 6902, the server can offer a lossy downgrade behind + a content negotiation header — join paths with slashes and `~0`/`~1` escaping, + materialise `append` and `truncate` into whole-value `replace`, expand `splice` + into `add`/`remove` runs. About fifteen lines. It reintroduces the quadratic text + cost, which is the point: conformance is available and it is the client paying for + it. + +Conforming by default would impose that cost on every consumer to serve a +hypothetical one, and would not even buy real interoperability, since Immer's output +was never conformant either (§9.2). + +### 15.4 What publishing commits you to + +A member in the catalogue is a compatibility promise in a way an internal token is not. +Members should carry a stability marker from the start so that publishing does not +silently mean freezing, and §16 lists version negotiation as unresolved. + +## 16. Deltas against `plugins.md` + +| topic | `plugins.md` | here | +| --- | --- | --- | +| dependency declaration | derived from `setup()` side effects | static `uses`/`provides`/`observes` | +| handles during setup | disconnected lazy proxies | real objects, after validation | +| validation | requires running facet code | pure manifest analysis | +| mode | declared per call site, validated | property of the token | +| cycles | tolerated via laziness | rejected; explicit `deferred()` escape | +| replication | `ReplicatedState` full-value; `DeltaState` deferred | one primitive; explicit root `r` + six-verb ops | +| hydration | separate atomic snapshot + buffering | base batch zero of the stream | +| presentation facets | loaded locally | delivered by server and worker | +| server↔worker | unspecified inversion | reporting registries; deps point upstream | +| authority | `requireClientIdentity` in method bodies | principal on context; filtered views + handle checks | +| per-client state | not addressed | `peer` mode: one instance per peer, consumed as a singleton | +| resource ownership | explicit `own()` | implicit; every handle is a self-disposing binding | +| state updates | hand-written patch unions | plain mutation on the provider; six-verb ops on the wire | +| UI mounting | facet-owned panels, unspecified host API | slots with `claim`/`add`; host unmounts on disposal | +| isolation | trusted code, unspecified | SES compartment per presentation facet | +| foreign clients | not addressed | opt-in `protocol` block; JSON Schema catalogue + HTTP/SSE | +| lifecycle | `setup` only | sync `construct`, async activate / deactivate | + +## 17. Open decisions + +- Whether cwd-sourced *session* facets need compartments too (§14.2). +- `CheckComplete` error message ergonomics. +- Where roles come from, and whether they are per-server or per-session. +- Whether handle tables are facet-owned or a kernel-provided utility. +- How an owner grants and revokes guest access at runtime. +- Protocol version negotiation, and how a state value definition's reducer participates + in source-generation skew (§9.2). +- Flow control for high-frequency state values; gap recovery is settled (§9.2). +- Discovering lanes created after startup (§9.2). +- Whether `reduceLaneSnapshot` becomes a draft mutator now or later; until it does, + lane state is replace-only. +- Whether `deferred()` survives contact with real plugins, or should be removed. +- Property tests for the tracker (`delta.md` §3.3) before any facet depends on it. +- Which durable values need an explicit periodic `rebase()` cadence to bound recovery work. diff --git a/packages/agent/docs/mobile-handoff/02-plugins/02-sandbox/README.md b/packages/agent/docs/mobile-handoff/02-plugins/02-sandbox/README.md new file mode 100644 index 000000000..fffda9b62 --- /dev/null +++ b/packages/agent/docs/mobile-handoff/02-plugins/02-sandbox/README.md @@ -0,0 +1,162 @@ +# Facet sandbox — isolated-vm + +**Unmodified pi facet code running in a V8 isolate with no ambient authority.** +Full JIT, so ~1.8× native rather than the 8–17× a WASM interpreter costs. + +```bash +npm install +npm run demo # a facet: contributions, components, commands, callbacks +npm run audit # escape audit — the interesting one +npm run bench # crossing cost, isolate cost, 300 components +npm test # 412 property assertions +``` + +Node 22+ (`--experimental-strip-types`). See `../facet-sandbox-poc` for the +QuickJS-WASM equivalent. + +## The safety claim, and why it is structural + +isolated-vm **can** be used unsafely, and that is what `facets.md` §14.2 objected +to. The footgun is handing the guest a live handle to a host object — via +`derefInto()`, or by returning a `Reference` from a host call — after which the +guest walks that object's prototype chain into the host realm. + +This membrane makes that impossible **by construction, not by care**: + +1. `encode()` is the only way a host value reaches the guest, and it is + `JSON.stringify` with a replacer. **Its output is a string.** A string cannot + carry a reference. +2. Host callables never cross. They become an integer id into `hostTable`, which + lives entirely host-side. The guest receives a **number**. +3. Exactly **one** `Reference` is ever given to the guest — `__invokeRef`, the + single call-in point. +4. `derefInto()` is used exactly once, on the guest's **own** global. No host + object is ever its argument. + +The guest's entire view of the host is `{ number, string }`. There is no object +graph, so there is nothing to walk. + +### Audit results (`npm run audit`) + +``` +Ambient authority: + require / process / fetch ["undefined","undefined","undefined"] + global names visible 64 globals + +Classic escape ladders: + Function('return process')() undefined + constructor walk on host data undefined + +The isolated-vm Reference footgun: + __invokeRef.deref() blocked: TypeError ("Cannot dereference + this from current isolate") + __invokeRef.copySync() blocked: TypeError + __invokeRef.getSync('constructor') blocked: TypeError + derefInto() is callable? inert (object) + invoke derefInto() result blocked: TypeError + host globals via derefInto() no host globals + +Resource bounds: + spinning guest interrupted after 205ms + isolate usable afterwards true +``` + +One probe gave a **false positive** in a first draft: testing `f.deref === undefined` +to prove a returned host function is a Proxy rather than a Reference. The +membrane's proxy returns a proxy for *every* property, so `f.deref` is truthy. +That is not a leak — calling it routes to `hostFn["deref"]`, which does not exist +and throws host-side. The audit now calls it instead of checking for absence. + +## Performance + +| | isolated-vm | QuickJS | native | +| --- | --- | --- | --- | +| 300 markdown components, small | **70 ms** | 494 ms | ~40 ms | +| 300 markdown components, typical | **108 ms** | 1118 ms | 64 ms | +| 300 markdown components, large | **419 ms** | 4260 ms | ~150 ms | +| 146 KB bundle load | **35 ms** | 104 ms | — | +| membrane crossing | **4.5 µs** | 7–17 µs | — | +| per compartment | 1080 KB, 5.5 ms | **77 KB, 0.8 ms** | — | + +**~1.8× native at typical size.** QuickJS is 8–17× and gets worse with input size. + +Isolates cost **14× more memory than a QuickJS runtime** (1080 KB vs 77 KB) and +are slower to create. For a handful of facets that is irrelevant; for hundreds it +would not be. + +`Intl.Segmenter` **exists** here, which is the single hard blocker for reusing +`packages/tui` components under QuickJS. That alone may decide it. + +## Limitations + +**Budgets do not nest.** A timeout bounds one `evalSync`. A guest function +re-entered *by the host* — a contributed callback, a component method — gets +`callGuestRef`'s own budget, not the outer one. Asserted explicitly in +`property-test.ts`; a runaway callback took 5005 ms under an outer budget of 50 ms. +Bounding total facet time needs separate accounting. + +**Async is a settle-callback, not a native promise.** `applySync` is synchronous, +so a host Promise cannot cross. The host returns a token, the guest builds a real +Promise against it, and the host settles it by calling back in. Verified: two +sequential guest `await`s while the host event loop ticked 22 times. + +**Native addon, and the ABI matrix is real.** `isolated-vm` ships prebuilds and +only falls back to `node-gyp rebuild` when none matches — which is why install +takes a second, not an hour. It is **not** building V8; V8 is already in the Node +binary. But coverage is narrow: + +| version | engines | prebuilds | +| --- | --- | --- | +| **6.2.0** | `>=22.0.0` | linux-x64/arm64, darwin-arm64, win32-x64 — abi127, abi137 | +| 7.0.1 | `>=24.0.0` | — | +| 7.0.0 | `>=26.0.0` | — | + +darwin-**x64** is absent everywhere. Installing `isolated-vm@7` on Node 22 falls +through to a source build and **fails without Python and a C++ toolchain** — +verified here. Note also that 6.2.0 is `latest` on npm despite 7.0.1 existing, +because it published one day later. + +The project is **not abandonware** — 6.0.1 (Jul 2025) through 7.0.1 (Aug 2026), +actively released. §14.2's "maintenance mode" claim is out of date and should be +corrected. The real risk is different: adopting it couples your minimum Node +version to theirs, and they dropped Node 20 in 6.x and Node 22 in 7.x within a +year. Shipping SEA builds moves that from every user's install to your CI. + +**Memory: references release, but lazily.** The `WeakRef` + `FinalizationRegistry` +interning table ports unchanged from the QuickJS membrane and works — 20 000 +crossings went to **0 live refs**. But finalization is not prompt: 5 000 refs were +still live 200 ms after a forced GC. Under sustained churn refs accumulate faster +than the collector reclaims them, and a 32 MB isolate did hit its limit. + +The consequence is an API rule, not a membrane fix: **do not create references in +a hot path.** `slots.claim(factory)` at construct is one reference forever; a +component method returning a fresh closure every frame is one per frame. + +**Hitting `memoryLimit` kills the facet cleanly.** It throws a catchable +`"Isolate was disposed during execution due to memory limit"` and the host +survives — but the isolate is dead and unrecoverable, so teardown must tolerate +an already-disposed isolate. `dispose()` on it throws `"Isolate is already +disposed"`; the membrane guards this. + +> **Do not call `isolate.getHeapStatisticsSync()` on a stressed isolate.** It +> **aborted the whole process** during testing — a hard crash, not an exception. +> A `heapMB()` helper was removed from the membrane rather than shipped. + +**Still one process, one engine.** A separate isolate is a far stronger boundary +than SES, but Figma's argument for QuickJS was that a *different VM* cannot +confuse objects because the representations differ. Here the guarantee is V8's +isolate boundary plus the membrane's discipline — strong, and structurally +enforced above, but not the same category of claim. + +## Files + +- `src/membrane.ts` — the membrane. The safety argument is in the header comment. +- `src/facet-example.js` — unmodified facet code: `ctx.use()`, `slots.claim()` + with a closure, a class instance returned to the host, a subscribe callback. +- `src/demo-facet.ts` — loads it and exercises four boundary crossings. +- `src/escape-audit.ts` — the audit above. +- `src/property-test.ts` — 412 assertions: random `JsonValue` round-trips both + directions, identity across crossings, callables, error propagation, reference + release under GC, prototype pollution both ways, budgets, async, disposal. +- `src/bench.ts` — the numbers above. +- `src/markdown-bundle.js` — the real pi `Markdown` component, esbuild-bundled. diff --git a/packages/agent/docs/mobile-handoff/02-plugins/02-sandbox/package.json b/packages/agent/docs/mobile-handoff/02-plugins/02-sandbox/package.json new file mode 100644 index 000000000..fb597ecc6 --- /dev/null +++ b/packages/agent/docs/mobile-handoff/02-plugins/02-sandbox/package.json @@ -0,0 +1,13 @@ +{ + "name": "facet-sandbox-ivm", + "private": true, + "type": "module", + "description": "Running pi facets in an isolated-vm isolate: full JIT, no ambient authority.", + "scripts": { + "demo": "node --experimental-strip-types src/demo-facet.ts", + "audit": "node --experimental-strip-types src/escape-audit.ts", + "bench": "node --experimental-strip-types src/bench.ts", + "test": "node --experimental-strip-types --expose-gc src/property-test.ts" + }, + "dependencies": { "isolated-vm": "6.2.0" } +} diff --git a/packages/agent/docs/mobile-handoff/02-plugins/02-sandbox/src/bench.ts b/packages/agent/docs/mobile-handoff/02-plugins/02-sandbox/src/bench.ts new file mode 100644 index 000000000..f917c767e --- /dev/null +++ b/packages/agent/docs/mobile-handoff/02-plugins/02-sandbox/src/bench.ts @@ -0,0 +1,48 @@ +/** + * Cost of the isolate, the membrane, and a real component at transcript scale. + * Compare against `../facet-sandbox-poc` (QuickJS) and native. + */ +import ivm from "isolated-vm"; +import { readFileSync } from "node:fs"; +import { createCompartment } from "./membrane.ts"; + +// ── membrane crossing ───────────────────────────────────────────────────── +{ + const c = await createCompartment(); + await c.endow("__h", { echo: (v: unknown) => v }); + c.load(`globalThis.spin = function (n) { for (var i = 0; i < n; i++) __h.echo("x"); };`); + c.load(`spin(2000);`); + const N = 20000, t = Date.now(); + c.load(`spin(${N});`, 60000); + console.log(`membrane crossing: ${((Date.now() - t) * 1000 / N).toFixed(1)} µs`); + c.dispose(); +} + +// ── isolate creation ────────────────────────────────────────────────────── +{ + const before = process.memoryUsage().rss, t = Date.now(); + const held = Array.from({ length: 50 }, () => { + const iso = new ivm.Isolate({ memoryLimit: 32 }); + const ctx = iso.createContextSync(); + ctx.evalSync(`var c = { render: function () { return ["x"]; } };`); + return iso; + }); + console.log(`50 isolates: ${(((process.memoryUsage().rss - before) / 50) / 1024).toFixed(0)} KB each, ${Date.now() - t} ms total`); + for (const i of held) i.dispose(); +} + +// ── real Markdown at transcript scale ───────────────────────────────────── +{ + const iso = new ivm.Isolate({ memoryLimit: 512 }); + const ctx = iso.createContextSync(); + ctx.global.setSync("global", ctx.global.derefInto()); + const t = Date.now(); + ctx.evalSync(readFileSync(new URL("./markdown-bundle.js", import.meta.url), "utf8"), { timeout: 60000 }); + console.log(`146 KB bundle load: ${Date.now() - t} ms`); + for (const [label, pts] of [["small", 2], ["typical", 8], ["large", 35]] as [string, number][]) { + ctx.evalSync(`__bench(${pts}, 20, 50)`, { timeout: 120000 }); // warm the JIT + const ms = ctx.evalSync(`__bench(${pts}, 300, 3)`, { timeout: 300000 }); + console.log(`300 markdown components, ${label.padEnd(8)} ${ms.toFixed(0)} ms per full repaint`); + } + iso.dispose(); +} diff --git a/packages/agent/docs/mobile-handoff/02-plugins/02-sandbox/src/demo-facet.ts b/packages/agent/docs/mobile-handoff/02-plugins/02-sandbox/src/demo-facet.ts new file mode 100644 index 000000000..eda12e7fa --- /dev/null +++ b/packages/agent/docs/mobile-handoff/02-plugins/02-sandbox/src/demo-facet.ts @@ -0,0 +1,22 @@ +import { readFileSync } from "node:fs"; +import { createCompartment } from "./membrane.ts"; +const mounted = new Map(), commands = new Map(), notifications = [], subs = []; +const Tui = { + slots: { claim: (s, f) => mounted.set(s, f) }, + commands: { add: (n, h) => commands.set(n, h) }, + notify: (m) => notifications.push(m), +}; +const Transcript = { tail: { subscribe: (f) => subs.push(f) } }; +const c = await createCompartment(); +await c.endow("__host", { use: (t) => (t === "Tui" ? Tui : Transcript) }); +c.load(readFileSync(new URL("./facet-example.js", import.meta.url), "utf8")); +c.load(`globalThis.__r = __facet.construct({ use: function (t) { return __host.use(t); } });`); +console.log("contributions:", { slots: [...mounted.keys()], commands: [...commands.keys()], subs: subs.length }); +const comp = mounted.get("footer")({ label: "presses" }); +console.log(comp.render(30).join("\n")); +comp.handleInput("abc"); +console.log(comp.render(30).join("\n")); +console.log("command ->", commands.get("footer.reset")()); +subs[0]({ entries: [1, 2, 3] }); +console.log("notifications:", notifications); +c.dispose(); diff --git a/packages/agent/docs/mobile-handoff/02-plugins/02-sandbox/src/escape-audit.ts b/packages/agent/docs/mobile-handoff/02-plugins/02-sandbox/src/escape-audit.ts new file mode 100644 index 000000000..5f3692435 --- /dev/null +++ b/packages/agent/docs/mobile-handoff/02-plugins/02-sandbox/src/escape-audit.ts @@ -0,0 +1,63 @@ +/** + * Escape audit. + * + * The claim the membrane makes is that the guest's entire view of the host is + * `{ number, string }` — no object graph, so nothing to walk. This tries to + * break that, including the specific `Reference` footgun that makes isolated-vm + * dangerous when used naively. + */ +import { createCompartment } from "./membrane.ts"; + +const c = await createCompartment(); +const seen: string[] = []; +await c.endow("__h", { + sink: (v: string) => seen.push(v), + give: () => ({ nested: { a: 1 } }), + giveFn: () => () => "host function result", +}); + +const probe = (name: string, src: string) => { + c.load(`globalThis.__p = String(${src});`); + c.load(`__h.sink(__p);`); + const got = seen.pop(); + const bad = got !== undefined && /GOT (?!undefined)|INVOKED|CALLABLE|^object$/.test(got) && !/blocked/.test(got); + console.log(` ${bad ? "!!" : "ok"} ${name.padEnd(34)} ${got}`); + return !bad; +}; + +console.log("Ambient authority:"); +let ok = true; +ok = probe("require / process / fetch", `JSON.stringify([typeof require, typeof process, typeof fetch])`) && ok; +ok = probe("global names visible", `Object.getOwnPropertyNames(globalThis).length + " globals"`) && ok; + +console.log("\nClassic escape ladders:"); +ok = probe("Function('return process')()", `(function(){ try { return String((function(){}).constructor("return typeof process")()); } catch(e){ return "blocked: "+e.name; } })()`) && ok; +ok = probe("constructor walk on host data", `(function(){ try { var o = __h.give(); return String(o.constructor.constructor("return typeof process")()); } catch(e){ return "blocked: "+e.name; } })()`) && ok; + +console.log("\nThe isolated-vm Reference footgun:"); +ok = probe("__invokeRef.deref()", `(function(){ try { return "GOT " + typeof __invokeRef.deref(); } catch(e){ return "blocked: "+e.name; } })()`) && ok; +ok = probe("__invokeRef.copySync()", `(function(){ try { return "GOT " + typeof __invokeRef.copySync(); } catch(e){ return "blocked: "+e.name; } })()`) && ok; +ok = probe("__invokeRef.getSync('constructor')", `(function(){ try { return "GOT " + typeof __invokeRef.getSync("constructor"); } catch(e){ return "blocked: "+e.name; } })()`) && ok; +ok = probe("derefInto() is callable?", `(function(){ var d = __invokeRef.derefInto(); return typeof d === "function" ? "CALLABLE" : "inert (" + typeof d + ")"; })()`) && ok; +ok = probe("invoke derefInto() result", `(function(){ try { __invokeRef.derefInto()(); return "INVOKED"; } catch(e){ return "blocked: "+e.name; } })()`) && ok; +ok = probe("host globals via derefInto()", `(function(){ var d = __invokeRef.derefInto(); return String(d.process || d.global || "no host globals"); })()`) && ok; + +console.log("\nWhat a host FUNCTION looks like to the guest:"); +ok = probe("typeof a returned host fn", `(function(){ var f = __h.giveFn(); return typeof f; })()`) && ok; +// NOTE: `f.deref` is NOT undefined — the membrane's proxy returns a proxy for +// every property. That is not a leak: calling it routes to hostFn["deref"], +// which does not exist, and throws host-side. Testing for the property's +// absence gave a false positive in a first draft of this audit. +ok = probe("calling f.deref() reaches nothing", `(function(){ try { var f = __h.giveFn(); f.deref(); return "INVOKED"; } catch(e){ return "blocked: "+e.name; } })()`) && ok; +ok = probe("calling the host fn works", `(function(){ var f = __h.giveFn(); return String(f()); })()`) && ok; + +console.log("\nResource bounds:"); +const t0 = Date.now(); +let interrupted = false; +try { c.load(`while(true){}`, 200); } catch { interrupted = true; } +console.log(` ${interrupted ? "ok" : "!!"} spinning guest interrupted after ${Date.now() - t0}ms`); +console.log(` ok isolate usable afterwards ${c.load(`1+1`) === 2}`); + +c.dispose(); +console.log(`\n${ok && interrupted ? "AUDIT PASSED" : "AUDIT FAILED"}`); +process.exit(ok && interrupted ? 0 : 1); diff --git a/packages/agent/docs/mobile-handoff/02-plugins/02-sandbox/src/facet-example.js b/packages/agent/docs/mobile-handoff/02-plugins/02-sandbox/src/facet-example.js new file mode 100644 index 000000000..0bdf0a9b4 --- /dev/null +++ b/packages/agent/docs/mobile-handoff/02-plugins/02-sandbox/src/facet-example.js @@ -0,0 +1,42 @@ +// ── REAL facet code. Nothing here knows it is sandboxed. ──────────────────── +// No imports rewritten, no string marshalling, no __dispatch. It uses the +// host API exactly as facets.md §11.1 describes it. + +class FooterComponent { + constructor(props) { this.label = props.label; this.count = 0; } + render(width) { + const text = ` ${this.label}: ${this.count} `; + const pad = Math.max(0, width - text.length - 2); + const bar = "\u2500".repeat(Math.max(0, width - 2)); + return [`\u250c${bar}\u2510`, `\u2502${text}${" ".repeat(pad)}\u2502`, `\u2514${bar}\u2518`]; + } + handleInput(data) { this.count += data.length; } +} + +const facet = { + id: "@demo/footer:tui", + uses: ["Tui", "Transcript"], + + construct(ctx) { + const tui = ctx.use("Tui"); + const transcript = ctx.use("Transcript"); + + // A factory crossing guest → host, by reference. + tui.slots.claim("footer", (props) => new FooterComponent(props)); + + // A command handler, likewise. + tui.commands.add("footer.reset", () => { + tui.notify("footer reset"); + return "ok"; + }); + + // A subscription: host calls back into the guest. + transcript.tail.subscribe((tail) => { + tui.notify(`transcript now has ${tail.entries.length} entries`); + }); + + return []; + }, +}; + +globalThis.__facet = facet; diff --git a/packages/agent/docs/mobile-handoff/02-plugins/02-sandbox/src/iso-vs-ctx.ts b/packages/agent/docs/mobile-handoff/02-plugins/02-sandbox/src/iso-vs-ctx.ts new file mode 100644 index 000000000..48cf8f27b --- /dev/null +++ b/packages/agent/docs/mobile-handoff/02-plugins/02-sandbox/src/iso-vs-ctx.ts @@ -0,0 +1,34 @@ +import ivm from "isolated-vm"; +const rss = () => process.memoryUsage().rss / 1024 / 1024; + +// 1. Are two CONTEXTS in one isolate isolated from each other? +{ + const iso = new ivm.Isolate({ memoryLimit: 64 }); + const a = iso.createContextSync(), b = iso.createContextSync(); + a.evalSync(`globalThis.secret = "from-A"`); + console.log("context B sees A's global:", b.evalSync(`typeof globalThis.secret`)); + // Can a spin in A be timed out without harming B? + try { a.evalSync(`while(true){}`, { timeout: 100 }); } catch { /* expected */ } + console.log("B usable after A was interrupted:", b.evalSync(`1+1`) === 2); + // Does A's memory bomb kill B? + try { a.evalSync(`var x=[]; for(;;) x.push(new Array(50000).fill(1));`, { timeout: 15000 }); } + catch (e: any) { console.log("A bomb:", e.message.slice(0, 45)); } + try { console.log("B after A's bomb:", b.evalSync(`1+1`)); } + catch (e: any) { console.log("B after A's bomb: DEAD -", e.message.slice(0, 40)); } + try { iso.dispose(); } catch {} +} + +// 2. Cost per facet, both ways. +for (const mode of ["one isolate, N contexts", "N isolates"]) { + const before = rss(), t = Date.now(); + const held: any[] = []; + const shared = mode.startsWith("one") ? new ivm.Isolate({ memoryLimit: 512 }) : null; + for (let i = 0; i < 50; i++) { + const iso = shared ?? new ivm.Isolate({ memoryLimit: 32 }); + const ctx = iso.createContextSync(); + ctx.evalSync(`var c = { render: function () { return ["x"]; } };`); + held.push(iso); + } + console.log(`50 facets, ${mode.padEnd(24)}: ${(((rss() - before) / 50) * 1024).toFixed(0)} KB each, ${Date.now() - t} ms`); + if (shared) { try { shared.dispose(); } catch {} } else for (const i of held) { try { i.dispose(); } catch {} } +} diff --git a/packages/agent/docs/mobile-handoff/02-plugins/02-sandbox/src/leak.ts b/packages/agent/docs/mobile-handoff/02-plugins/02-sandbox/src/leak.ts new file mode 100644 index 000000000..d6c5b42f7 --- /dev/null +++ b/packages/agent/docs/mobile-handoff/02-plugins/02-sandbox/src/leak.ts @@ -0,0 +1,27 @@ +import { createCompartment } from "./membrane.ts"; +const c = await createCompartment(); +let seen = 0; +await c.endow("__l", { drop: (_fn: unknown) => { seen++; } }); +c.load(`globalThis.churn = function (n) { for (var i = 0; i < n; i++) __l.drop(function () { return i; }); };`); + +const rss = () => process.memoryUsage().rss / 1024 / 1024; +const before = rss(); +c.load(`churn(20000);`, 60000); +const peak = c.debugRefCount(); +const afterChurn = rss(); + +globalThis.gc?.(); await new Promise((r) => setTimeout(r, 100)); +globalThis.gc?.(); await new Promise((r) => setTimeout(r, 100)); +const settled = c.debugRefCount(); + +console.log(`20000 crossings: host called ${seen}x`); +console.log(` guest refs: ${peak.guestRefs} -> ${settled.guestRefs}`); +console.log(` host table: ${peak.hostTable} -> ${settled.hostTable}`); +console.log(` rss: ${before.toFixed(0)} -> ${afterChurn.toFixed(0)} -> ${rss().toFixed(0)} MB`); + +// Cost of the release path: one evalSync crossing per finalized ref. +const t = Date.now(); +c.load(`churn(5000);`, 60000); +globalThis.gc?.(); await new Promise((r) => setTimeout(r, 200)); +console.log(` 5000 more + gc: ${Date.now() - t} ms, refs now ${c.debugRefCount().guestRefs}`); +c.dispose(); diff --git a/packages/agent/docs/mobile-handoff/02-plugins/02-sandbox/src/markdown-bundle.js b/packages/agent/docs/mobile-handoff/02-plugins/02-sandbox/src/markdown-bundle.js new file mode 100644 index 000000000..b43c46c82 --- /dev/null +++ b/packages/agent/docs/mobile-handoff/02-plugins/02-sandbox/src/markdown-bundle.js @@ -0,0 +1,3888 @@ +"use strict"; +var __m = (() => { + var __defProp = Object.defineProperty; + var __getOwnPropDesc = Object.getOwnPropertyDescriptor; + var __getOwnPropNames = Object.getOwnPropertyNames; + var __hasOwnProp = Object.prototype.hasOwnProperty; + var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value; + var __export = (target, all) => { + for (var name in all) + __defProp(target, name, { get: all[name], enumerable: true }); + }; + var __copyProps = (to, from, except, desc) => { + if (from && typeof from === "object" || typeof from === "function") { + for (let key of __getOwnPropNames(from)) + if (!__hasOwnProp.call(to, key) && key !== except) + __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable }); + } + return to; + }; + var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod); + var __publicField = (obj, key, value) => __defNormalProp(obj, typeof key !== "symbol" ? key + "" : key, value); + + // packages/tui/bundle-bench.ts + var bundle_bench_exports = {}; + __export(bundle_bench_exports, { + bench: () => bench + }); + + // node_modules/marked/lib/marked.esm.js + function M() { + return { async: false, breaks: false, extensions: null, gfm: true, hooks: null, pedantic: false, renderer: null, silent: false, tokenizer: null, walkTokens: null }; + } + var T = M(); + function N(l3) { + T = l3; + } + var _ = { exec: () => null }; + function E(l3) { + let e = []; + return (t) => { + let n = Math.max(0, Math.min(3, t - 1)), s = e[n]; + return s || (s = l3(n), e[n] = s), s; + }; + } + function d(l3, e = "") { + let t = typeof l3 == "string" ? l3 : l3.source, n = { replace: (s, r) => { + let i = typeof r == "string" ? r : r.source; + return i = i.replace(m.caret, "$1"), t = t.replace(s, i), n; + }, getRegex: () => new RegExp(t, e) }; + return n; + } + var Te = ((l3 = "") => { + try { + return !!new RegExp("(?<=1)(?/, blockquoteSetextReplace: /\n {0,3}((?:=+|-+) *)(?=\n|$)/g, blockquoteSetextReplace2: /^ {0,3}>[ \t]?/gm, listReplaceNesting: /^ {1,4}(?=( {4})*[^ ])/g, listIsTask: /^\[[ xX]\] +\S/, listReplaceTask: /^\[[ xX]\] +/, listTaskCheckbox: /\[[ xX]\]/, anyLine: /\n.*\n/, hrefBrackets: /^<(.*)>$/, tableDelimiter: /[:|]/, tableAlignChars: /^\||\| *$/g, tableRowBlankLine: /\n[ \t]*$/, tableAlignRight: /^ *-+: *$/, tableAlignCenter: /^ *:-+: *$/, tableAlignLeft: /^ *:-+ *$/, startATag: /^/i, startPreScriptTag: /^<(pre|code|kbd|script)(\s|>)/i, endPreScriptTag: /^<\/(pre|code|kbd|script)(\s|>)/i, startAngleBracket: /^$/, pedanticHrefTitle: /^([^'"]*[^\s])\s+(['"])(.*)\2/, unicodeAlphaNumeric: /[\p{L}\p{N}]/u, escapeTest: /[&<>"']/, escapeReplace: /[&<>"']/g, escapeTestNoEncode: /[<>"']|&(?!(#\d{1,7}|#[Xx][a-fA-F0-9]{1,6}|\w+);)/, escapeReplaceNoEncode: /[<>"']|&(?!(#\d{1,7}|#[Xx][a-fA-F0-9]{1,6}|\w+);)/g, caret: /(^|[^\[])\^/g, percentDecode: /%25/g, findPipe: /\|/g, splitPipe: / \|/, slashPipe: /\\\|/g, carriageReturn: /\r\n|\r/g, spaceLine: /^ +$/gm, notSpaceStart: /^\S*/, endingNewline: /\n$/, listItemRegex: (l3) => new RegExp(`^( {0,3}${l3})((?:[ ][^\\n]*)?(?:\\n|$))`), nextBulletRegex: E((l3) => new RegExp(`^ {0,${l3}}(?:[*+-]|\\d{1,9}[.)])((?:[ ][^\\n]*)?(?:\\n|$))`)), hrRegex: E((l3) => new RegExp(`^ {0,${l3}}((?:- *){3,}|(?:_ *){3,}|(?:\\* *){3,})(?:\\n+|$)`)), fencesBeginRegex: E((l3) => new RegExp(`^ {0,${l3}}(?:\`\`\`|~~~)`)), headingBeginRegex: E((l3) => new RegExp(`^ {0,${l3}}#`)), htmlBeginRegex: E((l3) => new RegExp(`^ {0,${l3}}<(?:[a-z].*>|!--)`, "i")), blockquoteBeginRegex: E((l3) => new RegExp(`^ {0,${l3}}>`)) }; + var Oe = /^(?:[ \t]*(?:\n|$))+/; + var we = /^((?: {4}| {0,3}\t)[^\n]+(?:\n(?:[ \t]*(?:\n|$))*)?)+/; + var ye = /^ {0,3}(`{3,}(?=[^`\n]*(?:\n|$))|~{3,})([^\n]*)(?:\n|$)(?:|([\s\S]*?)(?:\n|$))(?: {0,3}\1[~`]* *(?=\n|$)|$)/; + var B = /^ {0,3}((?:-[\t ]*){3,}|(?:_[ \t]*){3,}|(?:\*[ \t]*){3,})(?:\n+|$)/; + var Pe = /^ {0,3}(#{1,6})(?=\s|$)(.*)(?:\n+|$)/; + var j = / {0,3}(?:[*+-]|\d{1,9}[.)])/; + var oe = /^(?!bull |blockCode|fences|blockquote|heading|html|table)((?:.|\n(?!\s*?\n|bull |blockCode|fences|blockquote|heading|html|table))+?)\n {0,3}(=+|-+) *(?:\n+|$)/; + var ae = d(oe).replace(/bull/g, j).replace(/blockCode/g, /(?: {4}| {0,3}\t)/).replace(/fences/g, / {0,3}(?:`{3,}|~{3,})/).replace(/blockquote/g, / {0,3}>/).replace(/heading/g, / {0,3}#{1,6}/).replace(/html/g, / {0,3}<[^\n>]+>\n/).replace(/\|table/g, "").getRegex(); + var Se = d(oe).replace(/bull/g, j).replace(/blockCode/g, /(?: {4}| {0,3}\t)/).replace(/fences/g, / {0,3}(?:`{3,}|~{3,})/).replace(/blockquote/g, / {0,3}>/).replace(/heading/g, / {0,3}#{1,6}/).replace(/html/g, / {0,3}<[^\n>]+>\n/).replace(/table/g, / {0,3}\|?(?:[:\- ]*\|)+[\:\- ]*\n/).getRegex(); + var F = /^([^\n]+(?:\n(?!hr|heading|lheading|blockquote|fences|list|html|table| +\n)[^\n]+)*)/; + var $e = /^[^\n]+/; + var U = /(?!\s*\])(?:\\[\s\S]|[^\[\]\\])+/; + var Le = d(/^ {0,3}\[(label)\]: *(?:\n[ \t]*)?([^<\s][^\s]*|<.*?>)(?:(?: +(?:\n[ \t]*)?| *\n[ \t]*)(title))? *(?:\n+|$)/).replace("label", U).replace("title", /(?:"(?:\\"?|[^"\\])*"|'[^'\n]*(?:\n[^'\n]+)*\n?'|\([^()]*\))/).getRegex(); + var _e = d(/^(bull)([ \t][^\n]*?)?(?:\n|$)/).replace(/bull/g, j).getRegex(); + var H = "address|article|aside|base|basefont|blockquote|body|caption|center|col|colgroup|dd|details|dialog|dir|div|dl|dt|fieldset|figcaption|figure|footer|form|frame|frameset|h[1-6]|head|header|hr|html|iframe|legend|li|link|main|menu|menuitem|meta|nav|noframes|ol|optgroup|option|p|param|search|section|summary|table|tbody|td|tfoot|th|thead|title|tr|track|ul"; + var K = /|$))/; + var ze = d("^ {0,3}(?:<(script|pre|style|textarea)[\\s>][\\s\\S]*?(?:[^\\n]*\\n+|$)|comment[^\\n]*(\\n+|$)|<\\?[\\s\\S]*?(?:\\?>\\n*|$)|\\n*|$)|\\n*|$)|)[\\s\\S]*?(?:(?:\\n[ ]*)+\\n|$)|<(?!script|pre|style|textarea)([a-z][\\w-]*)(?:attribute)*? */?>(?=[ \\t]*(?:\\n|$))[\\s\\S]*?(?:(?:\\n[ ]*)+\\n|$)|(?=[ \\t]*(?:\\n|$))[\\s\\S]*?(?:(?:\\n[ ]*)+\\n|$))", "i").replace("comment", K).replace("tag", H).replace("attribute", / +[a-zA-Z:_][\w.:-]*(?: *= *"[^"\n]*"| *= *'[^'\n]*'| *= *[^\s"'=<>`]+)?/).getRegex(); + var le = d(F).replace("hr", B).replace("heading", " {0,3}#{1,6}(?:\\s|$)").replace("|lheading", "").replace("|table", "").replace("blockquote", " {0,3}>").replace("fences", " {0,3}(?:`{3,}(?=[^`\\n]*\\n)|~{3,})[^\\n]*\\n").replace("list", " {0,3}(?:[*+-]|1[.)])[ \\t]+[^ \\t\\n]").replace("html", ")|<(?:script|pre|style|textarea|!--)").replace("tag", H).getRegex(); + var Me = d(/^( {0,3}> ?(paragraph|[^\n]*)(?:\n|$))+/).replace("paragraph", le).getRegex(); + var W = { blockquote: Me, code: we, def: Le, fences: ye, heading: Pe, hr: B, html: ze, lheading: ae, list: _e, newline: Oe, paragraph: le, table: _, text: $e }; + var se = d("^ *([^\\n ].*)\\n {0,3}((?:\\| *)?:?-+:? *(?:\\| *:?-+:? *)*(?:\\| *)?)(?:\\n((?:(?! *\\n|hr|heading|blockquote|code|fences|list|html).*(?:\\n|$))*)\\n*|$)").replace("hr", B).replace("heading", " {0,3}#{1,6}(?:\\s|$)").replace("blockquote", " {0,3}>").replace("code", "(?: {4}| {0,3} )[^\\n]").replace("fences", " {0,3}(?:`{3,}(?=[^`\\n]*\\n)|~{3,})[^\\n]*\\n").replace("list", " {0,3}(?:[*+-]|1[.)])[ \\t]").replace("html", ")|<(?:script|pre|style|textarea|!--)").replace("tag", H).getRegex(); + var Ee = { ...W, lheading: Se, table: se, paragraph: d(F).replace("hr", B).replace("heading", " {0,3}#{1,6}(?:\\s|$)").replace("|lheading", "").replace("table", se).replace("blockquote", " {0,3}>").replace("fences", " {0,3}(?:`{3,}(?=[^`\\n]*\\n)|~{3,})[^\\n]*\\n").replace("list", " {0,3}(?:[*+-]|1[.)])[ \\t]+[^ \\t\\n]").replace("html", ")|<(?:script|pre|style|textarea|!--)").replace("tag", H).getRegex() }; + var Ie = { ...W, html: d(`^ *(?:comment *(?:\\n|\\s*$)|<(tag)[\\s\\S]+? *(?:\\n{2,}|\\s*$)|\\s]*)*?/?> *(?:\\n{2,}|\\s*$))`).replace("comment", K).replace(/tag/g, "(?!(?:a|em|strong|small|s|cite|q|dfn|abbr|data|time|code|var|samp|kbd|sub|sup|i|b|u|mark|ruby|rt|rp|bdi|bdo|span|br|wbr|ins|del|img)\\b)\\w+(?!:|[^\\w\\s@]*@)\\b").getRegex(), def: /^ *\[([^\]]+)\]: *]+)>?(?: +(["(][^\n]+[")]))? *(?:\n+|$)/, heading: /^(#{1,6})(.*)(?:\n+|$)/, fences: _, lheading: /^(.+?)\n {0,3}(=+|-+) *(?:\n+|$)/, paragraph: d(F).replace("hr", B).replace("heading", ` *#{1,6} *[^ +]`).replace("lheading", ae).replace("|table", "").replace("blockquote", " {0,3}>").replace("|fences", "").replace("|list", "").replace("|html", "").replace("|tag", "").getRegex() }; + var Ae = /^\\([!"#$%&'()*+,\-./:;<=>?@\[\]\\^_`{|}~])/; + var Ce = /^(`+)([^`]|[^`][\s\S]*?[^`])\1(?!`)/; + var ue = /^( {2,}|\\)\n(?!\s*$)/; + var Be = /^(`+|[^`])(?:(?= {2,}\n)|[\s\S]*?(?:(?=[\\`+)[^`]+\k(?!`))*?\]\((?:\\[\s\S]|[^\\\(\)]|\((?:\\[\s\S]|[^\\\(\)])*\))*\)/).replace("precode-", Te ? "(?`+)[^`]+\k(?!`)/).replace("html", /<(?! )[^<>]*?>/).getRegex(); + var ce = /^(?:\*+(?:((?!\*)punct)|([^\s*]))?)|^_+(?:((?!_)punct)|([^\s_]))?/; + var Ze = d(ce, "u").replace(/punct/g, I).getRegex(); + var Ge = d(ce, "u").replace(/punct/g, pe).getRegex(); + var he = "^[^_*]*?__[^_*]*?\\*[^_*]*?(?=__)|[^*]+(?=[^*])|(?!\\*)punct(\\*+)(?=[\\s]|$)|notPunctSpace(\\*+)(?!\\*)(?=punctSpace|$)|(?!\\*)punctSpace(\\*+)(?=notPunctSpace)|[\\s](\\*+)(?!\\*)(?=punct)|(?!\\*)punct(\\*+)(?!\\*)(?=punct)|notPunctSpace(\\*+)(?=notPunctSpace)"; + var Ne = d(he, "gu").replace(/notPunctSpace/g, X).replace(/punctSpace/g, Z).replace(/punct/g, I).getRegex(); + var Qe = d(he, "gu").replace(/notPunctSpace/g, ve).replace(/punctSpace/g, qe).replace(/punct/g, pe).getRegex(); + var je = d("^[^_*]*?\\*\\*[^_*]*?_[^_*]*?(?=\\*\\*)|[^_]+(?=[^_])|(?!_)punct(_+)(?=[\\s]|$)|notPunctSpace(_+)(?!_)(?=punctSpace|$)|(?!_)punctSpace(_+)(?=notPunctSpace)|[\\s](_+)(?!_)(?=punct)|(?!_)punct(_+)(?!_)(?=punct)", "gu").replace(/notPunctSpace/g, X).replace(/punctSpace/g, Z).replace(/punct/g, I).getRegex(); + var Fe = d(/^~~?(?:((?!~)punct)|[^\s~])/, "u").replace(/punct/g, I).getRegex(); + var Ue = "^[^~]+(?=[^~])|(?!~)punct(~~?)(?=[\\s]|$)|notPunctSpace(~~?)(?!~)(?=punctSpace|$)|(?!~)punctSpace(~~?)(?=notPunctSpace)|[\\s](~~?)(?!~)(?=punct)|(?!~)punct(~~?)(?!~)(?=punct)|notPunctSpace(~~?)(?=notPunctSpace)"; + var Ke = d(Ue, "gu").replace(/notPunctSpace/g, X).replace(/punctSpace/g, Z).replace(/punct/g, I).getRegex(); + var We = d(/\\(punct)/, "gu").replace(/punct/g, I).getRegex(); + var Xe = d(/^<(scheme:[^\s\x00-\x1f<>]*|email)>/).replace("scheme", /[a-zA-Z][a-zA-Z0-9+.-]{1,31}/).replace("email", /[a-zA-Z0-9.!#$%&'*+/=?^_`{|}~-]+(@)[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)+(?![-_])/).getRegex(); + var Je = d(K).replace("(?:-->|$)", "-->").getRegex(); + var Ve = d("^comment|^|^<[a-zA-Z][\\w-]*(?:attribute)*?\\s*/?>|^<\\?[\\s\\S]*?\\?>|^|^").replace("comment", Je).replace("attribute", /\s+[a-zA-Z:_][\w.:-]*(?:\s*=\s*"[^"]*"|\s*=\s*'[^']*'|\s*=\s*[^\s"'=<>`]+)?/).getRegex(); + var v = /(?:\[(?:\\[\s\S]|[^\[\]\\])*\]|\\[\s\S]|`+(?!`)[^`]*?`+(?!`)|``+(?=\])|[^\[\]\\`])*?/; + var Ye = d(/^!?\[(label)\]\(\s*(href)(?:(?:[ \t]+(?:\n[ \t]*)?|\n[ \t]*)(title))?\s*\)/).replace("label", v).replace("href", /<(?:\\.|[^\n<>\\])+>|[^ \t\n\x00-\x1f]*/).replace("title", /"(?:\\"?|[^"\\])*"|'(?:\\'?|[^'\\])*'|\((?:\\\)?|[^)\\])*\)/).getRegex(); + var ke = d(/^!?\[(label)\]\[(ref)\]/).replace("label", v).replace("ref", U).getRegex(); + var de = d(/^!?\[(ref)\](?:\[\])?/).replace("ref", U).getRegex(); + var et = d("reflink|nolink(?!\\()", "g").replace("reflink", ke).replace("nolink", de).getRegex(); + var ie = /[hH][tT][tT][pP][sS]?|[fF][tT][pP]/; + var J = { _backpedal: _, anyPunctuation: We, autolink: Xe, blockSkip: He, br: ue, code: Ce, del: _, delLDelim: _, delRDelim: _, emStrongLDelim: Ze, emStrongRDelimAst: Ne, emStrongRDelimUnd: je, escape: Ae, link: Ye, nolink: de, punctuation: De, reflink: ke, reflinkSearch: et, tag: Ve, text: Be, url: _ }; + var tt = { ...J, link: d(/^!?\[(label)\]\((.*?)\)/).replace("label", v).getRegex(), reflink: d(/^!?\[(label)\]\s*\[([^\]]*)\]/).replace("label", v).getRegex() }; + var Q = { ...J, emStrongRDelimAst: Qe, emStrongLDelim: Ge, delLDelim: Fe, delRDelim: Ke, url: d(/^((?:protocol):\/\/|www\.)(?:[a-zA-Z0-9\-]+\.?)+[^\s<]*|^email/).replace("protocol", ie).replace("email", /[A-Za-z0-9._+-]+(@)[a-zA-Z0-9-_]+(?:\.[a-zA-Z0-9-_]*[a-zA-Z0-9])+(?![-_])/).getRegex(), _backpedal: /(?:[^?!.,:;*_'"~()&]+|\([^)]*\)|&(?![a-zA-Z0-9]+;$)|[?!.,:;*_'"~)]+(?!$))+/, del: /^(~~?)(?=[^\s~])((?:\\[\s\S]|[^\\])*?(?:\\[\s\S]|[^\s~\\]))\1(?=[^~]|$)/, text: d(/^([`~]+|[^`~])(?:(?= {2,}\n)|(?=[a-zA-Z0-9.!#$%&'*+\/=?_`{\|}~-]+@)|[\s\S]*?(?:(?=[\\": ">", '"': """, "'": "'" }; + var ge = (l3) => rt[l3]; + function O(l3, e) { + if (e) { + if (m.escapeTest.test(l3)) return l3.replace(m.escapeReplace, ge); + } else if (m.escapeTestNoEncode.test(l3)) return l3.replace(m.escapeReplaceNoEncode, ge); + return l3; + } + function V(l3) { + try { + l3 = encodeURI(l3).replace(m.percentDecode, "%"); + } catch { + return null; + } + return l3; + } + function Y(l3, e) { + let t = l3.replace(m.findPipe, (r, i, o) => { + let u = false, a = i; + for (; --a >= 0 && o[a] === "\\"; ) u = !u; + return u ? "|" : " |"; + }), n = t.split(m.splitPipe), s = 0; + if (n[0].trim() || n.shift(), n.length > 0 && !n.at(-1)?.trim() && n.pop(), e) if (n.length > e) n.splice(e); + else for (; n.length < e; ) n.push(""); + for (; s < n.length; s++) n[s] = n[s].trim().replace(m.slashPipe, "|"); + return n; + } + function $(l3, e, t) { + let n = l3.length; + if (n === 0) return ""; + let s = 0; + for (; s < n; ) { + let r = l3.charAt(n - s - 1); + if (r === e && !t) s++; + else if (r !== e && t) s++; + else break; + } + return l3.slice(0, n - s); + } + function ee(l3) { + let e = l3.split(` +`), t = e.length - 1; + for (; t >= 0 && m.blankLine.test(e[t]); ) t--; + return e.length - t <= 2 ? l3 : e.slice(0, t + 1).join(` +`); + } + function fe(l3, e) { + if (l3.indexOf(e[1]) === -1) return -1; + let t = 0; + for (let n = 0; n < l3.length; n++) if (l3[n] === "\\") n++; + else if (l3[n] === e[0]) t++; + else if (l3[n] === e[1] && (t--, t < 0)) return n; + return t > 0 ? -2 : -1; + } + function me(l3, e = 0) { + let t = e, n = ""; + for (let s of l3) if (s === " ") { + let r = 4 - t % 4; + n += " ".repeat(r), t += r; + } else n += s, t++; + return n; + } + function xe(l3, e, t, n, s) { + let r = e.href, i = e.title || null, o = l3[1].replace(s.other.outputLinkReplace, "$1"); + n.state.inLink = true; + let u = { type: l3[0].charAt(0) === "!" ? "image" : "link", raw: t, href: r, title: i, text: o, tokens: n.inlineTokens(o) }; + return n.state.inLink = false, u; + } + function st(l3, e, t) { + let n = l3.match(t.other.indentCodeCompensation); + if (n === null) return e; + let s = n[1]; + return e.split(` +`).map((r) => { + let i = r.match(t.other.beginningSpace); + if (i === null) return r; + let [o] = i; + return o.length >= s.length ? r.slice(s.length) : r; + }).join(` +`); + } + var w = class { + constructor(e) { + __publicField(this, "options"); + __publicField(this, "rules"); + __publicField(this, "lexer"); + this.options = e || T; + } + space(e) { + let t = this.rules.block.newline.exec(e); + if (t && t[0].length > 0) return { type: "space", raw: t[0] }; + } + code(e) { + let t = this.rules.block.code.exec(e); + if (t) { + let n = this.options.pedantic ? t[0] : ee(t[0]), s = n.replace(this.rules.other.codeRemoveIndent, ""); + return { type: "code", raw: n, codeBlockStyle: "indented", text: s }; + } + } + fences(e) { + let t = this.rules.block.fences.exec(e); + if (t) { + let n = t[0], s = st(n, t[3] || "", this.rules); + return { type: "code", raw: n, lang: t[2] ? t[2].trim().replace(this.rules.inline.anyPunctuation, "$1") : t[2], text: s }; + } + } + heading(e) { + let t = this.rules.block.heading.exec(e); + if (t) { + let n = t[2].trim(); + if (this.rules.other.endingHash.test(n)) { + let s = $(n, "#"); + (this.options.pedantic || !s || this.rules.other.endingSpaceChar.test(s)) && (n = s.trim()); + } + return { type: "heading", raw: $(t[0], ` +`), depth: t[1].length, text: n, tokens: this.lexer.inline(n) }; + } + } + hr(e) { + let t = this.rules.block.hr.exec(e); + if (t) return { type: "hr", raw: $(t[0], ` +`) }; + } + blockquote(e) { + let t = this.rules.block.blockquote.exec(e); + if (t) { + let n = $(t[0], ` +`).split(` +`), s = "", r = "", i = []; + for (; n.length > 0; ) { + let o = false, u = [], a; + for (a = 0; a < n.length; a++) if (this.rules.other.blockquoteStart.test(n[a])) u.push(n[a]), o = true; + else if (!o) u.push(n[a]); + else break; + n = n.slice(a); + let c = u.join(` +`), p = c.replace(this.rules.other.blockquoteSetextReplace, ` + $1`).replace(this.rules.other.blockquoteSetextReplace2, ""); + s = s ? `${s} +${c}` : c, r = r ? `${r} +${p}` : p; + let k = this.lexer.state.top; + if (this.lexer.state.top = true, this.lexer.blockTokens(p, i, true), this.lexer.state.top = k, n.length === 0) break; + let h = i.at(-1); + if (h?.type === "code") break; + if (h?.type === "blockquote") { + let R = h, f = R.raw + ` +` + n.join(` +`), S = this.blockquote(f); + i[i.length - 1] = S, s = s.substring(0, s.length - R.raw.length) + S.raw, r = r.substring(0, r.length - R.text.length) + S.text; + break; + } else if (h?.type === "list") { + let R = h, f = R.raw + ` +` + n.join(` +`), S = this.list(f); + i[i.length - 1] = S, s = s.substring(0, s.length - h.raw.length) + S.raw, r = r.substring(0, r.length - R.raw.length) + S.raw, n = f.substring(i.at(-1).raw.length).split(` +`); + continue; + } + } + return { type: "blockquote", raw: s, tokens: i, text: r }; + } + } + list(e) { + let t = this.rules.block.list.exec(e); + if (t) { + let n = t[1].trim(), s = n.length > 1, r = { type: "list", raw: "", ordered: s, start: s ? +n.slice(0, -1) : "", loose: false, items: [] }; + n = s ? `\\d{1,9}\\${n.slice(-1)}` : `\\${n}`, this.options.pedantic && (n = s ? n : "[*+-]"); + let i = this.rules.other.listItemRegex(n), o = false; + for (; e; ) { + let a = false, c = "", p = ""; + if (!(t = i.exec(e)) || this.rules.block.hr.test(e)) break; + c = t[0], e = e.substring(c.length); + let k = me(t[2].split(` +`, 1)[0], t[1].length), h = e.split(` +`, 1)[0], R = !k.trim(), f = 0; + if (this.options.pedantic ? (f = 2, p = k.trimStart()) : R ? f = t[1].length + 1 : (f = k.search(this.rules.other.nonSpaceChar), f = f > 4 ? 1 : f, p = k.slice(f), f += t[1].length), R && this.rules.other.blankLine.test(h) && (c += h + ` +`, e = e.substring(h.length + 1), a = true), !a) { + let S = this.rules.other.nextBulletRegex(f), te = this.rules.other.hrRegex(f), ne = this.rules.other.fencesBeginRegex(f), re = this.rules.other.headingBeginRegex(f), be = this.rules.other.htmlBeginRegex(f), Re = this.rules.other.blockquoteBeginRegex(f); + for (; e; ) { + let G = e.split(` +`, 1)[0], C; + if (h = G, this.options.pedantic ? (h = h.replace(this.rules.other.listReplaceNesting, " "), C = h) : C = h.replace(this.rules.other.tabCharGlobal, " "), ne.test(h) || re.test(h) || be.test(h) || Re.test(h) || S.test(h) || te.test(h)) break; + if (C.search(this.rules.other.nonSpaceChar) >= f || !h.trim()) p += ` +` + C.slice(f); + else { + if (R || k.replace(this.rules.other.tabCharGlobal, " ").search(this.rules.other.nonSpaceChar) >= 4 || ne.test(k) || re.test(k) || te.test(k)) break; + p += ` +` + h; + } + R = !h.trim(), c += G + ` +`, e = e.substring(G.length + 1), k = C.slice(f); + } + } + r.loose || (o ? r.loose = true : this.rules.other.doubleBlankLine.test(c) && (o = true)), r.items.push({ type: "list_item", raw: c, task: !!this.options.gfm && this.rules.other.listIsTask.test(p), loose: false, text: p, tokens: [] }), r.raw += c; + } + let u = r.items.at(-1); + if (u) u.raw = u.raw.trimEnd(), u.text = u.text.trimEnd(); + else return; + r.raw = r.raw.trimEnd(); + for (let a of r.items) { + this.lexer.state.top = false, a.tokens = this.lexer.blockTokens(a.text, []); + let c = a.tokens[0]; + if (a.task && (c?.type === "text" || c?.type === "paragraph")) { + a.text = a.text.replace(this.rules.other.listReplaceTask, ""), c.raw = c.raw.replace(this.rules.other.listReplaceTask, ""), c.text = c.text.replace(this.rules.other.listReplaceTask, ""); + for (let k = this.lexer.inlineQueue.length - 1; k >= 0; k--) if (this.rules.other.listIsTask.test(this.lexer.inlineQueue[k].src)) { + this.lexer.inlineQueue[k].src = this.lexer.inlineQueue[k].src.replace(this.rules.other.listReplaceTask, ""); + break; + } + let p = this.rules.other.listTaskCheckbox.exec(a.raw); + if (p) { + let k = { type: "checkbox", raw: p[0] + " ", checked: p[0] !== "[ ]" }; + a.checked = k.checked, r.loose ? a.tokens[0] && ["paragraph", "text"].includes(a.tokens[0].type) && "tokens" in a.tokens[0] && a.tokens[0].tokens ? (a.tokens[0].raw = k.raw + a.tokens[0].raw, a.tokens[0].text = k.raw + a.tokens[0].text, a.tokens[0].tokens.unshift(k)) : a.tokens.unshift({ type: "paragraph", raw: k.raw, text: k.raw, tokens: [k] }) : a.tokens.unshift(k); + } + } else a.task && (a.task = false); + if (!r.loose) { + let p = a.tokens.filter((h) => h.type === "space"), k = p.length > 0 && p.some((h) => this.rules.other.anyLine.test(h.raw)); + r.loose = k; + } + } + if (r.loose) for (let a of r.items) { + a.loose = true; + for (let c of a.tokens) c.type === "text" && (c.type = "paragraph"); + } + return r; + } + } + html(e) { + let t = this.rules.block.html.exec(e); + if (t) { + let n = ee(t[0]); + return { type: "html", block: true, raw: n, pre: t[1] === "pre" || t[1] === "script" || t[1] === "style", text: n }; + } + } + def(e) { + let t = this.rules.block.def.exec(e); + if (t) { + let n = t[1].toLowerCase().replace(this.rules.other.multipleSpaceGlobal, " "), s = t[2] ? t[2].replace(this.rules.other.hrefBrackets, "$1").replace(this.rules.inline.anyPunctuation, "$1") : "", r = t[3] ? t[3].substring(1, t[3].length - 1).replace(this.rules.inline.anyPunctuation, "$1") : t[3]; + return { type: "def", tag: n, raw: $(t[0], ` +`), href: s, title: r }; + } + } + table(e) { + let t = this.rules.block.table.exec(e); + if (!t || !this.rules.other.tableDelimiter.test(t[2])) return; + let n = Y(t[1]), s = t[2].replace(this.rules.other.tableAlignChars, "").split("|"), r = t[3]?.trim() ? t[3].replace(this.rules.other.tableRowBlankLine, "").split(` +`) : [], i = { type: "table", raw: $(t[0], ` +`), header: [], align: [], rows: [] }; + if (n.length === s.length) { + for (let o of s) this.rules.other.tableAlignRight.test(o) ? i.align.push("right") : this.rules.other.tableAlignCenter.test(o) ? i.align.push("center") : this.rules.other.tableAlignLeft.test(o) ? i.align.push("left") : i.align.push(null); + for (let o = 0; o < n.length; o++) i.header.push({ text: n[o], tokens: this.lexer.inline(n[o]), header: true, align: i.align[o] }); + for (let o of r) i.rows.push(Y(o, i.header.length).map((u, a) => ({ text: u, tokens: this.lexer.inline(u), header: false, align: i.align[a] }))); + return i; + } + } + lheading(e) { + let t = this.rules.block.lheading.exec(e); + if (t) { + let n = t[1].trim(); + return { type: "heading", raw: $(t[0], ` +`), depth: t[2].charAt(0) === "=" ? 1 : 2, text: n, tokens: this.lexer.inline(n) }; + } + } + paragraph(e) { + let t = this.rules.block.paragraph.exec(e); + if (t) { + let n = t[1].charAt(t[1].length - 1) === ` +` ? t[1].slice(0, -1) : t[1]; + return { type: "paragraph", raw: t[0], text: n, tokens: this.lexer.inline(n) }; + } + } + text(e) { + let t = this.rules.block.text.exec(e); + if (t) return { type: "text", raw: t[0], text: t[0], tokens: this.lexer.inline(t[0]) }; + } + escape(e) { + let t = this.rules.inline.escape.exec(e); + if (t) return { type: "escape", raw: t[0], text: t[1] }; + } + tag(e) { + let t = this.rules.inline.tag.exec(e); + if (t) return !this.lexer.state.inLink && this.rules.other.startATag.test(t[0]) ? this.lexer.state.inLink = true : this.lexer.state.inLink && this.rules.other.endATag.test(t[0]) && (this.lexer.state.inLink = false), !this.lexer.state.inRawBlock && this.rules.other.startPreScriptTag.test(t[0]) ? this.lexer.state.inRawBlock = true : this.lexer.state.inRawBlock && this.rules.other.endPreScriptTag.test(t[0]) && (this.lexer.state.inRawBlock = false), { type: "html", raw: t[0], inLink: this.lexer.state.inLink, inRawBlock: this.lexer.state.inRawBlock, block: false, text: t[0] }; + } + link(e) { + let t = this.rules.inline.link.exec(e); + if (t) { + let n = t[2].trim(); + if (!this.options.pedantic && this.rules.other.startAngleBracket.test(n)) { + if (!this.rules.other.endAngleBracket.test(n)) return; + let i = $(n.slice(0, -1), "\\"); + if ((n.length - i.length) % 2 === 0) return; + } else { + let i = fe(t[2], "()"); + if (i === -2) return; + if (i > -1) { + let u = (t[0].indexOf("!") === 0 ? 5 : 4) + t[1].length + i; + t[2] = t[2].substring(0, i), t[0] = t[0].substring(0, u).trim(), t[3] = ""; + } + } + let s = t[2], r = ""; + if (this.options.pedantic) { + let i = this.rules.other.pedanticHrefTitle.exec(s); + i && (s = i[1], r = i[3]); + } else r = t[3] ? t[3].slice(1, -1) : ""; + return s = s.trim(), this.rules.other.startAngleBracket.test(s) && (this.options.pedantic && !this.rules.other.endAngleBracket.test(n) ? s = s.slice(1) : s = s.slice(1, -1)), xe(t, { href: s && s.replace(this.rules.inline.anyPunctuation, "$1"), title: r && r.replace(this.rules.inline.anyPunctuation, "$1") }, t[0], this.lexer, this.rules); + } + } + reflink(e, t) { + let n; + if ((n = this.rules.inline.reflink.exec(e)) || (n = this.rules.inline.nolink.exec(e))) { + let s = (n[2] || n[1]).replace(this.rules.other.multipleSpaceGlobal, " "), r = t[s.toLowerCase()]; + if (!r) { + let i = n[0].charAt(0); + return { type: "text", raw: i, text: i }; + } + return xe(n, r, n[0], this.lexer, this.rules); + } + } + emStrong(e, t, n = "") { + let s = this.rules.inline.emStrongLDelim.exec(e); + if (!s || !s[1] && !s[2] && !s[3] && !s[4] || s[4] && n.match(this.rules.other.unicodeAlphaNumeric)) return; + if (!(s[1] || s[3] || "") || !n || this.rules.inline.punctuation.exec(n)) { + let i = [...s[0]].length - 1, o, u, a = i, c = 0, p = s[0][0] === "*" ? this.rules.inline.emStrongRDelimAst : this.rules.inline.emStrongRDelimUnd; + for (p.lastIndex = 0, t = t.slice(-1 * e.length + i); (s = p.exec(t)) !== null; ) { + if (o = s[1] || s[2] || s[3] || s[4] || s[5] || s[6], !o) continue; + if (u = [...o].length, s[3] || s[4]) { + a += u; + continue; + } else if ((s[5] || s[6]) && i % 3 && !((i + u) % 3)) { + c += u; + continue; + } + if (a -= u, a > 0) continue; + u = Math.min(u, u + a + c); + let k = [...s[0]][0].length, h = e.slice(0, i + s.index + k + u); + if (Math.min(i, u) % 2) { + let f = h.slice(1, -1); + return { type: "em", raw: h, text: f, tokens: this.lexer.inlineTokens(f) }; + } + let R = h.slice(2, -2); + return { type: "strong", raw: h, text: R, tokens: this.lexer.inlineTokens(R) }; + } + } + } + codespan(e) { + let t = this.rules.inline.code.exec(e); + if (t) { + let n = t[2].replace(this.rules.other.newLineCharGlobal, " "), s = this.rules.other.nonSpaceChar.test(n), r = this.rules.other.startingSpaceChar.test(n) && this.rules.other.endingSpaceChar.test(n); + return s && r && (n = n.substring(1, n.length - 1)), { type: "codespan", raw: t[0], text: n }; + } + } + br(e) { + let t = this.rules.inline.br.exec(e); + if (t) return { type: "br", raw: t[0] }; + } + del(e, t, n = "") { + let s = this.rules.inline.delLDelim.exec(e); + if (!s) return; + if (!(s[1] || "") || !n || this.rules.inline.punctuation.exec(n)) { + let i = [...s[0]].length - 1, o, u, a = i, c = this.rules.inline.delRDelim; + for (c.lastIndex = 0, t = t.slice(-1 * e.length + i); (s = c.exec(t)) !== null; ) { + if (o = s[1] || s[2] || s[3] || s[4] || s[5] || s[6], !o || (u = [...o].length, u !== i)) continue; + if (s[3] || s[4]) { + a += u; + continue; + } + if (a -= u, a > 0) continue; + u = Math.min(u, u + a); + let p = [...s[0]][0].length, k = e.slice(0, i + s.index + p + u), h = k.slice(i, -i); + return { type: "del", raw: k, text: h, tokens: this.lexer.inlineTokens(h) }; + } + } + } + autolink(e) { + let t = this.rules.inline.autolink.exec(e); + if (t) { + let n, s; + return t[2] === "@" ? (n = t[1], s = "mailto:" + n) : (n = t[1], s = n), { type: "link", raw: t[0], text: n, href: s, tokens: [{ type: "text", raw: n, text: n }] }; + } + } + url(e) { + let t; + if (t = this.rules.inline.url.exec(e)) { + let n, s; + if (t[2] === "@") n = t[0], s = "mailto:" + n; + else { + let r; + do + r = t[0], t[0] = this.rules.inline._backpedal.exec(t[0])?.[0] ?? ""; + while (r !== t[0]); + n = t[0], t[1] === "www." ? s = "http://" + t[0] : s = t[0]; + } + return { type: "link", raw: t[0], text: n, href: s, tokens: [{ type: "text", raw: n, text: n }] }; + } + } + inlineText(e) { + let t = this.rules.inline.text.exec(e); + if (t) { + let n = this.lexer.state.inRawBlock; + return { type: "text", raw: t[0], text: t[0], escaped: n }; + } + } + }; + var x = class l { + constructor(e) { + __publicField(this, "tokens"); + __publicField(this, "options"); + __publicField(this, "state"); + __publicField(this, "inlineQueue"); + __publicField(this, "tokenizer"); + this.tokens = [], this.tokens.links = /* @__PURE__ */ Object.create(null), this.options = e || T, this.options.tokenizer = this.options.tokenizer || new w(), this.tokenizer = this.options.tokenizer, this.tokenizer.options = this.options, this.tokenizer.lexer = this, this.inlineQueue = [], this.state = { inLink: false, inRawBlock: false, top: true }; + let t = { other: m, block: D.normal, inline: A.normal }; + this.options.pedantic ? (t.block = D.pedantic, t.inline = A.pedantic) : this.options.gfm && (t.block = D.gfm, this.options.breaks ? t.inline = A.breaks : t.inline = A.gfm), this.tokenizer.rules = t; + } + static get rules() { + return { block: D, inline: A }; + } + static lex(e, t) { + return new l(t).lex(e); + } + static lexInline(e, t) { + return new l(t).inlineTokens(e); + } + lex(e) { + e = e.replace(m.carriageReturn, ` +`), this.blockTokens(e, this.tokens); + for (let t = 0; t < this.inlineQueue.length; t++) { + let n = this.inlineQueue[t]; + this.inlineTokens(n.src, n.tokens); + } + return this.inlineQueue = [], this.tokens; + } + blockTokens(e, t = [], n = false) { + this.tokenizer.lexer = this, this.options.pedantic && (e = e.replace(m.tabCharGlobal, " ").replace(m.spaceLine, "")); + let s = 1 / 0; + for (; e; ) { + if (e.length < s) s = e.length; + else { + this.infiniteLoopError(e.charCodeAt(0)); + break; + } + let r; + if (this.options.extensions?.block?.some((o) => (r = o.call({ lexer: this }, e, t)) ? (e = e.substring(r.raw.length), t.push(r), true) : false)) continue; + if (r = this.tokenizer.space(e)) { + e = e.substring(r.raw.length); + let o = t.at(-1); + r.raw.length === 1 && o !== void 0 ? o.raw += ` +` : t.push(r); + continue; + } + if (r = this.tokenizer.code(e)) { + e = e.substring(r.raw.length); + let o = t.at(-1); + o?.type === "paragraph" || o?.type === "text" ? (o.raw += (o.raw.endsWith(` +`) ? "" : ` +`) + r.raw, o.text += ` +` + r.text, this.inlineQueue.at(-1).src = o.text) : t.push(r); + continue; + } + if (r = this.tokenizer.fences(e)) { + e = e.substring(r.raw.length), t.push(r); + continue; + } + if (r = this.tokenizer.heading(e)) { + e = e.substring(r.raw.length), t.push(r); + continue; + } + if (r = this.tokenizer.hr(e)) { + e = e.substring(r.raw.length), t.push(r); + continue; + } + if (r = this.tokenizer.blockquote(e)) { + e = e.substring(r.raw.length), t.push(r); + continue; + } + if (r = this.tokenizer.list(e)) { + e = e.substring(r.raw.length), t.push(r); + continue; + } + if (r = this.tokenizer.html(e)) { + e = e.substring(r.raw.length), t.push(r); + continue; + } + if (r = this.tokenizer.def(e)) { + e = e.substring(r.raw.length); + let o = t.at(-1); + o?.type === "paragraph" || o?.type === "text" ? (o.raw += (o.raw.endsWith(` +`) ? "" : ` +`) + r.raw, o.text += ` +` + r.raw, this.inlineQueue.at(-1).src = o.text) : this.tokens.links[r.tag] || (this.tokens.links[r.tag] = { href: r.href, title: r.title }, t.push(r)); + continue; + } + if (r = this.tokenizer.table(e)) { + e = e.substring(r.raw.length), t.push(r); + continue; + } + if (r = this.tokenizer.lheading(e)) { + e = e.substring(r.raw.length), t.push(r); + continue; + } + let i = e; + if (this.options.extensions?.startBlock) { + let o = 1 / 0, u = e.slice(1), a; + this.options.extensions.startBlock.forEach((c) => { + a = c.call({ lexer: this }, u), typeof a == "number" && a >= 0 && (o = Math.min(o, a)); + }), o < 1 / 0 && o >= 0 && (i = e.substring(0, o + 1)); + } + if (this.state.top && (r = this.tokenizer.paragraph(i))) { + let o = t.at(-1); + n && o?.type === "paragraph" ? (o.raw += (o.raw.endsWith(` +`) ? "" : ` +`) + r.raw, o.text += ` +` + r.text, this.inlineQueue.pop(), this.inlineQueue.at(-1).src = o.text) : t.push(r), n = i.length !== e.length, e = e.substring(r.raw.length); + continue; + } + if (r = this.tokenizer.text(e)) { + e = e.substring(r.raw.length); + let o = t.at(-1); + o?.type === "text" ? (o.raw += (o.raw.endsWith(` +`) ? "" : ` +`) + r.raw, o.text += ` +` + r.text, this.inlineQueue.pop(), this.inlineQueue.at(-1).src = o.text) : t.push(r); + continue; + } + if (e) { + this.infiniteLoopError(e.charCodeAt(0)); + break; + } + } + return this.state.top = true, t; + } + inline(e, t = []) { + return this.inlineQueue.push({ src: e, tokens: t }), t; + } + inlineTokens(e, t = []) { + this.tokenizer.lexer = this; + let n = e, s = null; + if (this.tokens.links) { + let a = Object.keys(this.tokens.links); + if (a.length > 0) for (; (s = this.tokenizer.rules.inline.reflinkSearch.exec(n)) !== null; ) a.includes(s[0].slice(s[0].lastIndexOf("[") + 1, -1)) && (n = n.slice(0, s.index) + "[" + "a".repeat(s[0].length - 2) + "]" + n.slice(this.tokenizer.rules.inline.reflinkSearch.lastIndex)); + } + for (; (s = this.tokenizer.rules.inline.anyPunctuation.exec(n)) !== null; ) n = n.slice(0, s.index) + "++" + n.slice(this.tokenizer.rules.inline.anyPunctuation.lastIndex); + let r; + for (; (s = this.tokenizer.rules.inline.blockSkip.exec(n)) !== null; ) r = s[2] ? s[2].length : 0, n = n.slice(0, s.index + r) + "[" + "a".repeat(s[0].length - r - 2) + "]" + n.slice(this.tokenizer.rules.inline.blockSkip.lastIndex); + n = this.options.hooks?.emStrongMask?.call({ lexer: this }, n) ?? n; + let i = false, o = "", u = 1 / 0; + for (; e; ) { + if (e.length < u) u = e.length; + else { + this.infiniteLoopError(e.charCodeAt(0)); + break; + } + i || (o = ""), i = false; + let a; + if (this.options.extensions?.inline?.some((p) => (a = p.call({ lexer: this }, e, t)) ? (e = e.substring(a.raw.length), t.push(a), true) : false)) continue; + if (a = this.tokenizer.escape(e)) { + e = e.substring(a.raw.length), t.push(a); + continue; + } + if (a = this.tokenizer.tag(e)) { + e = e.substring(a.raw.length), t.push(a); + continue; + } + if (a = this.tokenizer.link(e)) { + e = e.substring(a.raw.length), t.push(a); + continue; + } + if (a = this.tokenizer.reflink(e, this.tokens.links)) { + e = e.substring(a.raw.length); + let p = t.at(-1); + a.type === "text" && p?.type === "text" ? (p.raw += a.raw, p.text += a.text) : t.push(a); + continue; + } + if (a = this.tokenizer.emStrong(e, n, o)) { + e = e.substring(a.raw.length), t.push(a); + continue; + } + if (a = this.tokenizer.codespan(e)) { + e = e.substring(a.raw.length), t.push(a); + continue; + } + if (a = this.tokenizer.br(e)) { + e = e.substring(a.raw.length), t.push(a); + continue; + } + if (a = this.tokenizer.del(e, n, o)) { + e = e.substring(a.raw.length), t.push(a); + continue; + } + if (a = this.tokenizer.autolink(e)) { + e = e.substring(a.raw.length), t.push(a); + continue; + } + if (!this.state.inLink && (a = this.tokenizer.url(e))) { + e = e.substring(a.raw.length), t.push(a); + continue; + } + let c = e; + if (this.options.extensions?.startInline) { + let p = 1 / 0, k = e.slice(1), h; + this.options.extensions.startInline.forEach((R) => { + h = R.call({ lexer: this }, k), typeof h == "number" && h >= 0 && (p = Math.min(p, h)); + }), p < 1 / 0 && p >= 0 && (c = e.substring(0, p + 1)); + } + if (a = this.tokenizer.inlineText(c)) { + e = e.substring(a.raw.length), a.raw.slice(-1) !== "_" && (o = a.raw.slice(-1)), i = true; + let p = t.at(-1); + p?.type === "text" ? (p.raw += a.raw, p.text += a.text) : t.push(a); + continue; + } + if (e) { + this.infiniteLoopError(e.charCodeAt(0)); + break; + } + } + return t; + } + infiniteLoopError(e) { + let t = "Infinite loop on byte: " + e; + if (this.options.silent) console.error(t); + else throw new Error(t); + } + }; + var y = class { + constructor(e) { + __publicField(this, "options"); + __publicField(this, "parser"); + this.options = e || T; + } + space(e) { + return ""; + } + code({ text: e, lang: t, escaped: n }) { + let s = (t || "").match(m.notSpaceStart)?.[0], r = e.replace(m.endingNewline, "") + ` +`; + return s ? '

' + (n ? r : O(r, true)) + `
+` : "
" + (n ? r : O(r, true)) + `
+`; + } + blockquote({ tokens: e }) { + return `
+${this.parser.parse(e)}
+`; + } + html({ text: e }) { + return e; + } + def(e) { + return ""; + } + heading({ tokens: e, depth: t }) { + return `${this.parser.parseInline(e)} +`; + } + hr(e) { + return `
+`; + } + list(e) { + let t = e.ordered, n = e.start, s = ""; + for (let o = 0; o < e.items.length; o++) { + let u = e.items[o]; + s += this.listitem(u); + } + let r = t ? "ol" : "ul", i = t && n !== 1 ? ' start="' + n + '"' : ""; + return "<" + r + i + `> +` + s + " +`; + } + listitem(e) { + return `
  • ${this.parser.parse(e.tokens)}
    • +`; + } + checkbox({ checked: e }) { + return " '; + } + paragraph({ tokens: e }) { + return `

    ${this.parser.parseInline(e)}

    +`; + } + table(e) { + let t = "", n = ""; + for (let r = 0; r < e.header.length; r++) n += this.tablecell(e.header[r]); + t += this.tablerow({ text: n }); + let s = ""; + for (let r = 0; r < e.rows.length; r++) { + let i = e.rows[r]; + n = ""; + for (let o = 0; o < i.length; o++) n += this.tablecell(i[o]); + s += this.tablerow({ text: n }); + } + return s && (s = `${s}`), ` + +` + t + ` +` + s + `
    +`; + } + tablerow({ text: e }) { + return ` +${e} +`; + } + tablecell(e) { + let t = this.parser.parseInline(e.tokens), n = e.header ? "th" : "td"; + return (e.align ? `<${n} align="${e.align}">` : `<${n}>`) + t + ` +`; + } + strong({ tokens: e }) { + return `${this.parser.parseInline(e)}`; + } + em({ tokens: e }) { + return `${this.parser.parseInline(e)}`; + } + codespan({ text: e }) { + return `${O(e, true)}`; + } + br(e) { + return "
    "; + } + del({ tokens: e }) { + return `${this.parser.parseInline(e)}`; + } + link({ href: e, title: t, tokens: n }) { + let s = this.parser.parseInline(n), r = V(e); + if (r === null) return s; + e = r; + let i = '    ", i; + } + image({ href: e, title: t, text: n, tokens: s }) { + s && (n = this.parser.parseInline(s, this.parser.textRenderer)); + let r = V(e); + if (r === null) return O(n); + e = r; + let i = `${O(n)} { + let o = r[i].flat(1 / 0); + n = n.concat(this.walkTokens(o, t)); + }) : r.tokens && (n = n.concat(this.walkTokens(r.tokens, t))); + } + } + return n; + } + use(...e) { + let t = this.defaults.extensions || { renderers: {}, childTokens: {} }; + return e.forEach((n) => { + let s = { ...n }; + if (s.async = this.defaults.async || s.async || false, n.extensions && (n.extensions.forEach((r) => { + if (!r.name) throw new Error("extension name required"); + if ("renderer" in r) { + let i = t.renderers[r.name]; + i ? t.renderers[r.name] = function(...o) { + let u = r.renderer.apply(this, o); + return u === false && (u = i.apply(this, o)), u; + } : t.renderers[r.name] = r.renderer; + } + if ("tokenizer" in r) { + if (!r.level || r.level !== "block" && r.level !== "inline") throw new Error("extension level must be 'block' or 'inline'"); + let i = t[r.level]; + i ? i.unshift(r.tokenizer) : t[r.level] = [r.tokenizer], r.start && (r.level === "block" ? t.startBlock ? t.startBlock.push(r.start) : t.startBlock = [r.start] : r.level === "inline" && (t.startInline ? t.startInline.push(r.start) : t.startInline = [r.start])); + } + "childTokens" in r && r.childTokens && (t.childTokens[r.name] = r.childTokens); + }), s.extensions = t), n.renderer) { + let r = this.defaults.renderer || new y(this.defaults); + for (let i in n.renderer) { + if (!(i in r)) throw new Error(`renderer '${i}' does not exist`); + if (["options", "parser"].includes(i)) continue; + let o = i, u = n.renderer[o], a = r[o]; + r[o] = (...c) => { + let p = u.apply(r, c); + return p === false && (p = a.apply(r, c)), p || ""; + }; + } + s.renderer = r; + } + if (n.tokenizer) { + let r = this.defaults.tokenizer || new w(this.defaults); + for (let i in n.tokenizer) { + if (!(i in r)) throw new Error(`tokenizer '${i}' does not exist`); + if (["options", "rules", "lexer"].includes(i)) continue; + let o = i, u = n.tokenizer[o], a = r[o]; + r[o] = (...c) => { + let p = u.apply(r, c); + return p === false && (p = a.apply(r, c)), p; + }; + } + s.tokenizer = r; + } + if (n.hooks) { + let r = this.defaults.hooks || new P(); + for (let i in n.hooks) { + if (!(i in r)) throw new Error(`hook '${i}' does not exist`); + if (["options", "block"].includes(i)) continue; + let o = i, u = n.hooks[o], a = r[o]; + P.passThroughHooks.has(i) ? r[o] = (c) => { + if (this.defaults.async && P.passThroughHooksRespectAsync.has(i)) return (async () => { + let k = await u.call(r, c); + return a.call(r, k); + })(); + let p = u.call(r, c); + return a.call(r, p); + } : r[o] = (...c) => { + if (this.defaults.async) return (async () => { + let k = await u.apply(r, c); + return k === false && (k = await a.apply(r, c)), k; + })(); + let p = u.apply(r, c); + return p === false && (p = a.apply(r, c)), p; + }; + } + s.hooks = r; + } + if (n.walkTokens) { + let r = this.defaults.walkTokens, i = n.walkTokens; + s.walkTokens = function(o) { + let u = []; + return u.push(i.call(this, o)), r && (u = u.concat(r.call(this, o))), u; + }; + } + this.defaults = { ...this.defaults, ...s }; + }), this; + } + setOptions(e) { + return this.defaults = { ...this.defaults, ...e }, this; + } + lexer(e, t) { + return x.lex(e, t ?? this.defaults); + } + parser(e, t) { + return b.parse(e, t ?? this.defaults); + } + parseMarkdown(e) { + return (n, s) => { + let r = { ...s }, i = { ...this.defaults, ...r }, o = this.onError(!!i.silent, !!i.async); + if (this.defaults.async === true && r.async === false) return o(new Error("marked(): The async option was set to true by an extension. Remove async: false from the parse options object to return a Promise.")); + if (typeof n > "u" || n === null) return o(new Error("marked(): input parameter is undefined or null")); + if (typeof n != "string") return o(new Error("marked(): input parameter is of type " + Object.prototype.toString.call(n) + ", string expected")); + if (i.hooks && (i.hooks.options = i, i.hooks.block = e), i.async) return (async () => { + let u = i.hooks ? await i.hooks.preprocess(n) : n, c = await (i.hooks ? await i.hooks.provideLexer(e) : e ? x.lex : x.lexInline)(u, i), p = i.hooks ? await i.hooks.processAllTokens(c) : c; + i.walkTokens && await Promise.all(this.walkTokens(p, i.walkTokens)); + let h = await (i.hooks ? await i.hooks.provideParser(e) : e ? b.parse : b.parseInline)(p, i); + return i.hooks ? await i.hooks.postprocess(h) : h; + })().catch(o); + try { + i.hooks && (n = i.hooks.preprocess(n)); + let a = (i.hooks ? i.hooks.provideLexer(e) : e ? x.lex : x.lexInline)(n, i); + i.hooks && (a = i.hooks.processAllTokens(a)), i.walkTokens && this.walkTokens(a, i.walkTokens); + let p = (i.hooks ? i.hooks.provideParser(e) : e ? b.parse : b.parseInline)(a, i); + return i.hooks && (p = i.hooks.postprocess(p)), p; + } catch (u) { + return o(u); + } + }; + } + onError(e, t) { + return (n) => { + if (n.message += ` +Please report this to https://github.com/markedjs/marked.`, e) { + let s = "

    An error occurred:

    " + O(n.message + "", true) + "
    "; + return t ? Promise.resolve(s) : s; + } + if (t) return Promise.reject(n); + throw n; + }; + } + }; + var z = new q(); + function g(l3, e) { + return z.parse(l3, e); + } + g.options = g.setOptions = function(l3) { + return z.setOptions(l3), g.defaults = z.defaults, N(g.defaults), g; + }; + g.getDefaults = M; + g.defaults = T; + g.use = function(...l3) { + return z.use(...l3), g.defaults = z.defaults, N(g.defaults), g; + }; + g.walkTokens = function(l3, e) { + return z.walkTokens(l3, e); + }; + g.parseInline = z.parseInline; + g.Parser = b; + g.parser = b.parse; + g.Renderer = y; + g.TextRenderer = L; + g.Lexer = x; + g.lexer = x.lex; + g.Tokenizer = w; + g.Hooks = P; + g.parse = g; + var Ft = g.options; + var Ut = g.setOptions; + var Kt = g.use; + var Wt = g.walkTokens; + var Xt = g.parseInline; + var Vt = b.parse; + var Yt = x.lex; + + // node_modules/get-east-asian-width/lookup-data.js + var ambiguousMinimalCodePoint = 161; + var ambiguousMaximumCodePoint = 1114109; + var ambiguousRanges = [161, 161, 164, 164, 167, 168, 170, 170, 173, 174, 176, 180, 182, 186, 188, 191, 198, 198, 208, 208, 215, 216, 222, 225, 230, 230, 232, 234, 236, 237, 240, 240, 242, 243, 247, 250, 252, 252, 254, 254, 257, 257, 273, 273, 275, 275, 283, 283, 294, 295, 299, 299, 305, 307, 312, 312, 319, 322, 324, 324, 328, 331, 333, 333, 338, 339, 358, 359, 363, 363, 462, 462, 464, 464, 466, 466, 468, 468, 470, 470, 472, 472, 474, 474, 476, 476, 593, 593, 609, 609, 708, 708, 711, 711, 713, 715, 717, 717, 720, 720, 728, 731, 733, 733, 735, 735, 768, 879, 913, 929, 931, 937, 945, 961, 963, 969, 1025, 1025, 1040, 1103, 1105, 1105, 8208, 8208, 8211, 8214, 8216, 8217, 8220, 8221, 8224, 8226, 8228, 8231, 8240, 8240, 8242, 8243, 8245, 8245, 8251, 8251, 8254, 8254, 8308, 8308, 8319, 8319, 8321, 8324, 8364, 8364, 8451, 8451, 8453, 8453, 8457, 8457, 8467, 8467, 8470, 8470, 8481, 8482, 8486, 8486, 8491, 8491, 8531, 8532, 8539, 8542, 8544, 8555, 8560, 8569, 8585, 8585, 8592, 8601, 8632, 8633, 8658, 8658, 8660, 8660, 8679, 8679, 8704, 8704, 8706, 8707, 8711, 8712, 8715, 8715, 8719, 8719, 8721, 8721, 8725, 8725, 8730, 8730, 8733, 8736, 8739, 8739, 8741, 8741, 8743, 8748, 8750, 8750, 8756, 8759, 8764, 8765, 8776, 8776, 8780, 8780, 8786, 8786, 8800, 8801, 8804, 8807, 8810, 8811, 8814, 8815, 8834, 8835, 8838, 8839, 8853, 8853, 8857, 8857, 8869, 8869, 8895, 8895, 8978, 8978, 9312, 9449, 9451, 9547, 9552, 9587, 9600, 9615, 9618, 9621, 9632, 9633, 9635, 9641, 9650, 9651, 9654, 9655, 9660, 9661, 9664, 9665, 9670, 9672, 9675, 9675, 9678, 9681, 9698, 9701, 9711, 9711, 9733, 9734, 9737, 9737, 9742, 9743, 9756, 9756, 9758, 9758, 9792, 9792, 9794, 9794, 9824, 9825, 9827, 9829, 9831, 9834, 9836, 9837, 9839, 9839, 9886, 9887, 9919, 9919, 9926, 9933, 9935, 9939, 9941, 9953, 9955, 9955, 9960, 9961, 9963, 9969, 9972, 9972, 9974, 9977, 9979, 9980, 9982, 9983, 10045, 10045, 10102, 10111, 11094, 11097, 12872, 12879, 57344, 63743, 65024, 65039, 65533, 65533, 127232, 127242, 127248, 127277, 127280, 127337, 127344, 127373, 127375, 127376, 127387, 127404, 917760, 917999, 983040, 1048573, 1048576, 1114109]; + var fullwidthMinimalCodePoint = 12288; + var fullwidthMaximumCodePoint = 65510; + var fullwidthRanges = [12288, 12288, 65281, 65376, 65504, 65510]; + var wideMinimalCodePoint = 4352; + var wideMaximumCodePoint = 262141; + var wideRanges = [4352, 4447, 8986, 8987, 9001, 9002, 9193, 9196, 9200, 9200, 9203, 9203, 9725, 9726, 9748, 9749, 9776, 9783, 9800, 9811, 9855, 9855, 9866, 9871, 9875, 9875, 9889, 9889, 9898, 9899, 9917, 9918, 9924, 9925, 9934, 9934, 9940, 9940, 9962, 9962, 9970, 9971, 9973, 9973, 9978, 9978, 9981, 9981, 9989, 9989, 9994, 9995, 10024, 10024, 10060, 10060, 10062, 10062, 10067, 10069, 10071, 10071, 10133, 10135, 10160, 10160, 10175, 10175, 11035, 11036, 11088, 11088, 11093, 11093, 11904, 11929, 11931, 12019, 12032, 12245, 12272, 12287, 12289, 12350, 12353, 12438, 12441, 12543, 12549, 12591, 12593, 12686, 12688, 12773, 12783, 12830, 12832, 12871, 12880, 42124, 42128, 42182, 43360, 43388, 44032, 55203, 63744, 64255, 65040, 65049, 65072, 65106, 65108, 65126, 65128, 65131, 94176, 94180, 94192, 94198, 94208, 101589, 101631, 101662, 101760, 101874, 110576, 110579, 110581, 110587, 110589, 110590, 110592, 110882, 110898, 110898, 110928, 110930, 110933, 110933, 110948, 110951, 110960, 111355, 119552, 119638, 119648, 119670, 126980, 126980, 127183, 127183, 127374, 127374, 127377, 127386, 127488, 127490, 127504, 127547, 127552, 127560, 127568, 127569, 127584, 127589, 127744, 127776, 127789, 127797, 127799, 127868, 127870, 127891, 127904, 127946, 127951, 127955, 127968, 127984, 127988, 127988, 127992, 128062, 128064, 128064, 128066, 128252, 128255, 128317, 128331, 128334, 128336, 128359, 128378, 128378, 128405, 128406, 128420, 128420, 128507, 128591, 128640, 128709, 128716, 128716, 128720, 128722, 128725, 128728, 128732, 128735, 128747, 128748, 128756, 128764, 128992, 129003, 129008, 129008, 129292, 129338, 129340, 129349, 129351, 129535, 129648, 129660, 129664, 129674, 129678, 129734, 129736, 129736, 129741, 129756, 129759, 129770, 129775, 129784, 131072, 196605, 196608, 262141]; + + // node_modules/get-east-asian-width/utilities.js + var isInRange = (ranges, codePoint) => { + let low = 0; + let high = Math.floor(ranges.length / 2) - 1; + while (low <= high) { + const mid = Math.floor((low + high) / 2); + const i = mid * 2; + if (codePoint < ranges[i]) { + high = mid - 1; + } else if (codePoint > ranges[i + 1]) { + low = mid + 1; + } else { + return true; + } + } + return false; + }; + + // node_modules/get-east-asian-width/lookup.js + var commonCjkCodePoint = 19968; + var [wideFastPathStart, wideFastPathEnd] = /* @__PURE__ */ findWideFastPathRange(wideRanges); + function findWideFastPathRange(ranges) { + let fastPathStart = ranges[0]; + let fastPathEnd = ranges[1]; + for (let index = 0; index < ranges.length; index += 2) { + const start = ranges[index]; + const end = ranges[index + 1]; + if (commonCjkCodePoint >= start && commonCjkCodePoint <= end) { + return [start, end]; + } + if (end - start > fastPathEnd - fastPathStart) { + fastPathStart = start; + fastPathEnd = end; + } + } + return [fastPathStart, fastPathEnd]; + } + var isAmbiguous = (codePoint) => { + if (codePoint < ambiguousMinimalCodePoint || codePoint > ambiguousMaximumCodePoint) { + return false; + } + return isInRange(ambiguousRanges, codePoint); + }; + var isFullWidth = (codePoint) => { + if (codePoint < fullwidthMinimalCodePoint || codePoint > fullwidthMaximumCodePoint) { + return false; + } + return isInRange(fullwidthRanges, codePoint); + }; + var isWide = (codePoint) => { + if (codePoint >= wideFastPathStart && codePoint <= wideFastPathEnd) { + return true; + } + if (codePoint < wideMinimalCodePoint || codePoint > wideMaximumCodePoint) { + return false; + } + return isInRange(wideRanges, codePoint); + }; + + // node_modules/get-east-asian-width/index.js + function validate(codePoint) { + if (!Number.isSafeInteger(codePoint)) { + throw new TypeError(`Expected a code point, got \`${typeof codePoint}\`.`); + } + } + function eastAsianWidth(codePoint, { ambiguousAsWide = false } = {}) { + validate(codePoint); + if (isFullWidth(codePoint) || isWide(codePoint) || ambiguousAsWide && isAmbiguous(codePoint)) { + return 2; + } + return 1; + } + + // packages/tui/src/utils.ts + var graphemeSegmenter = new Intl.Segmenter(void 0, { granularity: "grapheme" }); + var wordSegmenter = new Intl.Segmenter(void 0, { granularity: "word" }); + function couldBeEmoji(segment) { + const cp = segment.codePointAt(0); + return cp >= 126976 && cp <= 130047 || // Emoji and Pictograph + cp >= 8960 && cp <= 9215 || // Misc technical + cp >= 9728 && cp <= 10175 || // Misc symbols, dingbats + cp >= 11088 && cp <= 11093 || // Specific stars/circles + segment.includes("\uFE0F") || // Contains VS16 (emoji presentation selector) + segment.length > 2; + } + var zeroWidthRegex = new RegExp("^(?:\\p{Default_Ignorable_Code_Point}|\\p{Control}|\\p{Mark}|\\p{Surrogate})+$", "v"); + var leadingNonPrintingRegex = new RegExp("^[\\p{Default_Ignorable_Code_Point}\\p{Control}\\p{Format}\\p{Mark}\\p{Surrogate}]+", "v"); + var nonPrintingCharRegex = new RegExp("^(?:\\p{Default_Ignorable_Code_Point}|\\p{Control}|\\p{Format}|\\p{Mark}|\\p{Surrogate})$", "v"); + var markCharRegex = new RegExp("^\\p{Mark}$", "v"); + var terminalSpacingMarkRegex = new RegExp("^(?:[\\p{Spacing_Mark}--[\\u1734\\u302E\\u302F]]|[\\u065F\\u0F7F\\u102B\\u102C\\u1031\\u1033-\\u1035\\u1038\\u103A-\\u103E])+$", "v"); + var rgiEmojiRegex = new RegExp("^\\p{RGI_Emoji}$", "v"); + var WIDTH_CACHE_SIZE = 512; + var widthCache = /* @__PURE__ */ new Map(); + var cjkBreakRegex = /[\p{Script_Extensions=Han}\p{Script_Extensions=Hiragana}\p{Script_Extensions=Katakana}\p{Script_Extensions=Hangul}\p{Script_Extensions=Bopomofo}]/u; + function isPrintableAscii(str) { + for (let i = 0; i < str.length; i++) { + const code = str.charCodeAt(i); + if (code < 32 || code > 126) { + return false; + } + } + return true; + } + function graphemeWidth(segment) { + if (segment === " ") { + return 3; + } + if (terminalSpacingMarkRegex.test(segment)) { + return [...segment].length; + } + if (zeroWidthRegex.test(segment)) { + return 0; + } + if (couldBeEmoji(segment) && rgiEmojiRegex.test(segment)) { + return 2; + } + const base = segment.replace(leadingNonPrintingRegex, ""); + const cp = base.codePointAt(0); + if (cp === void 0) { + return 0; + } + if (cp >= 127462 && cp <= 127487) { + return 2; + } + let width = eastAsianWidth(cp); + let followsMark = false; + const chars = [...base]; + for (const char of chars.slice(1)) { + if (terminalSpacingMarkRegex.test(char)) { + width += 1; + followsMark = false; + } else if (markCharRegex.test(char)) { + followsMark = true; + } else if (!nonPrintingCharRegex.test(char)) { + const c = char.codePointAt(0); + if (followsMark || c >= 65280 && c <= 65519) { + width += eastAsianWidth(c); + } else if (c === 3635 || c === 3763) { + width += 1; + } + followsMark = false; + } + } + return width; + } + function visibleWidth(str) { + if (str.length === 0) { + return 0; + } + if (isPrintableAscii(str)) { + return str.length; + } + const cached = widthCache.get(str); + if (cached !== void 0) { + return cached; + } + let clean = str; + if (str.includes(" ")) { + clean = clean.replace(/\t/g, " "); + } + if (clean.includes("\x1B")) { + let stripped = ""; + let i = 0; + while (i < clean.length) { + const ansi = extractAnsiCode(clean, i); + if (ansi) { + i += ansi.length; + continue; + } + stripped += clean[i]; + i++; + } + clean = stripped; + } + let width = 0; + for (const { segment } of graphemeSegmenter.segment(clean)) { + width += graphemeWidth(segment); + } + if (widthCache.size >= WIDTH_CACHE_SIZE) { + const firstKey = widthCache.keys().next().value; + if (firstKey !== void 0) { + widthCache.delete(firstKey); + } + } + widthCache.set(str, width); + return width; + } + function extractAnsiCode(str, pos) { + if (pos >= str.length || str[pos] !== "\x1B") return null; + const next = str[pos + 1]; + if (next === "[") { + let j2 = pos + 2; + while (j2 < str.length && !/[mGKHJ]/.test(str[j2])) j2++; + if (j2 < str.length) return { code: str.substring(pos, j2 + 1), length: j2 + 1 - pos }; + return null; + } + if (next === "]") { + let j2 = pos + 2; + while (j2 < str.length) { + if (str[j2] === "\x07") return { code: str.substring(pos, j2 + 1), length: j2 + 1 - pos }; + if (str[j2] === "\x1B" && str[j2 + 1] === "\\") return { code: str.substring(pos, j2 + 2), length: j2 + 2 - pos }; + j2++; + } + return null; + } + if (next === "_") { + let j2 = pos + 2; + while (j2 < str.length) { + if (str[j2] === "\x07") return { code: str.substring(pos, j2 + 1), length: j2 + 1 - pos }; + if (str[j2] === "\x1B" && str[j2 + 1] === "\\") return { code: str.substring(pos, j2 + 2), length: j2 + 2 - pos }; + j2++; + } + return null; + } + return null; + } + function parseOsc8Hyperlink(ansiCode) { + if (!ansiCode.startsWith("\x1B]8;")) { + return void 0; + } + const terminator = ansiCode.endsWith("\x07") ? "\x07" : "\x1B\\"; + const body = ansiCode.slice(4, terminator === "\x07" ? -1 : -2); + const separatorIndex = body.indexOf(";"); + if (separatorIndex === -1) { + return void 0; + } + const params = body.slice(0, separatorIndex); + const url = body.slice(separatorIndex + 1); + if (!url) { + return null; + } + return { params, url, terminator }; + } + function formatOsc8Hyperlink(hyperlink2) { + return `\x1B]8;${hyperlink2.params};${hyperlink2.url}${hyperlink2.terminator}`; + } + function formatOsc8Close(terminator) { + return `\x1B]8;;${terminator}`; + } + var AnsiCodeTracker = class { + constructor() { + // Track individual attributes separately so we can reset them specifically + this.bold = false; + this.dim = false; + this.italic = false; + this.underline = false; + this.blink = false; + this.inverse = false; + this.hidden = false; + this.strikethrough = false; + this.fgColor = null; + // Stores the full code like "31" or "38;5;240" + this.bgColor = null; + // Stores the full code like "41" or "48;5;240" + this.activeHyperlink = null; + } + process(ansiCode) { + const hyperlink2 = parseOsc8Hyperlink(ansiCode); + if (hyperlink2 !== void 0) { + this.activeHyperlink = hyperlink2; + return; + } + if (!ansiCode.endsWith("m")) { + return; + } + const match = ansiCode.match(/\x1b\[([\d;]*)m/); + if (!match) return; + const params = match[1]; + if (params === "" || params === "0") { + this.reset(); + return; + } + const parts = params.split(";"); + let i = 0; + while (i < parts.length) { + const code = Number.parseInt(parts[i], 10); + if (code === 38 || code === 48) { + if (parts[i + 1] === "5" && parts[i + 2] !== void 0) { + const colorCode = `${parts[i]};${parts[i + 1]};${parts[i + 2]}`; + if (code === 38) { + this.fgColor = colorCode; + } else { + this.bgColor = colorCode; + } + i += 3; + continue; + } else if (parts[i + 1] === "2" && parts[i + 4] !== void 0) { + const colorCode = `${parts[i]};${parts[i + 1]};${parts[i + 2]};${parts[i + 3]};${parts[i + 4]}`; + if (code === 38) { + this.fgColor = colorCode; + } else { + this.bgColor = colorCode; + } + i += 5; + continue; + } + } + switch (code) { + case 0: + this.reset(); + break; + case 1: + this.bold = true; + break; + case 2: + this.dim = true; + break; + case 3: + this.italic = true; + break; + case 4: + this.underline = true; + break; + case 5: + this.blink = true; + break; + case 7: + this.inverse = true; + break; + case 8: + this.hidden = true; + break; + case 9: + this.strikethrough = true; + break; + case 21: + this.bold = false; + break; + case 22: + this.bold = false; + this.dim = false; + break; + case 23: + this.italic = false; + break; + case 24: + this.underline = false; + break; + case 25: + this.blink = false; + break; + case 27: + this.inverse = false; + break; + case 28: + this.hidden = false; + break; + case 29: + this.strikethrough = false; + break; + case 39: + this.fgColor = null; + break; + case 49: + this.bgColor = null; + break; + default: + if (code >= 30 && code <= 37 || code >= 90 && code <= 97) { + this.fgColor = String(code); + } else if (code >= 40 && code <= 47 || code >= 100 && code <= 107) { + this.bgColor = String(code); + } + break; + } + i++; + } + } + reset() { + this.bold = false; + this.dim = false; + this.italic = false; + this.underline = false; + this.blink = false; + this.inverse = false; + this.hidden = false; + this.strikethrough = false; + this.fgColor = null; + this.bgColor = null; + } + /** Clear all state for reuse. */ + clear() { + this.reset(); + this.activeHyperlink = null; + } + getActiveCodes() { + const codes = []; + if (this.bold) codes.push("1"); + if (this.dim) codes.push("2"); + if (this.italic) codes.push("3"); + if (this.underline) codes.push("4"); + if (this.blink) codes.push("5"); + if (this.inverse) codes.push("7"); + if (this.hidden) codes.push("8"); + if (this.strikethrough) codes.push("9"); + if (this.fgColor) codes.push(this.fgColor); + if (this.bgColor) codes.push(this.bgColor); + let result = codes.length > 0 ? `\x1B[${codes.join(";")}m` : ""; + if (this.activeHyperlink) { + result += formatOsc8Hyperlink(this.activeHyperlink); + } + return result; + } + hasActiveCodes() { + return this.bold || this.dim || this.italic || this.underline || this.blink || this.inverse || this.hidden || this.strikethrough || this.fgColor !== null || this.bgColor !== null || this.activeHyperlink !== null; + } + /** + * Get reset codes for attributes that need to be turned off at line end. + * Underline must be closed to prevent bleeding into padding. + * Active OSC 8 hyperlinks must be closed and re-opened on the next line. + * Returns empty string if no attributes need closing. + */ + getLineEndReset() { + let result = ""; + if (this.underline) { + result += "\x1B[24m"; + } + if (this.activeHyperlink) { + result += formatOsc8Close(this.activeHyperlink.terminator); + } + return result; + } + }; + function updateTrackerFromText(text, tracker) { + let i = 0; + while (i < text.length) { + const ansiResult = extractAnsiCode(text, i); + if (ansiResult) { + tracker.process(ansiResult.code); + i += ansiResult.length; + } else { + i++; + } + } + } + function splitIntoTokensWithAnsi(text) { + const tokens = []; + let current = ""; + let pendingAnsi = ""; + let currentKind = null; + let i = 0; + const flushCurrent = () => { + if (!current) { + return; + } + tokens.push(current); + current = ""; + currentKind = null; + }; + while (i < text.length) { + const ansiResult = extractAnsiCode(text, i); + if (ansiResult) { + pendingAnsi += ansiResult.code; + i += ansiResult.length; + continue; + } + let end = i; + while (end < text.length && !extractAnsiCode(text, end)) { + end++; + } + for (const { segment } of graphemeSegmenter.segment(text.slice(i, end))) { + const segmentIsSpace = segment === " "; + if (!segmentIsSpace && cjkBreakRegex.test(segment)) { + flushCurrent(); + const token = pendingAnsi + segment; + pendingAnsi = ""; + tokens.push(token); + continue; + } + const segmentKind = segmentIsSpace ? "space" : "word"; + if (current && currentKind !== segmentKind) { + flushCurrent(); + } + if (pendingAnsi) { + current += pendingAnsi; + pendingAnsi = ""; + } + currentKind = segmentKind; + current += segment; + } + i = end; + } + if (pendingAnsi) { + if (current) { + current += pendingAnsi; + } else if (tokens.length > 0) { + tokens[tokens.length - 1] += pendingAnsi; + } else { + current = pendingAnsi; + } + } + if (current) { + tokens.push(current); + } + return tokens; + } + function wrapTextWithAnsi(text, width) { + if (!text) { + return [""]; + } + const inputLines = text.split(/\r\n|\r|\n/); + const result = []; + const tracker = new AnsiCodeTracker(); + for (const inputLine of inputLines) { + const prefix = result.length > 0 ? tracker.getActiveCodes() : ""; + const wrappedLines = wrapSingleLine(prefix + inputLine, width); + for (const wrappedLine of wrappedLines) { + result.push(wrappedLine); + } + updateTrackerFromText(inputLine, tracker); + } + return result.length > 0 ? result : [""]; + } + function wrapSingleLine(line, width) { + if (!line) { + return [""]; + } + const visibleLength = visibleWidth(line); + if (visibleLength <= width) { + return [line]; + } + const wrapped = []; + const tracker = new AnsiCodeTracker(); + const tokens = splitIntoTokensWithAnsi(line); + let currentLine = ""; + let currentVisibleLength = 0; + for (const token of tokens) { + const tokenVisibleLength = visibleWidth(token); + const isWhitespace = token.trim() === ""; + if (tokenVisibleLength > width && !isWhitespace) { + if (currentLine) { + const lineEndReset = tracker.getLineEndReset(); + if (lineEndReset) { + currentLine += lineEndReset; + } + wrapped.push(currentLine); + currentLine = ""; + currentVisibleLength = 0; + } + const broken = breakLongWord(token, width, tracker); + for (let i = 0; i < broken.length - 1; i++) { + wrapped.push(broken[i]); + } + currentLine = broken[broken.length - 1]; + currentVisibleLength = visibleWidth(currentLine); + continue; + } + const totalNeeded = currentVisibleLength + tokenVisibleLength; + if (totalNeeded > width && currentVisibleLength > 0) { + let lineToWrap = currentLine.trimEnd(); + const lineEndReset = tracker.getLineEndReset(); + if (lineEndReset) { + lineToWrap += lineEndReset; + } + wrapped.push(lineToWrap); + if (isWhitespace) { + currentLine = tracker.getActiveCodes(); + currentVisibleLength = 0; + } else { + currentLine = tracker.getActiveCodes() + token; + currentVisibleLength = tokenVisibleLength; + } + } else { + currentLine += token; + currentVisibleLength += tokenVisibleLength; + } + updateTrackerFromText(token, tracker); + } + if (currentLine) { + wrapped.push(currentLine); + } + return wrapped.length > 0 ? wrapped.map((line2) => line2.trimEnd()) : [""]; + } + function breakLongWord(word, width, tracker) { + const lines = []; + let currentLine = tracker.getActiveCodes(); + let currentWidth = 0; + let i = 0; + const segments = []; + while (i < word.length) { + const ansiResult = extractAnsiCode(word, i); + if (ansiResult) { + segments.push({ type: "ansi", value: ansiResult.code }); + i += ansiResult.length; + } else { + let end = i; + while (end < word.length) { + const nextAnsi = extractAnsiCode(word, end); + if (nextAnsi) break; + end++; + } + const textPortion = word.slice(i, end); + for (const seg of graphemeSegmenter.segment(textPortion)) { + segments.push({ type: "grapheme", value: seg.segment }); + } + i = end; + } + } + for (const seg of segments) { + if (seg.type === "ansi") { + currentLine += seg.value; + tracker.process(seg.value); + continue; + } + const grapheme = seg.value; + if (!grapheme) continue; + const graphemeWidth2 = visibleWidth(grapheme); + if (currentWidth + graphemeWidth2 > width) { + const lineEndReset = tracker.getLineEndReset(); + if (lineEndReset) { + currentLine += lineEndReset; + } + lines.push(currentLine); + currentLine = tracker.getActiveCodes(); + currentWidth = 0; + } + currentLine += grapheme; + currentWidth += graphemeWidth2; + } + if (currentLine) { + lines.push(currentLine); + } + return lines.length > 0 ? lines : [""]; + } + function applyBackgroundToLine(line, width, bgFn) { + const visibleLen = visibleWidth(line); + const paddingNeeded = Math.max(0, width - visibleLen); + const padding = " ".repeat(paddingNeeded); + const withPadding = line + padding; + return bgFn(withPadding); + } + var pooledStyleTracker = new AnsiCodeTracker(); + + // packages/tui/src/latex.ts + var SYMBOLS = { + alpha: "\u03B1", + beta: "\u03B2", + gamma: "\u03B3", + delta: "\u03B4", + epsilon: "\u03F5", + varepsilon: "\u03B5", + zeta: "\u03B6", + eta: "\u03B7", + theta: "\u03B8", + vartheta: "\u03D1", + iota: "\u03B9", + kappa: "\u03BA", + varkappa: "\u03F0", + lambda: "\u03BB", + mu: "\u03BC", + nu: "\u03BD", + xi: "\u03BE", + pi: "\u03C0", + varpi: "\u03D6", + rho: "\u03C1", + varrho: "\u03F1", + sigma: "\u03C3", + varsigma: "\u03C2", + tau: "\u03C4", + upsilon: "\u03C5", + phi: "\u03D5", + varphi: "\u03C6", + chi: "\u03C7", + psi: "\u03C8", + omega: "\u03C9", + Gamma: "\u0393", + Delta: "\u0394", + Theta: "\u0398", + Lambda: "\u039B", + Xi: "\u039E", + Pi: "\u03A0", + Sigma: "\u03A3", + Upsilon: "\u03A5", + Phi: "\u03A6", + Psi: "\u03A8", + Omega: "\u03A9", + pm: "\xB1", + mp: "\u2213", + times: "\xD7", + div: "\xF7", + cdot: "\xB7", + ast: "\u2217", + star: "\u22C6", + circ: "\u2218", + bullet: "\u2022", + oplus: "\u2295", + ominus: "\u2296", + otimes: "\u2297", + oslash: "\u2298", + odot: "\u2299", + bigcirc: "\u25CB", + dagger: "\u2020", + ddagger: "\u2021", + amalg: "\u2A3F", + uplus: "\u228E", + sqcap: "\u2293", + sqcup: "\u2294", + triangleleft: "\u25C1", + triangleright: "\u25B7", + wr: "\u2240", + cap: "\u2229", + cup: "\u222A", + bigcap: "\u22C2", + bigcup: "\u22C3", + bigwedge: "\u22C0", + bigvee: "\u22C1", + bigsqcup: "\u2A06", + biguplus: "\u2A04", + bigoplus: "\u2A01", + bigotimes: "\u2A02", + bigodot: "\u2A00", + setminus: "\u2216", + in: "\u2208", + notin: "\u2209", + ni: "\u220B", + subset: "\u2282", + supset: "\u2283", + subseteq: "\u2286", + supseteq: "\u2287", + sqsubset: "\u228F", + sqsupset: "\u2290", + sqsubseteq: "\u2291", + sqsupseteq: "\u2292", + prec: "\u227A", + preceq: "\u227C", + succ: "\u227B", + succeq: "\u227D", + ll: "\u226A", + gg: "\u226B", + le: "\u2264", + leq: "\u2264", + leqslant: "\u2264", + ge: "\u2265", + geq: "\u2265", + geqslant: "\u2265", + ne: "\u2260", + neq: "\u2260", + equiv: "\u2261", + approx: "\u2248", + sim: "\u223C", + simeq: "\u2243", + cong: "\u2245", + asymp: "\u224D", + doteq: "\u2250", + propto: "\u221D", + parallel: "\u2225", + perp: "\u22A5", + mid: "\u2223", + vdash: "\u22A2", + dashv: "\u22A3", + models: "\u22A8", + Vdash: "\u22A9", + Vvdash: "\u22AA", + nvdash: "\u22AC", + nvDash: "\u22AD", + forall: "\u2200", + exists: "\u2203", + nexists: "\u2204", + neg: "\xAC", + land: "\u2227", + wedge: "\u2227", + lor: "\u2228", + vee: "\u2228", + to: "\u2192", + rightarrow: "\u2192", + longrightarrow: "\u2192", + leftarrow: "\u2190", + longleftarrow: "\u2190", + gets: "\u2190", + leftrightarrow: "\u2194", + longleftrightarrow: "\u2194", + hookleftarrow: "\u21A9", + hookrightarrow: "\u21AA", + twoheadleftarrow: "\u219E", + twoheadrightarrow: "\u21A0", + leftharpoonup: "\u21BC", + leftharpoondown: "\u21BD", + rightharpoonup: "\u21C0", + rightharpoondown: "\u21C1", + rightleftharpoons: "\u21CC", + leftrightharpoons: "\u21CB", + nearrow: "\u2197", + searrow: "\u2198", + swarrow: "\u2199", + nwarrow: "\u2196", + rightsquigarrow: "\u21DD", + leadsto: "\u21DD", + Rightarrow: "\u21D2", + Longrightarrow: "\u21D2", + Leftarrow: "\u21D0", + Longleftarrow: "\u21D0", + Leftrightarrow: "\u21D4", + Longleftrightarrow: "\u21D4", + implies: "\u21D2", + iff: "\u21D4", + mapsto: "\u21A6", + longmapsto: "\u21A6", + uparrow: "\u2191", + downarrow: "\u2193", + partial: "\u2202", + nabla: "\u2207", + int: "\u222B", + iint: "\u222C", + iiint: "\u222D", + oint: "\u222E", + sum: "\u2211", + prod: "\u220F", + coprod: "\u2210", + infty: "\u221E", + emptyset: "\u2205", + varnothing: "\u2205", + angle: "\u2220", + therefore: "\u2234", + because: "\u2235", + aleph: "\u2135", + beth: "\u2136", + gimel: "\u2137", + daleth: "\u2138", + top: "\u22A4", + bot: "\u22A5", + triangle: "\u25B3", + square: "\u25A1", + lozenge: "\u25CA", + checkmark: "\u2713", + complement: "\u2201", + wp: "\u2118", + prime: "\u2032", + ldots: "\u2026", + dots: "\u2026", + cdots: "\u22EF", + vdots: "\u22EE", + ddots: "\u22F1", + ell: "\u2113", + hbar: "\u210F", + Im: "\u2111", + Re: "\u211C", + langle: "\u27E8", + rangle: "\u27E9", + vert: "|", + lvert: "|", + rvert: "|", + Vert: "\u2016", + lVert: "\u2016", + rVert: "\u2016", + lbrace: "{", + rbrace: "}", + backslash: "\\", + lfloor: "\u230A", + rfloor: "\u230B", + lceil: "\u2308", + rceil: "\u2309", + colon: ":" + }; + var NAMED_OPERATORS = /* @__PURE__ */ new Set([ + "arccos", + "arcsin", + "arctan", + "arg", + "cos", + "cosh", + "cot", + "coth", + "csc", + "deg", + "det", + "dim", + "exp", + "gcd", + "hom", + "inf", + "ker", + "lg", + "lim", + "liminf", + "limsup", + "ln", + "log", + "max", + "min", + "Pr", + "sec", + "sin", + "sinh", + "sup", + "tan", + "tanh" + ]); + var LIMIT_OPERATORS = /* @__PURE__ */ new Set([ + "argmax", + "argmin", + "inf", + "injlim", + "lim", + "liminf", + "limsup", + "max", + "min", + "projlim", + "sup" + ]); + var DISPLAY_LIMIT_SYMBOLS = /* @__PURE__ */ new Set([ + "bigcap", + "bigcup", + "bigodot", + "bigoplus", + "bigotimes", + "bigsqcup", + "biguplus", + "bigvee", + "bigwedge", + "coprod", + "int", + "iint", + "iiint", + "oint", + "prod", + "sum" + ]); + var RELATION_COMMANDS = /* @__PURE__ */ new Set([ + "Leftarrow", + "Leftrightarrow", + "Longleftarrow", + "Longleftrightarrow", + "Longrightarrow", + "Rightarrow", + "Vdash", + "Vvdash", + "approx", + "asymp", + "cong", + "dashv", + "doteq", + "downarrow", + "equiv", + "ge", + "geq", + "geqslant", + "gets", + "gg", + "hookleftarrow", + "hookrightarrow", + "iff", + "implies", + "in", + "leadsto", + "le", + "leftarrow", + "leftharpoondown", + "leftharpoonup", + "leftrightarrow", + "leftrightharpoons", + "leq", + "leqslant", + "ll", + "longleftarrow", + "longleftrightarrow", + "longmapsto", + "longrightarrow", + "mapsto", + "mid", + "models", + "ne", + "nearrow", + "neq", + "ni", + "notin", + "nvdash", + "nvDash", + "nwarrow", + "parallel", + "perp", + "prec", + "preceq", + "propto", + "rightharpoondown", + "rightharpoonup", + "rightleftharpoons", + "rightarrow", + "rightsquigarrow", + "searrow", + "sim", + "simeq", + "sqsubset", + "sqsubseteq", + "sqsupset", + "sqsupseteq", + "subset", + "subseteq", + "succ", + "succeq", + "supset", + "supseteq", + "swarrow", + "to", + "triangleleft", + "triangleright", + "twoheadleftarrow", + "twoheadrightarrow", + "uparrow", + "vdash" + ]); + var NEGATED_SYMBOLS = { + "<": "\u226E", + ">": "\u226F", + "=": "\u2260", + "\u2208": "\u2209", + "\u220B": "\u220C", + "\u2223": "\u2224", + "\u2225": "\u2226", + "\u223C": "\u2241", + "\u2243": "\u2244", + "\u2245": "\u2247", + "\u2248": "\u2249", + "\u2261": "\u2262", + "\u2264": "\u2270", + "\u2265": "\u2271", + "\u227A": "\u2280", + "\u227B": "\u2281", + "\u2282": "\u2284", + "\u2283": "\u2285", + "\u2286": "\u2288", + "\u2287": "\u2289", + "\u22A2": "\u22AC", + "\u22A8": "\u22AD", + "\u2194": "\u21AE", + "\u2190": "\u219A", + "\u2192": "\u219B", + "\u21D2": "\u21CF", + "\u21D0": "\u21CD", + "\u21D4": "\u21CE", + "\u227C": "\u22E0", + "\u227D": "\u22E1" + }; + var BLACKBOARD = { + C: "\u2102", + H: "\u210D", + N: "\u2115", + P: "\u2119", + Q: "\u211A", + R: "\u211D", + Z: "\u2124" + }; + var SUPERSCRIPTS = { + "0": "\u2070", + "1": "\xB9", + "2": "\xB2", + "3": "\xB3", + "4": "\u2074", + "5": "\u2075", + "6": "\u2076", + "7": "\u2077", + "8": "\u2078", + "9": "\u2079", + "+": "\u207A", + "-": "\u207B", + "=": "\u207C", + "(": "\u207D", + ")": "\u207E", + a: "\u1D43", + b: "\u1D47", + c: "\u1D9C", + d: "\u1D48", + e: "\u1D49", + f: "\u1DA0", + g: "\u1D4D", + h: "\u02B0", + i: "\u2071", + j: "\u02B2", + k: "\u1D4F", + l: "\u02E1", + m: "\u1D50", + n: "\u207F", + o: "\u1D52", + p: "\u1D56", + r: "\u02B3", + s: "\u02E2", + t: "\u1D57", + u: "\u1D58", + v: "\u1D5B", + w: "\u02B7", + x: "\u02E3", + y: "\u02B8", + z: "\u1DBB" + }; + var SUBSCRIPTS = { + "0": "\u2080", + "1": "\u2081", + "2": "\u2082", + "3": "\u2083", + "4": "\u2084", + "5": "\u2085", + "6": "\u2086", + "7": "\u2087", + "8": "\u2088", + "9": "\u2089", + "+": "\u208A", + "-": "\u208B", + "=": "\u208C", + "(": "\u208D", + ")": "\u208E", + a: "\u2090", + e: "\u2091", + h: "\u2095", + i: "\u1D62", + j: "\u2C7C", + k: "\u2096", + l: "\u2097", + m: "\u2098", + n: "\u2099", + o: "\u2092", + p: "\u209A", + r: "\u1D63", + s: "\u209B", + t: "\u209C", + u: "\u1D64", + v: "\u1D65", + x: "\u2093" + }; + var SPACING_COMMANDS = /* @__PURE__ */ new Set([ + ",", + ":", + ";", + " ", + ">", + "enspace", + "enskip", + "medspace", + "quad", + "qquad", + "thickspace", + "thinspace" + ]); + var NEGATIVE_SPACING_COMMANDS = /* @__PURE__ */ new Set(["!", "negmedspace", "negthickspace", "negthinspace"]); + var NEGATIVE_SPACE = "\0"; + var IGNORED_COMMANDS = /* @__PURE__ */ new Set([ + "displaystyle", + "limits", + "nolimits", + "scriptstyle", + "scriptscriptstyle", + "textstyle" + ]); + var SIZE_COMMANDS = /* @__PURE__ */ new Set([ + "big", + "Big", + "bigg", + "Bigg", + "bigl", + "Bigl", + "biggl", + "Biggl", + "bigr", + "Bigr", + "biggr", + "Biggr" + ]); + var PLAIN_WRAPPERS = /* @__PURE__ */ new Set([ + "emph", + "mathcal", + "mathbf", + "mathfrak", + "mathit", + "mathrm", + "mathnormal", + "mathscr", + "mathsf", + "mathtt", + "mathup", + "mbox", + "overbrace", + "pmb", + "smash", + "substack", + "text", + "textbf", + "textit", + "textmd", + "textnormal", + "textrm", + "textsc", + "textsf", + "textsl", + "texttt", + "textup", + "underbrace", + "bm", + "boldsymbol" + ]); + var ACCENTS = { + acute: "\u0301", + bar: "\u0305", + breve: "\u0306", + check: "\u030C", + ddot: "\u0308", + dot: "\u0307", + grave: "\u0300", + hat: "\u0302", + mathring: "\u030A", + overleftarrow: "\u20D6", + overleftrightarrow: "\u20E1", + overline: "\u0305", + overrightarrow: "\u20D7", + tilde: "\u0303", + underline: "\u0332", + vec: "\u20D7", + widehat: "\u0302", + widetilde: "\u0303" + }; + function replaceCharacters(value, replacements) { + let result = ""; + for (const character of value) { + const replacement = replacements[character]; + if (replacement === void 0) { + return void 0; + } + result += replacement; + } + return result; + } + function formatScript(value, kind) { + value = value.trim(); + const replacements = kind === "sub" ? SUBSCRIPTS : SUPERSCRIPTS; + const unicode = replaceCharacters(value.replace(/\s*([=+-])\s*/g, "$1"), replacements); + if (unicode !== void 0) { + return unicode; + } + const prefix = kind === "sub" ? "_" : "^"; + if (Array.from(value).length === 1 || kind === "sub" && /^[A-Za-z]+$/.test(value)) { + return `${prefix}${value}`; + } + return `${prefix}(${value})`; + } + function formatFraction(numerator, denominator) { + numerator = numerator.trim(); + denominator = denominator.trim(); + const simpleNumerator = /^[\p{L}\p{N}.]+$/u.test(numerator); + const simpleDenominator = /^[\p{N}.]+$/u.test(denominator) || Array.from(denominator).length === 1; + return `${simpleNumerator ? numerator : `(${numerator})`}/${simpleDenominator ? denominator : `(${denominator})`}`; + } + function formatRoot(value, symbol = "\u221A") { + value = value.trim(); + return /^[\p{L}\p{N}.]+$/u.test(value) ? `${symbol}${value}` : `${symbol}(${value})`; + } + var NAMED_OPERATOR_START = "\u{F0004}"; + var NAMED_OPERATOR_END = "\u{F0005}"; + var NAMED_OPERATOR_LEFT_SPACING_PATTERN = /(?<=[\p{L}\p{N})\]}\u{f0001}])\u{f0004}/gu; + var NAMED_OPERATOR_RIGHT_SPACING_PATTERN = /\u{f0005}(?=[\p{L}\p{N}√\u{f0000}])/gu; + function normalizeOutput(value) { + return value.replace(NAMED_OPERATOR_LEFT_SPACING_PATTERN, " ").replaceAll(NAMED_OPERATOR_START, "").replace(NAMED_OPERATOR_RIGHT_SPACING_PATTERN, " ").replaceAll(NAMED_OPERATOR_END, "").split("\n").map((line) => line.replace(/[ \t]+/g, " ").trim()).filter((line, index, lines) => line.length > 0 || index > 0 && index < lines.length - 1).join("\n").trim(); + } + var LAYOUT_MARKER_START = "\u{F0000}"; + var LAYOUT_MARKER_END = "\u{F0001}"; + var LAYOUT_MARKER_PATTERN = /\u{f0000}(\d+)\u{f0001}/gu; + var TRAILING_LAYOUT_MARKER_PATTERN = /\u{f0000}(\d+)\u{f0001}$/u; + var PROTECTED_SPACE = "\u{F0002}"; + function padLayoutLine(line, width, centered = false) { + const padding = Math.max(0, width - visibleWidth(line)); + const left = centered ? Math.floor(padding / 2) : 0; + return `${" ".repeat(left)}${line}${" ".repeat(padding - left)}`; + } + function joinLayouts(layouts) { + if (layouts.length === 0) { + return { lines: [""], width: 0, baseline: 0 }; + } + const baseline = Math.max(...layouts.map((layout) => layout.baseline)); + const below = Math.max(...layouts.map((layout) => layout.lines.length - layout.baseline - 1)); + const lines = []; + for (let row = 0; row <= baseline + below; row++) { + let line = ""; + for (const layout of layouts) { + const sourceRow = row - baseline + layout.baseline; + line += sourceRow >= 0 && sourceRow < layout.lines.length ? padLayoutLine(layout.lines[sourceRow] ?? "", layout.width) : " ".repeat(layout.width); + } + lines.push(line.trimEnd()); + } + return { + lines, + width: layouts.reduce((width, layout) => width + layout.width, 0), + baseline + }; + } + function renderLayout(source, nodes) { + const renderedLines = []; + let firstBaseline = 0; + for (const sourceLine of source.split("\n")) { + const layouts = []; + let position = 0; + let previousNode; + for (const match of sourceLine.matchAll(LAYOUT_MARKER_PATTERN)) { + const index = match.index; + const node = nodes[Number(match[1])]; + if (!node) { + continue; + } + if (index > position) { + const sliced = sourceLine.slice(position, index); + const trimmed = (previousNode ? sliced.trimStart() : sliced).trimEnd(); + const preserveLeadingSpace = previousNode?.type === "matrix" && /^\s/.test(sliced); + const preserveTrailingSpace = node.type === "matrix" && /\s$/.test(sliced); + const text = trimmed ? `${preserveLeadingSpace ? " " : ""}${trimmed}${preserveTrailingSpace ? " " : ""}` : preserveLeadingSpace || preserveTrailingSpace ? " " : ""; + layouts.push({ lines: [text], width: visibleWidth(text), baseline: 0 }); + } + if (node.type === "fraction") { + const numerator = renderLayout(node.numerator, nodes); + const denominator = renderLayout(node.denominator, nodes); + const contentWidth = Math.max(numerator.width, denominator.width, 1); + const width = contentWidth + 2; + layouts.push({ + lines: [ + ...numerator.lines.map((line) => padLayoutLine(line, width, true)), + ` ${"\u2500".repeat(contentWidth)} `, + ...denominator.lines.map((line) => padLayoutLine(line, width, true)) + ], + width, + baseline: numerator.lines.length + }); + } else if (node.type === "operator") { + const contentWidth = Math.max( + visibleWidth(node.operator), + node.lower === void 0 ? 0 : visibleWidth(node.lower), + node.upper === void 0 ? 0 : visibleWidth(node.upper) + ); + const lines = []; + if (node.upper !== void 0) { + lines.push(`${padLayoutLine(node.upper, contentWidth, true)} `); + } + lines.push(`${padLayoutLine(node.operator, contentWidth, true)} `); + if (node.lower !== void 0) { + lines.push(`${padLayoutLine(node.lower, contentWidth, true)} `); + } + layouts.push({ + lines, + width: contentWidth + 1, + baseline: node.upper === void 0 ? 0 : 1 + }); + } else { + const width = Math.max(0, ...node.lines.map((line) => visibleWidth(line))); + layouts.push({ + lines: node.lines.map((line) => padLayoutLine(line, width)), + width, + baseline: node.baseline + }); + } + position = index + match[0].length; + previousNode = node; + } + if (position < sourceLine.length) { + const sliced = sourceLine.slice(position); + const trimmed = previousNode ? sliced.trimStart() : sliced; + const text = previousNode?.type === "matrix" && /^\s/.test(sliced) ? ` ${trimmed}` : trimmed; + layouts.push({ lines: [text], width: visibleWidth(text), baseline: 0 }); + } + const lineLayout = joinLayouts(layouts); + if (renderedLines.length === 0) { + firstBaseline = lineLayout.baseline; + } + renderedLines.push(...lineLayout.lines); + } + return { + lines: renderedLines, + width: Math.max(0, ...renderedLines.map((line) => visibleWidth(line))), + baseline: firstBaseline + }; + } + var LatexParser = class _LatexParser { + constructor(source, layoutNodes, display) { + this.position = 0; + this.supported = true; + this.stackFractions = true; + this.source = source; + this.layoutNodes = layoutNodes; + this.display = display; + } + render() { + const rendered = this.parseSequence(); + if (!this.supported || this.position !== this.source.length) { + return void 0; + } + return normalizeOutput(rendered); + } + parseSequence(endCharacter) { + let result = ""; + while (this.position < this.source.length) { + const character = this.source[this.position]; + if (endCharacter && character === endCharacter) { + this.position++; + return result; + } + if (character === "}") { + this.supported = false; + return result; + } + if (character === "{") { + this.position++; + result += this.parseSequence("}"); + continue; + } + if (character === "\\") { + const command = this.parseCommand(); + if (command === NEGATIVE_SPACE) { + result = result.trimEnd(); + if (result.endsWith(NAMED_OPERATOR_END)) { + result = result.slice(0, -NAMED_OPERATOR_END.length); + } + } else { + result += command; + } + continue; + } + if (character === "^" || character === "_") { + this.position++; + result = result.trimEnd(); + const script = formatScript(this.parseRequiredArgument(false), character === "_" ? "sub" : "sup"); + if (result.endsWith(NAMED_OPERATOR_END)) { + result = `${result.slice(0, -NAMED_OPERATOR_END.length)}${script}${NAMED_OPERATOR_END}`; + } else { + result += script; + } + continue; + } + if (/\s/.test(character)) { + result += this.parseWhitespace(); + continue; + } + if (character === "=" || character === "<" || character === ">") { + result = `${result.trimEnd()} ${character} `; + this.position++; + continue; + } + if (character === "&") { + this.position++; + continue; + } + if (character === "~") { + this.position++; + result += " "; + continue; + } + if (character === ".") { + const marker = TRAILING_LAYOUT_MARKER_PATTERN.exec(result); + const node = marker ? this.layoutNodes[Number(marker[1])] : void 0; + if (node?.type === "matrix") { + const lastLine = node.lines.length - 1; + node.lines[lastLine] = `${node.lines[lastLine] ?? ""}${character}`; + this.position++; + continue; + } + } + result += character; + this.position++; + } + if (endCharacter) { + this.supported = false; + } + return result; + } + parseWhitespace() { + while (this.position < this.source.length && /\s/.test(this.source[this.position] ?? "")) { + this.position++; + } + return " "; + } + parseCommand() { + this.position++; + if (this.position >= this.source.length) { + this.supported = false; + return ""; + } + let command = ""; + const first = this.source[this.position] ?? ""; + if (first === "\n" || first === "\r") { + this.position++; + if (first === "\r" && this.source[this.position] === "\n") { + this.position++; + } + return " "; + } + if (/[A-Za-z]/.test(first)) { + const start = this.position; + while (this.position < this.source.length && /[A-Za-z]/.test(this.source[this.position] ?? "")) { + this.position++; + } + command = this.source.slice(start, this.position); + } else { + command = first; + this.position++; + } + if (command === "\\") { + return "\n"; + } + if (SPACING_COMMANDS.has(command)) { + return " "; + } + if (NEGATIVE_SPACING_COMMANDS.has(command)) { + return NEGATIVE_SPACE; + } + if (IGNORED_COMMANDS.has(command)) { + return ""; + } + if (command === "{" || command === "}" || command === "$" || command === "%" || command === "#" || command === "_" || command === "&") { + return command; + } + if (command === "|") { + return "\u2016"; + } + if (command === "not") { + const value = this.parseRequiredArgument(false).trim(); + const negated = NEGATED_SYMBOLS[value]; + if (negated !== void 0) { + return ` ${negated} `; + } + const characters = Array.from(value); + if (characters.length === 0) { + this.supported = false; + return ""; + } + return ` ${characters[0]}\u0338${characters.slice(1).join("")} `; + } + if (LIMIT_OPERATORS.has(command)) { + return this.parseOperator(command, "bracket", true, true); + } + const symbol = SYMBOLS[command]; + if (symbol !== void 0) { + if (DISPLAY_LIMIT_SYMBOLS.has(command)) { + return this.parseOperator(symbol, "script", true); + } + return command === "cdot" || command === "times" || RELATION_COMMANDS.has(command) ? ` ${symbol} ` : symbol; + } + if (NAMED_OPERATORS.has(command)) { + return `${NAMED_OPERATOR_START}${command}${NAMED_OPERATOR_END}`; + } + if (SIZE_COMMANDS.has(command)) { + return ""; + } + if (command === "left" || command === "middle" || command === "right") { + if (this.source[this.position] === ".") { + this.position++; + } + return ""; + } + if (command === "frac" || command === "dfrac" || command === "tfrac") { + const shouldStack = this.display && this.stackFractions && command !== "tfrac"; + const numerator = this.parseRequiredArgument(!shouldStack); + const denominator = this.parseRequiredArgument(!shouldStack); + if (shouldStack) { + const index = this.layoutNodes.push({ + type: "fraction", + numerator: normalizeOutput(numerator), + denominator: normalizeOutput(denominator) + }) - 1; + return `${LAYOUT_MARKER_START}${index}${LAYOUT_MARKER_END}`; + } + return formatFraction(numerator, denominator); + } + if (command === "sqrt") { + const degree = this.parseOptionalArgument()?.trim(); + const value = this.parseRequiredArgument(); + if (degree === void 0 || degree === "2") { + return formatRoot(value); + } + if (degree === "3") { + return formatRoot(value, "\u221B"); + } + if (degree === "4") { + return formatRoot(value, "\u221C"); + } + return `${formatScript(degree, "sup")}${formatRoot(value)}`; + } + if (command === "boxed" || command === "fbox") { + return `[${this.parseRequiredArgument().trim()}]`; + } + if (command === "binom" || command === "dbinom" || command === "tbinom") { + return `(${this.parseRequiredArgument()} choose ${this.parseRequiredArgument()})`; + } + const accent = ACCENTS[command]; + if (accent !== void 0) { + const value = this.parseRequiredArgument(); + return Array.from(value).length === 1 ? `${value}${accent}` : `${command}(${value})`; + } + if (command === "mathbb") { + const value = this.parseRequiredArgument(); + return Array.from(value, (character) => BLACKBOARD[character] ?? character).join(""); + } + if (command === "operatorname") { + const starred = this.source[this.position] === "*"; + if (starred) { + this.position++; + } + const operator = normalizeOutput(this.parseRequiredArgument()).trim(); + return this.parseOperator(operator, "bracket", starred, true); + } + if (command === "mod" || command === "bmod") { + return " mod "; + } + if (command === "pmod" || command === "pod") { + const value = this.parseRequiredArgument().trim(); + return command === "pmod" ? ` (mod ${value})` : ` (${value})`; + } + if (command === "overset" || command === "stackrel") { + const upper = this.parseRequiredArgument(); + const value = this.parseRequiredArgument().trim(); + return `${value}${formatScript(upper, "sup")}`; + } + if (command === "underset") { + const lower = this.parseRequiredArgument(); + const value = this.parseRequiredArgument().trim(); + return `${value}${formatScript(lower, "sub")}`; + } + if (PLAIN_WRAPPERS.has(command)) { + const value = this.parseRequiredArgument(); + return command.startsWith("text") || command === "mbox" ? value : value.trim(); + } + if (command === "begin") { + return this.parseEnvironment(); + } + if (command === "end") { + this.supported = false; + return ""; + } + this.supported = false; + return `\\${command}`; + } + parseOperator(operator, inlineLowerStyle, displayLimits, spaced = false) { + let useDisplayLimits = displayLimits; + let modifierPosition = this.position; + while (modifierPosition < this.source.length && /[ \t]/.test(this.source[modifierPosition] ?? "")) { + modifierPosition++; + } + const modifier = /^\\(limits|nolimits)(?![A-Za-z])/.exec(this.source.slice(modifierPosition)); + if (modifier) { + useDisplayLimits = modifier[1] === "limits"; + this.position = modifierPosition + modifier[0].length; + } + let lower; + let upper; + while (true) { + let scriptPosition = this.position; + while (scriptPosition < this.source.length && /[ \t]/.test(this.source[scriptPosition] ?? "")) { + scriptPosition++; + } + const kind = this.source[scriptPosition]; + if (kind !== "_" && kind !== "^") { + break; + } + this.position = scriptPosition + 1; + const value = normalizeOutput(this.parseRequiredArgument(false)).replaceAll(" ", ""); + if (kind === "_") { + if (lower !== void 0) { + this.supported = false; + } + lower = value; + } else { + if (upper !== void 0) { + this.supported = false; + } + upper = value; + } + } + if (this.display && useDisplayLimits && (lower !== void 0 || upper !== void 0)) { + const index = this.layoutNodes.push({ type: "operator", operator, lower, upper }) - 1; + return `${LAYOUT_MARKER_START}${index}${LAYOUT_MARKER_END}`; + } + let rendered = operator; + if (lower !== void 0) { + rendered += inlineLowerStyle === "bracket" ? `[${lower}]` : formatScript(lower, "sub"); + } + if (upper !== void 0) { + rendered += formatScript(upper, "sup"); + } + return spaced ? ` ${rendered} ` : rendered; + } + parseRequiredArgument(stackFractions = true) { + const previousStackFractions = this.stackFractions; + this.stackFractions = previousStackFractions && stackFractions; + const value = this.parseRequiredArgumentValue(); + this.stackFractions = previousStackFractions; + return value; + } + parseRequiredArgumentValue() { + while (this.position < this.source.length && /\s/.test(this.source[this.position] ?? "")) { + this.position++; + } + if (this.position >= this.source.length) { + this.supported = false; + return ""; + } + if (this.source[this.position] === "{") { + this.position++; + return this.parseSequence("}"); + } + if (this.source[this.position] === "\\") { + return this.parseCommand(); + } + const value = this.source[this.position] ?? ""; + this.position++; + return value; + } + parseOptionalArgument() { + while (this.position < this.source.length && /[ \t]/.test(this.source[this.position] ?? "")) { + this.position++; + } + if (this.source[this.position] !== "[") { + return void 0; + } + const end = this.source.indexOf("]", this.position + 1); + if (end < 0) { + this.supported = false; + return void 0; + } + const value = this.source.slice(this.position + 1, end); + this.position = end + 1; + return this.renderNested(value); + } + readRawGroup() { + while (this.position < this.source.length && /[ \t]/.test(this.source[this.position] ?? "")) { + this.position++; + } + if (this.source[this.position] !== "{") { + this.supported = false; + return void 0; + } + const start = ++this.position; + let depth = 1; + while (this.position < this.source.length) { + const character = this.source[this.position]; + if (character === "\\") { + this.position += 2; + continue; + } + if (character === "{") depth++; + if (character === "}") depth--; + if (depth === 0) { + const value = this.source.slice(start, this.position); + this.position++; + return value; + } + this.position++; + } + this.supported = false; + return void 0; + } + splitEnvironmentRows(body) { + return body.split(/\\\\(?:\[[^\]\n]*\])?/); + } + parseEnvironment() { + const environment = this.readRawGroup(); + if (!environment) { + return ""; + } + const endMarker = `\\end{${environment}}`; + const end = this.source.indexOf(endMarker, this.position); + if (end < 0) { + this.supported = false; + return ""; + } + const body = this.source.slice(this.position, end); + this.position = end + endMarker.length; + if (environment === "equation" || environment === "equation*" || environment === "displaymath") { + return this.renderNested(body).trim(); + } + if (environment === "aligned" || environment === "align" || environment === "align*" || environment === "alignedat" || environment === "alignat" || environment === "alignat*" || environment === "gather" || environment === "gathered" || environment === "multline" || environment === "multline*" || environment === "split") { + const alignedAt = ["alignedat", "alignat", "alignat*"].includes(environment); + const alignedBody = alignedAt ? body.replace(/^\s*\{[^}]*\}/, "") : body; + return this.splitEnvironmentRows(alignedBody).map((row) => { + const cells = row.split("&"); + const source = alignedAt ? Array.from( + { length: Math.ceil(cells.length / 2) }, + (_2, index) => cells.slice(index * 2, index * 2 + 2).join("") + ).join(" ") : cells.join(""); + return this.renderNested(source).trim(); + }).filter(Boolean).join("\n"); + } + if (environment === "cases" || environment === "cases*") { + const rows = this.splitEnvironmentRows(body).map((row) => row.split("&").map((cell) => this.renderNested(cell, false).trim())).filter((row) => row.some(Boolean)); + return rows.map((row, index) => { + const value = (row[0] ?? "").replace(/,\s*$/, ""); + const condition = row[1] ?? ""; + const delimiter = index === 0 ? "\u23A7" : index === rows.length - 1 ? "\u23A9" : "\u23A8"; + const conditionPrefix = /^(?:if|when|for|otherwise)\b/i.test(condition) ? " " : " if "; + return `${delimiter} ${value}${condition ? `${conditionPrefix}${condition}` : ""}`; + }).join("\n"); + } + if (["array", "matrix", "smallmatrix", "pmatrix", "bmatrix", "Bmatrix", "vmatrix", "Vmatrix"].includes(environment)) { + const matrixBody = environment === "array" ? body.replace(/^\s*\{[^}]*\}/, "") : body; + return this.renderMatrix(environment, matrixBody); + } + this.supported = false; + return body; + } + renderMatrix(environment, body) { + const matrix = this.splitEnvironmentRows(body).map((row) => row.split("&").map((cell) => this.renderNested(cell, false).trim())).filter((row) => row.some(Boolean)); + const columnCount = Math.max(0, ...matrix.map((row) => row.length)); + const columnWidths = Array.from( + { length: columnCount }, + (_2, column) => Math.max(0, ...matrix.map((row) => visibleWidth(row[column] ?? ""))) + ); + const rows = matrix.map( + (row) => Array.from({ length: columnCount }, (_2, column) => { + const cell = row[column] ?? ""; + return `${cell}${PROTECTED_SPACE.repeat(Math.max(0, (columnWidths[column] ?? 0) - visibleWidth(cell)))}`; + }).join(" \u2502 ") + ); + let lines; + if (environment === "array" || environment === "matrix" || environment === "smallmatrix") { + lines = rows; + } else { + const delimiters = { + pmatrix: ["\u239B", "\u239E", "\u239C", "\u239F", "\u239D", "\u23A0"], + bmatrix: ["\u23A1", "\u23A4", "\u23A2", "\u23A5", "\u23A3", "\u23A6"], + Bmatrix: ["\u23A7", "\u23AB", "\u23A8", "\u23AC", "\u23A9", "\u23AD"], + vmatrix: ["\u2502", "\u2502", "\u2502", "\u2502", "\u2502", "\u2502"], + Vmatrix: ["\u2551", "\u2551", "\u2551", "\u2551", "\u2551", "\u2551"] + }; + const delimiter = delimiters[environment]; + if (!delimiter) { + this.supported = false; + return rows.join("\n"); + } + lines = rows.map((row, index2) => { + const left = index2 === 0 ? delimiter[0] : index2 === rows.length - 1 ? delimiter[4] : delimiter[2]; + const right = index2 === 0 ? delimiter[1] : index2 === rows.length - 1 ? delimiter[5] : delimiter[3]; + return `${left} ${row} ${right}`; + }); + } + if (lines.length <= 1) { + return lines[0] ?? ""; + } + const index = this.layoutNodes.push({ type: "matrix", lines, baseline: 0 }) - 1; + return `${LAYOUT_MARKER_START}${index}${LAYOUT_MARKER_END}`; + } + renderNested(source, stackFractions = true) { + const rendered = new _LatexParser(source, this.layoutNodes, this.display && stackFractions).render(); + if (rendered === void 0) { + this.supported = false; + return source; + } + return rendered; + } + }; + function renderLatex(source, options = {}) { + const layoutNodes = []; + const rendered = new LatexParser(source, layoutNodes, options.display === true).render(); + if (rendered === void 0) { + return void 0; + } + if (layoutNodes.length === 0) { + return rendered.replaceAll(PROTECTED_SPACE, " "); + } + const lines = renderLayout(rendered, layoutNodes).lines; + const indentation = Math.min( + ...lines.filter((line) => line.trim()).map((line) => line.length - line.trimStart().length) + ); + return lines.map((line) => line.slice(indentation).trimEnd()).join("\n").trimEnd().replaceAll(PROTECTED_SPACE, " "); + } + + // ../../../tmp/qtest/shim/terminal-image.ts + var getCapabilities = () => ({ images: false, hyperlinks: false }); + var hyperlink = (text, _url) => text; + var isImageLine = () => false; + + // packages/tui/src/components/markdown.ts + var STRICT_STRIKETHROUGH_REGEX = /^(~~)(?=[^\s~])((?:\\.|[^\\])*?(?:\\.|[^\s~\\]))\1(?=[^~]|$)/; + var StrictStrikethroughTokenizer = class extends w { + del(src) { + const match = STRICT_STRIKETHROUGH_REGEX.exec(src); + if (!match) { + return void 0; + } + const text = match[2]; + return { + type: "del", + raw: match[0], + text, + tokens: this.lexer.inlineTokens(text) + }; + } + }; + function isEscaped(source, index) { + let backslashes = 0; + for (let position = index - 1; position >= 0 && source[position] === "\\"; position--) { + backslashes++; + } + return backslashes % 2 === 1; + } + function findClosingDelimiter(source, closing, start) { + let index = source.indexOf(closing, start); + while (index >= 0 && isEscaped(source, index)) { + index = source.indexOf(closing, index + closing.length); + } + return index; + } + function looksLikePendingDollarMath(source) { + return /\\[A-Za-z]+|[_^=+*/<>()[\]|±≤≥≠≈∈→⇒∞∫∑√-]/.test(source); + } + function tokenizeInlineLatex(source) { + let opening = ""; + let closing = ""; + if (source.startsWith("$$")) { + opening = "$$"; + closing = "$$"; + } else if (source.startsWith("\\(")) { + opening = "\\("; + closing = "\\)"; + } else if (source.startsWith("\\[")) { + opening = "\\["; + closing = "\\]"; + } else if (source.startsWith("$") && !/^\$\s/.test(source)) { + opening = "$"; + closing = "$"; + } else { + return void 0; + } + const closingIndex = findClosingDelimiter(source, closing, opening.length); + if (closingIndex >= 0 && opening === "$" && (/\s$/.test(source.slice(opening.length, closingIndex)) || /^\d/.test(source.slice(closingIndex + 1)) || /^[A-Z_][A-Z0-9_]*(?:[^A-Za-z0-9_\s])?$/.test(source.slice(opening.length, closingIndex)) && /^[A-Za-z_][A-Za-z0-9_]*/.test(source.slice(closingIndex + 1)) || source.slice(opening.length, closingIndex).includes("`"))) { + return void 0; + } + if (closingIndex < 0) { + const pendingSource = source.slice(opening.length); + if (opening.startsWith("\\") || looksLikePendingDollarMath(pendingSource)) { + return { type: "latex", raw: source, text: pendingSource, pending: true }; + } + return void 0; + } + const text = source.slice(opening.length, closingIndex); + if (!text || text.includes("\n")) { + return void 0; + } + const raw = source.slice(0, closingIndex + closing.length); + return { type: "latex", raw, text }; + } + function tokenizeBlockLatex(source) { + const dollarMatch = /^ {0,3}\$\$[ \t]*(?:\n)?([\s\S]*?)\$\$[ \t]*(?:\n|$)/.exec(source); + if (dollarMatch?.[1]) { + return { type: "latexBlock", raw: dollarMatch[0], text: dollarMatch[1].trim() }; + } + const bracketMatch = /^ {0,3}\\\[[ \t]*(?:\n)?([\s\S]*?)\\\][ \t]*(?:\n|$)/.exec(source); + if (bracketMatch?.[1]) { + return { type: "latexBlock", raw: bracketMatch[0], text: bracketMatch[1].trim() }; + } + const pendingBracket = /^ {0,3}\\\[[ \t]*(?:\n)?([\s\S]*)$/.exec(source); + if (pendingBracket) { + return { type: "latexBlock", raw: pendingBracket[0], text: pendingBracket[1], pending: true }; + } + const pendingDollar = /^ {0,3}\$\$[ \t]*(?:\n)?([\s\S]*)$/.exec(source); + if (pendingDollar?.[1] && looksLikePendingDollarMath(pendingDollar[1])) { + return { type: "latexBlock", raw: pendingDollar[0], text: pendingDollar[1], pending: true }; + } + return void 0; + } + var LATEX_MARKDOWN_EXTENSIONS = [ + { + name: "latexBlock", + level: "block", + start(source) { + const match = /(?:^|\n) {0,3}(?:\$\$|\\\[)/.exec(source); + return match ? match.index + (match[0].startsWith("\n") ? 1 : 0) : void 0; + }, + tokenizer: tokenizeBlockLatex + }, + { + name: "latex", + level: "inline", + start(source) { + const indices = [source.indexOf("$"), source.indexOf("\\("), source.indexOf("\\[")].filter( + (index) => index >= 0 + ); + return indices.length > 0 ? Math.min(...indices) : void 0; + }, + tokenizer: tokenizeInlineLatex + } + ]; + function trimPartialClosingFences(tokens) { + const token = tokens[tokens.length - 1]; + if (token?.type === "list") { + trimPartialClosingFences(token.items[token.items.length - 1]?.tokens ?? []); + return; + } + if (token?.type === "blockquote") { + trimPartialClosingFences(token.tokens ?? []); + return; + } + if (token?.type !== "code") { + return; + } + const marker = /^(`{3,}|~{3,})/.exec(token.raw)?.[1]; + const lastLine = token.raw.split("\n").pop(); + if (!marker || !lastLine || lastLine.length >= marker.length || lastLine !== marker[0]?.repeat(lastLine.length)) { + return; + } + token.text = token.text.slice(0, -lastLine.length).replace(/\n$/, ""); + } + var markdownParser = new q(); + markdownParser.setOptions({ + tokenizer: new StrictStrikethroughTokenizer() + }); + markdownParser.use({ extensions: [...LATEX_MARKDOWN_EXTENSIONS] }); + var Markdown = class { + constructor(text, paddingX, paddingY, theme, defaultTextStyle, options) { + this.text = text; + this.paddingX = paddingX; + this.paddingY = paddingY; + this.theme = theme; + this.defaultTextStyle = defaultTextStyle; + this.options = options ? { ...options } : {}; + } + setText(text) { + this.text = text; + this.invalidate(); + } + invalidate() { + this.cachedText = void 0; + this.cachedWidth = void 0; + this.cachedLines = void 0; + } + render(width) { + if (this.cachedLines && this.cachedText === this.text && this.cachedWidth === width) { + return this.cachedLines; + } + const contentWidth = Math.max(1, width - this.paddingX * 2); + const text = this.options.transform?.(this.text, contentWidth) ?? this.text; + if (!text || text.trim() === "") { + const result2 = []; + this.cachedText = this.text; + this.cachedWidth = width; + this.cachedLines = result2; + return result2; + } + const normalizedText = text.replace(/\t/g, " "); + const tokens = markdownParser.lexer(normalizedText); + trimPartialClosingFences(tokens); + const renderedLines = []; + for (let i = 0; i < tokens.length; i++) { + const token = tokens[i]; + const nextToken = tokens[i + 1]; + const tokenLines = this.renderToken(token, contentWidth, nextToken?.type); + for (const tokenLine of tokenLines) { + renderedLines.push(tokenLine); + } + } + const wrappedLines = []; + for (const line of renderedLines) { + if (isImageLine(line)) { + wrappedLines.push(line); + } else { + for (const wrappedLine of wrapTextWithAnsi(line, contentWidth)) { + wrappedLines.push(wrappedLine); + } + } + } + const leftMargin = " ".repeat(this.paddingX); + const rightMargin = " ".repeat(this.paddingX); + const bgFn = this.defaultTextStyle?.bgColor; + const contentLines = []; + for (const line of wrappedLines) { + if (isImageLine(line)) { + contentLines.push(line); + continue; + } + const lineWithMargins = leftMargin + line + rightMargin; + if (bgFn) { + contentLines.push(applyBackgroundToLine(lineWithMargins, width, bgFn)); + } else { + const visibleLen = visibleWidth(lineWithMargins); + const paddingNeeded = Math.max(0, width - visibleLen); + contentLines.push(lineWithMargins + " ".repeat(paddingNeeded)); + } + } + const emptyLine = " ".repeat(width); + const emptyLines = []; + for (let i = 0; i < this.paddingY; i++) { + const line = bgFn ? applyBackgroundToLine(emptyLine, width, bgFn) : emptyLine; + emptyLines.push(line); + } + const result = emptyLines.concat(contentLines, emptyLines); + this.cachedText = this.text; + this.cachedWidth = width; + this.cachedLines = result; + return result.length > 0 ? result : [""]; + } + /** + * Apply default text style to a string. + * This is the base styling applied to all text content. + * NOTE: Background color is NOT applied here - it's applied at the padding stage + * to ensure it extends to the full line width. + */ + applyDefaultStyle(text) { + if (!this.defaultTextStyle) { + return text; + } + let styled = text; + if (this.defaultTextStyle.color) { + styled = this.defaultTextStyle.color(styled); + } + if (this.defaultTextStyle.bold) { + styled = this.theme.bold(styled); + } + if (this.defaultTextStyle.italic) { + styled = this.theme.italic(styled); + } + if (this.defaultTextStyle.strikethrough) { + styled = this.theme.strikethrough(styled); + } + if (this.defaultTextStyle.underline) { + styled = this.theme.underline(styled); + } + return styled; + } + getDefaultStylePrefix() { + if (!this.defaultTextStyle) { + return ""; + } + if (this.defaultStylePrefix !== void 0) { + return this.defaultStylePrefix; + } + const sentinel = "\0"; + let styled = sentinel; + if (this.defaultTextStyle.color) { + styled = this.defaultTextStyle.color(styled); + } + if (this.defaultTextStyle.bold) { + styled = this.theme.bold(styled); + } + if (this.defaultTextStyle.italic) { + styled = this.theme.italic(styled); + } + if (this.defaultTextStyle.strikethrough) { + styled = this.theme.strikethrough(styled); + } + if (this.defaultTextStyle.underline) { + styled = this.theme.underline(styled); + } + const sentinelIndex = styled.indexOf(sentinel); + this.defaultStylePrefix = sentinelIndex >= 0 ? styled.slice(0, sentinelIndex) : ""; + return this.defaultStylePrefix; + } + getStylePrefix(styleFn) { + const sentinel = "\0"; + const styled = styleFn(sentinel); + const sentinelIndex = styled.indexOf(sentinel); + return sentinelIndex >= 0 ? styled.slice(0, sentinelIndex) : ""; + } + getDefaultInlineStyleContext() { + return { + applyText: (text) => this.applyDefaultStyle(text), + stylePrefix: this.getDefaultStylePrefix() + }; + } + renderToken(token, width, nextTokenType, styleContext) { + const lines = []; + switch (token.type) { + case "heading": { + const headingLevel = token.depth; + const headingPrefix = `${"#".repeat(headingLevel)} `; + let headingStyleFn; + if (headingLevel === 1) { + headingStyleFn = (text) => this.theme.heading(this.theme.bold(this.theme.underline(text))); + } else { + headingStyleFn = (text) => this.theme.heading(this.theme.bold(text)); + } + const headingStyleContext = { + applyText: headingStyleFn, + stylePrefix: this.getStylePrefix(headingStyleFn) + }; + const headingText = this.renderInlineTokens(token.tokens || [], headingStyleContext); + const styledHeading = headingLevel >= 3 ? headingStyleFn(headingPrefix) + headingText : headingText; + lines.push(styledHeading); + if (nextTokenType && nextTokenType !== "space") { + lines.push(""); + } + break; + } + case "paragraph": { + const paragraphText = this.renderInlineTokens(token.tokens || [], styleContext); + lines.push(paragraphText); + if (nextTokenType && nextTokenType !== "list" && nextTokenType !== "space") { + lines.push(""); + } + break; + } + case "text": + lines.push(this.renderInlineTokens([token], styleContext)); + break; + case "latexBlock": { + const latexToken = token; + const rendered = !latexToken.pending && this.options.renderLatex !== false ? renderLatex(latexToken.text, { display: true }) ?? latexToken.raw.trim() : latexToken.raw.trim(); + for (const line of rendered.split("\n")) { + lines.push(this.applyDefaultStyle(line)); + } + if (nextTokenType && nextTokenType !== "space") { + lines.push(""); + } + break; + } + case "code": { + const indent = this.theme.codeBlockIndent ?? " "; + lines.push(this.theme.codeBlockBorder(`\`\`\`${token.lang || ""}`)); + if (this.theme.highlightCode) { + const highlightedLines = this.theme.highlightCode(token.text, token.lang); + for (const hlLine of highlightedLines) { + lines.push(`${indent}${hlLine}`); + } + } else { + const codeLines = token.text.split("\n"); + for (const codeLine of codeLines) { + lines.push(`${indent}${this.theme.codeBlock(codeLine)}`); + } + } + lines.push(this.theme.codeBlockBorder("```")); + if (nextTokenType && nextTokenType !== "space") { + lines.push(""); + } + break; + } + case "list": { + const listLines = this.renderList(token, 0, width, styleContext); + lines.push(...listLines); + break; + } + case "table": { + const tableLines = this.renderTable(token, width, nextTokenType, styleContext); + lines.push(...tableLines); + break; + } + case "blockquote": { + const quoteStyle = (text) => this.theme.quote(this.theme.italic(text)); + const quoteStylePrefix = this.getStylePrefix(quoteStyle); + const applyQuoteStyle = (line) => { + if (!quoteStylePrefix) { + return quoteStyle(line); + } + const lineWithReappliedStyle = line.replace(/\x1b\[0m/g, `\x1B[0m${quoteStylePrefix}`); + return quoteStyle(lineWithReappliedStyle); + }; + const quoteContentWidth = Math.max(1, width - 2); + const quoteInlineStyleContext = { + applyText: (text) => text, + stylePrefix: quoteStylePrefix + }; + const quoteTokens = token.tokens || []; + const renderedQuoteLines = []; + for (let i = 0; i < quoteTokens.length; i++) { + const quoteToken = quoteTokens[i]; + const nextQuoteToken = quoteTokens[i + 1]; + renderedQuoteLines.push( + ...this.renderToken(quoteToken, quoteContentWidth, nextQuoteToken?.type, quoteInlineStyleContext) + ); + } + while (renderedQuoteLines.length > 0 && renderedQuoteLines[renderedQuoteLines.length - 1] === "") { + renderedQuoteLines.pop(); + } + for (const quoteLine of renderedQuoteLines) { + const styledLine = applyQuoteStyle(quoteLine); + const wrappedLines = wrapTextWithAnsi(styledLine, quoteContentWidth); + for (const wrappedLine of wrappedLines) { + lines.push(this.theme.quoteBorder("\u2502 ") + wrappedLine); + } + } + if (nextTokenType && nextTokenType !== "space") { + lines.push(""); + } + break; + } + case "hr": + lines.push(this.theme.hr("\u2500".repeat(Math.min(width, 80)))); + if (nextTokenType && nextTokenType !== "space") { + lines.push(""); + } + break; + case "html": + if ("raw" in token && typeof token.raw === "string") { + lines.push(this.applyDefaultStyle(token.raw.trim())); + } + break; + case "space": + lines.push(""); + break; + default: + if ("text" in token && typeof token.text === "string") { + lines.push(token.text); + } + } + return lines; + } + renderInlineTokens(tokens, styleContext) { + let result = ""; + const resolvedStyleContext = styleContext ?? this.getDefaultInlineStyleContext(); + const { applyText, stylePrefix } = resolvedStyleContext; + const applyTextWithNewlines = (text) => { + const segments = text.split("\n"); + return segments.map((segment) => applyText(segment)).join("\n"); + }; + for (const token of tokens) { + switch (token.type) { + case "latex": { + const latexToken = token; + const rendered = !latexToken.pending && this.options.renderLatex !== false ? renderLatex(latexToken.text) ?? latexToken.raw : latexToken.raw; + result += applyTextWithNewlines(rendered); + break; + } + case "escape": + result += applyTextWithNewlines(this.options.preserveBackslashEscapes ? token.raw : token.text); + break; + case "text": + if (token.tokens && token.tokens.length > 0) { + result += this.renderInlineTokens(token.tokens, resolvedStyleContext); + } else { + result += applyTextWithNewlines(token.text); + } + break; + case "paragraph": + result += this.renderInlineTokens(token.tokens || [], resolvedStyleContext); + break; + case "strong": { + const boldContent = this.renderInlineTokens(token.tokens || [], resolvedStyleContext); + result += this.theme.bold(boldContent) + stylePrefix; + break; + } + case "em": { + const italicContent = this.renderInlineTokens(token.tokens || [], resolvedStyleContext); + result += this.theme.italic(italicContent) + stylePrefix; + break; + } + case "codespan": + result += this.theme.code(token.text) + stylePrefix; + break; + case "link": { + const linkText = this.renderInlineTokens(token.tokens || [], resolvedStyleContext); + const styledLink = this.theme.link(this.theme.underline(linkText)); + if (getCapabilities().hyperlinks) { + result += hyperlink(styledLink, token.href) + stylePrefix; + } else { + const hrefForComparison = token.href.startsWith("mailto:") ? token.href.slice(7) : token.href; + if (token.text === token.href || token.text === hrefForComparison) { + result += styledLink + stylePrefix; + } else { + result += styledLink + this.theme.linkUrl(` (${token.href})`) + stylePrefix; + } + } + break; + } + case "br": + result += "\n"; + break; + case "del": { + const delContent = this.renderInlineTokens(token.tokens || [], resolvedStyleContext); + result += this.theme.strikethrough(delContent) + stylePrefix; + break; + } + case "html": + if ("raw" in token && typeof token.raw === "string") { + result += applyTextWithNewlines(token.raw); + } + break; + default: + if ("text" in token && typeof token.text === "string") { + result += applyTextWithNewlines(token.text); + } + } + } + while (stylePrefix && result.endsWith(stylePrefix)) { + result = result.slice(0, -stylePrefix.length); + } + return result; + } + getOrderedListMarker(item) { + const match = /^(?: {0,3})(\d{1,9}[.)])[ \t]+/.exec(item.raw); + return match ? `${match[1]} ` : void 0; + } + getUnorderedListMarker(item) { + const match = /^(?: {0,3})([-+*])(?:[ \t]+|(?=\r?\n|$))/.exec(item.raw); + return match ? `${match[1]} ` : void 0; + } + /** + * Render a list with proper nesting support + */ + renderList(token, depth, width, styleContext) { + const lines = []; + const indent = " ".repeat(depth); + const startNumber = typeof token.start === "number" ? token.start : 1; + for (let i = 0; i < token.items.length; i++) { + const item = token.items[i]; + const isLastItem = i === token.items.length - 1; + const bullet = token.ordered ? this.options.preserveOrderedListMarkers ? this.getOrderedListMarker(item) ?? `${startNumber + i}. ` : `${startNumber + i}. ` : this.options.preserveOrderedListMarkers ? this.getUnorderedListMarker(item) ?? "- " : "- "; + const taskMarker = item.task ? `[${item.checked ? "x" : " "}] ` : ""; + const marker = bullet + taskMarker; + const firstPrefix = indent + this.theme.listBullet(marker); + const continuationPrefix = indent + " ".repeat(visibleWidth(marker)); + const itemWidth = Math.max(1, width - visibleWidth(firstPrefix)); + let renderedAnyLine = false; + for (const itemToken of item.tokens) { + if (itemToken.type === "list") { + lines.push(...this.renderList(itemToken, depth + 1, width, styleContext)); + renderedAnyLine = true; + continue; + } + const itemLines = this.renderToken(itemToken, itemWidth, void 0, styleContext); + for (const line of itemLines) { + for (const wrappedLine of wrapTextWithAnsi(line, itemWidth)) { + const linePrefix = renderedAnyLine ? continuationPrefix : firstPrefix; + lines.push(linePrefix + wrappedLine); + renderedAnyLine = true; + } + } + } + if (!renderedAnyLine) { + lines.push(firstPrefix); + } + if (token.loose && !isLastItem) { + lines.push(""); + } + } + return lines; + } + /** + * Get the visible width of the longest word in a string. + */ + getLongestWordWidth(text, maxWidth) { + const words = text.split(/\s+/).filter((word) => word.length > 0); + let longest = 0; + for (const word of words) { + longest = Math.max(longest, visibleWidth(word)); + } + if (maxWidth === void 0) { + return longest; + } + return Math.min(longest, maxWidth); + } + /** + * Wrap a table cell to fit into a column. + * + * Delegates to wrapTextWithAnsi() so ANSI codes + long tokens are handled + * consistently with the rest of the renderer. + */ + wrapCellText(text, maxWidth, stylePrefix = "") { + const lines = wrapTextWithAnsi(text, Math.max(1, maxWidth)); + return lines.map((line, index) => { + const styleReset = index < lines.length - 1 ? "\x1B[22;23;24;25;27;28;29;39m" : ""; + return `${line}${styleReset}${stylePrefix}`; + }); + } + /** + * Render a table with width-aware cell wrapping. + * Cells that don't fit are wrapped to multiple lines. + */ + renderTable(token, availableWidth, nextTokenType, styleContext) { + const lines = []; + const numCols = token.header.length; + if (numCols === 0) { + return lines; + } + const borderOverhead = 3 * numCols + 1; + const availableForCells = availableWidth - borderOverhead; + if (availableForCells < numCols) { + const fallbackLines = token.raw ? wrapTextWithAnsi(token.raw, availableWidth) : []; + if (nextTokenType && nextTokenType !== "space") { + fallbackLines.push(""); + } + return fallbackLines; + } + const maxUnbrokenWordWidth = 30; + const naturalWidths = []; + const minWordWidths = []; + for (let i = 0; i < numCols; i++) { + const headerText = this.renderInlineTokens(token.header[i].tokens || [], styleContext); + naturalWidths[i] = visibleWidth(headerText); + minWordWidths[i] = Math.max(1, this.getLongestWordWidth(headerText, maxUnbrokenWordWidth)); + } + for (const row of token.rows) { + for (let i = 0; i < row.length; i++) { + const cellText = this.renderInlineTokens(row[i].tokens || [], styleContext); + naturalWidths[i] = Math.max(naturalWidths[i] || 0, visibleWidth(cellText)); + minWordWidths[i] = Math.max( + minWordWidths[i] || 1, + this.getLongestWordWidth(cellText, maxUnbrokenWordWidth) + ); + } + } + let minColumnWidths = minWordWidths; + let minCellsWidth = minColumnWidths.reduce((a, b2) => a + b2, 0); + if (minCellsWidth > availableForCells) { + minColumnWidths = new Array(numCols).fill(1); + const remaining = availableForCells - numCols; + if (remaining > 0) { + const totalWeight = minWordWidths.reduce((total, width) => total + Math.max(0, width - 1), 0); + const growth = minWordWidths.map((width) => { + const weight = Math.max(0, width - 1); + return totalWeight > 0 ? Math.floor(weight / totalWeight * remaining) : 0; + }); + for (let i = 0; i < numCols; i++) { + minColumnWidths[i] += growth[i] ?? 0; + } + const allocated = growth.reduce((total, width) => total + width, 0); + let leftover = remaining - allocated; + for (let i = 0; leftover > 0 && i < numCols; i++) { + minColumnWidths[i]++; + leftover--; + } + } + minCellsWidth = minColumnWidths.reduce((a, b2) => a + b2, 0); + } + const totalNaturalWidth = naturalWidths.reduce((a, b2) => a + b2, 0) + borderOverhead; + let columnWidths; + if (totalNaturalWidth <= availableWidth) { + columnWidths = naturalWidths.map((width, index) => Math.max(width, minColumnWidths[index])); + } else { + const totalGrowPotential = naturalWidths.reduce((total, width, index) => { + return total + Math.max(0, width - minColumnWidths[index]); + }, 0); + const extraWidth = Math.max(0, availableForCells - minCellsWidth); + columnWidths = minColumnWidths.map((minWidth, index) => { + const naturalWidth = naturalWidths[index]; + const minWidthDelta = Math.max(0, naturalWidth - minWidth); + let grow = 0; + if (totalGrowPotential > 0) { + grow = Math.floor(minWidthDelta / totalGrowPotential * extraWidth); + } + return minWidth + grow; + }); + const allocated = columnWidths.reduce((a, b2) => a + b2, 0); + let remaining = availableForCells - allocated; + while (remaining > 0) { + let grew = false; + for (let i = 0; i < numCols && remaining > 0; i++) { + if (columnWidths[i] < naturalWidths[i]) { + columnWidths[i]++; + remaining--; + grew = true; + } + } + if (!grew) { + break; + } + } + } + const topBorderCells = columnWidths.map((w2) => "\u2500".repeat(w2)); + lines.push(`\u250C\u2500${topBorderCells.join("\u2500\u252C\u2500")}\u2500\u2510`); + const headerCellLines = token.header.map((cell, i) => { + const text = this.renderInlineTokens(cell.tokens || [], styleContext); + return this.wrapCellText(text, columnWidths[i], styleContext?.stylePrefix); + }); + const headerLineCount = Math.max(...headerCellLines.map((c) => c.length)); + for (let lineIdx = 0; lineIdx < headerLineCount; lineIdx++) { + const rowParts = headerCellLines.map((cellLines, colIdx) => { + const text = cellLines[lineIdx] || ""; + const padded = text + " ".repeat(Math.max(0, columnWidths[colIdx] - visibleWidth(text))); + return this.theme.bold(padded); + }); + lines.push(`\u2502 ${rowParts.join(" \u2502 ")} \u2502`); + } + const separatorCells = columnWidths.map((w2) => "\u2500".repeat(w2)); + const separatorLine = `\u251C\u2500${separatorCells.join("\u2500\u253C\u2500")}\u2500\u2524`; + lines.push(separatorLine); + for (let rowIndex = 0; rowIndex < token.rows.length; rowIndex++) { + const row = token.rows[rowIndex]; + const rowCellLines = row.map((cell, i) => { + const text = this.renderInlineTokens(cell.tokens || [], styleContext); + return this.wrapCellText(text, columnWidths[i], styleContext?.stylePrefix); + }); + const rowLineCount = Math.max(...rowCellLines.map((c) => c.length)); + for (let lineIdx = 0; lineIdx < rowLineCount; lineIdx++) { + const rowParts = rowCellLines.map((cellLines, colIdx) => { + const text = cellLines[lineIdx] || ""; + return text + " ".repeat(Math.max(0, columnWidths[colIdx] - visibleWidth(text))); + }); + lines.push(`\u2502 ${rowParts.join(" \u2502 ")} \u2502`); + } + if (rowIndex < token.rows.length - 1) { + lines.push(separatorLine); + } + } + const bottomBorderCells = columnWidths.map((w2) => "\u2500".repeat(w2)); + lines.push(`\u2514\u2500${bottomBorderCells.join("\u2500\u2534\u2500")}\u2500\u2518`); + if (nextTokenType && nextTokenType !== "space") { + lines.push(""); + } + return lines; + } + }; + + // packages/tui/bundle-bench.ts + var sty = (n) => (t) => `\x1B[${n}m${t}\x1B[0m`; + function bench(nPoints, count, reps) { + const theme = { + heading: sty(36), + bold: sty(1), + italic: sty(3), + code: sty(33), + codeBlock: sty(33), + codeBlockBorder: sty(33), + quote: sty(3), + quoteBorder: sty(3), + hr: sty(3), + listBullet: sty(1), + link: sty(34), + linkUrl: sty(34), + strikethrough: sty(3), + underline: sty(1) + }; + let text = "## Section\n\nHere is an explanation with **bold** and `inline code` plus a [link](http://x).\n\n"; + for (let i = 0; i < nPoints; i++) text += `- point ${i} with some trailing prose that wraps at narrow widths +`; + text += "\n```ts\nconst x = 1;\n```\n"; + const comps = Array.from({ length: count }, () => new Markdown(text, 0, 0, theme)); + for (let i = 0; i < 20; i++) comps[0].render(40 + i % 20); + const t0 = Date.now(); + for (let r = 0; r < reps; r++) for (const c of comps) c.render(40 + r % 60); + return (Date.now() - t0) / reps; + } + return __toCommonJS(bundle_bench_exports); +})(); + +globalThis.__bench = __m.bench; \ No newline at end of file diff --git a/packages/agent/docs/mobile-handoff/02-plugins/02-sandbox/src/membrane.ts b/packages/agent/docs/mobile-handoff/02-plugins/02-sandbox/src/membrane.ts new file mode 100644 index 000000000..d8a2f211e --- /dev/null +++ b/packages/agent/docs/mobile-handoff/02-plugins/02-sandbox/src/membrane.ts @@ -0,0 +1,197 @@ +import ivm from "isolated-vm"; + +/** + * A membrane between the host realm and an isolated-vm isolate. + * + * ── The safety argument, which is structural, not a convention ────────────── + * + * isolated-vm CAN be used unsafely. The documented footgun is handing the guest + * a live handle to a host object — via `derefInto()`, or by returning a + * `Reference` from a host call — after which the guest can walk that object's + * prototype chain into the host realm. + * + * This membrane makes that impossible by construction rather than by care: + * + * 1. `encode()` is the ONLY way a host value reaches the guest, and it is + * `JSON.stringify` with a replacer. Its output is a string. A string + * cannot carry a reference. + * 2. Host callables do not cross. They are replaced by an integer id into + * `hostTable`, which lives entirely host-side. The guest receives a number. + * 3. Exactly ONE `Reference` is ever given to the guest — `__invokeRef`, the + * single call-in point — and its `deref()` throws cross-isolate ("Cannot + * dereference this from current isolate"). Its `derefInto()` yields an + * inert `Dereference` marker: not callable, no host globals. + * 4. `derefInto()` is used exactly once, on the guest's OWN global. No host + * object is ever an argument to it. + * + * The consequence: the guest's entire view of the host is `{ number, string }`. + * There is no object graph to walk, so there is nothing to walk it into. This is + * the same property the QuickJS membrane has, and it is why the choice between + * them is about performance and packaging rather than about safety. + * + * Verified by `src/escape-audit.ts`. + */ +export async function createCompartment({ memoryLimit = 256 } = {}) { + const isolate = new ivm.Isolate({ memoryLimit }); + const ctx = await isolate.createContext(); + await ctx.global.set("global", ctx.global.derefInto()); // guest's own global only + + const hostTable = new Map(); + let nextHostId = 1; + const exposeHost = (o) => { const id = nextHostId++; hostTable.set(id, o); return id; }; + + const guestProxies = new Map(); + const finalizer = new FinalizationRegistry((id) => { + guestProxies.delete(id); + if (!released) ctx.evalSync(`delete __refs[${JSON.stringify(String(id))}]`); + }); + let released = false; + + // Async: `applySync` is synchronous, so a host Promise cannot itself cross. + // Instead the host returns a token, the guest builds a real Promise against + // it, and the host settles that Promise by calling back in when it resolves. + let nextPending = 1; + function encodePending(promise) { + const token = nextPending++; + promise.then( + (v) => settle(token, true, v === undefined ? "" : encode(v)), + (e) => settle(token, false, JSON.stringify(String(e?.message ?? e))), + ); + return JSON.stringify({ __pending: token }); + } + function settle(token, ok, json) { + if (released) return; + ctx.evalSync(`__settle(${token}, ${ok}, ${JSON.stringify(json)})`); + } + + function encode(v) { + if (v && typeof v.then === "function") return encodePending(v); + return JSON.stringify(v, (_k, val) => { + if (typeof val === "function") return { __ref: exposeHost(val) }; + if (val && typeof val === "object" && !Array.isArray(val) && hasMethods(val)) + return { __obj: exposeHost(val), keys: methodNames(val) }; + return val; + }); + } + const decode = (json) => JSON.parse(json, (_k, v) => reviveGuest(v)); + + function intern(id, build) { + const hit = guestProxies.get(id)?.deref(); + if (hit !== undefined) return hit; + const p = build(); + guestProxies.set(id, new WeakRef(p)); + finalizer.register(p, id); + return p; + } + function reviveGuest(v) { + if (!v || typeof v !== "object") return v; + if (typeof v.__guestRef === "number") { const r = v.__guestRef; return intern(r, () => (...a) => callGuestRef(r, a)); } + if (typeof v.__guestObj === "number") { + const r = v.__guestObj; + return intern(r, () => { const o = {}; for (const k of v.keys) o[k] = (...a) => callGuestRef(r, [k, a]); return o; }); + } + return v; + } + + function callGuestRef(ref, args, timeout = 5000) { + const out = ctx.evalSync( + `__refs[${JSON.stringify(String(ref))}](${JSON.stringify(JSON.stringify(args))})`, + { timeout }, + ); + return out === "" || out === undefined ? undefined : decode(out); + } + + // The single host entry point. A Reference, applied synchronously with copied + // arguments — no host object graph is ever exposed. + const invoke = new ivm.Reference((idJson, pathJson, argsJson) => { + const path = JSON.parse(pathJson); + let target = hostTable.get(JSON.parse(idJson)); + for (let i = 0; i < path.length - 1; i++) target = target[path[i]]; + const result = path.length ? target[path[path.length - 1]](...decode(argsJson)) : target(...decode(argsJson)); + return result === undefined ? "" : encode(result); + }); + await ctx.global.set("__invokeRef", invoke); + + await ctx.eval(` + globalThis.__refs = {}; globalThis.__nextRef = 1; + globalThis.__outgoing = new WeakMap(); + globalThis.__methodNames = function (o) { + var out = [], p = o; + while (p && p !== Object.prototype) { + for (const k of Object.getOwnPropertyNames(p)) + if (k !== "constructor" && typeof o[k] === "function" && out.indexOf(k) < 0) out.push(k); + p = Object.getPrototypeOf(p); + } + return out; + }; + globalThis.__mkRef = function (fn) { + var prior = __outgoing.get(fn); if (prior !== undefined) return { __guestRef: prior }; + var id = __nextRef++; __outgoing.set(fn, id); + __refs[String(id)] = function (j) { var r = fn.apply(null, JSON.parse(j)); return r === undefined ? "" : __encode(r); }; + return { __guestRef: id }; + }; + globalThis.__mkObjRef = function (obj) { + var prior = __outgoing.get(obj); + if (prior !== undefined) return { __guestObj: prior, keys: __methodNames(obj) }; + var id = __nextRef++; __outgoing.set(obj, id); + __refs[String(id)] = function (j) { var c = JSON.parse(j); var r = obj[c[0]].apply(obj, c[1]); return r === undefined ? "" : __encode(r); }; + return { __guestObj: id, keys: __methodNames(obj) }; + }; + globalThis.__encode = function (v) { + return JSON.stringify(v, function (k, val) { + if (typeof val === "function") return __mkRef(val); + if (val && typeof val === "object" && !Array.isArray(val) && __methodNames(val).length) return __mkObjRef(val); + return val; + }); + }; + globalThis.__pending = {}; + globalThis.__settle = function (token, ok, json) { + var d = __pending[token]; + if (!d) return; + delete __pending[token]; + var value = json === "" ? undefined : JSON.parse(json); + ok ? d.resolve(__revive(value)) : d.reject(new Error(value)); + }; + globalThis.__revive = function (v) { + if (v && typeof v === "object" && typeof v.__pending === "number") { + var token = v.__pending; + return new Promise(function (res, rej) { __pending[token] = { resolve: res, reject: rej }; }); + } + if (v && typeof v === "object") { + if (typeof v.__ref === "number") return __wrap(v.__ref, []); + if (typeof v.__obj === "number") return __wrap(v.__obj, []); + for (var k in v) v[k] = __revive(v[k]); + } + return v; + }; + globalThis.__wrap = function (id, path) { + return new Proxy(function () {}, { + get: function (t, prop) { + if (typeof prop !== "string") return undefined; + if (prop === "then" || prop === "toJSON" || prop === "inspect") return undefined; + if (prop === "length" || prop === "name") return t[prop]; + return __wrap(id, path.concat([prop])); + }, + apply: function (_t, _this, args) { + var out = __invokeRef.applySync(undefined, [JSON.stringify(id), JSON.stringify(path), __encode(args)]); + return out === "" ? undefined : __revive(JSON.parse(out)); + }, + }); + }; + `); + + return { + async endow(name, obj) { await ctx.eval(`globalThis[${JSON.stringify(name)}] = __wrap(${exposeHost(obj)}, []);`); }, + load: (src, timeout = 30000) => ctx.evalSync(src, { timeout }), + debugRefCount: () => ({ hostTable: hostTable.size, guestRefs: ctx.evalSync(`Object.keys(__refs).length`) }), + dispose() { + released = true; + // An isolate that hit its memory limit is already disposed, and calling + // dispose() again throws "Isolate is already disposed". + try { isolate.dispose(); } catch { /* already gone */ } + }, + }; +} +const RESERVED = new Set(["constructor"]); +const methodNames = (o) => { const out = []; for (let p = o; p && p !== Object.prototype; p = Object.getPrototypeOf(p)) for (const k of Object.getOwnPropertyNames(p)) if (!RESERVED.has(k) && typeof o[k] === "function" && !out.includes(k)) out.push(k); return out; }; +const hasMethods = (o) => methodNames(o).length > 0; diff --git a/packages/agent/docs/mobile-handoff/02-plugins/02-sandbox/src/property-test.ts b/packages/agent/docs/mobile-handoff/02-plugins/02-sandbox/src/property-test.ts new file mode 100644 index 000000000..a9574edaf --- /dev/null +++ b/packages/agent/docs/mobile-handoff/02-plugins/02-sandbox/src/property-test.ts @@ -0,0 +1,137 @@ +/** + * Property tests for the membrane. + * + * Every bug found by hand while building it failed SILENTLY — wrong output, no + * throw. That is the worst possible failure mode for a boundary, and the reason + * these exist. Run with: node --experimental-strip-types --expose-gc property-test.ts + */ +import { createCompartment } from "./membrane.ts"; + +let pass = 0; +const failures: string[] = []; +const check = (name: string, ok: boolean, detail = "") => { + if (ok) pass++; + else failures.push(`${name}${detail ? ` — ${detail}` : ""}`); +}; + +/** Random JsonValue. The membrane claims to round-trip exactly these. */ +function randomJson(depth = 0): unknown { + const r = Math.random(); + if (depth > 3 || r < 0.25) return Math.floor(Math.random() * 1000) - 500; + if (r < 0.35) return Math.random() * 1e6 - 5e5; + if (r < 0.45) return Math.random() < 0.5; + if (r < 0.5) return null; + if (r < 0.65) { + const alphabet = ['a', 'z', '0', ' ', '"', '\\', '\n', '\u00e9', '\u4e2d', '\u{1f600}']; + let s = ""; + const n = Math.floor(Math.random() * 12); + for (let i = 0; i < n; i++) s += alphabet[Math.floor(Math.random() * alphabet.length)]; + return s; + } + if (r < 0.8) return Array.from({ length: Math.floor(Math.random() * 5) }, () => randomJson(depth + 1)); + const o: Record = {}; + for (const k of ["a", "b", "c", "d", "__proto__x", "0", "length"]) { + if (Math.random() < 0.5) o[k] = randomJson(depth + 1); + } + return o; +} + +const c = await createCompartment(); + +// ── 1. Value round-trip, both directions ────────────────────────────────── +const echoed: unknown[] = []; +await c.endow("__h", { echo: (v: unknown) => v, sink: (v: unknown) => { echoed.push(v); } }); +c.load(`globalThis.roundTrip = function (v) { return __h.echo(v); }; + globalThis.sendBack = function (v) { __h.sink(v); };`); + +for (let i = 0; i < 400; i++) { + const value = randomJson(); + const json = JSON.stringify(value); + if (json === undefined) continue; + // host -> guest -> host + c.load(`globalThis.__t = roundTrip(${json});`); + c.load(`sendBack(__t);`); + const got = echoed.pop(); + check("value round-trip", JSON.stringify(got) === json, `${json} became ${JSON.stringify(got)}`); +} + +// ── 2. Identity: the same guest object must map to the same host proxy ──── +const seen: unknown[] = []; +await c.endow("__id", { take: (o: unknown) => { seen.push(o); } }); +c.load(`globalThis.shared = { m: function () { return 1; } }; + globalThis.sendShared = function () { __id.take(shared); __id.take(shared); };`); +c.load(`sendShared();`); +check("identity preserved across crossings", seen[0] === seen[1]); +c.load(`sendShared();`); +check("identity stable across separate calls", seen[0] === seen[2]); + +// ── 3. Callables survive and remain callable ────────────────────────────── +let held: any; +await c.endow("__f", { keep: (fn: unknown) => { held = fn; } }); +c.load(`__f.keep(function (a, b) { return a + b; });`); +check("guest function callable from host", held(2, 3) === 5, `got ${held?.(2, 3)}`); + +// ── 4. Errors cross with name and message ───────────────────────────────── +c.load(`__f.keep(function () { throw new TypeError("boom"); });`); +let msg = ""; +try { held(); } catch (e: any) { msg = e.message; } +check("guest throw reaches host", msg.includes("boom"), `got "${msg}"`); + +// ── 5. Reference release: crossings must not leak ───────────────────────── +await c.endow("__l", { drop: (_fn: unknown) => {} }); +c.load(`globalThis.churn = function () { for (var i = 0; i < 500; i++) __l.drop(function () { return i; }); };`); +c.load(`churn();`); +const beforeGc = c.debugRefCount().guestRefs; +globalThis.gc?.(); +await new Promise((r) => setTimeout(r, 60)); +globalThis.gc?.(); +await new Promise((r) => setTimeout(r, 60)); +const afterGc = c.debugRefCount().guestRefs; +check("dropped references are released", afterGc < beforeGc / 2, `${beforeGc} -> ${afterGc}`); + +// ── 5b. Prototype pollution must not cross ──────────────────────────────── +// JSON.parse is safe (it makes __proto__ an own property), but a membrane that +// assigns parsed keys onto a fresh object is not. +const before = ({} as any).polluted; +c.load(`__h.sink(JSON.parse('{"__proto__":{"polluted":true}}'));`); +check("guest cannot pollute host Object.prototype", ({} as any).polluted === before); +await c.endow("__p", { give: () => JSON.parse('{"__proto__":{"hostPolluted":true}}') }); +await c.endow("__q", { show: (v: string) => check("host cannot pollute guest prototype", v === "undefined", v) }); +c.load(`var got = __p.give(); __q.show(String(({}).hostPolluted));`); + +// ── 6. Interrupt budget, and the VM survives it ─────────────────────────── +// Direct: the budget on this call bounds it. +{ + const t0 = Date.now(); + try { c.load(`while (true) {}`, 100); } catch { /* expected */ } + check("runaway guest code is interrupted", Date.now() - t0 < 600, `took ${Date.now() - t0}ms`); +} +// Re-entrant: a guest function invoked BY THE HOST gets `callGuestRef`'s own +// budget, not the outer one. Budgets do not nest — see README "Limitations". +c.load(`globalThis.spin = function () { while (true) {} };`); +await c.endow("__s", { run: (fn: any) => { try { fn(); return "returned"; } catch { return "threw"; } } }); +{ + const t0 = Date.now(); + try { c.load(`globalThis.__x = __s.run(spin);`, 50); } catch { /* expected */ } + const took = Date.now() - t0; + check("re-entrant call uses its OWN budget, not the outer one", took > 1000, + `took ${took}ms — if this is now <1s the nesting behaviour changed`); +} +c.load(`globalThis.__alive = 1 + 1;`); +check("VM usable after an interrupt", true); + +// ── 7. Async, and clean teardown with a pending promise ─────────────────── +await c.endow("__a", { never: () => new Promise(() => {}), soon: () => Promise.resolve("v") }); +c.load(`globalThis.out = null; __a.soon().then(function (v) { out = v; });`); +await new Promise((r) => setTimeout(r, 30)); +await c.endow("__o", { show: (v: string) => check("async host result reaches guest", v === '"v"', v) }); +c.load(`__o.show(JSON.stringify(out));`); +c.load(`__a.never().then(function () {}, function () {});`); // deliberately unsettled + +let disposeOk = true; +try { c.dispose(); } catch { disposeOk = false; } +check("dispose with a pending promise does not crash", disposeOk); + +console.log(`\n${pass} passed, ${failures.length} failed`); +for (const f of failures.slice(0, 10)) console.log(" FAIL", f); +process.exit(failures.length === 0 ? 0 : 1); diff --git a/packages/agent/docs/mobile-handoff/README.md b/packages/agent/docs/mobile-handoff/README.md new file mode 100644 index 000000000..8ac7c5c1f --- /dev/null +++ b/packages/agent/docs/mobile-handoff/README.md @@ -0,0 +1,81 @@ +# pi — design handoff + +Work in numbered order. Each unit is self-contained and independently testable; +later units consume earlier ones. + +``` +01-harness/ + 01-delta/ op vocabulary, tracker, applier, codec [LANDED IN CHORD] + 02-scopes/ storage scopes and list tags [STEP 1 ACTIONABLE] + 03-execenv/ bounded Shell output, capture, spill [PRODUCTION CODE + TESTS] + 04-tool-output/ the ToolOutput sink [SPEC ONLY] + 05-assistant-output/ assistant partials, symmetric with 04 [SPEC ONLY] +02-plugins/ + 01-facets/ the facet system [SPEC ONLY] + 02-sandbox/ isolated-vm membrane [CODE + 412 tests] +``` + +Base everything on a clean checkout of `origin/dev`. + +## Read this first + +**Three units ship working code. Four are specifications.** The table above says +which. Do not assume a doc describes something that exists. + +**`01-delta/FINDINGS.md` is historical evidence, not an implementation queue.** Production lives in `packages/chord/src/delta/index.ts`: flush-time dirty tracking fixed D1, and production remeasurement closed D2. The explicit append/truncate API is rejected; see [`01-harness/01-delta/append-decision.md`](01-harness/01-delta/append-decision.md). The code beside the handoff remains prototype and benchmark evidence. + +**If a doc and the code disagree, the code wins** — fix the doc and say so in the +commit. + +**Port the tests before the implementation.** Each group's comment explains the +failure it guards against, and several of those failures are silent: wrong output, +no exception. + +**Benchmark with `node --experimental-strip-types`, never through a transpiler.** +Measuring this module through `tsx` inflates results 2.6x. `FINDINGS.md` D5 lists +five more measurement traps, each of which produced a confident wrong conclusion. + +## Unit status + +| unit | ships | state | +| --- | --- | --- | +| **01-delta** | production implementation and tests in `packages/chord`; prototype evidence here | landed; D1 fixed, explicit text API rejected after production remeasurement | +| **02-scopes** | spec + [actionable Step 1 handoff](01-harness/02-scopes/implementation-handoff.md) + `scopes.variance.ts` | Step 1 scopes/list tags actionable, not implemented; JSONL Chord encoding/address interning deferred to separately approved Step 2 | +| **03-execenv** | production implementation in `packages/agent`; prototype evidence here | source-bounded adaptive output, lazy spill backpressure, and bash migration landed; bash's temporary checkpoint cadence moves to `ToolOutput` next | +| **04-tool-output** | spec + design notes | **not built.** The piece every measurement of the op encoding depends on | +| **05-assistant-output** | spec | not built. Same shape as 04; do it after | +| **02-plugins/01-facets** | spec, ~1800 lines | not built. §14 rewritten to match the sandbox PoC | +| **02-plugins/02-sandbox** | working PoC, 412 assertions | `npm install && npm run audit` | + +## Suggested order + +1. **`02-scopes` Step 1** — follow the [actionable implementation handoff](01-harness/02-scopes/implementation-handoff.md); stop for approval before its separate Step 2. +2. **`04-tool-output`** — reuse the landed adaptive publisher for generic tools, Chord event/durable batches, terminal flushes, and atomic memo checkpoints. +3. **05**, then **02-plugins**. + +## Live bugs on `origin/dev`, independent of this design + +- `drive/tools.ts:257` — `clearReplayCheckpoint` deletes `pendingToolOutput` before + re-executing a replay-safe tool. Memos exist so a replayed tool skips work, and + skipped work emits nothing, so output for memoised work is lost today. Seed from + it instead (`harness-tools.md` §7.4). +- `runtime/progress.ts:44` — `commitWrite(item)` captures the value at call time + and writes fire-and-forget, so an older checkpoint can land after a newer one. +- memo and checkpoint are two transactions (`drive/tools.ts:112` vs + `progress.ts:44`). They must be one (`harness-tools.md` §7.5). + +**Expected test churn:** nine tests assert the old cleanup write set and will fail +once `retireScope` replaces the per-address deletes. That is the change landing. + +## Environment + +- Node 22+ for every shipped `.ts` file. They run under + `node --experimental-strip-types` with no build step and no dependencies. +- In the pi repo, tests run from the package: + `cd packages/agent && npx vitest run --config vitest.harness.config.ts`. + The root vitest config does **not** alias `@earendil-works/pi-ai`; the + per-package harness config does. +- Typecheck with `npx tsgo --noEmit` from the repo root. **Baseline is ~788 + pre-existing errors**, almost all in `packages/ai/test`. Count only: + `grep "error TS" | grep -E "packages/(agent|session-backends)/src"`. +- `packages/ai` cannot be built offline — model data is fetched at build time. diff --git a/packages/agent/docs/pico-v3.md b/packages/agent/docs/pico-v3.md new file mode 100644 index 000000000..1189fd3ec --- /dev/null +++ b/packages/agent/docs/pico-v3.md @@ -0,0 +1,2113 @@ +# pico v3 + +Design under discussion. Code and traces illustrate the proposed behavior; they are not existing +package exports. + +## 1. Goals and assumptions + +### 1.1 Small concepts, replaceable behavior + +Run agents using conversations, immutable entries, durable tasks, and scoped values/lists. +Built-in agent behavior is composed from tasks, not embedded in a fixed scheduler state machine. + +```text + ┌─ tool A ─┐ +generation ──────────┼─ tool B ─┼─ post_tools ─ generation + └─ tool C ─┘ + +replace a task definition + → change that behavior + → keep the scheduler and storage +``` + +The scheduler understands task lifecycle, dependencies, timing and cancellation. It does not +understand prompts, tool arguments or summaries. Storage understands stored objects and atomic +mutations, not task behavior. + +### 1.2 One writer per session + +Concurrent execution is allowed. Concurrent independent writers to the same session are not. + +```text +caller ───────┐ +tool ─────────┼─ session owner ─ serialized commands ─ storage +background ──┘ + +one command: + read committed state + construct mutations and assign numeric IDs + commit the complete batch + publish committed changes +``` + +All mutations, including working-state writes, pass through that owner. IDs are unique within the +session and are persisted unchanged. Uncommitted IDs cannot escape to callers or external effects. + +Different sessions may have different owners and share a SQLite database. SQLite still serializes +its write transactions. After a crash, a replacement owner resumes from durable state; it must not +run alongside the previous owner. + +### 1.3 Durable work, explicit recovery + +Durable storage preserves work that has been accepted, including work that has not started yet. +Opening a session starts no task effects. + +```text +commit pending task + → crash + → open: task is still pending + → drive: execute it + +commit external-effect intent + → perform external effect + → commit outcome and required successor tasks together +``` + +A crash during an external effect leaves uncertainty, not proof that nothing happened. Each task +definition must decide how to recover: retry safely, adopt existing work, or report interruption. +Durability does not promise exactly-once external effects. + +```text +close → stop local execution; leave unfinished durable tasks resumable +abort → durably cancel selected work; drive its cancellation to settlement +``` + +### 1.4 Long-lived sessions without residency machinery + +Old transcript entries and terminal tasks remain queryable. The harness should not retain them +merely because it encountered them earlier. + +```text +check a dependency: + read the named task + inspect its terminal status + return + // no harness-owned reference needs to survive the check +``` + +Active executions naturally retain their inputs and working data. Other query results are ordinary +local variables. No pin/unpin API, residency manager, or sweep that maintains a second resident +model. Routine execution should depend on current work and context, not scan accumulated history. + +Backend memory is a separate choice: + +| Backend | Storage-owned memory | Persistence | +|---|---|---| +| Memory | All stored data and indexes | None; useful for tests | +| JSONL | All stored data and indexes, loaded on open | Main file plus working sidecars | +| SQLite | Read results and database caches; no required full-history JS copy | Database | + +JSONL loading everything is intentional, not a violation of the working-set goal. Choose SQLite +when minimizing resident process memory matters. No fixed session-length limit is imposed; storage +capacity, numeric sequence limits and the amount of simultaneously live work remain real limits. + +### 1.5 Efficiency must be demonstrated + +```text +same faux-provider workload on each harness/backend + → CPU time + → total process memory + → actual disk bytes, including working data and auxiliary files +``` + +Prefer small atomic writes, indexed queries and batched reads. Do not turn a point lookup into a +history scan or reload every object to answer a small question. Measure hot paths before adding +caches or specialized machinery. A faster spike is evidence to investigate, not proof that its +interfaces or correctness shortcuts belong in this design. + +## 2. Conversations + +A conversation has a linear transcript of immutable entries and a context list selecting entries +for the model. Tasks and state are associated with it, but do not implicitly become model input. + +### 2.1 Entries and the transcript + +Minimal shapes; `JsonValue` means strict JSON. Historical-position and ownership metadata are +introduced separately. + +```ts +interface Entry { + readonly id: number; + readonly conversationId: number; + readonly kind: string; + readonly content: JsonValue; +} + +interface Conversation { + readonly id: number; + readonly context: readonly number[]; +} +``` + +These are read snapshots. Changing context produces a new conversation snapshot; an existing +entry's content never changes. + +```text +conversation A + +entry ID kind content +10 user "Inspect the parser" +20 assistant "The parser has two problems ..." +30 user "Explain the first one" +40 assistant "The first problem is ..." + +transcript: [10, 20, 30, 40] +``` + +- New entries append; existing entries are never patched, reordered or renumbered. +- IDs increase in transcript order, but need not be consecutive numbers. +- A note can belong to the transcript without belonging to model context. + +### 2.2 Context selects; projection converts + +```text +transcript context IDs provider messages +what happened what is selected what the model receives + +[10, 20, 30, 40] ──────────> [10, 20, 30, 40] ──────> user, assistant, user, assistant + projection +``` + +The context stores IDs, not copied messages or a second transcript. + +```text +entryById = readEntries(conversation.context) // batched read +messages = project selected entries in context order +normalize tool-result groups into their call order +``` + +An entry's kind defines its message contribution. A summary can project as a user message; a +selected custom entry can project nothing. Projection reads content and returns messages. It +writes no entries, state or tasks. + +Tool completion order need not equal call order: + +```text +assistant calls: [A, B] +transcript: assistant → result B → result A +model messages: assistant → result A → result B +``` + +The transcript remains unchanged by that normalization. + +### 2.3 Three context operations + +```text +appendContext(ids) + extend with newly appended selected entries, in transcript order + +replaceContextPrefix(through, replacement) + replace the prefix ending at through; preserve the existing suffix + +resetContext(ids) + replace the whole context with an empty or bootstrap context +``` + +Appending selected content and appending its context ID happen in the same commit. Context-only +changes do not create generation tasks. + +**Compaction:** append a summary entry, then replace a prefix in the same commit. + +```text +before: + transcript [10, 20, 30, 40] + context [10, 20, 30, 40] + +commit: + append summary 50 describing the prefix through 20 + replaceContextPrefix(through=20, replacement=50) + +after: + transcript [10, 20, 30, 40, 50] + context [50, 30, 40] +``` + +Summary 50 is newer than entries 30 and 40, but appears before them in model context. The entries +it summarizes remain in the transcript. Repeated compaction summarizes the previous summary too, +not an ever-growing chain of summary heads. + +**Reset:** discard the selected context, not the transcript. + +```text +commit: + append handoff entry 70 + resetContext([70]) + +after: + transcript [10, 20, 30, 40, 50, 70] + context [70] + +resetContext([]) + → empty model context + → same transcript +``` + +Baseline constraints: + +```text +context shape: optional summary/handoff head + selected entries in transcript order +IDs: existing entries in this conversation or its inherited prefix; no duplicates +replacement: a contiguous PREFIX of the current context, not a consecutive numeric ID range +exchange: replacement/reset must not split an assistant's tool-call/result exchange +concurrency: no replacement/reset during a provider request or unresolved tool exchange +``` + +A summary prepared against an old prefix may still replace it after later appends. A competing +replacement/reset invalidates that prepared prefix. The commit must check this before publishing. + +These operations are not an arbitrary list editor: no middle deletion or unrestricted reordering. +Reset accepts an empty context or a bootstrap head, not an arbitrary permutation of old entries. +Reset alone neither cancels existing tasks nor requests a new model response. + +### 2.4 Forks share history, not future changes + +A historical fork creates a new conversation at a selected historical position. Its transcript +shares the source prefix; new entries belong to the new conversation. + +```text +A transcript: 10 ─ 20 ─ 30 ─ 40 ─ 50 ─ 70 + │ + └────────────── 90 ─ 100 B + +B transcript: [10, 20, 90, 100] + shared local +``` + +Entries 10 and 20 retain their original IDs and owning conversation. There are no copied entry +rows for them. Each conversation still presents a linear transcript. + +The fork inherits the context that existed at its selected position, not today's context truncated +by entry ID: + +```text +P = historical position after the first exchange + +A context at P: [10, 20] +A context now: [70] + +create B from A at P: + B context starts as [10, 20] + NOT [] obtained by filtering today's [70] +``` + +B has its own context list. Later appends, compaction and resets in A cannot change B. Live tasks +are not inherited; continuing B requires explicit new work. + +Here P is a historical position already selected by the caller. §3 defines how selecting a +transcript entry resolves to that position, including the entry-versus-commit distinction. State +initialization and historical inheritance are defined in §4. + +### 2.5 Fork history and task ownership are different relationships + +```text +historical relationship: + conversation A ── fork at P ──> conversation B + +execution ownership: + conversation A ── task T ── owns ──> conversation C +``` + +B is an independent fork. C is an owned child, such as a subagent conversation. A child can start +with fresh or inherited context; that choice is separate from which state gets initialized. + +```text +historical source → where inherited content/state came from +owning task → which execution owns this conversation +``` + +A historical relationship alone does not pull B into A's drive or cancellation scope. Ownership +is explicit; scheduling and cancellation follow the task relationships defined in §§5–6. + +## 3. Identity, commits and historical positions + +One session-global sequence orders mutations. A creation mutation's sequence is the new object's +numeric ID. A complete commit, not an individual mutation, is a historical snapshot boundary. + +### 3.1 Allocate while constructing the command + +```text +last committed sequence: 99 + +build command: + 100 append user entry → entry ID 100 + 101 append context ID 100 + 102 set planMode = true + 103 create generation task → task ID 103 + +persist ONE batch [100–103] +publish the committed changes +return entry ID 100 to caller +``` + +The command can reference IDs it just created. It cannot call external services with them or +return them to callers before persistence succeeds. Storage preserves the supplied numbers. + +```text +command A constructs [100–103] ─ commits ─ command B starts at 104 +command A rejects ─ no write ─ command B starts at 100 +``` + +Numbers are positive safe integers; exhaustion rejects. There are no UUID columns on ordinary +entries, conversations or tasks, and no separately reserved ID ranges. + +### 3.2 Commit boundaries are part of the read result + +Extend the entry snapshot from §2 with its creation commit's end: + +```ts +interface Entry { + readonly commitEnd: number; +} +``` + +The end is known once construction finishes. The caller does not need to enumerate the other +mutations in the commit. + +```text +getEntry(100) + → { id:100, conversationId:1, commitEnd:103, ... } + +getConversation(1, asOf=103) + → context as of that boundary + +getValue(conversation=1, planMode, asOf=103) + → true +``` + +```text +Memory entry metadata contains commitEnd=103 +JSONL commit envelope contains [100,103]; replay derives the entry metadata +SQLite entry row contains commit_end=103, inserted in the same SQL transaction +``` + +Task creation at 103 belongs to the source's durable work. It does not mean a fork inherits that +task. Context and rewindable state are historical; task execution is not rewound. + +### 3.3 Proposed fork contract: after the containing commit + +```text +fork(A, atEntry=100) + → resolve entry 100 to commitEnd 103 + → fork A's transcript/context/rewindable state through 103 + → store source A and boundary 103 + → inherit no live tasks +``` + +This deliberately means **after the commit containing the selected entry**, not an intermediate +state after only mutation 100. No second filter truncates context to IDs <= 100. + +```text +commit [200–203]: + 200 append entry X + 201 append entry Y + 202 append context IDs [200,201] + 203 set value V + +fork at X ─┐ + ├─ same snapshot: includes X, Y and V +fork at Y ─┘ +``` + +This is a proposed semantic choice to review, not merely an indexing trick. A UI must not imply +that X and Y select different snapshots. Strict entry-prefix forks would need a different rule +for associating context/state changes with entries; they must not silently mix a truncated +transcript with later state from the same commit. + +A caller can also select a complete main-commit boundary directly: + +```text +commit [204]: set V = 2 // no transcript entry +fork(A, asOf=204) // selects this state-only change +``` + +Entry and explicit-boundary targets use the same historical query path. A position inside a +commit rejects. A fork requiring usable model context must also select a complete tool exchange; +reject an incomplete exchange rather than silently moving to a different historical position. + +### 3.4 Sequence gaps are normal + +Working writes use the same allocator but do not become retained conversation history. + +```text +100–103 main commit +104–109 working progress +110–112 main settlement, including retirement of that working scope + +retained main history: 100,101,102,103,110,111,112 +next sequence: 113 +``` + +Never use an array offset as the durable identity. A deleted working file does not free its IDs. +Working commits are not fork targets. The backend preserves the global high-water mark even when +working contents are retired; main historical boundaries remain independently identifiable. + +### 3.5 Failure before or after admission + +```text +construction/validation rejection: + no durable writes; no IDs consumed; session remains usable + +persistence outcome uncertain: + stop using this session handle + reopen storage + recover the last complete durable batch and its high-water mark + continue from recovered state, not the old process's guess +``` + +No observer may see half a commit. No effect may treat an uncertain write failure as permission +to retry the same external action with newly allocated IDs. + +### 3.6 Caller-provided identities use ordinary state + +External request keys are optional session-scoped values, not alternate internal IDs. + +```text +accept(input, requestId="client-42"), ONE command: + read session value (pi.request-input, "client-42") + if present: resolve the existing acceptance + otherwise: + create accepted input and its required task/queue changes + set session value (pi.request-input, "client-42") = input ID + +commit succeeds, reply is lost + → caller retries "client-42" + → same accepted input ID, no second input +``` + +Lookup and insertion happen inside the serialized command. Core owns these mappings; normal +callers cannot overwrite them. Conflicting key reuse must reject rather than silently accept a +different request; the exact input-equivalence check belongs with acceptance in §8. Mapping a key +to an input does not answer which eventual result belongs to that input. + +## 4. State + +State is addressed independently of transcript content. Its mutations share the commit sequence, +so a historical boundary selects context and rewindable state together. + +### 4.1 Scope and rewind behavior + +| Address scope | Rewind behavior | Lifetime | +|---|---|---| +| Session | Sticky; one current value/list | Session | +| Conversation | Rewindable or sticky | Conversation and required fork history | +| Named working scope | Never rewound or inherited | Until explicit retirement | + +```ts +const sessionName = sessionValue("pi.session.name"); +const plan = conversationValue("plugin.plan", { rewind: true }); +const expanded = conversationValue("ui.expanded", { rewind: false }); +const moves = conversationList("game.moves", { rewind: true }); + +await conversation.setValue(plan, true); +const enabled = await conversation.getValue(plan) ?? false; +await conversation.appendList(moves, move); +``` + +Addresses bind namespace, optional key, scope and rewind policy. Payload types come from the +address; writes must not infer a different type to accommodate the supplied value. + +```ts +// Value and ValueList denote typed addresses, not wrappers around current data. +interface StateWrites { + setValue(address: Value, value: NoInfer): Promise; + appendList(address: ValueList, value: NoInfer): Promise; +} +``` + +The final interfaces also distinguish address scopes. A conversation address cannot be used as a +session address. Constructors require no global registration; defaults belong at the read site. +Storage persists scope/rewind information rather than depending on token objects surviving restart. +All persisted payloads are strict JSON; callers cannot mutate stored data through returned aliases. + +### 4.2 Small mutation vocabulary + +```text +value: set(value), delete() +list: append(value), clear() +read: getValue(address), readList(address, cursor, limit) +``` + +An append gets a numeric element ID from its mutation sequence. Baseline lists preserve append +order; they do not support replacing individual elements, inserting in the middle or moving them. +Structured editing uses delta values, as illustrated in §4.5. + +```text +rewindable list: + 40 append A + 50 append B + 60 clear + 70 append C + +read through 50 → [A, B] +read through 60 → [] +read through 70 → [C] +``` + +Clear hides older elements at and after its position; it does not erase history required by earlier +forks. Deleting a rewindable scalar similarly records absence. JSON null remains a stored value. + +Session/sticky state exposes only its current contents. JSONL may still contain previous mutations; +the harness does not interpret them as rewindable history. + +### 4.3 Initialization is explicit; a historical fork preserves history + +Creating a new subagent conversation chooses initial state independently of context: + +```ts +await conversation.spawn({ + prompt: "Inspect the parser", + context: "inherit", + values: { + inherit: [model, thinkingLevel], + set: [setValue(activeTools, ["read", "grep"])], + }, +}); +``` + +```text +ONE creation command: + read selected current source values + create child conversation + copy selected present values, then apply overrides + append prompt and create initial generation + +unselected plan mode → absent in child +inherited context → does not imply inherited plugin state +``` + +This initialization copies values, not a live link to parent state. Subsequent reads have no +unselected parent fallback. An absent selected value stays absent unless overridden. The baseline +selection is scalar-only; lists require an explicit initialization policy, not an accidental copy +of an unbounded log. + +A historical fork has a different purpose: + +```text +fork A at P: + rewindable conversation values/lists → inherit all visible history through P + sticky conversation state → copy only explicitly selected current values/overrides + session state → same session-wide state, not copied + working state → never inherited +``` + +The proposed sticky default is no inheritance; callers opt in at creation. Sticky values selected +for copying come from the current source, not P: sticky state is outside the historical snapshot +promise. This is a separate choice from the fork's automatic rewindable inheritance. + +### 4.4 Rewindable lookup follows the fork boundary + +Only local changes need new records. A historical fork does not copy every scalar version or list +prefix into its own storage rows. + +```text +A: plan=false at 40; plan=true at 80 +B: fork A through 60 + +read B.plan: + local version? no + follow source A, capped at 60 + newest version <= 60 is false + +B deletes plan: + local deletion found → absent; do not fall back to A + +B sets plan=true: + local set found → true +``` + +At every ancestor, retain the tighter cutoff: + +```text +readValue(C, address, cutoff): + local = newest local version at or before cutoff + if local is a set: return its value + if local is a deletion: return absent + if C has no historical-state source: return absent + return readValue(C.source, address, min(cutoff, C.sourceBoundary)) +``` + +Lists combine inherited visible ranges and local appends, honoring clear operations and the same +ancestor cutoffs. A list cursor identifies an element position, not an array offset. Fixed asOf +plus cursor/limit gives stable pages while later writes continue. + +Memory/JSONL use indexed arrays; SQLite uses indexed version/range queries. Cost may include fork +depth, but not a scan of unrelated session history. An independently initialized child terminates +fallback; a later historical fork of that child inherits its local history normally. + +### 4.5 Structured state uses deltas and checkpoints + +```text +checkpoint value: { through: 500, state: ... } +delta list: ... 501, 507, 510, 518 ... + +hydrate at P=510: + read checkpoint visible at P + read visible deltas after checkpoint.through through P + apply deltas to privately owned state +``` + +```text +update, ONE command: + read/derive from committed state + append delta batch + optionally set checkpoint including that batch +commit +publish the updated state +``` + +The consumer owns the delta vocabulary, reducer and checkpoint cadence. Chord's delta machinery is +a candidate, not a storage dependency decided here. Storage does not execute plugin reducers. +Checkpointing bounds replay only when its cadence is maintained; it does not authorize deleting +older history while arbitrary historical forks remain supported. A live consumer can retain its +current state; it need not hydrate on every edit. + +### 4.6 Working state is ordinary state with a named lifetime + +```ts +const work = workingScope(String(taskId)); +const frames = list("pi.pending.frames", { scope: work }); +const checkpoint = value("pi.pending.checkpoint", { scope: work }); +``` + +```text +working-only commit: + append frame + set checkpoint + +main settlement commit: + append immutable result assembled from working data + settle task + retireWorkingScope(work) +``` + +Working values/lists survive restart until explicit retirement. They are not transcript entries, +rewindable state, or a separate scratch-storage API. Keys distinguish attempts so recovery cannot +mistake an earlier request's frames for the current request's output. + +```text +allowed commit: main writes, including retirement of working scopes +allowed commit: value/list writes in ONE working scope +rejected: direct main + working writes, or writes in two working scopes +``` + +This lets JSONL persist each batch to one file. Main retirement is authoritative; unlink happens +afterward. SQLite can delete working rows inside the main settlement transaction. + +```text +crash before settlement → task and working data remain recoverable +crash after settlement → result and retirement both exist; leftover file is invisible +close → no retirement +``` + +Before settlement, stop admitting progress from that execution and drain already admitted writes. +Late callbacks must be rejected, not allowed to recreate retired task data. Generic scope names +can be reused deliberately; retirement ends the old contents, not every future use of the name. +Storage never guesses retirement from a missing task, task kind or owner status. + +## 5. Tasks + +A task is durable work associated with one conversation. Its definition supplies behavior; its +stored record supplies the inputs, progress and eventual outcome needed across process lifetimes. + +### 5.1 Task records and lifecycle + +```ts +type TaskRole = "ready" | "inflight" | "waiting" | "terminal"; + +interface Task { + readonly id: number; + readonly conversationId: number; + readonly kind: string; + readonly status: string; + readonly role: TaskRole; + readonly state: State; + readonly after: readonly number[]; + readonly foreground: boolean; + readonly spawnedBy?: number; + readonly ownedConversationId?: number; +} +``` + +Timing and cancellation metadata are added in §6. State is validated JSON, not a promise, closure +or process handle. External handles must have a durable representation that recovery understands. + +| Role | Meaning | Action when covered by a drive and not executing locally | +|---|---|---| +| ready | Can execute once dependencies and timing/permissions allow | Execute | +| inflight | External-effect intent was committed; outcome may be unknown | Recover | +| waiting | A child or external observer supplies the next transition | Restore observation when needed | +| terminal | Immutable outcome | None | + +A definition maps its statuses to these generic roles: + +```text +generation: + pending, retry_wait, deferred → ready + streaming, polling → inflight + done, failed, aborted → terminal + +job: + planned → ready + spawning → inflight + running → waiting + exited, killed, lost → terminal +``` + +The harness derives role from status when validating a write. Storage persists/indexes that generic +metadata so querying live work does not decode every historical task's state or execute kind code. +Role is not independently editable. Terminal records accept no further patches. + +### 5.2 Definitions are replaceable units of behavior + +```ts +interface TaskDefinition { + readonly kind: string; + readonly initialStatus: string; + readonly roles: Readonly>; + readonly transitions: Readonly>; + validateState(value: unknown): State; + execute(task: Task, ctx: TaskExecution): Promise; + recover(task: Task, ctx: TaskExecution): Promise; + abort(task: Task, ctx: TaskExecution): Promise; +} +``` + +TaskExecution supplies the execution signal, working scope and serialized command access. Its +concrete surface belongs in §§6 and 9; it is not another durable object. + +```text +registry: + "generation" → custom generation definition + "tool" → built-in tool definition + "post_tools" → custom exchange policy + "job" → application process integration +``` + +The scheduler does not change when a definition changes. A replacement must understand the +persisted statuses/state of its live tasks; incompatible durable formats need an explicit migration. +Unknown live kinds or invalid live statuses reject restoration before execution begins. + +A task handles ordinary provider/tool failures according to its policy. Storage failures and +invariant violations are not ordinary tool errors: they fault the session instead of being +swallowed and retried indefinitely. + +### 5.3 Dependencies mean terminal, not successful + +These traces assume T has a ready-role status and its timing/permissions allow execution. + +```text +T.after = [A, B] + +A running, B done → T blocked +A failed, B done → T eligible +A aborted, B done → T eligible unless T itself was cancelled +``` + +T interprets the outcomes it requires. The scheduler does not propagate success/failure policy +along edges. Referenced tasks must exist, and dependency edits must preserve an acyclic graph. +Dependencies stay within one execution-ownership tree; independent forks do not become runnable +merely because another tree refers to them. + +```text +spawnedBy = who created this task // provenance +after = which tasks must settle // execution dependency +``` + +These are different. A generation can create a collapse task and then depend on it; that provenance +link does not create a dependency cycle. + +### 5.4 A generation publishes tools and their join atomically + +```text +G returns assistant with calls [A, B] + +ONE settlement command: + append assistant entry + create tool A + create tool B + create P = post_tools, after:[A,B] + settle G + +A settles its own result ─┐ + ├─ P becomes eligible ─ { settle P; create G2 } +B settles its own result ─┘ +``` + +Each terminal command also records known usage and retires its working scope when present. A tool +settles only itself: no sibling inspection, queue draining or next-generation creation. P owns the +exchange decision and remains foreground even after both tools have settled. + +```text +parallel: + A; B; C; P.after=[A,B,C] + +sequential: + A; B.after=[A]; C.after=[B]; P.after=[A,B,C] +``` + +Same tools, same join, different dependencies. No tool-specific scheduler lock is required. + +```text +crash before G settlement → G remains live; normal recovery applies +crash after G settlement → assistant, tools and P all exist +crash after last tool → P already exists; no tail scan or successor repair +``` + +A final-answer generation either creates its explicit continuation in its terminal command or +ends the chain. An idle user-shaped context does not create work by inference. + +### 5.5 A durable wait is not a task-lifetime promise + +```text +DO NOT: + create child task + await child task's entire lifetime inside this effect + +DO: + commit dependency and continuation state + return from this effect + execute the continuation when its dependencies become terminal +``` + +A foreground tool delegating to a subagent can reuse its own task: + +```text +tool execute: + ONE command: + create subagent owner S + set tool status=finishing, after=[S] + return + +S starts child conversation, then returns in waiting status +child final settlement also settles S +existing tool.finishing becomes eligible + → read S's result + → run result hooks + → commit own tool result and terminal status +``` + +Recovery uses stored tasks/dependencies, not reconstructed chains of waitFor promises. Joining this +process's current execution of the SAME task during cancellation is different; §6 defines that +ordering so effect and abort handlers cannot write concurrently. + +### 5.6 Foreground work and detached background work + +Foreground is chosen when work is created, not inferred from a tool name or the transcript tail. + +```text +local foreground roots: + generation, its tools, post_tools, automatic collapse + +required foreground work: + those roots + unfinished dependencies + + required foreground work in their owned conversations + +detached background work: + other live tasks, such as background jobs, schedules and background subagents +``` + +A task needed by a foreground dependency is part of that required work even if it was not created +as a local foreground root. To detach a launch, return its ID instead of creating such a dependency. +Detailed cancellation selection is defined in §6. + +```text +foreground launch: + { create S; tool finishing after:[S] } + parent exchange cannot finish until S supplies a result + +background launch: + { create S; tool result contains S.id; settle tool } + parent exchange can finish while S continues +``` + +Normal conversation abort targets foreground/required work, not detached background work. The +public API also needs cancellation of a selected background operation and all-background shutdown; +these select cancellation scopes, not arbitrary edits to task status. + +### 5.7 Ownership links a task to a child conversation + +```text +conversation A + task S ── owns ──> conversation C + generation, tools, post_tools, ... +``` + +```ts +interface Conversation { + readonly ownerTaskId?: number; +} +``` + +S.ownedConversationId and C.ownerTaskId are reciprocal and commit together. Fork provenance does +not establish ownership. Child completion settles its existing owner in the same command when +appropriate; it does not depend on a later notification manufacturing missing work. + +An ordinary fork has no owner and no copied tasks. A task can create a child, initialize its chosen +state/context, append its prompt and create its first generation atomically. Starting any of those +effects waits until that creation command is durable. + +### 5.8 What every definition must specify + +```text +creation inputs/settings captured; initial status; foreground/dependencies +execution durable intent before external work; bounded working-state writes +parking persisted continuation/dependency/observer state; effect returns +settlement output + usage + terminal state + successors + working retirement +recovery retry/adopt/interrupted policy for every nonterminal status +cancellation stop/join own execution; required result/cleanup; no normal successors +``` + +Normal successor-creating commands re-check current task status and cancellation before committing. +Task-specific cancellation must preserve required result records, such as an error result for an +unexecuted tool call. No definition may depend on the scheduler knowing its domain semantics. + +## 6. Scheduling and cancellation + +The scheduler executes durable task records. A drive call selects where execution is allowed; +it does not create work or become a second owner of tasks already executing. + +### 6.1 Generic scheduling metadata + +```ts +interface Task { + readonly notBefore?: number; // earliest execution time + readonly requiredPermit?: string; // optional host permission, e.g. deferred polling + readonly abortRequested?: true; // durable cancellation membership +} + +interface DriveOptions { + readonly permits?: readonly string[]; + readonly signal?: AbortSignal; // cancels this caller's wait, not durable tasks +} +``` + +The scheduler tests a permit name, not a provider's polling semantics. A retry stores a timestamp +on the existing task, not a promise that must survive restart. + +```text +ready to execute = ready role + + every after dependency terminal + + notBefore reached + + requiredPermit supplied by a covering drive + + not cancelled or already executing +``` + +### 6.2 Drive scopes and outcomes + +```text +conversation.drive() → that conversation and descendants reached through task ownership +harness.drive() → all independent conversation trees + +fork provenance → never expands a drive scope +``` + +Overlapping callers share one scheduler. Permissions apply only inside the caller's scope; a +polling permit for one tree must not authorize unrelated trees. + +| Outcome | Conversation drive | Session drive | +|---|---|---| +| idle | No required foreground work or selected cancellation remains in its tree | No live tasks anywhere | +| suspended | Work remains but needs an outside event/permission | Same | +| closed | Host closed before the requested condition | Same | + +A future timer is local progress: wait for it rather than report suspension. An installed human +or process observer can leave the drive suspended. The observer may commit its task's outcome; +subsequent task execution requires a covering drive again. + +Foreground completion is rooted at the requested conversation: foreground tasks inside a detached +child do not make its parent busy. Driving that child directly does wait for its own foreground. +A conversation drive services background tasks while active, but does not wait solely for detached +work. Hosts supervising background work drive the session and resume after relevant external events. + +### 6.3 One execution claim per task + +```ts +interface ExecutionClaim { + readonly controller: AbortController; + readonly promise: Promise; +} +const executing = new Map(); +``` + +This map owns actual executions, not a mirror of stored tasks. Waiting observers and their re-arm +markers exist only for live waiting phases; discard them on phase exit, settlement or close. +Determine scope membership from live task queries and ownership ancestor point reads, not by +walking every completed child conversation ever created. Temporary ancestor lookups can be shared +within a pass without becoming a permanent resident model. + +```text +scheduler pass: + read live tasks in covered scopes, using bounded indexed queries + signal EVERY affected executing cancellation target + reconcile cancellation targets under exclusive claims + + for each unmarked, unclaimed task: + ready → test dependencies/time/permit; claim and execute + inflight → claim and recover + waiting → if not armed for this waiting phase: claim and recover to re-arm + + derive caller outcomes from committed state + await commit | execution completion | next timer | outside notification +``` + +Claim before launching. Do not await an entire effect inside the scan before considering the next +task. Initial execution can install its observer; recovery must not install a duplicate. A waiting +owner whose child supplies completion needs no subscription and can simply mark re-arming complete. + +Effects and observers re-check current task status, cancellation and their execution validity on +the command line before writing. Events wake checks; they are not durable continuation messages. +An unchanged ready task must not be relaunched forever after an unexpected exception: report it +and fault the handle. Ordinary retry belongs in a committed task transition. + +### 6.4 Cancellation selects work, then drives cleanup + +```text +conversation.abort(): + select current required foreground work (§5.6) + +conversation.abortBackground(taskId): + require a detached background root in this conversation + select it, unfinished dependencies, and all live work in their owned subtrees + +conversation.abortBackground(): + apply that selection to detached background work in this ownership tree +``` + +The background handle is a task ID, but this is scope cancellation, not permission to terminalize +an arbitrary node or skip its cleanup. A task currently required by foreground work is not detached; +use conversation abort for that chain. Further dependency-sharing policies remain a review point. + +```text +ONE cancellation command: + capture selected task IDs; mark every nonterminal target abortRequested + for foreground abort, cancel currently queued steer/followUp in affected conversations + keep context-only writes and nextRun input + +repeated abort while the same foreground cancellation is outstanding: + join existing cancellation; do not drain newly arrived input again +``` + +The marks survive restart. Do not shrink selection as earlier tasks settle. New foreground input +queues while cancellation is outstanding; normal successors and owned-child creation cannot escape +a marked task. Background cancellation also prevents new work beneath its marked owner. + +### 6.5 Join an execution before invoking its abort handler + +```text +for each marked task: + keep/acquire ONE exclusive execution claim + signal and await its current local execution, if any + report its rejection; storage faults forbid further writes + re-read committed task + if already terminal: release claim + otherwise: invoke definition.abort under the SAME claim + release only after cleanup returns +``` + +Signal all affected executions before waiting on a slow one. Unstarted tasks can abort without +waiting for their normal after dependencies. An uncooperative effect can delay its own cleanup; +a timeout must not silently admit a second writer for that task. + +A marked child-conversation owner settles only after its selected child work is terminal. The +scheduler waits for that condition, not an effect-held child-lifetime promise. Child completion +checks the owner's cancellation mark before publishing success. Direct foreground abort of an +owned child must also complete its owner as cancelled. That child drive may finish cancellation +of its marked owning task after selected child work is terminal, even though the owner record is +in the parent. It must not start other parent effects; parent-side continuation requires a covering +drive of the parent. + +```text +post_tools settlement wins: + { settle P; create G2 } → abort selects G2 + +abort wins: + { mark P } → P.abort settles without G2 +``` + +Task-specific error/result records still belong to task handlers. Cancellation does not synthesize +provider messages or interpret tool arguments in the scheduler. + +### 6.6 Open, close, shutdown and deletion + +```text +open: + open storage; validate live task definitions + expose inspect(), state and transcript queries + start no provider, tool or process effects + +close: + seal public/effect admission + signal local executions; stop observers/progress admission + wait for cooperative executions and admitted persistence + release storage; write no cancellation or terminal outcomes + +harness.shutdown: + seal public admission but allow internal cancellation commands + mark ALL live session work, including background and owned descendants + cancel all queued inputs, including write/nextRun; preserve their immutable history + drive selected cancellation to settlement + close +``` + +Close itself adds no outcomes; commands already admitted before its barrier can still complete, +including terminal settlements. It interrupts local execution, not necessarily an external process +that can be adopted later. Shutdown uses each kind's abort policy to stop external work where supported. A crash during +shutdown leaves durable cancellation marks for the next owner to reconcile. + +Observer-wait cancellation only removes that caller's drive interest; it never signals shared +executions as if the conversation were aborted. Close remains cooperative and may block on a +broken external integration. Bounded-close/forced-process policies need an explicit host contract. + +Deleting a conversation is a separate atomic mutation: reject while its ownership subtree has +live work. Once deleted, reject new work there. Preserve immutable history and source metadata +needed by independent forks; logical deletion is not permission to erase their inherited data. + +## 7. Storage + +One storage interface answers entity/state queries and commits atomic mutation batches. There is +no public journal reader, separate read-index interface, scratch store or payload-address system. + +### 7.1 Store the information queries need + +Complete the conversation snapshot with its historical source and context-replacement revision: + +```ts +interface Conversation { + readonly source?: { readonly conversationId: number; readonly asOf: number }; + readonly inheritState: boolean; + readonly prefixRevision: number; +} + +interface Entry { + readonly metadata?: JsonObject; // small preview/attribution/exchange fields + readonly key?: string; // optional indexed content key, not a state address + readonly byTaskId?: number; +} +type EntryHeader = Omit; +``` + +Source links cap inherited transcript access. Historical forks also set inheritState=true; +independently initialized children set it false even when inheriting context. Owner links remain +separate. Source and ownership identity are not arbitrary mutable configuration. + +Current context is materialized. Its historical edits are retained. Prefix replacement/reset bumps +prefixRevision; append does not. Task role, conversation ID and scheduling fields are queryable +without decoding kind-specific state. Entry metadata permits exchange checks without loading large +message/image content. + +### 7.2 Queries, motivated by callers + +| Caller | Required read | Must not do | +|---|---|---| +| Scheduler | Live tasks in a scope; dependency IDs in one batch | Load terminal task history | +| Generation | Current context IDs, then those entries in a batch | Reconstruct context by scanning transcript | +| UI | Transcript page before/after an ID, with a limit | Load the whole conversation | +| Validation | Named task/conversation records and entry headers | Load unrelated message content | +| Fork | Entry commit boundary; historical context/state | Truncate today's context | +| State consumer | Latest value/version; bounded list range | Replay unrelated journal records | +| Background status | Task and named result entry | Keep an ancient task resident forever | +| Reopen on SQLite | Live tasks and required conversations | Replay all historical task transitions | + +Filters apply before limits. Latest matching record means an indexed descending query with limit=1, +not loading a collection and taking its last item. Conversation batch reads matter just as entry +and task batch reads do. + +```ts +interface CursorPage { + readonly after?: number; + readonly before?: number; + readonly order?: "asc" | "desc"; + readonly limit: number; +} +interface Page { + readonly items: readonly T[]; + readonly asOf: number; + readonly next?: number; +} +interface ConversationQuery extends CursorPage { + readonly sourceConversationId?: number; + readonly ownerTaskId?: number; +} +interface EntryQuery extends CursorPage { + readonly conversationId: number; + readonly kind?: string; + readonly key?: string; + readonly asOf?: number; +} +interface TaskQuery extends CursorPage { + readonly conversationIds?: readonly number[]; + readonly live?: boolean; + readonly role?: TaskRole; + readonly kind?: string; + readonly foreground?: boolean; + readonly abortRequested?: boolean; +} +interface ListQuery extends CursorPage { readonly asOf?: number } +interface StoredValue { readonly seq: number; readonly value: T } +interface ListElement { readonly id: number; readonly value: T } +interface CommitBoundary { readonly first: number; readonly last: number } +``` + +Transcript queries return the effective inherited prefix plus local entries; each entry retains +its original conversationId. Tasks are never inherited by that query convention. Historical state +uses §4's capped source traversal. A page cutoff is not a generic timestamp for every data class: +sticky state and task scans expose current state, not historical snapshots. Transcript/rewindable +pages carry a main boundary; current task/sticky/working pages carry a global observation cutoff, +which is not a fork target. Historical reads on sticky/working addresses reject. Coherent current +multi-page reads use the command line; a cursor alone does not freeze mutable sticky contents. + +### 7.3 One interface, no second transaction layer + +```ts +interface Storage { + readonly head: number; // global allocator high-water mark + readonly mainHead: number; // last main commit, not another allocator + commit(batch: CommitBatch): Promise; + + getConversations(ids: readonly number[], asOf?: number): Promise>; + scanConversations(query: ConversationQuery): Promise>; + getEntries(ids: readonly number[]): Promise>; + getEntries(ids: readonly number[], options: { content: false }): Promise>; + scanEntries(query: EntryQuery): Promise>; + getTasks(ids: readonly number[]): Promise>; + scanTasks(query: TaskQuery): Promise>; + + getValue(address: Value, asOf?: number): Promise | undefined>; + scanValues(query: ValueQuery): Promise; + readList(address: ValueList, query: ListQuery): Promise>>; + close(): Promise; +} +``` + +ValueQuery is a bounded scope/namespace scan ordered by address key; its cursor is an address key, +not a numeric entity cursor. ValuePage returns addresses with their current visible values. No +arbitrary predicates, plugin reducers, user comparators or general SQL expressions cross this API. +State tokens arrive with their conversation/working identity bound; scope-specific receiver shapes +are shown in §9. + +Each logical read, including ancestor traversal, observes one committed state. The session command +line supplies coherent multi-read operations and read-modify-write; no second public storage +transaction callback is needed. Historical context reads affect context, not current sticky state. +Tasks remain current even when their originating entries are old. + +### 7.4 Atomic mutations + +```text +main mutation vocabulary: + createConversation, deleteConversation + appendEntry, createTask, patchTask + appendContext, replaceContextPrefix, resetContext + setValue, deleteValue, appendList, clearList + retireWorkingScope + +working mutation vocabulary: + setValue, deleteValue, appendList, clearList +``` + +```ts +interface CommitBatch { + readonly scope: "main" | { readonly working: string }; + readonly writes: readonly Write[]; // tagged mutations above; each has its final seq +} +``` + +```text +harness command: + read committed state; construct mutations and IDs + validate against committed state + earlier writes in this batch + no mutations → return builder result without a storage commit + otherwise persist one batch + notify observers; resolve caller + +storage commit: + check scope, supplied sequences and structural constraints + prepare only touched data/index changes + persist atomically + publish new query-visible state and heads together +``` + +Task definitions validate status transitions/state; the harness enforces ownership, dependency, +context and admission rules. Storage stores the validated generic records without running kinds. +Rejected construction changes nothing. An admitted failure faults the handle (§3.5). Cancelling a +caller after persistence admission must not abandon the transaction or expose half a batch. + +### 7.5 Memory and JSONL: the same queries, with optional files + +```text +current maps: conversations, tasks, scoped values/lists +ordered data: entries per conversation, state versions, context edits +lookup maps: IDs, addresses, live-task membership, source/owner relationships +``` + +Memory applies prepared changes without exposing intermediate state. Indexes may reference the +same immutable payload objects; they need not duplicate content. JSONL uses that representation +and loads it fully on open. + +```json +{"first":100,"last":103,"writes":[ + {"type":"appendEntry","conversationId":1,"kind":"user","content":"Inspect"}, + {"type":"appendContext","conversationId":1,"ids":[100]}, + {"type":"setValue","conversationId":1,"namespace":"plugin.plan","value":true}, + {"type":"createTask","conversationId":1,"kind":"generation","status":"pending"} +]} +``` + +Shown across lines for reading; the file contains one complete line per batch. This is illustrative +encoding, omitting address/task fields. A compact encoding may derive each stored seq from first+i; +logical writes already carried their final IDs before encoding. + +```text +JSONL commit → encode whole batch → write all bytes + newline → publish in-memory changes +JSONL open → validate complete batches → rebuild maps/indexes → expose queries +``` + +Do not publish in-memory changes before the write completes. Handle short writes. Discard only an +unterminated final batch; malformed complete lines fail open rather than silently deleting history. + +```text +session.jsonl main batches +session.scopes/ working batches for one named scope +``` + +On open, compute global high-water from all complete main/working batches before cleanup. Main +retirement hides that scope's records through its retirement seq. Later deliberate reuse survives; +serialize old unlink before reuse so delayed cleanup cannot delete a new lifetime. Only explicit +retirement authorizes cleanup. Process-crash durability is required; fsync/power-loss policy is an +explicit backend option, not a claim that ordinary writes are immune to power loss. + +### 7.6 SQLite: queryable rows, not mandatory raw-journal replay + +```text +conversations / current_context_elements +entries +tasks // current records, including terminal outcomes +current_values / value_versions +list_elements / historical_clear_markers +context_edits +commit_boundaries / session_metadata +``` + +Every shared-container key starts with session ID. Suggested indexed suffixes: + +```text +entries (conversation, id), (conversation, kind, key, id) +tasks (role, id), (conversation, role, id) +values (scope, namespace, key) +value_versions (conversation, namespace, key, seq) +list_elements (scope, namespace, key, id) +context_edits (conversation, seq) +conversations (sourceConversationId), (ownerTaskId) +``` + +Live scans also need ordered partial indexes, for example (session_id, id) WHERE role != 'terminal', +so scanning across nonterminal roles does not sort all task history. Additional combinations follow +actual query plans, not an index for every possible filter. + +```sql +-- One local step in a rewindable scalar lookup. +SELECT seq, deleted, value +FROM value_versions +WHERE session_id = :session AND conversation_id = :conversation + AND namespace = :namespace AND key = :key AND seq <= :cutoff +ORDER BY seq DESC LIMIT 1; +``` + +```text +BEGIN + validate global head + insert entry 100 with commit_end=103 + append context element 100; retain context edit 101 + insert rewindable value version 102 + insert current task 103 with indexed role + record main boundary 103; advance global head +COMMIT +``` + +No full transcript/context rewrite per append. Current context can use ordered element rows; +prefix replacement removes the selected prefix and inserts its head without rewriting the retained +suffix. A historical fork stores a source link and its initial context, not copies of source entry +or value/list inventories. Sticky updates overwrite current contents; working retirement deletes +working rows inside the settlement transaction. Retain only the histories the API promises. + +### 7.7 Historical context is not today's context minus later IDs + +```text +contextAsOf(C, P): + start from latest reset <= P, otherwise C's initial context + page later context edits through P in order + append → extend + replace → replace named prefix, keeping the existing suffix + reset → replace whole list +``` + +A replacement is not a full checkpoint: it says nothing about the retained suffix's earlier +appends. The baseline replay may span from creation/reset; it is correct but not necessarily cheap +for ancient forks. Indexed context checkpoints can optimize it later if measurements justify them. +Current generation requests use the materialized context instead of this historical path. + +## 8. Built-in task flows + +These are task definitions and boundary policies, not scheduler special cases. Braces below group +one main commit. Every attempt outcome includes its reported usage; terminal outcomes retire their +working data. Hooks and external calls run outside the command line. + +### 8.1 Acceptance, placement and result ownership + +Proposed input representation: use an immutable admission entry as the input identity. Idle input +can be a selected user entry directly; busy input is unselected until a later placement entry. + +```text +idle accept A: + { user A with inline message; select A; G.inputs=[A]; result[A]=running } + +busy accept B: + { unselected input B with inline message; enqueue B; result[B]=queued } + +later consume B: + { user placement U referring to B; select U; dequeue B; + create G2.inputs=[B]; result[B]=running } +``` + +The placement is new, so context remains in transcript order. Its message is resolved from B's +immutable content, not copied into another stored payload. Projection batch-loads such references; +placement references point directly to admission entries, never chains of placements. UI rendering +can distinguish arrival from placement rather than displaying the same message twice. + +Queues use ordinary sticky values under conversation namespaces, one address per queued input. +Separate namespaces identify queue modes; fixed-width decimal input-ID keys preserve numeric +admission order in bounded key scans. Dequeue deletes the current queue value; it does not require +rewriting a growing inbox array or retaining consumed queue rows. +Session-scoped input-result values preserve each acceptance's outcome independently of that queue. + +```text +G1.inputs=[A] +B queued while G1 runs +G1 produces final answer R + +{ result[A]=done(R); settle G1; + place B; dequeue B; G2.inputs=[B]; result[B]=running } + +result(B) is NOT R just because R was appended after B's admission +``` + +At a tool boundary, newly consumed steering joins the current pending input group. A yield hook's +continuation also retains that group. Only a completed answer/stopping boundary resolves it. +Follow-ups consumed after an answer start a new group. Several inputs can legitimately share one +answer. Carry input IDs through generation/post_tools state, not through a new durable run object. + +```ts +type InputResult = + | { readonly status: "queued" | "running" } + | { readonly status: "done" | "placed"; readonly entryId: number } + | { readonly status: "cancelled" | "failed" | "stopped"; readonly reason?: string; readonly entryId?: number }; +``` + +Placed is the terminal receipt for a context-only write, not an assistant answer. Cancelling a +queued input records cancelled; cancelling an active chain resolves its pending inputs in the +terminal boundary command. Retrying/deferred tasks leave them running. Session-wide request keys +resolve the original admission; reusing a key for another conversation or different normalized +semantic input rejects. Harness-generated metadata is not part of input equivalence. + +This representation/result contract is proposed for review. It avoids chronological attribution, +a new input table, and duplicate message payloads; it does add placement records for queued input. + +### 8.2 Queue boundaries + +| Input mode | When it can be placed | Requests generation? | +|---|---|---| +| write | Next safe context boundary | No | +| steer | Next tool or final-answer boundary; explicit idle acceptance | Yes | +| followUp | Normal final-answer boundary; explicit idle acceptance | Yes | +| nextRun | Next explicit idle acceptance only | Yes | + +```text +idle acceptance: + place eligible queued items, then new input + create ONE generation with all selected trigger IDs + +post_tools: + place safe writes and selected steering + continue current input group unless cancelled/terminated + +final-answer generation: + if yield hook continues: retain current input group + otherwise resolve current group with this answer + eligible steer/followUp → place them and create next group/generation + +failure/termination: + resolve current group; do not consume triggers with no successor + +foreground abort: + cancel queued steer/followUp; preserve write/nextRun +``` + +Steering/follow-up modes select all eligible items or one oldest item of each mode. Preserve +admission order among selected items. Safe write placement never becomes a hidden run request. +Selection, placement, dequeue, result-state changes and required generation creation share a commit. +Queued input takes precedence over an old yield decision; stale hook preparation is discarded. + +### 8.3 Generation + +```text +create: + capture model, thinking, tools, provider options and retry policy + persist inputs and pending status + +pending/retry execution: + check current status/cancellation and request/exchange exclusion + completed captured collapse → interpret its outcome once; do not recreate it on every retry + live collapse → attach dependency and return + automatic collapse needed → create C and G.after=[C] together; return + capture main historical context boundary; commit streaming intent + project context; apply before_request; re-check execution validity + call provider; persist frames under this attempt's working keys + +outcome: + calls → { assistant; tool tasks; post_tools with inputs; settle G } + final answer → { assistant; resolve/continue inputs; settle G; required successor } + deferred → { usage; same G.deferred(handle), requiredPermit="deferred-poll" } + retryable → { usage; same G.retry_wait, attempt+1, notBefore } + overflow → { usage; same G waiting on one captured collapse retry } + failure → { partial/error if needed; resolve inputs failed; settle G } +``` + +A generation task exists throughout preflight/collapse/retry. Retries do not create a replacement +with a fresh budget. Polling records intent before fetching the stored handle; another deferred +response updates the same task. Transitions clear permit/timing metadata that no longer applies; +a normal retry must not accidentally retain a deferred-poll requirement. Poll permits and retry +timestamps remain generic scheduler data. +Known usage commits even for failed, deferred or discarded attempts; no report means unknown cost, +not zero and not an exactly-once billing guarantee. + +```text +recover polling → deferred; retain handle and budget +recover streaming → committed partial: publish interrupted outcome + required call errors + no partial: retry within budget or fail +abort → drain frames; best-effort provider cancellation; required partial/results; + resolve pending inputs cancelled; no tools or normal successor +``` + +The interrupted-stream policy and provider-specific retry/overflow classification need parity +review. Do not catch persistence/invariant failures as provider errors. A never-started generation +has no fabricated assistant response. Stop reasons containing unusable/truncated calls still need +error results without invoking those tools. + +### 8.4 Tool execution and post_tools + +```text +tool execute: + validate offered-tool authorization and arguments + before_tool → allow | block | hold + allow → commit running intent with effective args and replay policy + invoke tool with execution signal, identity and bounded sink + block/truncated/missing → prepare own error result without external invocation + run after_tool outside line + { own result; usage; active-tool additions; control intent; settle tool } +``` + +Registry presence is not authorization: the generation must have offered the tool. Persist the +actual arguments/replay decision before invocation. Ordinary throws produce error results, not +cancellation. Capture terminate/handoff intent in the outcome; the tool does not reset context or +inspect siblings. Foreground delegation uses the finishing phase in §5.5. + +```text +recover → replay only when captured AND current tool policy permit it; + otherwise interrupted result from checkpoint, no uncertain control side effects +abort → join own execution; drain working writes; own aborted error result +``` + +```text +post_tools, ONE command: + require all calls terminal; re-check its cancellation + read outcomes and choose exchange policy + stopped → resolve current inputs stopped/cancelled; no successor + continue → place eligible writes/steering; apply valid handoff; create G2 with inputs + settle post_tools + +post_tools.abort: + { resolve its pending input group cancelled; settle post_tools } + no context edit, trigger consumption or successor +``` + +A new_context tool returns handoff intent. The join appends/selects that handoff and resets context +only after the exchange is complete. Ordinary errors can still be followed by generation. The draft +uses any terminate request to stop; conflicting handoffs reject the handoff rather than choosing +completion order. These policies are reviewable independently of the join mechanism. + +### 8.5 Collapse, including speculative completion + +```text +create C: + capture complete prefix, through, prefixRevision, historical context and model/settings + allow one live collapse per conversation; generation can depend on that existing C + +execute: + commit summarizing intent → call summarizer + persist complete candidate, including reported usage, in working scope + { usage; candidate reference/phase identifying that the attempt outcome was recorded } + editable context + matching prefix → { summary; replace prefix; settle C } + stale prefix/cancelled → terminal outcome without context publication + context busy → park C waiting for an editable-context predicate +``` + +The parked task restores an observer that checks committed context/request state, then makes C +ready to publish. Events only wake that check. Re-check prefix and editability in the publishing +command; do not record usage twice. + +**Do not wait for a generation's terminal status to make context editable:** a retry can keep the +same generation alive while it waits for C. That would form a deadlock. The context predicate, +not the generation's lifetime, controls speculative publication. + +```text +G retry_wait after:[C] → no provider request in flight +C sees editable context → publish and settle +G becomes eligible → retry with new context +``` + +Recovery reuses a durable complete candidate or retries within budget. Abort records known usage +and settles without replacement. C never creates G. Failed/declined overflow compaction ends the +bounded overflow retry; threshold/manual failure can leave G using unchanged context. + +### 8.6 Subagent and approval + +```text +subagent owner S: + { child with owner=S; selected initial state; prompt; first generation; S waiting } + return + +child terminal foreground boundary: + { its output/input outcomes; terminal child task; settle S with result reference } +``` + +For inherited subagent context, the proposed default is the last complete-exchange boundary before +the launching exchange, not its unresolved calls. Selected initial values still come from the +creation command. This is initialization, not a historical fork claiming one state/context snapshot. +Recovery drives the existing child tasks; it does not recreate the child or await a child-lifetime +promise. Cancellation completion follows §6.5 and must not report success for a marked owner. + +```text +hold tool: + { create approval A; tool waiting after:[A] } + +grant: + { A granted; matching tool ready with approved arguments } + +deny: + { A denied; matching tool finishing with denial outcome } + tool runs its result hooks, writes own result and settles +``` + +Approval recovery restores the human decision surface; it never grants automatically. Cancellation +marks the approval and its required tool independently. Grant/deny versus abort is serialized; +late decisions cannot restart terminal work. + +### 8.7 Jobs and schedules + +```text +job: + { spawning intent with durable adoption/log identity } + spawn or reconcile existing process + { running with verified process identity } + install output/exit observers; return + +exit: + drain output → { immutable result; usage if reported; terminal job } +``` + +Recovery adopts the same process, or records lost if its outcome cannot be recovered. PID alone is +not identity. Abort stops the process through its host service, drains output and records killed. +Close detaches local observation; an adoptable external job can survive it. Process launch/exit +reconciliation and log retention are explicit host capabilities, not storage guesses. + +A recurring schedule needs a stable cancellation handle, not a trail of successor IDs: + +```text +schedule S ready at notBefore: + { create action J; S collecting after:[J] } + return + +J terminal → S collecting: + { record outcome reference; + S scheduled with next notBefore and after:[], or terminal if complete } +``` + +The action can be a job or subagent owner. S remains the background root; cancellation includes +its unfinished dependency J and owned work. Recurrence and missed-run policy are persisted; +positive intervals and bounded catch-up prevent a restart from launching an unbounded backlog. +Recovery uses the stored timestamp/dependency; it must not create a second J while collecting. +Abort settles S without recurrence. Captured cancellation marks keep its action/owned work selected +even if S settles first; S does not hold an abort promise across J's lifetime. + +### 8.8 Hooks are preparation or observation, not another scheduler + +| Point | Decision | +|---|---| +| before_request | Transform the prepared request | +| after_response | Observe provider outcome | +| before_tool | Allow, block with result, or hold for approval | +| after_tool | Process/observe this tool's result | +| on_yield | Stop or request explicit continuation | +| before_collapse | Allow with instructions or decline | + +Run hooks outside the command line, then revalidate their decisions before committing. Hooks may +repeat after a crash; external hook side effects need their own idempotence. Commit listeners, +unlike hooks, must not await another command on the same serialized line. + +Hook ordering, transformation rights and exception policy need an explicit compatibility decision; +the task chain does not settle those questions. Cancellation suppresses yield/normal successor +creation even when a hook was already preparing a continuation. + +## 9. Public API and observation + +The earlier shapes describe stored snapshots and behavior. Handles issue commands; they are not +stored conversation objects. These sketches assemble those surfaces without requiring a second +state model or exposing backend layout. + +### 9.1 Conversation and session handles + +```ts +interface Accepted { readonly inputId: number } +type HistoryTarget = { readonly atEntry: number } | { readonly asOf: number }; +type DriveOutcome = "idle" | "suspended" | "closed"; + +interface ConversationHandle { + readonly id: number; + snapshot(): Promise; + accept(input: AgentInput, options?: { requestId?: string }): Promise; + result(inputId: number): Promise; + drive(options?: DriveOptions): Promise; + prompt(input: AgentInput, options?: { requestId?: string }): Promise<{ inputId: number; result: InputResult }>; + + appendMessage(message: AgentMessage): Promise; // context only; may queue + steer(input: AgentInput): Promise; + followUp(input: AgentInput): Promise; + nextRun(input: AgentInput): Promise; + cancelQueued(inputId: number): Promise<"cancelled" | "already_consumed" | "not_found">; + + abort(): Promise; // durable intent; drive performs cleanup + abortBackground(taskId?: number): Promise; + collapse(options?: { instructions?: string }): Promise; + resetContext(ids: readonly number[]): Promise; + fork(target: HistoryTarget, options?: { sticky?: Initialization }): Promise; + rewind(target: HistoryTarget, options?: { keepRunning?: boolean; sticky?: Initialization }): Promise; + spawn(options: SpawnOptions): Promise; // subagent owner ID; no effect starts here + + getValue(address: ConversationValue, asOf?: number): Promise; + setValue(address: ConversationValue, value: NoInfer): Promise; + deleteValue(address: ConversationValue): Promise; + appendList(address: ConversationList, value: NoInfer): Promise; + clearList(address: ConversationList): Promise; + readList(address: ConversationList, query: ListQuery): Promise>>; + command(build: (tx: ConversationCommand) => T | Promise): Promise; +} + +interface Harness { + root(): Promise; + conversation(id: number): Promise; + conversations(query: ConversationQuery): Promise>; + inspect(): Promise<{ ready: readonly Task[]; inflight: readonly Task[]; waiting: readonly Task[] }>; + drive(options?: DriveOptions): Promise; + entry(id: number): Promise; + entries(conversationId: number, query: CursorPage): Promise>; + task(id: number): Promise; + getValue(address: SessionValue): Promise; + setValue(address: SessionValue, value: NoInfer): Promise; + deleteValue(address: SessionValue): Promise; + appendList(address: SessionList, value: NoInfer): Promise; + clearList(address: SessionList): Promise; + readList(address: SessionList, query: ListQuery): Promise>>; + command(build: (tx: Command) => T | Promise): Promise; + command(scope: WorkingScope, build: (tx: WorkingCommand) => T | Promise): Promise; + watch(conversationId: number, options: { limit: number }, listener: (event: WatchEvent) => void): Promise<() => void>; + decideApproval(taskId: number, decision: "grant" | "deny"): Promise; + deleteConversation(id: number): Promise; + shutdown(): Promise; + close(): Promise; +} +``` + +AgentInput is string/user-message/message-batch input normalized at acceptance. AgentMessage and +provider model/usage types reuse the existing agent/pi-ai boundary. Initialization is the typed +scalar selection/override shape in §4.3; SpawnOptions adds prompt and fresh/inherit context. +These are proposed signatures, not new copies of provider message types. Direct spawn creates a +detached owner; foreground tool delegation makes that work required through its dependency. + +Prompt accepts, drives while observing that input, and returns its result when terminal or progress +suspends/closes. It never substitutes another input's answer. Rewind normally requests/drains source +foreground cancellation before forking; keepRunning leaves source work alone. Validate the target +before cancellation so an invalid history request does not abort otherwise valid work. + +### 9.2 Commands return results only after commit + +Command exposes the mutations from §7.4 and read-only storage queries. Creation/list-append methods +return final numeric IDs immediately inside the builder; the outer command resolves only after +persistence and publication. ConversationCommand binds the conversation automatically. WorkingCommand +exposes only value/list writes, checked against one exact scope ID. + +```ts +await conversation.command(async tx => { + const current = await tx.getValue(plan) ?? false; + tx.setValue(plan, !current); +}); + +const entryId = await harness.command(tx => { + const id = tx.appendEntry(conversationId, "note", content); + tx.setValue(labelAddress(id), "checkpoint"); + return id; +}); +// entryId can escape here, not during the builder +``` + +Reads see pre-command committed state; validation applies buffered mutations in order. Buffer +construction does not perform external effects or publish plugin state. A throwing builder discards +its mutations. Runtime checks supplement token typing for strict JSON, address policy and exact +working-scope identity; changing an established address's rewind policy is not an implicit migration. + +### 9.3 Task and tool integration + +```ts +interface TaskExecution { + readonly signal: AbortSignal; + readonly workingScope: WorkingScope; + readonly services: TaskServices; + command(build: (tx: Command) => T | Promise): Promise; + command(scope: WorkingScope, build: (tx: WorkingCommand) => T | Promise): Promise; + observe( + subscribe: (notify: (event: Event) => void) => () => void, + reconcile: (event: Event | undefined, ctx: TaskExecution) => Promise, + ): void; +} + +interface Tool { + readonly name: string; + readonly description: string; + readonly parameters: JsonObject; // provider-compatible JSON Schema + readonly replay?: "safe" | "never"; + execute(args: JsonObject, ctx: ToolExecution): Promise; +} +``` + +TaskServices supplies models, registered tools, hooks and host resources. ToolExecution supplies +invocation/call identity, the signal and bounded output/checkpoint/memo access using ordinary working +values/lists. The tool schema validates arguments before invocation. ToolDelegation identifies a +subagent/job launch and foreground/background choice; it does not contain an in-process promise. + +```text +tool output surface: + write text / add image → durable bounded progress; capped final result + set details / report usage → final result/attempt metadata + addTools / terminate / handoff → control intent interpreted at settlement/boundary + checkpoint / typed memo → ordinary working values, retired with that execution +``` + +Output backpressure must be explicit: await admitted progress or use a bounded buffer drained before +settlement. Delta frames/checkpoints must not silently rewrite an entire growing assistant message +for every token. Caps, image handling and durable log references are part of tool policy. + +Observe installs a live-phase subscription, not another executor. Each reconciliation receives a +fresh execution context/signal under the same per-task claim used by effects/recovery/abort. Register +before an initial reconciliation with event=undefined, so an already-satisfied condition is not lost. +Events arriving during a claim queue/coalesce within a bounded policy. Phase exit/cancellation seals +and drains admitted callbacks. A stale callback cannot write through an expired context. Waiting +recovery restores missing subscriptions, not duplicates. Teardown and settlement are distinct. + +### 9.4 Watch committed state without gaps + +```text +ONE serialized step: + capture main/global observation boundaries + read conversation, bounded transcript page, live task metadata, + queued-input state and requested reduced working previews + register listener + +then: + deliver relevant main commits as whole batches + deliver committed working previews and transient notifications separately +``` + +```text +base { conversation, entries, referencedInputs, tasks, queued, previews, asOf } +commit { firstSeq, lastSeq, changes:[...] } +preview { taskId, kind, value } +signal { kind:"fault" | "handler_error" | "idle", ... } +``` + +This union is WatchEvent. Entry pages use immutable IDs and a cutoff for deduplication. Commit +filtering can omit unrelated changes but must preserve batch boundaries; clients publish after +reducing the complete delivered batch. A terminal task and its successor must not appear as an +observable idle gap. + +Batch-load admission payloads referenced by visible placement entries into referencedInputs; they +need not appear as extra transcript rows. Arbitrary plugin state is hydrated by its explicit state +consumer, not automatically by a transcript watch. A commit listener cannot await a new command on +the same line; schedule it after delivery. Listener exceptions do not undo commits. Slow-consumer +buffering is bounded; a disconnected/lagging client can rebind from a fresh base. + +If a Chord adapter needs consecutive revisions, those belong to its live binding: + +```text +main commits 104, 117, 130 → binding updates 1, 2, 3 +rebind/rewind → fresh base, update counter starts again +``` + +Journal gaps are not transport gaps. Do not persist another state-sequence allocator. The adapter +publishes only after durable commit and rejects events from retired bindings; its concrete codec +and private-draft integration remain to be chosen. + +### 9.5 End-to-end usage + +```ts +const h = await Harness.open(storage, hostOptions); +const c = await h.root(); +const off = await h.watch(c.id, { limit: 100 }, render); + +const accepted = await c.accept("Inspect the parser", { requestId: "web-42" }); +const outcome = await c.drive(); +const result = await c.result(accepted.inputId); +if (result?.status === "done") show(await h.entry(result.entryId)); +// suspended: wait for the needed event/permission, then drive again +``` + +```ts +// Cancellation is durable intent; keep or start a drive for cleanup. +const driving = c.drive(); +await c.abort(); +await driving; +await c.drive(); // also covers a previously idle/suspended drive + +const child = await c.spawn({ + prompt: "Inspect only tests", + context: "fresh", + values: { inherit: [model], set: [setValue(activeTools, ["read"])] }, +}); +await h.drive(); // supervise detached work too; may suspend on outside events +console.log(await h.task(child)); + +const alternate = await c.fork({ atEntry: completedAnswerId }); +await alternate.accept("Try a different implementation"); +await alternate.drive(); // source and unrelated forks remain parked + +off(); +await h.close(); +``` + +Harness.open receives storage, task definitions, tools/model services, hooks/resources and initial +root values. It creates root plus initial values only for empty storage. Reopening existing storage +creates no new work. Host invocation cancellation/telemetry should be threaded through these calls, +not replaced with a separate tracing or cancellation framework. + +## 10. Validation, performance and review decisions + +This is a design, not a claim of implementation or benchmark parity. Tests should prove behavior +from committed state, then benchmarks should measure equivalent work under the chosen guarantees. + +### 10.1 Backend conformance from the same mutation stream + +```text +validated batches ─┬─ reference replay used only by tests + ├─ Memory + ├─ JSONL → close → reopen + └─ SQLite → close → reopen + +compare: + current objects and generic live-task metadata + transcript pages and inherited prefixes + historical contexts, scalar tombstones, list clears/ranges + input/request mappings and result outcomes + visible working lifetimes and next allocated sequence +``` + +Reference replay is a test oracle, not a required public storage API. Use randomized small histories +and explicit deep-fork cases. Include value-only commits, multi-entry commits and independent child +initialization so message-only tests cannot accidentally pass a broken history implementation. + +```text +fork before/after a value delete → correct absence/fallback +fork before/after list clear → correct inherited ranges +fork before a later summary → earlier context, not today's filtered context +nested source cutoffs → no later ancestor version leaks +sticky state → unchanged by historical reads; explicit copy on creation +``` + +### 10.2 Failure and concurrency matrix + +| Scenario | Required observation | +|---|---| +| Persistence paused after construction | Queries/watchers still see the old complete state | +| Commit durable but caller reply lost | Reopen sees the whole batch; keyed retry reuses the input | +| Main + working writes in one batch | Reject without partial mutation | +| Torn final JSONL batch | Discard only that incomplete tail | +| Malformed complete JSONL batch | Fail open, never truncate valid later history silently | +| Retirement committed, unlink fails | Old working contents remain invisible | +| Reused scope, delayed old cleanup | New lifetime survives | +| Effect returns while abort starts | One claim covers effect join and abort reconciliation | +| Parallel tools settle in either order | Existing join becomes ready; exactly one continuation | +| Abort versus join/final settlement | No unmarked successor escapes the selected cancellation | +| Child finishes during owner cancellation | No success outcome overwrites cancellation | +| Observer notification during close | Drain/reject by admission order; no late terminal overwrite | +| Two overlapping drives | One execution per task; permissions stay scoped | +| One drive caller cancels its wait | Other callers and durable task intent are unaffected | +| Queued B precedes A's answer in transcript | B remains pending until its own group completes | +| Provider retry while collapse waits to publish | No generation/collapse dependency deadlock | +| Watch registration races a commit | Base includes it or the stream delivers it, never neither | + +Use faux providers, fake processes/clocks and storage barriers. No real providers or paid tokens. +Exercise both durable orders, not a test that happens to win the same race every time. Validate +that failed/no-op commands consume no IDs and that old task IDs remain queryable after compaction. + +### 10.3 Performance measurements + +```text +workloads: + many short turns, with/without tools + frequent compaction plus cold reopen + parallel/sequential tools and held approvals + large streaming output/checkpoints + deep forks and repeated historical state reads + many completed child conversations, few live tasks + many genuinely live background tasks + queued input, cancellation and result lookup + +measure separately: + CPU user/system time; wall latency distributions + process RSS, heap, external/array-buffer memory; peaks and post-GC retained size + actual main/sidecar/database/WAL/auxiliary-file bytes + query count, rows decoded, logical mutations and physical bytes written +``` + +Same provider messages, tool output, frame cadence, compaction boundaries, hooks, durability mode +and observation workload on both implementations. Warm-up and measurement instrumentation must be +comparable. Include fault handling costs required by the final contract, not just a happy-path spike. + +Memory/JSONL history growth is expected. SQLite should avoid full-history JS copies. In every case, +separate backend-owned data, live execution data and faux-provider/test bookkeeping before claiming +anything about the harness working set. Do not hide backend memory when reporting total process use. + +The exported pico2 spike reports 9.7 s versus 35.9 s for the lane harness on 2,000 faux turns with +two tools per turn. That motivates investigation, not a validated result for this design. Its +four-way test measures wall time and serialized main-mutation character counts; pico2 scratch and +usage writes bypass that counter. The export has no SQLite implementation. Its projection cache +also means the timing difference cannot be attributed solely to storage interfaces. + +The useful lessons are cheap current/live queries, batched ID reads, small mutations and avoiding +history scans on routine execution—not adopting its types, residency machinery or unsafe shortcuts. + +### 10.4 Decisions to review before implementation + +| Decision | Current proposal / remaining work | +|---|---| +| Historical position | Entry selects its whole commit; reject incomplete exchange targets (§3.3) | +| Sticky conversation inheritance | Copy only explicitly selected current scalars; lists need a deliberate policy (§4.3) | +| State/list vocabulary | Set/delete and append/clear; structured edits as deltas, not arbitrary mutable-list operations | +| Input identity and results | Admission/placement entries plus ordinary result values and input groups (§8.1); finalize normalization/queue policies | +| Background dependency sharing | Required dependencies join foreground cancellation; review interaction with previously detached work | +| Recovery and shutdown | Cooperative close; explicit external adoption/idempotence and bounded host policies | +| Compaction | Correct historical replay first; review checkpoints, token budgeting and oversized/split turns | +| Hooks and tool policy | Confirm transformation/error rules, effective arguments, termination aggregation and conflicting handoffs | +| Observation/replication | Finalize bounded buffering, requested-state hydration and Chord adapter/codec boundaries | +| Backend details | Concrete schema/query plans, snapshot ownership and power-loss policy; no inferred crash guarantees | +| Old-harness capabilities | Explicit parity decisions for provider options/cache behavior, telemetry, usage adjustments, navigation/import and host context | + +Do not silently remove an existing capability because the smaller core does not mention it. Decide +whether it belongs in a task, a host integration, or the public contract before implementing that +part. The next step is feedback on this design, not copying the exported spike into the repository. diff --git a/packages/agent/docs/pico/pico-handoff-v2.md b/packages/agent/docs/pico/pico-handoff-v2.md new file mode 100644 index 000000000..a18dcb521 --- /dev/null +++ b/packages/agent/docs/pico/pico-handoff-v2.md @@ -0,0 +1,2937 @@ +# Pico handoff v2 + +Pico is a durable agent harness. This document is the complete implementation specification for +`packages/agent/src/harness/pico/`. Everything is specified except one intentional gap: the exact ordinary +tool API in section 11.1 (how a `ToolDefinition` executes, what facade it receives, how it reaches its output, +memos and children). Its requirements are fixed; its public shape is not, and must not be guessed. Each rule +that is not obvious carries its reason. +Rules that exist only to catch Pico's own bugs are marked as assertions. + +Size is a design constraint: production Pico code (everything under `src/harness/pico/` except the memory, +JSONL and SQLite storage implementations and tests) targets 3k–4k lines. The line and the kernel are small by +intent; if an implementation grows materially past 4k, the response is to simplify the design, not to add +caches, index abstractions or helper layers. Storage backends are excluded because their correctness +(fsync, replay, locking) is not compressible. + +Read order: glossary, sections 1–4 (data), 5 (commit line), 6–8 (tasks, scheduler, cancellation), 9 (input), +10–12 (core kinds, tools and hooks, system prompt), 13–15 (harness, watch, backends), 16 (state adapter), +17 (tests), 18 (work packages), appendix A (type index), appendix B (changes from v1). + +## 0. Glossary + +- **Session**: one open storage plus the process-local kernel over it. One process owns a session + at a time. +- **Harness**: the public object a host uses. One Harness per Session. +- **Conversation**: an ordered transcript. The root conversation has ID 1. A fork has a `parent` pointer + into its source. A subagent has an `owner` task and may independently choose whether to inherit any + transcript (`parent`) or start empty. +- **Entry**: one immutable transcript record. Never edited, reordered or deleted. +- **Context**: the message list sent to the model, derived from entries. +- **Task**: one recoverable async operation with durable status `pending | running | terminal`. +- **Kind**: the code for a task (`execute`, `recover`, `abort`) or the type witness for an entry. +- **Core kind**: one of Pico's four privileged built-in task kinds (`pi.generation`, `pi.tool`, + `pi.post_tools`, `pi.collapse`). Core kinds have direct append and boundary authority; ordinary kinds have + only the boundary-safe passive `write`. +- **Turn task**: a live `pi.generation`, `pi.tool` or `pi.post_tools`. A conversation with a live turn task + is *busy* for input admission. +- **Line**: the single FIFO async mutex through which every commit passes. +- **Commit**: one callback that buffers a `Write[]`, followed by exactly one atomic `Storage.commit`. +- **Kick**: a coalesced "look for eligible tasks" signal to the scheduler. +- **Mark**: durable `abort: true` on a task. Marking starts cancellation. +- **Fresh abort**: the new invocation of `kind.abort` that runs after the old invocation has returned. +- **Scratch**: private per-task state, deleted when the task terminates. +- **Output**: shared live state of a task (streaming assistant text, tool progress), kept as Chord deltas. +- **Sticky**: conversation state with only a current value. **Rewindable**: conversation state whose value at + any historical entry can be read. +- **Inbox**: the per-conversation queue of input that arrived while the conversation was busy. +- **Boundary**: a point in a turn where queued input may be placed into the transcript. +- **Managed entry**: a `pi.system` entry written by system preparation (section 12). +- **Ctx**: Chord's `Context` (cancellation signal, telemetry, invocation identity). Every async operation takes + it last. + +## 1. Data model + +All durable payloads are strict JSON: + +```ts +type JsonPrimitive = null | boolean | number | string; +type JsonValue = JsonPrimitive | JsonObject | readonly JsonValue[]; +interface JsonObject { readonly [key: string]: JsonValue } +type Id = number; // stable object identity; positive safe integer +type Seq = number; // storage write order; positive safe integer + +import type { JsonRepresentation } from "@earendil-works/chord"; +type Stored = JsonRepresentation; // the durable form of any imported pi-ai type, including unions with primitive arms (UserInput) +``` + +pi-ai's `AssistantMessage`, `ToolCall`, `Usage`, `DeferredHandle` and friends are interfaces: they have no +index signature and `ToolCall.arguments` is `any`, so they are not assignable to `JsonValue` and cannot be +used directly as task payloads, checkpoints, output state or entry `data`. Pico stores them as `Stored`, +Chord's structural strict-JSON representation (`any`/`unknown` become `JsonValue`, methods disappear, +optional keys stay optional). Mapped and literal object types with optional properties are assignable to the +strict `{ readonly [key: string]: JsonValue }` index signature (verified under strict TypeScript, including +optional, nested and `Record` fields), so `JsonObject` stays strict and `{ value: undefined }` +remains a compile error. Strict JSON is validated once at provider/tool ingress; ordinary Pico writes do not +normalize or strip anything. Nothing is duplicated: `Stored` is derived, not redeclared. Public *hydrated* surfaces +(`InputOutcome.answer.message`, hook payloads) hand out the pi-ai type; only durable positions use `Stored`. +Every durable position that carries a pi-ai type uses `Stored`: `Entry.model` and the entry witnesses +(`Stored`, `Stored`, `Stored`, `Stored`), +`ContextEdit.replace.messages`, `QueuedInput.input` and `StoredEntryDraft` (`Stored`, +`Stored`), `ToolInput.call`, `GenerationOutput.message`, the deferred handle, `UsageEntryData.usage`, +`Prep.retry`/`Base.retry`, and the `retry` config token. Public surfaces stay pi-ai typed (`SendInput.content: +UserInput`, `InputOutcome.answer.message: AssistantMessage`, hook payloads) and are converted and validated at +admission or hydration. Compile assertions required in WP1, one per durable position: `Stored`, +`Stored`, `Stored`, `Stored`, `Stored`, +`Stored`, `Stored`, `Stored` each `extends JsonObject`; `Stored +extends JsonValue`; `{ phase: "deferred"; handle: Stored } extends TaskCheckpoint`; +`GenerationOutput extends OutputState`; `ToolInput`, `UsageEntryData`, `QueuedInput`, `StoredEntryDraft`, +`SystemEntryData`, `AssistantEntryData`, `ToolResultData`, `Acceptance`, `SectionRecord`, `ToolControl`, +`ToolDiagnostic` each `extends JsonValue`; `ToolOutputState extends OutputState`, `JobOutput extends OutputState` and `o.stdout = s.stdout` compiles inside a `JobOutput` mutate; every `Task` checkpoint union +in section 10 `extends TaskCheckpoint`; and every entry witness `extends Entry`. In addition, every declaration block in this document is represented in the WP1 compile-only test +(`test/harness/pico/types.compile.ts`, or a dedicated `state-task.compile.ts` for section 16); the compact +payload listings in section 10 are transcribed exactly into named aliases. They must type-check under the +repository `tsconfig` with no `any`: prose that claims exact TypeScript compiles or is wrong. The compile matrix also includes the usage-guide entry example as written (`tx.write(noteKind, { data: +{ text } })` with no `kind`, `tx.write(pinnedKind, { data, model })`, and a `head: "self"` input for a head +witness) and asserts that supplying `kind` or `id` in an `EntryInput` is a compile error. Scope: `defineValue({ +type: "shared", id: 1 }, "x")` and `defineList({ type: "shared", id: 1 }, "x")` are `@ts-expect-error` fixtures. +Task creation: `tx.task(kind, { input: { prompt, extra: 1 } })`, `const spec = { input: {...}, extra: 1 }; +tx.task(kind, spec)`, `tx.task(kind, { ...spec })` and an `input` with a nested extra top-level field are all +`@ts-expect-error`; the exact literal, variable and spread each compile; `tx.task(coreKind, …)` on a +`BaseTaskTx` is an error while the same call on `CoreTaskTx` compiles. Runtime output: +`TaskRuntime` has no `output` key (`rt.output` is a compile error) and +`TaskRuntime["output"]` is `TaskOutput<{ text: string }>`; same for the abort runtime with +`ReadonlyTaskOutput`, and for `FinalTx`/`RunningTask`. Negative fixtures +(`// @ts-expect-error`) for the bundle guards: a fake kind map with two kinds sharing a config key, one with a +config key named `set` or `get`, and two kinds sharing a hook point name, each making the corresponding +`Assert>` or +`Assert>` fail; plus positive fixtures over the real `CoreConfigs` and `CoreHooks`. + +`Id` and `Seq` are separate positive safe-integer domains. IDs are minted by `Storage.nextId()` on the line +before a commit and become valid object identities only when the containing commit succeeds; an ID minted in a +callback that throws must not escape that callback. `Storage.commit` returns exactly one `Seq` per submitted +write, contiguous and ascending, positionally aligned with the `Write[]`, and strictly greater than every +sequence of every earlier commit. Reason: a callback that fails after minting IDs must not leave holes +in the sequence, and a state-only write consumes a sequence without creating an object. Storage keeps a +durable **committed-ID high-water**: after every successful `commit` it raises it to the greatest object ID +(conversation, entry, task, list element) in that batch, and persists it with the batch. On reopen the +allocator starts after the high-water; IDs minted but never committed never reached `commit` and may be +reused, committed IDs never are, including IDs whose only record was in a sidecar that has since been retired +(section 15). + +Values passed into Pico are treated as immutable and are not cloned. External boundaries (RPC, plugins) +validate strict JSON before calling Pico. + +### 1.1 Conversations + +```ts +interface Conversation { + readonly id: Id; + readonly parent?: { readonly conversationId: Id; readonly at: Id }; // fork point + readonly owner?: Id; // task that created it (subagent); absent for host-created +} +// storage record = Conversation & { readonly sectionSeed?: readonly SectionRecord[] }: private immutable creation metadata +// (rendered section seed, section 8.1) carried by the `conversation.create` write; not part of the public projection. +``` + +A conversation's logical transcript is its parent's transcript up to and including `parent.at`, recursively, +followed by its own entries. Later parent entries are invisible. A seed with `parent.at: "start"` stores no +`parent` at all: the child has no inherited entries and no fork link (ownership, if any, is `owner`). Tasks, +inbox and scratch are never inherited. `owner` is a capability and cancellation link (section +8), not an execution link. + +### 1.2 Entries + +```ts +type ContextEdit = + | { readonly target: Id; readonly action: "omit" } + | { readonly target: Id; readonly action: "replace"; readonly messages: readonly Stored[] }; + +interface Entry { + readonly id: Id; + readonly conversationId: Id; + readonly kind: string; + readonly byTaskId?: Id; // set by Pico when a task appended it + readonly data?: JsonValue; // kind-specific payload + readonly model?: readonly Stored[]; // pi-ai messages this entry contributes, in durable form + readonly head?: Id; // context starts here (see 1.3) + readonly edits?: readonly ContextEdit[]; // changes to earlier entries' projection +} + +interface EntryKind { readonly kind: string; is(e: Entry | undefined): e is E } +type EntryIdentity = Pick; // supplied by Pico and the witness, never by the caller +type EntryInput = Omit & + (E extends { head: Id } ? { readonly head: Id | "self" } : { readonly head?: never }); // `kind` comes from the EntryKind argument +``` + +The four facets `data`, `model`, `head`, `edits` are materialized. Reason: context derivation must never call +kind code, so an entry of an unknown kind still projects correctly. + +```ts +declare function defineEntry(kind: string): EntryKind; // typed witness; `is` narrows by kind string +type EntryBase = EntryIdentity; +// facet aliases for composing witnesses (as in the current WP1 code) +interface EntryData { readonly data: D } +interface ModelProjection { readonly model: readonly Stored[] } +interface ContextHead { readonly head: Id } +interface ContextEdits { readonly edits: readonly ContextEdit[] } +type UserEntry = EntryBase & { readonly model: readonly [Stored]; readonly data?: { readonly continuation: true; readonly from: Id } }; // data only on yield continuations +type AssistantEntry = EntryBase & { readonly model: readonly [Stored]; readonly data?: AssistantEntryData }; +type ToolResultEntry = EntryBase & { readonly model: readonly [Stored]; readonly data: ToolResultData }; +type SystemEntry = EntryBase & { readonly model: readonly [] | readonly [Stored]; readonly data: SystemEntryData; readonly edits?: readonly ContextEdit[] }; +type NoticeEntry = EntryBase & { readonly model: readonly [Stored]; readonly data?: JsonValue }; +type SummaryEntry = EntryBase & { readonly model: readonly [Stored]; readonly data: { readonly through: Id }; readonly head: Id }; +type HandoffEntry = EntryBase & { readonly model: readonly [Stored]; readonly head: Id }; +type ResetEntry = EntryBase & { readonly head: Id }; +``` + +Built-in kinds: + +| kind | model | head | data | +|---|---|---|---| +| `pi.user` | `[UserMessage]` | | `{ continuation: true; from: Id }` on yield continuations only | +| `pi.assistant` | `[AssistantMessage]` | | `AssistantEntryData` (attempt count; usage lives on the message) | +| `pi.tool_result` | `[ToolResultMessage]` | | `ToolResultData` | +| `pi.system` | `[SystemMessage]` or `[]` | | `SystemEntryData` (section 12) | +| `pi.notice` | `[UserMessage]` | | optional | +| `pi.usage` | `[]` | | `UsageEntryData = { attempt: number; usage?: Stored; error: string }` (failed generation attempt) | +| `pi.summary` | `[UserMessage]` | yes | `{ through: Id }` | +| `pi.handoff` | `[UserMessage]` | yes | | +| `pi.reset` | `[]` | yes | | + +Assistant entries with `stopReason` `"error"` or `"aborted"` may be stored for display but never enter a +model request and never create tool work. `"pending"`/`"deferred"` are never stored as entries. + +### 1.3 Context derivation + +For conversation `c` and inclusive target `T`: + +```text +H = newest fork-visible entry at or before T with a head +from = H ? H.head : transcript start +range = fork-visible entries from `from` through T +edits = per target, newest edit in range wins +entries = H ? [H, ...range without any head entries] : range +model = concat(entries.map(e => edits[e.id] ? apply : e.model)), then reorder tool results +request = { messages: model } // no top-level systemPrompt or tools; see section 12 +``` + +`head: "self"` on append means "context starts at this entry" (reset, handoff). A summary's head points at +the first retained entry after the summarized prefix, so the summary appears, then the tail. + +Tool-result ordering: results are appended as tools finish, so transcript chronology may not match the +assistant's call order. Projection reorders results into call order. If a fork cut an exchange before all +results landed, projection synthesizes `ToolResultMessage { isError: true, text: "Tool result unavailable: +history ends before this call completed.", details: { reason: "missing_after_fork" } }` per missing call. +Synthesized results are never stored. + +A model request captures the entry ID list and effective messages at a durable cutoff and does not change +afterward. Reason: a generation must know exactly what it sent for recovery and for hooks. A current-context +cache is disposable derived state; historical derivations for older cutoffs read storage and never rewind the +live cache. + +### 1.4 Who appends, and where + +Only core task kinds append entries directly (`CoreTaskTx.entry`). Everyone else, host and ordinary tasks +included, goes through `write` (section 9): the entry is appended immediately if the conversation is not +busy, otherwise it is queued and placed at the next boundary. There is no host `entry()`. + +Reason: every boundary sits between complete exchanges, so an entry placed at a boundary can never split an +assistant/tool-result block. Removing direct host appends removes the only path by which that could happen. + +Transcript invariants, maintained by core kinds and asserted before persistence (section 5.2): + +- Heads only move forward: a new head never points before the previous head's target. Otherwise context + could grow back after a collapse. +- A head never splits an exchange: it keeps an assistant with all its results or omits both. Collapse + guarantees this by choosing `through` at an exchange end. +- Edits target earlier visible entries. Edits never target managed `pi.system` entries, except the omission + edits a fresh baseline carries for superseded managed entries (section 12.4). +- Tool-call IDs within one assistant are unique; at most one result per call, in the same conversation. + +The one runtime check on non-core data is on a queued `write` that carries a `head` (reset, handoff): at +placement, the head target must still be visible and not before the current newest head. A collapse +committed while the write was queued can violate that. The write is then dropped with result +`unanswered/stale`. Edits whose targets have left the context are no-ops and do not stale. + +## 2. Scoped state + +```ts +type Scope = + | { readonly type: "session" } + | { readonly type: "conversation"; readonly conversationId: Id } + | { readonly type: "task"; readonly taskId: Id } // scratch + | { readonly type: "shared"; readonly id: Id }; // task output + +declare const addressType: unique symbol; // compile-only payload witness; never present at runtime, never serialized +interface Address { + readonly scope: Scope; readonly namespace: string; readonly key?: string; + readonly kind: "value" | "list"; readonly rewind: boolean; + readonly [addressType]?: T; // makes Value and Value distinct types +} +interface Value extends Address { readonly kind: "value"; readonly default?: T } +interface List extends Address { readonly kind: "list" } +interface Element { readonly id: Id; readonly value: T } + +type PublicScope = Exclude; // exported constructors never accept shared scope +type StickyScope = Extract; +type ConversationScope = Extract; +// `defineValue({ type: "shared", id }, ...)` is a compile error and runtime rejection; the task-output list is built by an internal constructor +// external constructors reject every `pi.*` namespace at runtime +type DefaultedUnboundValue = Omit, "bind" | "default"> & { readonly default: T; bind(conversationId: Id): Value & { readonly default: T } }; +declare function defineValue(scope: StickyScope, namespace: string, options: { readonly key?: string; readonly default: T }): Value & { readonly default: T }; +declare function defineValue(scope: StickyScope, namespace: string, options?: { readonly key?: string }): Value; +declare function defineValue(scope: ConversationScope, namespace: string, options: { readonly key?: string; readonly rewind: boolean; readonly default: T }): Value & { readonly default: T }; +declare function defineValue(scope: ConversationScope, namespace: string, options: { readonly key?: string; readonly rewind: boolean }): Value; +declare function defineList(scope: StickyScope, namespace: string, options?: { readonly key?: string }): List; +declare function defineList(scope: ConversationScope, namespace: string, options: { readonly key?: string; readonly rewind: boolean }): List; +// one public type argument (T); rewind is a literal picked by overload, so `conversationValue("x", { rewind: true })` infers UnboundValue +declare function conversationValue(namespace: string, options: { readonly key?: string; readonly rewind: true; readonly default: T }): DefaultedUnboundValue; +declare function conversationValue(namespace: string, options: { readonly key?: string; readonly rewind: false; readonly default: T }): DefaultedUnboundValue; +declare function conversationValue(namespace: string, options: { readonly key?: string; readonly rewind: true }): UnboundValue; +declare function conversationValue(namespace: string, options: { readonly key?: string; readonly rewind: false }): UnboundValue; +declare function conversationList(namespace: string, options: { readonly key?: string; readonly rewind: true }): UnboundList; +declare function conversationList(namespace: string, options: { readonly key?: string; readonly rewind: false }): UnboundList; +declare function sessionValue(namespace: string, options: { readonly key?: string; readonly default: T }): Value & { readonly default: T }; +declare function sessionValue(namespace: string, options?: { readonly key?: string }): Value; +``` + +`default` is resolution metadata, not identity: every reader that resolves this definition (`c.value`, +`c.config.`, `view.config`, core kinds) returns `default` when no value is stored or the value was +deleted. It lives on the token, so there is exactly one place a default is spelled. + +Storage and every `Write` carry fully bound addresses; the unbound form exists only at the public surface. +`ConversationHandle.value`/`list`, `ConversationTx.value`/`list`, `TaskConversation.value`/`list`, +`SeedWrite` in `rootValues`/`ConversationSeed.values`, and `ConversationWatchOptions.values` accept a +bound address or an unbound one, which they bind to their own or target conversation before anything reaches +the kernel. A bound address naming a different conversation than the handle's is a `ScopeViolation`. Reason: +plugins and core kinds declare one token and use it against any conversation; `generationKind.config.model` is +one definition, not one per conversation. + +Identity is `[scope.type, scope id, kind, rewind, namespace, key ?? null]`. Omitted `key` and `key: undefined` +are the same unkeyed singleton; `key: ""` is a distinct keyed member. Only conversation scope may be +`rewind: true`. Public constructors create session, conversation and (for a task's own runtime) task-scope +addresses only; shared scope is internal to `TaskOutput` in v1 and has no public constructor. + +The namespace prefix `pi.` is reserved for built-ins: external constructors reject it, so a plugin cannot +define `pi.anything`. Within it there are exactly two sets: (1) the fixed kinds' config bundle definitions +(`generationKind.config.*` and the other core bundles, section 2.1) are exported tokens usable through every +generic handle and automatically part of `view.config`; they are rejected in requested watch `values` because +they already have their own field and event. (2) The internals `pi.inbox`, `pi.input_result`, `pi.request`, +and `pi.output` are protected: generic read/write/watch/`rootValues` reject them and they surface only through +`view.inbox`, `InputHandle` and `view.taskOutputs`. Plugin and +application definitions (any non-`pi.` namespace) are public and appear in a watch only when requested. There +is no access metadata on definitions and no access registry. + +Values: set / delete. Lists: append / remove / clear. Rewindable state records every write with its `Seq`. +`get(address, at)` with an entry ID resolves `at` to that entry's `Seq` and returns the newest write at or +before it, recursing into the parent capped at `min(at, parent.at)`. Sticky state has only a current value; +historical reads of sticky/session/task/shared state reject `InvalidHistoryPosition`. Forks inherit +rewindable state through the parent walk; sticky *values* are inherited only by an explicit copy at creation +(`copySticky`, section 8.1); sticky lists are never copied in v1. + +**Rewindable writes precede entry appends within one commit.** Reason: a fork at that entry must see the +state written "with" it and not state written after it. The builder throws on violation. + +Who may write which scope: + +| writer | session | own conversation subtree | other conversation | own scratch | shared | +|---|---|---|---|---|---| +| host | yes | yes (all) | yes | no | no | +| task (execute/recover) | yes | yes | no | yes | via `output` only | +| task (abort) | yes | yes | no | read only | read only | + +### 2.1 Typed configuration bundles + +There is no untyped settings object. Each core kind exports a typed bundle of unbound conversation addresses. +The public `conversation.config` facade is assembled from those bundles mechanically, so core tasks and the +public API read and write identical addresses and nothing is declared twice. + +```ts +// the bundles the fixed kinds attach as `.config` (definition objects; the internal constructor is allowed to use pi.* names) +const generationConfig = { + model: conversationValue("pi.model", { rewind: true }), // no default -> no_model + thinkingLevel: conversationValue("pi.thinking", { rewind: true, default: "off" }), + selectedTools: conversationValue("pi.tools.selected", { rewind: true, default: [] }), + profile: conversationValue("pi.prompt.profile", { rewind: true, default: "default" }), + retry: conversationValue>("pi.generation.retry", { rewind: false, + default: { enabled: true, maxRetries: 3, baseDelayMs: 2000, maxAgentDelayMs: 60_000 } }), // pi-ai RetryPolicy; coding-agent defaults +} satisfies ConfigBundle; +const postToolsConfig = { + steeringMode: conversationValue<"all" | "one-at-a-time">("pi.queue.steering", { rewind: false, default: "one-at-a-time" }), + followUpMode: conversationValue<"all" | "one-at-a-time">("pi.queue.follow_up", { rewind: false, default: "one-at-a-time" }), +} satisfies ConfigBundle; +const toolConfig = {} satisfies ConfigBundle; // the tool time budget belongs to the gated ordinary tool API (11.1); nothing in v1 +const collapseConfig = { + threshold: conversationValue("pi.collapse.threshold", { rewind: true, default: 0 }), // context tokens; 0 = never + keepRecent: conversationValue("pi.collapse.keep_recent", { rewind: true, default: 20_000 }), // tokens retained after `through` +} satisfies ConfigBundle; +// generationKind.config === generationConfig, etc. (section 10 kinds are defined with these as `config`) + +// derived from the fixed bundle map, not declared. The map is over the bundle constants, not the kind tokens: +// hook payloads mention `Settings`, so deriving Settings from `typeof generationKind` (which carries hooks) would be circular. +interface CoreConfigs { readonly generation: typeof generationConfig; readonly tool: typeof toolConfig; readonly postTools: typeof postToolsConfig; readonly collapse: typeof collapseConfig; readonly job: {} } +type UnionToIntersection = (U extends unknown ? (x: U) => void : never) extends (x: infer I) => void ? I : never; +type CoreConfigBundle = UnionToIntersection; +type _ConfigsMatchKinds = Assert<{ [K in keyof CoreConfigs]: CoreTaskKinds[K]["config"] extends CoreConfigs[K] ? true : false }[keyof CoreConfigs]>; // the kinds attach exactly these bundles + +type BundleValues = { [K in keyof B]: B[K] extends ConversationValueDefinition ? T : never }; // payload types (for set patches) +type ResolvedValue = D extends { readonly default: infer V } ? V : PayloadOf | undefined; // bound or unbound; defaulted members never undefined +type ResolvedValues = { readonly [K in keyof B]: ResolvedValue }; +type Settings = BundleValues; + +// compile-time guards, checked where CoreTaskKinds is assembled: +type ReservedConfigKeys = "set" | "get"; +type Assert = T; // the only way to make a type-level check a compile error +// M is a map of bundles (CoreConfigs or CoreHooks); each member is one kind's bundle +type KeysOf = U extends unknown ? keyof U : never; // distribute over the other bundles; `keyof (A | B)` alone is only their common keys +type Collides = // true iff bundle N shares a key with another bundle (or uses a reserved name) + [Extract]> | Reserved>] extends [never] ? false : true; +type AnyCollision = true extends { [N in keyof M]: Collides }[keyof M] ? true : false; +type DisjointBundles = AnyCollision extends false ? true : false; // boolean, never `never` +type _AssertDisjointConfig = Assert>; // compile error on any duplicate or set/get; also asserted at install time + +type ConfigFacade = { + readonly [K in Exclude]: B[K] extends ConversationValueDefinition + ? (R extends true ? PublicValue> : StickyPublicValue>) // bound to this conversation; get() returns the resolved type + : never; +} & { + set

    >>(patch: NoExtra>, P>, ctx: Context): Promise; // one commit + get(ctx: Context): Promise>>; // one batched read, current, defaults resolved +}; + +// ConversationHandle.config: ConfigFacade. Usage: +// c.config.model.get(ctx) current +// c.config.model.get(entryId, ctx) historical: rewindable members only, typed +// c.config.followUpMode.get(ctx) sticky: no historical form +// c.config.model.set(ref, ctx) one value, one commit +// c.config.set({ model, thinkingLevel, followUpMode }, ctx) exact typed partial, one commit +``` + +Each property of `c.config` is the conversation-bound public value for that address, identical to +`c.value(generationKind.config.model)`; core kind code reads the same definitions through `tx.value( +generationKind.config.model)` in its own conversation. Bundle members are reusable +`ConversationValueDefinition` tokens (namespace, key, rewind, payload type; no conversation ID); the +handle or transaction binds them, and storage only ever sees bound addresses. Flattened property names must be +unique across the fixed kinds and must not be `set` or `get`; both are compile errors (`Assert>` +resolves to `false`, which does not satisfy `extends true`) and are re-asserted when the core kinds are installed +at open. Plugin configuration is not merged into `c.config`; +plugins keep their own bundles and use `c.value(token)`. `ConversationSeed.values` (section 8.1) accepts the +same tokens, so an owned child is seeded with exactly these definitions before its first input. + +Model, thinking level, selected tools, profile and collapse thresholds are rewindable: a fork at yesterday's +answer gets yesterday's model. Queue modes and the retry policy are sticky: they describe how the UI wants +the present handled, not history. Defaults are on the tokens above and are what `get`, `view.config` and the +core kinds observe when nothing is stored; queue modes default to `"one-at-a-time"` (matching the current +`Agent`/coding-agent defaults). `model` has no default: a missing model is a typed `no_model` generation +failure, not a fault. + +Generation tasks never copy configuration into their input; they read it at preparation and record in their +checkpoint only what recovery needs to resend the same request (`Prep`, 10.1: model, thinking level, offered +tool names, retry policy, cutoff, system entry, attempt). + +## 3. Tasks + +### 3.1 Records + +```ts +interface TaskCheckpoint extends JsonObject { readonly phase: string } + +type TaskOutcome = + | { readonly status: "completed"; readonly result: R } | { readonly status: "failed"; readonly failure: F } + | { readonly status: "aborted"; readonly result: A } | { readonly status: "orphaned" }; + +interface StoredTaskOutputRef { readonly id: Id; readonly kind: string } +interface TaskBase { + readonly id: Id; readonly conversationId: Id; readonly kind: string; + readonly input: I; // immutable + readonly checkpoint?: C; // full replacement each time + readonly after: readonly Id[]; // dependencies, immutable, acyclic + readonly background?: true; // excluded from idle and foreground cancellation + readonly owns: readonly Id[]; // conversations this task created + readonly output?: StoredTaskOutputRef; + readonly abort?: true; // the mark +} +type Task = + TaskBase & ( + | { readonly status: "pending" | "running"; readonly outcome?: never } // live: never carries an outcome + | { readonly status: "terminal"; readonly outcome: TaskOutcome } // terminal: always carries one + ); +``` + +Bare `Task` (all defaults) is the erased form used by storage, scans, the watch view and host inspection; +status and outcome are correlated by the union, so a pending task cannot carry an outcome and a terminal task +cannot omit one. + +A checkpoint is the *entire* recovery state; each update replaces it whole. `phase` is a kind-defined tag the +kernel never interprets. There is no event log or replay: only the latest checkpoint is durable. Reason: a +full snapshot is trivially recoverable and inspectable; partial patches need a merge algorithm and break on +shape changes. + +### 3.2 Lifecycle + +```text +1. create task.create {status: pending} durable +2. reserve task.set {status: running} durable, before any effect +3. run kind.execute(task, runtime, ctx) -- or kind.recover if step 2 was found on reopen + may commit checkpoints, scratch, output deltas, child tasks +4. close execute returns a closure; kernel runs it on the line; it buffers writes and returns + completed|failed; kernel appends the terminal task.set; one atomic commit +5. cancel mark -> revoke old writes -> signal ctx -> join old invocation -> reserve fresh abort + kind.abort(task, abortRuntime, ctx) returns a closure; aborted outcome; one atomic commit +``` + +Recoverability means: a `running` task found at reopen gets `recover`, with or without a checkpoint. It does +not mean exactly-once external effects. A crash after an unkeyed external call but before recording its +result is uncertain; kinds that need certainty record a request key in a checkpoint *before* the call +(`phase` naming an in-flight effect) and reconcile in `recover`. + +### 3.3 Kinds + +```ts +type OutputState = JsonObject | JsonValue[]; // mutable JSON container; Chord tracks it, storage persists its deltas +type ConfigBundle = Record>; +declare const taskKindBrand: unique symbol; + +interface TaskKindMethods { + readonly kind: string; + readonly hooks?: H; // typed hook points this kind invokes (section 11.2) + readonly config?: ConfigBundle; // conversation-value definitions this kind reads (section 2.1) + execute(task: RunningTask, rt: TaskRuntime, ctx: Context): Promise>; + recover(task: RunningTask, rt: TaskRuntime, ctx: Context): Promise>; + abort (task: RunningTask, rt: AbortTaskRuntime, ctx: Context): Promise>; +} +interface CoreTaskKindMethods { + readonly kind: string; readonly hooks?: H; readonly config?: ConfigBundle; + execute(task: RunningTask, rt: CoreTaskRuntime, ctx: Context): Promise>; + recover(task: RunningTask, rt: CoreTaskRuntime, ctx: Context): Promise>; + abort (task: RunningTask, rt: CoreAbortTaskRuntime, ctx: Context): Promise>; +} + +type TaskKind = + TaskKindMethods & TaskOutputDefinition & { readonly [taskKindBrand]: "ordinary" }; +type CoreTaskKind = + CoreTaskKindMethods & TaskOutputDefinition & { readonly [taskKindBrand]: "core" }; + +// shared constraint list, used verbatim by every task-typed alias below and in sections 16 and appendix A: +// I extends JsonValue, C extends TaskCheckpoint, R extends JsonValue, F extends JsonValue, A extends JsonValue, +// O extends OutputState, H extends HookPoints +type TaskKindDefinition = Omit, typeof taskKindBrand>; +type CoreTaskKindDefinition = Omit, typeof taskKindBrand>; + +declare function defineTask(): + >(definition: NoExtra, D>) + => D & TaskKind; // H and D["config"] inferred from the literal; brand added +declare function defineCoreTask(): + >(definition: NoExtra, D>) + => D & CoreTaskKind; // internal; not a plugin API +``` + +`defineTask` returns an ordinary kind, `defineCoreTask` a core kind; the returned type is `D & TaskKind<...>`, +so `kind.hooks` and `kind.config` keep the exact literal bundle types (that exactness is what `c.config`, +`c.hook`, `ConfigEvent` and `CoreTaskKinds` derive from). Only the fixed core bundles flatten onto +`c.config`/`view.config`; a plugin kind's bundle is accessed through `c.value(kind.config.x)`. Core kinds +receive `CoreTaskTx` (direct entry append, core child creation, admission and boundary helpers); ordinary kinds +receive `BaseTaskTx` (passive `write` only). The private `taskKindBrand` symbol makes the two non-interchangeable. +Reason for the split: plugins must not be able to append mid-exchange, resolve input groups or create turn +tasks; a type-level brand plus separate runtime constructors makes that unforgeable without runtime checks. The +passive `write` is safe to hand out because it only ever lands at a boundary. + +`TaskOutputDefinition` (appendix A) makes `output` required when `O` is declared and forbidden otherwise. +Task creation goes through `TaskCreator` (5.3): `NoExtra` on the spec and `ExactJsonInput` on `input` reject +visible extra fields whether the argument is a literal, a variable or a spread. Excess-property checking on task specs and inputs (`NoExtra`, `ExactJsonInput`) +is applied so that visible extra fields reject at compile time. Deep exactness, casts and erased fields remain +ordinary TypeScript limits. + +### 3.4 Output + +Output is the live, shared, observable state of a task: the assistant message being streamed, a tool's +progress, a job's stdout tail. It is one Chord tracker per output ID; every nonempty flush is appended as one +delta to the protected shared list built by Pico's internal address constructor (`pi.output`). + +- A kind declares `output: { kind, initial(input) }` with `O extends OutputState` (a plain JSON object or + array; no `Date`, `Map`, class instances or functions, because the durable form is Chord deltas over strict + JSON). On creation with no `output` ref, the task's output ID is its own ID and the first list element is + `track(initial(input)).flush()`, committed with `task.create`. `initial` is pure and synchronous; its return + value is transferred to the tracker and must not be retained. One boundary assertion: the kernel asserts that + `initial`'s result and every `replace` value are strict JSON before handing them to the tracker; a violation + faults the session (kind contract error). +- A task may be created with another live task's `output` ref (same `kind` string). Both write the same + tracker. Use: a tool times out, creates a background job with its own ref, and the job keeps writing where + the tool left off. +- The output lives while any live task references it and is deleted in the same commit that terminalizes the + last reference. The terminal closure of that last task receives the final value as `tx.output` so it can + materialize it (e.g. into the assistant entry) atomically. A retired output ID is never resurrected. +- Runtime facade: `output.read(ctx)`, `output.mutate(state => { ... }, ctx)`, `output.replace(value, ctx)`. + `read` and terminal `tx.output` return immutable snapshots, never the tracker target or a proxy. `mutate` + runs the callback synchronously on the line against the tracker proxy; the callback must not be + async, retain the proxy, or call other runtime methods. If it throws, or if flushing/strict-JSON validation + of the mutated state fails, the session faults: the tracker is now in an unknown state and the only safe + recovery is reopen from durable deltas. These are task-contract faults, not domain failures. Only a **nonempty** tracker + flush creates a `pi.output` list element, consumes a `Seq` and emits a `task_output` event; a `mutate` whose + callback changes nothing tracked yields an empty flush and returns normally with no storage commit and no + event: an empty `Write[]` is never passed to storage and an empty `Op[]` is never persisted. `replace(value)` + is an explicit replacement: after strict-JSON validation it always persists Chord's full base flush (`["r", + value]`), even if the value is structurally equal to the current state; Pico does no deep-equality check. +- After a task is marked, its `output` facade rejects. Fresh abort gets `read` only. +- Replay: fold list elements with `applyImmutable()`, hydrate one tracker, consume its synthetic first flush + locally (do not persist it). +- Each protected append emits one `task_output` watch event with that exact decoded delta. An output may be + referenced by tasks in different conversations (a tool in A hands its output to a job it created in owned + conversation B). A conversation watch tracks outputs by *local* reference: when a `task_start`/`task_update` + introduces an output ID that no live task in this conversation referenced before, the envelope carries a + synthetic full-base `task_output` (`[["r", currentValue]]`) for that ID before the task event, so the + reducer can populate `view.taskOutputs` without a persisted append; when the last local reference ends the + reducer drops the output even if another conversation still holds it. The synthetic base is derived on the + line from the live tracker and is never stored. + +Reason for deltas over snapshots: a long assistant message would otherwise be rewritten in full on every +token. V1 does not compact or rebase an active output log; measure first, and reclaim the whole log only when +its final live reference retires. + +## 4. Storage + +Storage is mechanical. It applies a `Write[]` atomically, assigns sequences, maintains indexes, and answers +queries. It never runs kind code and never validates application payloads. + +```ts +type Write = + | { type: "conversation.create"; conversation: Conversation & { readonly sectionSeed?: readonly SectionRecord[] } } // private metadata, see 1.1 + | { type: "entry.append"; entry: Entry } + | { type: "task.create"; task: NewTask } + | { type: "task.set"; task: Task } // complete replacement + | { type: "value.set"; address: Value; value: JsonValue } + | { type: "value.delete"; address: Value } + | { type: "list.append"; address: List; element: Element } + | { type: "list.remove"; address: List; elementId: Id } + | { type: "list.clear"; address: List }; + +interface Storage { + nextId(): Id; + commit(writes: readonly Write[], ctx: Context): Promise; // one Seq per write, aligned + getConversations(ids: readonly Id[], ctx: Context): Promise>; + scanConversations(query: ConversationScan, ctx: Context): Promise>; + getEntries(ids: readonly Id[], ctx: Context): Promise>; + scanEntries(query: EntryScan, ctx: Context): Promise>; + newestHead(conversationId: Id, at: Id, ctx: Context): Promise; + getTasks(ids: readonly Id[], ctx: Context): Promise>; + scanTasks(query: TaskScan, ctx: Context): Promise>; + getValue(address: Value, at: Id | undefined, ctx: Context): Promise; + readList(address: List, at: Id | undefined, ctx: Context): Promise[]>; + close(ctx: Context): Promise; // releases files/locks; idempotent; called by the Session on close/fault, never by hosts directly +} +``` + +Query rules: + +- Conversation and task scans ascend by ID; `cursor` is an exclusive lower bound. +- Entry scans are fork-aware and newest-first; `through` is an inclusive upper bound, `cursor` an exclusive + upper bound; `kind` filters before `limit`. `through`, `cursor`, and `newestHead`'s `at` must name a + fork-visible entry of the scanned conversation or the read rejects `InvalidHistoryPosition`. `newestHead` + is the dedicated "newest entry with a head at or before `at`" query so context derivation never tails a scan. +- `readList` returns the complete logical list ascending by element ID. No paging. +- Pages carry no snapshot sequence. Reason: reads happen on the line or against immutable history; a + multi-page snapshot protocol is unnecessary. +- List removal targets an element of the *logical* list at the write position: an element appended locally or + inherited from a fork parent within that list's lineage (same definition, ancestor conversation, at or before + the fork cap). Removing a visible inherited element is valid and records a local removal (the parent is + untouched); removing an element already removed in this lineage is idempotent and still consumes a sequence. + An ID that never existed, or that belongs to a different definition or an unrelated conversation's list, + rejects. Clear on an empty list is valid. + +Storage side effects on `task.set` to terminal: delete the task's scratch scope; delete each shared output +that no live task references after this batch. Reading retired scratch rejects `ScratchRetired`; reading a +retired output rejects `OutputRetired`, in every backend. + +Empty `Write[]` is never passed to storage. An uncertain commit failure (storage threw after possibly +writing) poisons the session: fail-stop, never retry the callback. + +## 5. The commit line + +One FIFO async mutex serializes every commit, every `nextId()` call, every reservation, mark, registry change, +watch capture and waiter registration. Only explicit transaction builders run while the line is held; kind +methods, hooks, signals/listeners and other application callbacks never do. + +```text +enter line (caller ctx checked before entering and again before the callback) +build a capability-bound tx with an empty Write[] +run callback + explicit reads (tx.getTask, tx.value().get, ...) await committed storage — allowed only before the first write + writes buffer synchronously; creations call Storage.nextId() synchronously + on throw: discard Write[], minted IDs stay burned, rethrow +validate and assert (section 5.2) +Storage.commit(Write[], internalCtx) -- exactly once, non-cancellable +apply to in-memory indexes; kick scheduler; resolve waiters +leave line +dispatch signals/listeners; resolve caller +``` + +### 5.1 Read before write + +A transaction may read committed state only until its first buffered write; a later explicit read rejects +`ReadAfterWrite`. Reason: there is no read-your-writes overlay. Allowing reads after writes would either +require the kernel to merge the pending `Write[]` into every query (a second storage engine) or return stale +answers that silently disagree with what the callback just wrote. Authors read first, decide, then write, +carrying the values they wrote in local variables. A builder validation failure poisons that transaction: +catching `ReadAfterWrite`, a pending-read write, or a rewindable-after-entry error does not permit earlier +buffered writes to commit. The builder does remember its own same-batch creations for validation (a task +created earlier in the batch is a valid `after` target), but does not expose them as reads. + +Reads while holding the line block every other commit. Keep them small and few. Entering any Session line +from inside an active Session callback rejects `NestedLineOperation` immediately; use only the supplied `tx`. + +### 5.2 What the Session checks + +**Validation** (capability or lifecycle; a failure rejects the commit, the session continues): + +| check | why | +|---|---| +| tx capability: ordinary tx has no `entry`/admission/core creation, only passive `write`; abort tx has no task creation or `write` | plugin isolation (by construction, not a check) | +| address scope: conversation address must be in writer's subtree; task address must be writer's own; protected `pi.*` internals rejected for non-internal writers | capability | +| invocation identity: task tx/runtime ctx carries the current invocation object; stale or foreign rejects | cancellation correctness (section 7) | +| task marked: normal commit/scratch/output from a marked invocation rejects | cancellation correctness | +| rewindable writes before entry appends | fork visibility (section 2) | +| queued `write` head still valid at placement, else `unanswered/stale` | a collapse may have moved the head meanwhile | +| `after` targets exist and form no cycle; task self-wait and dependency-path waits reject | deadlock prevention | +| output ref names a live output of the same kind | writes to nothing otherwise | +| token identity: the kind object passed to `tx.task(kind, …)` / `tx.write(entryKind, …)` / `hooks.on(kind, …)` is the object currently registered for that name (or the fixed built-in witness), compared by identity, else `StaleDefinition` | a redeclared or replaced token must not create durable input/facets under a different current implementation | +| `createConversation` parent is in the writer's subtree | a task cannot inherit an unrelated tree | +| session phase (open/stopping/closing) permits this operation | lifecycle | + +**Pre-persistence assertions** (invariants of kernel- and core-constructed writes; a failure faults the +session because persisting them would be durable corruption): + +- Task transitions: `task.create` only for an absent ID, pending, no checkpoint/outcome/abort, empty `owns`; + running only from pending with every `after` terminal (or marked, ignoring `after`); terminal only from + running and carrying an outcome, with one narrow exception: open-time orphan reconciliation (section 13 step + 5) may terminalize a `pending` *or* `running` task directly as `orphaned`, and only it may; marked snapshots + keep their lifecycle state; terminal is never replaced. +- Every owned `conversation.create` has a same-batch `task.set` whose `owns` gains exactly that ID; `owner` + matches; ownership is never changed later. +- One `pi.output` append per tracker flush, with provenance from a live task referencing that output. +- Transcript invariants of section 1.4 for core appends. + +### 5.3 Transaction surfaces + +```ts +interface TxReaders { + getConversation(id: Id): Promise; + getEntry(id: Id): Promise; + getEntry(kind: EntryKind, id: Id): Promise; + getEntries(ids: readonly Id[]): Promise>; + getTask(id: Id): Promise; + getTask(kind: K, id: Id): Promise | undefined>; + getTasks(ids: readonly Id[]): Promise>; +} +interface TxValue { get(at?: Id): Promise; set(v: T): void; delete(): void } +interface TxList { read(at?: Id): Promise[]>; append(v: T): Id; remove(id: Id): void; clear(): void } +interface TxState { // bound addresses only (host scope is unbound) + value>(a: D): TxValue, ResolvedValue>; + list(a: List): TxList; +} +interface BoundTxState { // binds unbound definitions to own conversation + value | UnboundValue>(a: D): TxValue, ResolvedValue>; + list(a: List | UnboundList): TxList; +} + +// exact task creation: visible extra fields in the spec or in `input` reject, for literals, variables and spreads alike +interface TaskCreator { + task & Extra>( + kind: K, + spec: NoExtra & Extra, S> & { readonly input: ExactJsonInput, S["input"]> }, + ): Id; +} + +interface HostTx extends TxReaders, TxState, TaskCreator { // host must name the conversation + createConversation(spec: ConversationCreateSpec): Id; // sync; no first input (section 8.1) + write(conversationId: Id, kind: EntryKind, input: EntryInput): Promise; // passive; returns the input id; no request key +} +interface ConversationTx extends TxReaders, BoundTxState, TaskCreator { // HostTx bound to one conversation + createConversation(spec: ConversationCreateSpec): Id; + write(kind: EntryKind, input: EntryInput): Promise; +} +interface BaseTaskTx extends TxReaders, BoundTxState, TaskCreator { // conversationId defaults to own + checkpoint(c: C): void; // buffers a full task.set with this checkpoint + createConversation(spec: ConversationCreateSpec): Id; // owner = this task; parent must be in subtree; no first input + write(kind: EntryKind, input: EntryInput): Promise; // own conversation; passive, boundary-safe; no request key +} +type CoreTaskTx = Omit, "task" | "write"> & TaskCreator & { // composed, not extended: `write` changes arity; creator accepts ordinary or fixed core children + entry(kind: EntryKind, conversationId: Id, input: EntryInput): Id; + accept(conversationId: Id, options: InternalAcceptOptions): Promise; // section 9 + queueInput(conversationId: Id, input: QueuedInput): Promise; + write(conversationId: Id, kind: EntryKind, input: EntryInput, requestId?: string): Promise; // private admission helper; request-keyed receipts are kind "write" +}; +interface AbortTx extends TxReaders, BoundTxState { // same own-conversation binding as BaseTaskTx; scratch stays task-bound via ScratchReader + checkpoint(c: C): void; + markTask(id: Id): Promise<"marked" | "terminal">; +} +interface CoreAbortTx extends AbortTx { + entry(kind: EntryKind, conversationId: Id, input: EntryInput): Id; // publish aborted result + write(conversationId: Id, kind: EntryKind, input: EntryInput, requestId?: string): Promise; +} +type FinalTx = BaseTaskTx & FinalOutput; +type CoreFinalTx = CoreTaskTx & FinalOutput; +type AbortFinalTx = AbortTx & FinalOutput; +type CoreAbortFinalTx = CoreAbortTx & FinalOutput; +``` + +`accept`/`queueInput`/`write` are async because a request-key lookup and the busy check are committed reads; +called before any write they read then buffer, called after a write they use only their same-batch memory +and reject if a committed read would be needed. `write` is the only way for non-core code to add to the +transcript (section 1.4); it is available to ordinary task kinds in normal commits and terminal closures +(`FinalTx`), so a task can queue its completion notice atomically with its own terminalization. The busy check +is prospective over the whole batch: an ordinary task never makes a conversation busy, and a core task's own +terminal `task.set` in the same batch counts as gone, so a `write` in a terminal closure is appended +immediately unless some *other* turn task is still live after the batch. Public and ordinary `write` take no +request key and return only the input ID: a terminal-closure notice is already exactly-once because closure +settlement is atomic, and a host write that must be idempotent can be keyed by the caller in its own state. +Only the core admission helper `CoreTaskTx.write`/`CoreAbortTx.write` accepts a `requestId`, and receipts it +creates are tagged `kind: "write"`. `Harness.input(requestId)` returns a handle only for receipts tagged +`kind: "send"`, so a passive write can never be retrieved as an `InputHandle`; `InputHandle` is exclusively +admitted user input. The placed entry of a write is observed through the watch (`entry` event); passive writes +have no public handle. Ordinary kinds never get direct `entry`, `accept`/`queueInput`, boundary helpers, +input-group resolution or core creation. `AbortTx` has no `write`. + +### 5.4 Runtime surfaces + +```ts +interface TaskRuntimeBase { + readonly taskId: Id; + scratch(build: (tx: TxState) => T | Promise, ctx: Context): Promise; // own task scope only + readonly hooks: HookRunner; // section 11.2; runs outside the line + conversation(id: Id, ctx: Context): Promise; // own subtree only + waitForTask(id: Id, ctx: Context): Promise; // own subtree or own deps only + abortTask(id: Id, ctx: Context): Promise<"marked" | "terminal">; + now(): number; sleep(untilMs: number, ctx: Context): Promise; +} +interface CommitRuntime { + commit(build: (tx: Tx, current: RunningTask) => T | Promise, ctx: Context): Promise; +} +type RuntimeOutput = [O] extends [never] ? Record : { readonly output: TaskOutput }; +type AbortRuntimeOutput = [O] extends [never] ? Record : { readonly output: ReadonlyTaskOutput }; + +type TaskRuntime = + TaskRuntimeBase & CommitRuntime> & RuntimeOutput; +type CoreTaskRuntime = + TaskRuntimeBase & CommitRuntime> & RuntimeOutput; + +interface AbortRuntimeBase { + readonly taskId: Id; + scratch(read: (s: ScratchReader) => T | Promise, ctx: Context): Promise; // read only + readonly hooks: HookRunner; + conversation(id: Id, ctx: Context): Promise; + waitForTask(id: Id, ctx: Context): Promise; + abortTask(id: Id, ctx: Context): Promise<"marked" | "terminal">; + now(): number; sleep(untilMs: number, ctx: Context): Promise; // cancellable wait for graceful cleanup; still no write/create/mutate authority +} +type AbortTaskRuntime = + AbortRuntimeBase & CommitRuntime> & AbortRuntimeOutput; +type CoreAbortTaskRuntime = + AbortRuntimeBase & CommitRuntime> & AbortRuntimeOutput; + +interface BoundPublicState { // shared by every conversation-bound public handle + value | UnboundValue>(a: D): PublicValue, ResolvedValue>; + list(a: List | UnboundList): PublicList; +} +interface TaskConversation extends BoundPublicState { // restricted; no commit, no entry + readonly id: Id; + send(input: SendInput, ctx: Context): Promise; +} +interface AbortTaskConversation extends BoundPublicState { readonly id: Id } +``` + +`current` passed to every `commit`/closure callback is a fresh read of the task on the line, not the stale +method argument. Reason: checkpoints and marks committed since the method started must be visible. + +Invocation identity: the scheduler installs a private `{ taskId, method, token: object }` in a derived ctx +through a Chord context key. Every runtime method and `TaskConversation`/`InputHandle` returned to a task +captures that object and compares it by identity. Metadata or RPC transport never confers authority; a task +that uses an unrelated host ctx is an in-process escape no type can prevent. + +Line reentry (calling `runtime.commit` inside a commit callback) rejects `NestedLineOperation` immediately +rather than deadlocking. + +## 6. Scheduler + +One scheduler per Session covers every conversation. Open constructs the indexes and leaves it inert; +`resume(ctx)` enables it. Indexes: all live tasks; reverse dependency edges; tasks per conversation with +turn/foreground subsets; conversation owner/parent ancestry; one invocation slot per running task; waiters. + +```text +kick: if disabled -> nothing (resume rescans everything); else dirty = true, ensure one drain runner +drain: while dirty { dirty = false; on line: reserve every eligible task; off line: dispatch each } +eligible: live, and (marked -> fresh abort, ignoring dependencies) + or (pending, unmarked, every `after` terminal -> task.set running, dispatch execute) + or (running, unmarked, loaded at open with no invocation -> dispatch recover) +``` + +Every successful main commit kicks after in-memory application. Scratch and output-only commits do not kick +(they cannot change eligibility). A commit racing with drain exit sets dirty; the drain loops. No polling. + +"Dependency terminal" includes failed, aborted and orphaned; the dependent reads the outcome and decides. +Reason: a failed dependency should not strand the dependent as pending forever. + +One invocation per task ID at a time. The slot is held from dispatch through method return, producer joins +(pending scratch/output writes) and closure acceptance or discard. The scheduler never awaits one task +before dispatching another; tasks in the root, forks and subagents run concurrently. + +`background: true` only affects idle observation and foreground cancellation. It does not affect +eligibility or execution. A foreground task depending on endless background work keeps foreground idle false +intentionally. + +Waits (`waitForTask`, `InputHandle.wait`, `waitForIdle`) ensure `resume`, return immediately if already +satisfied, else register one cancellable waiter on the line. A waiter observes; it never owns execution. +Cancelling a wait removes only that waiter. Deadlock guards: direct self-wait rejects; waiting on a task whose +unresolved dependency path reaches the caller rejects; `after` cycles reject at creation. + +## 7. Cancellation and lifecycle + +### 7.1 Task abort + +```text +commit task.set {abort: true} -- the mark; idempotent +revoke: old invocation's commit/scratch/output now reject before the callback +signal old invocation's ctx -- off line +await old method return + producer joins +on line: discard any normal closure it returned; release slot +reserve fresh abort: new invocation object, new ctx; kind.abort(task, abortRuntime, ctx) +abort may checkpoint (cleanup progress) and read scratch/output; may not create work +abort returns closure -> aborted outcome + scratch/output retirement, one commit +``` + +Why mark-then-fresh-abort rather than "let the old invocation handle its own cancellation"? The old +invocation may be mid-await on an external effect and may never return cleanly; its partial writes must not +be trusted. A durable mark survives a crash, so reopen runs `abort` (not `recover`) for a marked task even if +the old process died. Revoking before signaling guarantees no write from the old invocation lands after the +mark. + +A pending task that is marked before reservation never executes; the kernel writes `running` only as part of +reserving fresh abort so `abort` receives a `RunningTask`. A repeated mark on a task whose abort is already +running is a no-op and does not cancel the abort. An uncooperative effect (ignores ctx) delays abort +indefinitely; no forced isolation in v1. + +Cancellation is classified by Pico's own identity and known reason, never by error name alone. Local tool +deadlines are not marks; their errors are domain outcomes. + +### 7.2 Conversation abort + +`conversation.abort(ctx)`: + +1. Compute the foreground cancellation set: every live task directly in this conversation that is not + `background: true`, whatever its kind (turn tasks, threshold collapse, ordinary plugin tasks); for each, the + conversations it owns; recurse into their live non-background tasks. A background or terminal owner stops + the walk, so manual collapse, jobs and other background work survive. Fork parentage is not ownership and + grants no reach. +2. One commit: mark every task in the set; remove queued `steer`/`followUp` items in affected conversations + and write `unanswered/aborted` results; keep queued `write` items. +3. Track the set (plus any task marked transitively by those aborts). Resolve when all are terminal. + +Task abort (`abortTask`) preserves queues; conversation abort withdraws steer/followUp. Reason: aborting one +tool should not throw away the user's typed follow-up; aborting the whole turn should. + +### 7.3 Close, shutdown, fault + +- **close**: stop admission and reservation; let admitted line work finish; signal and join every + invocation; close the storage. Writes no marks or outcomes. Running tasks stay `running` durably and get + `recover` next open. +- **shutdown**: enter `stopping`; one commit marks every live task; only abort commits are accepted; wait + until no live tasks and no invocation slots; close. Queues are preserved. A failing abort faults. Explicit + `close` may interrupt shutdown, which then rejects; durable marks recover later. +- **fault**: on invariant/persistence/closure/contract error: reject all waiters, stop admission, signal and + join invocations, close storage. Never manufacture outcomes. Domain failures (provider errors, tool throws) + are typed outcomes, not faults. + +Repeated close/shutdown calls share one completion. Caller cancellation cannot abandon an admitted close or +shutdown. + +Phase machine and exact sequence: + +```text +open --close--> closing --> closed +open --shutdown--> stopping --(all live tasks terminal, no invocation slots)--> closing --> closed +any --fault--> faulted (terminal; storage closed after joins) + +closing: 1 close() queues on the line like any operation; the callback already admitted owns the line and + finishes and persists normally; close enters after it + 2 on line: phase = closing; every waiter (task/input/idle) rejects Closed; every watch is detached and + gets {type:"closed", reason:"session"} once; operations queued behind close reject Closed + 3 off line: signal every invocation ctx; await method returns and producer joins; + returned closures are discarded (nothing written); invocation commits reject Closed + 4 storage.close(ctx); phase = closed +faulted: the faulting operation holds the line; on line: phase = faulted; waiters reject Faulted; watches are + detached and get {type:"closed", reason:"session"}; then off line: signal and join invocations, + storage.close(ctx). Nothing is written after the fault. +closed/faulted: permanent for this Harness object; every method rejects Closed/Faulted except `close()` and + `shutdown()`, which are idempotent and return the shared completion; a new Harness.open on the + same storage path is the only way forward +``` + +| phase / method | normal commit | scratch write | terminal closure | abort commit/closure | host admission | +|---|---|---|---|---|---| +| open, execute/recover, unmarked | yes | yes | yes | no | yes | +| open, execute/recover, marked | reject | reject | discard | no | yes | +| open, abort invocation | no | read only | no | yes | yes | +| stopping | no | no | discard | yes | reject | +| closing / faulted / closed | no | no | discard | no | reject | + +## 8. Ownership + +A task that calls `tx.createConversation` becomes the conversation's `owner` and the conversation ID is added +to its `owns` in the same commit. Ownership grants: + +- capability: `runtime.conversation(id)`, `waitForTask`, `abortTask` accept targets in the owner's subtree; +- cancellation reach: conversation abort and `pi.tool` abort follow `owns`; +- foreground idle: a conversation is idle when its foreground cancellation set is empty. + +Ownership does not affect scheduling. A terminal owner's owned conversations keep running. A subagent is an +owned conversation; there is no "subagent task". One task cannot resolve a conversation owned by a sibling +task; the only exception is the `pi.tool` descendant resolver of section 11.1 for later sibling tool calls. + +### 8.1 Seeding owned conversations + +All model, system and tool configuration is conversation-scoped. An owned conversation does **not** copy its +parent's current configuration implicitly; it gets exactly what its creator seeds. Reason: a subagent with a +different model, a smaller tool set and its own identity is the normal case, and an implicit copy of a +parent's mutable settings would be a hidden dependency that changes under the child. + +```ts +type SeedWrite = + | { readonly address: Value | UnboundValue; readonly value: JsonValue } // set + | { readonly address: Value | UnboundValue; readonly delete: true }; // durable absence (cuts an inherited rewindable value) + +interface ConversationSeed { + readonly parent?: { readonly conversationId: Id; readonly at: Id | "start" }; // transcript inheritance; default none + readonly values?: readonly SeedWrite[]; // conversation-bound values: generationKind.config.*, postToolsKind.config.*, plugin state + readonly copySticky?: readonly UnboundValue[]; // copy parent sticky values at creation (async surfaces only: needs a parent read) + readonly sections?: readonly SectionSeed[]; // sectionSeed(section, value) to set; removeSection(key) to cut an inherited section (host forks) + readonly input?: UserInput; // first input; creates the first generation. Omit to create inert (path 1 below) + readonly requestId?: string; +} +type ConversationCreateSpec = Omit; // the synchronous tx form +type ForkOptions = { readonly at: Id | "start"; readonly abort?: boolean } & Omit; // parent is the handle's conversation +``` + +Creation is one transaction: `createConversation` (for a task creator: owner = current task, `owns` updated), +every `values` set/delete and sticky copy, the section seed (below), for owned children the configuration +isolation writes (below), and, when `input` is present, `accept` of the first input through the fixed +generation strategy plus the creator's own checkpoint recording the child conversation ID, input ID and +request key. Either all of it commits or none does. + +Which surface can do which part: `HostTx.createConversation(spec)` and `BaseTaskTx.createConversation(spec)` +are synchronous and take `ConversationCreateSpec`: no `input` (accepting one needs a committed request-key +read, which a builder cannot do after its first write) and no `copySticky` (copying needs a parent read; a +caller that wants it reads first and passes explicit `values`). The asynchronous conveniences +`Harness.createConversation(seed, ctx)` and `ConversationHandle.fork(options: ForkOptions, ctx)` (host; one +commit each) and the core-owned mediated path of section 11.1 (tools; internal `CoreTaskTx` performing the +request-key/busy/parent reads before its first write) accept the full seed. There is no other way to create a +conversation with a first input. + +Request key on creation: a seed with `input` and `requestId` cannot use first-key-wins, because the original +acceptance (if any) belongs to some other conversation and returning that handle would break fork/creation +semantics, while creating an empty conversation anyway would leave garbage. Rule: the async creation planner +checks `pi.request` on the line before minting anything; if a receipt for that key exists (of either kind), +creation rejects `RequestKindMismatch` and writes nothing, and the caller recovers the original through +`h.input(requestId)`. A fresh key, or no key, creates conversation and input atomically. + +Two planners, one seed shape: + +| | host fork / host `createConversation` with `parent` | task-owned child (subagent) | +|---|---|---| +| built-in rewindable config (`model`, `thinkingLevel`, `selectedTools`, `profile`, `threshold`, `keepRecent`) | inherited at the fork point, then seed overrides | **isolated**: never inherited (below) | +| built-in sticky config (`retry`, `steeringMode`, `followUpMode`) | absent (reads default) unless `copySticky`/`values` | absent (reads default) unless `copySticky`/`values` | +| system sections | desired state starts from inherited canonical state | desired state starts **empty** (below) | +| plugin rewindable values | inherited unless seed sets/deletes | inherited unless seed sets/deletes | +| `owner` | none | the creating task | + +A fork is the same agent continuing from an earlier point; an owned conversation is a different agent. + +**The seed must be sufficient on its own.** If `input` is present, the child's first generation may be +reserved by the scheduler in the same drain that follows the creation commit, so nothing registered +afterwards can be assumed to exist. Every child-specific configuration and system-section value must +therefore be durable in the creation transaction, either literally (`values`, `sections`) or through a durable +profile value (`generationKind.config.profile`) that already-registered handlers expand deterministically. + +**Hooks are code, not data.** A hook handler is a process-local callback; it cannot be serialized, persisted +or made part of an atomic commit, and there is no durable identity for a callback. `ConversationSeed` +therefore has no `hooks` field. Child-specific behaviour that lives in hooks reaches the child by exactly one +of two paths: + +1. **Inert creation, then activate.** The host creates the child without `input` (`tx.createConversation`, + `h.createConversation`, `c.fork`), obtains its `ConversationHandle`, registers child-scoped hooks with + `c.hook.*` / `c.hooks.on` and adjusts `c.config`, and only then calls `send`. Nothing runs before `send`, so + there is no race. This is the host path. +2. **Atomic creation with a durable profile.** A tool-mediated (or host) creation that includes `input` seeds + `generationKind.config.profile` (and any other selection values) in the same transaction. Hook + implementations for every profile were registered before `resume`, harness-wide or on the parent with + `subtree: true`, and dispatch on the child's profile value read from `info.conversationId`. No + post-creation registration is needed, so no race exists. This is the subagent path. + +On reopen, the host reinstalls hook code before `resume`; durable profile/config values select behaviour. A +child whose profile has no registered implementation gets the generic handlers, which must treat an unknown +profile as a plain default, never as an error. + +**Owned conversations never inherit built-in configuration.** For a task-owned child, even with `parent` +set (transcript inheritance), the creation transaction writes each seeded rewindable member of the full +`CoreConfigBundle` locally and a durable `value.delete` for each unseeded rewindable member (`model`, +`thinkingLevel`, `selectedTools`, `profile`, `threshold`, `keepRecent`; the set is derived from the bundle, so +a new rewindable built-in is covered automatically). Rewindable lookup therefore finds a local write for every +member and never walks into the parent for configuration. Sticky members cannot inherit by construction and +follow `copySticky` or `values` only. Absent `model` yields a typed `no_model` failure on the child's first +generation; every other absent member reads its default. Plugin rewindable values are *not* isolated: Pico +cannot know which unknown definitions should be cut, so they follow ordinary parent history at the fork point +unless the seed sets or deletes them (`SeedWrite` with `delete: true`). Host forks skip this block entirely. + +Section seed: set entries are rendered with the registered definitions at creation time (an unregistered +section rejects); removals (`removeSection(key)`) need no definition. The resulting `SectionRecord[]` (`set` +with value+rendered, or `remove`) is persisted as private immutable metadata on the `conversation.create` +storage record itself (section 1.1), not as an address: it is atomic with creation, survives inert setup and +reopen, is invisible to the public `Conversation` projection and to every generic handle, and needs no new +protected namespace. Preparation applies it while this conversation has no *local* managed `pi.system` entry +(inherited managed history from a fork parent does not count): for a host fork it is folded over the inherited +canonical state, for an owned child over the empty desired state (below). After the first local managed entry +is appended it is inert. The root uses the same metadata for `root.sections`. + +**First preparation, two planners.** The *comparison base* is always the inherited canonical section state +(fork-visible managed history if `parent` is set, empty otherwise); the delta or baseline is computed against +it (section 12.5). What differs is where the *desired* state starts: + +- Host fork / host `parent`: desired = inherited canonical state, then seed `set`/`remove`, then + `systemInstructions` handlers. Tools = the fork's inherited-then-overridden `selectedTools`. +- Task-owned child: desired = **empty**, then seed `set`, then `systemInstructions` handlers (subtree + handlers on ancestors provide defaults; child-local and profile-dispatched handlers run last). Tools = the + child's isolated `selectedTools`. Diffing desired against the inherited base therefore emits a `remove` for + every inherited section the child did not seed or re-set, and `toolsRemoved` for every inherited tool not in + the child's loadout. + +Inheriting the parent's `pi.system` entries and then sending without that delta would present the main agent's +identity and tool loadout to the child's model; that is a defect, not a configuration choice. The requirement +holds regardless of how the mediated `conversation()` API of section 11.1 is eventually shaped. Whichever path +created it, a child's system sections, configuration and profile are in place before its first generation +prepares; that is the invariant both paths exist to guarantee. + +## 9. Input admission + +### 9.1 Records + +```ts +type UserInput = string | readonly (TextContent | ImageContent)[]; + +type QueuedInput = + | { mode: "steer" | "followUp"; input: Stored; requestId?: string } // durable form; SendInput.content is converted at admission + | { mode: "write"; entry: StoredEntryDraft; requestId?: string }; // passive entry, e.g. a notice + +// durable, protected, ID-only +type StoredInputResult = + | { status: "queued"; requestId?: string } + | { status: "placed"; requestId?: string; entry: Id } + | { status: "done"; requestId?: string; entry: Id; answer?: Id } + | { status: "unanswered"; requestId?: string; entry?: Id; reason: "terminated"|"aborted"|"failed"|"stale"; detail?: string }; + +// public, hydrated by the handle before it resolves +type InputOutcome = + | { readonly status: "queued"; readonly requestId?: string } + | { readonly status: "placed"; readonly requestId?: string; readonly input: UserEntry } + | { readonly status: "done"; readonly requestId?: string; readonly input: UserEntry; + readonly answer?: { readonly entry: AssistantEntry; readonly message: AssistantMessage } } + | { readonly status: "unanswered"; readonly requestId?: string; readonly input?: UserEntry; + readonly reason: "terminated" | "aborted" | "failed" | "stale"; readonly detail?: string }; +type TerminalInputOutcome = Extract; + +interface SendInput { requestId?: string; content: UserInput; whenBusy?: "followUp" | "steer" | "reject" } +interface InputHandle { + readonly id: Id; readonly conversationId: Id; + result(ctx: Context): Promise; // point read plus entry hydration, no activation + wait(ctx: Context): Promise; + abort(ctx: Context): Promise<"aborted" | "already_placed" | "not_found">; +} +``` + +`StoredInputResult` is what admission writes, for user input and passive writes alike; `InputOutcome` is +what callers see, and only for admitted user input (`send`). The handle reads the stored record, then fetches +the referenced entries (`getEntries`, one round trip) and resolves with them attached. Reason: every caller of +`wait` wants the answer text, and entries are immutable, so hydration is a pure convenience with no staleness. +`answer.message` is `answer.entry.model[0]` handed out as the pi-ai `AssistantMessage` type (the stored +form is structurally identical after ingress validation), not the current edited context projection. Passive writes have stored results (for placement bookkeeping and `unanswered/stale`) but no public +handle; `Harness.input(requestId)` returns `undefined` for a `kind: "write"` receipt. + +Protected state (written only by admission helpers): + +- `pi.inbox`: sticky conversation list of `QueuedInput`. The element ID is the input ID; ascending IDs are + admission order. Not inherited by forks. Reads return one complete current collection; no backend replays + history per read. +- `pi.input_result`: sticky session value keyed by input ID → `StoredInputResult`. +- `pi.request`: sticky session value keyed by `requestId` → `Acceptance` (tagged `kind`; `Harness.input` ignores `"write"`). + +Result and receipt lookups are point reads, never list scans. + +### 9.2 Request keys + +A `requestId` names the first acceptance for the session's lifetime. Lookup happens before anything else. A +duplicate `send` returns a handle to the original acceptance and writes nothing, even through another +conversation with different content. Reason: a client that sent, lost the reply and retried must get the same +input, not a second one. `accept`/`queueInput`/`write` obey the same rule within their own `Acceptance.kind`. +Across kinds a key collision is internal misuse: a `send` whose key names a `"write"` receipt (or a core `write` +whose key names a `"send"` receipt) rejects `RequestKindMismatch` and writes nothing; there is no cross-kind +return. + +### 9.3 Busy + +`busy(c)` = at least one live `pi.generation`, `pi.tool` or `pi.post_tools` directly in conversation `c`, +foreground or background. Evaluated against committed live tasks plus the current batch's `Write[]`. +Ordinary tasks and `pi.collapse` never make a conversation busy. Reason: collapse runs speculatively next +to a turn; an idle `send` must not be blocked by it. + +### 9.4 Send + +```text +send when not busy: + one commit: place older queued writes; place older queued steer/followUp per queue mode plus the new + input as pi.user entries; create one pi.generation { inputs: [...placed ids] }; results -> placed; + receipt if requestId. `whenBusy` ignored. +send when busy: + followUp (default) or steer: one commit: inbox append; result queued; receipt. No task. + reject: throw ConversationBusy; write nothing. +write when not busy: append entry, result done (no task). When busy: queue. +``` + +After the commit, `send` ensures `resume` and returns the handle. It never waits for generation. + +Result transitions (only admission/boundary helpers write them): + +```text +absent -> queued -> placed -> done | unanswered(terminated|aborted|failed) + queued -> unanswered(aborted|stale) + queued write -> done +idle send collapses absent -> placed; idle write collapses absent -> done (still creates+removes inbox element) +``` + +### 9.5 Boundaries and queue modes + +Boundaries are helper calls inside a core task's terminal closure, not separate commits. Which queued items a +boundary selects is governed by `postToolsKind.config.steeringMode`/`followUpMode` (section 2.1): `"all"` takes +every queued item of that mode, `"one-at-a-time"` takes the oldest. All queued writes are always selected. +Selected items, whatever their mode, are placed in one pass strictly by ascending input ID (inbox element ID, +i.e. admission order); no mode has priority over another. Unselected items keep their relative order. + +- **post-tools** (in `pi.post_tools` closure): selected = all queued writes + steer per `steeringMode`; place + them by ID. Steer joins the active input group (the continuation generation owns them). FollowUp stays + queued. Reason: steer means "interrupt the current line of work"; followUp means "after you're done". +- **final answer** (in `pi.generation` closure when no tool calls): the closure has already appended the + assistant entry; then resolve the active group → `done(answer: assistant)`; selected = all queued writes + + steer per `steeringMode` + followUp per `followUpMode`; place them by ID after the assistant; if any + steer/followUp was placed, they form a new group and its generation is created. +- **yield continuation** (section 10.1): when an `onYield` hook asks to continue and no steer/followUp is + queued, the closure, in this order: appends the assistant entry (the exchange is now complete); places + selected queued writes in global inbox order (they land at the post-answer boundary, never inside the + exchange); appends a `pi.user` entry `{ model: [{ role: "user", content: , timestamp }], data: + { continuation: true, from: } }` that has **no** input result (it is not admitted input); and + creates the successor generation carrying the **same** `inputs`; the active group's results stay `placed`. + If one of the placed writes is a self-head (reset/handoff), the continuation is invalidated: the caller just + cleared the state the hook wanted to continue. The closure then behaves as a plain final answer: assistant + appended, writes placed, active inputs `done(answer)`, no continuation entry and no successor. +- **idle turn** (kernel, after any normal core closure): if this commit takes the conversation from busy to + not busy and creates no turn task, place queued items as at a final-answer boundary. Covers post_tools + ending the turn without a continuation. +- **abort**: no automatic drain. Conversation abort withdrew steer/followUp; task abort left them. A later + idle `send` consumes them. + +A queued `write` carrying a `head` is revalidated at placement (section 1.4). If the head is no longer valid, +the item is removed and its result set to `unanswered/stale`. Never a fault. + +Heads placed at a boundary interact with the group the boundary forms. Placement is still strictly by ID, but +after placing the selected items the boundary applies two rules so that no generation ever owns an input that +its own request cannot see: + +- Every steer/followUp admitted *before* a self-head write placed in this batch (a queued reset/handoff with + `head: "self"`) is now outside the context the caller asked for, whether or not the mode selected it: a + selected one whose entry now lies before the head is excluded from the successor group, an unselected one is + removed from the inbox, and both resolve `unanswered/stale`. Otherwise an unselected `one-at-a-time` leftover + would resurrect the cleared state on a later send. Items admitted after the final self-head keep their order + and are selected normally. Example: queued followUp B1, B2, then queued reset R with `one-at-a-time`: B1 is + placed then made invisible by R, B2 was never selected; both resolve `unanswered/stale` and no generation is + created for either. The same rule runs when a self-head `write` is admitted while the conversation is idle + (for example `abortTask` left followUps queued and the host then calls `c.reset`): the immediate append also + removes every older queued steer/followUp as `unanswered/stale`, in the same commit. Otherwise they would + resurrect on the next send into a context the reset just cleared. +- At a post-tools boundary, a placed self-head also ends the *old* active group: its inputs resolve + `unanswered/terminated` (the model can no longer see them), no ordinary continuation is created, and only + visible triggers placed after the final head (steer or followUp with entries after `R`) may form a fresh group + and start a generation. Example: queued reset R, then queued followUp B; placement gives `reset R; user B`; the + old group is terminated, B starts a fresh generation whose request begins at `R`. + +Both use existing result reasons; nothing new is stored. + +Trace: + +```text +idle send A TX[user A; generation G1{inputs:[A]}; A=placed; receipt] kick -> G1 +G1 calls X,Y TX[assistant; tool X; tool Y; post_tools P1{after:[X,Y], inputs:[A]}; G1 terminal] + kick -> X, Y +busy send B TX[inbox B; B=queued] +busy send S (steer) TX[inbox S; S=queued] +busy write W TX[inbox W; W=queued] +X done TX[tool_result X; X terminal] +Y done TX[tool_result Y; Y terminal] kick -> P1 +P1 closure TX[user S; S=placed; entry W; W=done; generation G2{inputs:[A,S]}; P1 terminal] (S admitted before W) +G2 final answer E2 TX[assistant E2; A,S=done(answer:E2); user B; B=placed; generation G3{inputs:[B]}; G2 terminal] +``` + +There is no fourth inbox mode (no `nextRun`): the modes are `steer`, `followUp` and `write`, and Pico does not +buffer non-triggering context for a later turn. An application that accumulates such context (notes, events, +partial instructions) keeps it in its own value or list and folds it into one later `UserInput` when it +actually wants a turn; `write` is only for content that should become a passive transcript entry now. + +Input ownership is explicit: `inputs: Id[]` in generation/post_tools input. Exactly one live turn task owns +each placed group; ownership transfers in the same commit that creates the successor. The kernel never +inspects a checkpoint to find owners. + +### 9.6 Withdraw and wait + +`InputHandle.abort`: on the line, if still queued: remove inbox element, result `unanswered/aborted`, return +`aborted`. If placed or terminal: return `already_placed`, change nothing. If unknown: `not_found`. A queued +result whose inbox element is missing is a corrupted-state fault. + +`InputHandle.wait`: reject `InputNotFound` if no result exists; ensure `resume`; return an existing terminal +result; else register one waiter, cleaned up exactly once on result, caller cancel, close or fault. Caller +cancellation never marks or removes the durable input. In both paths the handle hydrates the referenced +entries after observing the terminal stored record and before resolving; hydration is off the line and reads +immutable entries, so it cannot observe a different state than the record names. + +## 10. Core task kinds + +All five are installed at open and cannot be replaced or removed. Generation, tool, post_tools and collapse +use `defineCoreTask`; job uses ordinary `defineTask`. Payloads below are normative compact syntax; the code +uses named `type` aliases with exactly these shapes. Hook points are declared per kind (section 11.2). + +### 10.1 `pi.generation` + +```ts +input: { inputs: Id[] } +checkpoint: // `Prep` is captured once at `prepared`; every later phase carries it unchanged except `attempt` (see steps 2-3) and `tools` (step 2) + Prep = { cutoff: Id; system: Id | null; model: ModelRef; thinkingLevel: ThinkingLevel; tools: string[]; retry: Stored; attempt: number } + | { phase: "prepared" } & Prep + | { phase: "requesting"; requestKey: string } & Prep // effect in flight + | { phase: "retrying"; untilMs: number; lastError: string } & Prep + | { phase: "deferred"; handle: Stored } & Prep // provider-side async +result: { assistant: Id; tools: Id[]; postTools?: Id; successor?: Id } +failure: { reason: "provider" | "overflow" | "retries_exhausted" | "no_model"; detail: string; assistant?: Id } +aborted: { assistant?: Id } +output: GenerationOutput = { message?: Stored } // absent until the provider's `start` frame; reset to {} on retry +``` + +The task is created by admission before model, provider, API, timestamp or usage are known, so `initial(input)` +is `{}`. pi-ai's `event.partial` is one shared object the provider keeps mutating after `start`, so it must +never be inserted into or retained by the tracker. Generation runs every provider event through pi-ai's +`AssistantMessageFrameEncoder`, whose `start` frame carries a *clone* of the partial and whose later frames are +compact per-block deltas. Generation keeps only the small process-local per-block reducer state (which block +index is text/thinking/tool call, whether it has ended, the tool-call JSON so far) and applies each encoded frame +directly inside one `output.mutate` on the tracker's own `message`, using the same switch and invariants as +pi-ai's frame reducer. It never buffers prior frames, never re-reduces the sequence, never keeps a second +`AssistantMessage`, and never `replace`s the whole message per token: one frame, one incremental `mutate`, one +Chord delta. pi-ai's `reduceAssistantMessageFrames` is a test oracle only (the final tracker value must equal +its reduction of the same frames), not a runtime dependency. `output.replace({})` when a retry begins. The +terminal `AssistantMessage` from `done`/`error` or a deferred fetch is cloned and strict-JSON validated before +the closure stores it as the assistant entry; a terminal path that requires an entry but has no message is a +kind-contract fault. Watchers treat an absent `message` as "no response yet". +One task output, no second frame log. + +Execute: + +1. Read `generationKind.config` at the current position (`model` absent → `failed/no_model`). Run system + preparation (section 12), which may append a managed `pi.system` entry and returns its ID. Commit + `prepared` with `Prep`: the cutoff (newest visible entry ID after preparation), the system entry ID, the + resolved model, thinking level, the loadout's tool names and the retry policy as read now. Reason: the + cutoff fixes what is sent; the copied config makes recovery send the same request under the same cutoff and + attempt even if `c.config` changed meanwhile; later phases read `Prep`, never `c.config`. +2. Derive the immutable request snapshot from the cutoff. Run `beforeRequest` hooks on a private copy + (messages only). The transformed request is **not** persisted; a recovery or retry that re-enters this step + reruns the hooks (never step 1: `prepared` and system preparation run once per generation), so handlers must + be deterministic for a given (cutoff, config) or accept that a retried request may differ. Then, on the + *final* transformed request: (a) overflow check, `estimate(final) > model.contextWindow - model.maxTokens` → + the overflow path below, without calling the provider; (b) derive `tools` = the tool names actually declared + by the final request's `SystemMessage`s (added minus removed), whatever the hooks did; no subset rule is + enforced, and a model call naming a tool with no registered executor is already a missing-tool result in + `pi.tool`. Commit `requesting` with a fresh `requestKey`, that `tools`, and `attempt` (1 on the first pass; + `retrying.attempt + 1` when entered from step 3 or from recovery of `retrying`). Call the provider outside + the line with a derived ctx. Encode each event with `AssistantMessageFrameEncoder`; on the + `start` frame `output.mutate(o => { o.message = frame.partial })` (the encoder's clone, never the provider's + `partial`); apply every later frame inside `output.mutate` to `o.message`. +3. Every terminal provider `AssistantMessage` (stop, length, toolUse, error, aborted, or the result of a + deferred fetch) first goes to `afterResponse` outside the line with `{ message, usage, attempt }`, before + any classification; it observes, it does not decide. Then, on a provider error (`stopReason: "error"`): let + `policy = checkpoint.retry`. Classify with pi-ai's + `isRetryableAssistantError(message)`; if retryable, `policy.enabled` and `attempt <= policy.maxRetries`: + one commit buffers a data-only `pi.usage` entry `{ attempt, usage, error }` (no `model`; records the failed + attempt's usage) together with the `retrying` checkpoint (`attempt` = the attempt that just failed, + `untilMs = now + retryDelayMs(policy, attempt)`); then `sleep`, `output.replace({})`, go to 2, which commits + `requesting` with `attempt + 1`. Otherwise + `failed/provider` (not retryable or disabled) or `failed/retries_exhausted`. Provider returned `deferred`: + commit `deferred` with `Stored`; wait `handle.pollAfterMs` (fallback 5 s when absent), then + `models.fetchDeferred(model, handle)`; a message → step 4; another deferred handle → commit `deferred` with + the new handle and repeat; an error → step 3 as a provider error. Deferred polling is not the retry backoff. +3b. Terminal non-retryable outcome: a provider `stopReason: "aborted"` without a Pico mark (the provider ended + the stream on its own) and a final `"error"` (not retryable, retry disabled, or attempts exhausted) take the + same path: the closure appends the error/aborted `AssistantMessage` once as a display-only `pi.assistant` + entry (never in context, never creating tools), resolves the active inputs `unanswered/failed`, and + terminalizes `failed/provider` or `failed/retries_exhausted`. A Pico mark on the ctx does not reach here: it + unwinds the invocation and fresh abort runs (below). +4. Successful response (`stop`, `length`, `toolUse`): `afterResponse` has already run (step 3; the assistant + *entry* does not exist yet, hooks never run inside or after the closure). For a final answer (no tool + calls): run `onYield` outside the line **only if** no steer/followUp is queued at that moment. Closure then rechecks the inbox and cancellation on the line. If a trigger arrived meanwhile, + the yield decision is stale and discarded; normal final-answer boundary. Otherwise `{ continue }` → yield + continuation boundary (section 9.5) with a `pi.user` entry carrying the returned text; the successor + generation keeps the same `inputs`. No `continue` → final-answer boundary. +5. Tool calls: closure buffers the assistant entry, one `pi.tool` per call and one + `pi.post_tools{after: tools, inputs, assistant, tools}`. Every call gets a `pi.tool`, including one naming a + tool that was not offered; `pi.tool` decides (10.2) using the generation's own terminal checkpoint as the + evidence, so `ToolInput` carries no duplicate offered list. + +Recover: `prepared` → step 2 (hooks rerun; request built from `Prep`, not from current config; step 1 never +reruns). `requesting` +→ the effect may have happened and there is no result lookup: treat the interruption as a retryable failed +attempt with usage absent, evaluated against the persisted `checkpoint.retry` and `checkpoint.attempt` exactly +like step 3: if `retry.enabled` and `attempt <= retry.maxRetries`, one commit appends `pi.usage{ attempt, error: +"interrupted" }` and the `retrying` checkpoint, then sleep and step 2 with `attempt + 1`; otherwise terminalize +`failed/provider` (disabled) or `failed/retries_exhausted` with **no** assistant entry (there is no message to +display) and resolve inputs `unanswered/failed`. Never a silent increment, never an unconditional `retrying`. +`retrying` → sleep out `untilMs`, then step 2. `deferred` → +resume polling with the stored handle. Usage accounting: each failed attempt's usage is one `pi.usage` entry committed with its +`retrying` checkpoint; the successful attempt's usage lives once, on the final `pi.assistant` entry's +`AssistantMessage.usage`. Nothing is counted twice. Missing usage is unknown, not zero. + +Size estimation: `estimate(messages) = estimateContextTokens(messages).tokens` from pi-ai, applied to the +projected request messages (including the messages-only system/tool declarations once the pi-ai PR lands). +The same estimator is used for overflow, threshold and `through` selection. + +Overflow (request too large): detected in step 2(a) on the final transformed request, never on the pre-hook +snapshot, and always before `requesting` is committed or the provider is called (pi-ai `Model` fields; no +separate cap in v1): compute `through` (below). If a collapsible prefix exists, the +closure creates a **foreground** `pi.collapse C{reason: "overflow", through}` and a replacement +`pi.generation G{after: [C], inputs}`, transfers input ownership to `G`, and terminalizes `failed/overflow` +without resolving inputs. `G` reads `C`'s outcome first: completed → normal execution against the new head; +failed/declined/aborted/orphaned → `G`'s closure `write`s a `pi.notice` ("context too large; compaction +failed: "), resolves the inputs `unanswered/failed` and terminalizes `failed/overflow`. If no +collapsible prefix exists (the retained suffix is the whole context), there is no `C` and no `G`: the closure +`write`s the same notice, resolves inputs `unanswered/failed` and terminalizes `failed/overflow` directly. + +Input ownership, all generation outcomes: success → inputs `done` (final answer) or transferred to +`pi.post_tools` (tool calls); `no_model`, terminal provider error/abort, and interrupted `requesting` with no +retry left → `unanswered/failed`; overflow with +collapse → transferred to the replacement; replacement after failed collapse → `unanswered/failed`; Pico abort +→ `unanswered/aborted`. + +Threshold (context large but the request succeeded): if `collapseKind.config.threshold > 0` and +`max(usage.input + usage.output, estimate(request))` exceeds it, the normal closure additionally creates a +**foreground** `pi.collapse C{reason: "threshold", through}`; the generation completes normally. Automatic +collapse is foreground because it is part of the turn's work: conversation abort reaches it and `waitForIdle` +waits for it. Manual `c.collapse` is background and speculative. Neither makes the conversation busy (9.3). + +Choosing `through` (all cases, including manual): let the context be the sequence of complete exchanges +(user/notice entries count as their own one-message exchanges). Walk from the newest exchange backwards, +accumulating `estimate` of the retained suffix; keep extending the suffix while its estimate is ≤ +`collapseKind.config.keepRecent`. `through` is the last entry of the exchange immediately before the retained +suffix. If the whole context fits in `keepRecent`, there is nothing to collapse and no task is created (manual +`c.collapse` then rejects `NothingToCollapse`). If even the newest exchange exceeds `keepRecent`, the suffix is +that single exchange. The summary's `head` is the first entry after `through`. + +Abort: if the checkpoint is `deferred`, fresh abort first calls `models.cancelDeferred(model, handle)` best +effort (errors ignored); the closure stores the partial message as an `aborted` assistant entry if any content +exists (display only) and resolves inputs `unanswered/aborted`. + +### 10.2 `pi.tool` + +```ts +input: ToolInput = { assistant: Id; call: Stored } +checkpoint: + | { phase: "started"; replay: "safe" | "unsafe"; call: Stored } // final identity-preserving, schema-validated call + | { phase: "children"; replay; call: Stored; ... } // keyed child/owned-conversation records; exact remaining schema gated (11.1) +result: { entry: Id; control?: ToolControl } +failure: never // tool errors are model-visible results, not task failures +aborted: { entry: Id } +output: ToolOutputState +``` + +Execute: first the offered-set check. Read the assistant entry (`input.assistant`), follow its `byTaskId` to the +generation task, and read `tools` from that task's retained checkpoint (terminal records keep their last +checkpoint). The task must be a completed `pi.generation` whose result names this assistant entry, and its last +checkpoint phase must be `requesting` or `deferred` (both carry `Prep.tools` as actually offered; a deferred +response can end with tool calls). Anything else (no `byTaskId`, task absent, not completed, different assistant, +or a `prepared`/`retrying` checkpoint) is corruption and faults: Pico itself wrote those records. `call.name` absent from that list → synthetic +`pi.tool_result` `isError: true, "tool not offered"` without consulting the registry. Then look up the tool by +`call.name` in the registry and validate the original arguments. A missing tool or invalid arguments produces +the corresponding synthetic error result. Otherwise run `beforeTool` outside the line, before any `started` +checkpoint; a block (including a hook throw under this point's policy) produces a synthetic error result. +Validate the final rewritten arguments again. Only the invocation branch commits `started` with the tool's +`replay` policy and the complete final call, immediately before invoking outside the line through the gated +execution API (section 11.1); in WP10 that is an internal test executor. A tool throw becomes an error result. +Run `afterTool`. Closure: re-read any children on the line (never trust a +stale `Promise.race`); buffer exactly one `pi.tool_result` entry whose `model` message is the bounded output +followed by diagnostics, and whose `data` carries details, usage, control and truncation metadata; return +completed with `control`. + +Recover: no checkpoint → the crash happened between reservation and the `started` commit, and since `started` +is always committed before the tool is invoked, no external effect can have occurred: repeat lookup, argument +validation, `beforeTool`, final validation and then start normally. A `started` checkpoint never reruns +`beforeTool`: its stored final call is the effect evidence. Resolve the current executor by name and replay +only when both the persisted and current declarations say `replay: "safe"` and the persisted arguments validate +against the current schema. Missing/current-unsafe/schema-incompatible → a model-visible interrupted or +unavailable error result, never a fault; safe/current-safe/valid → invoke the persisted call. This avoids +running old evidence under an incompatible deployment without adding definition fingerprints. In-process +invocations retain the definition they started with. `children` resumes the recorded children and retains the +same final call. + +Abort: mark each child with `abortWithTool: true` and every live foreground task in each subagent with +`abortWithTool: true`, atomically mark-if-live; an already terminal target counts as cleaned. Do not withdraw +subagent queues. Closure appends an aborted `pi.tool_result` through `CoreAbortTx.entry`. + +### 10.3 `pi.post_tools` + +```ts +input: { inputs: Id[]; assistant: Id; tools: Id[] } +checkpoint: never +result: { successor?: Id; ended?: "terminate" | "handoff" } +failure: never +aborted: null +``` + +Runs after every tool is terminal. Execute: through one read-only `runtime.commit` (reads only, no writes) +take an immutable snapshot of the terminal tool tasks and their result entries; terminal records never change, +so the snapshot cannot go stale. Run `afterTools` outside the line on that snapshot. Closure, one commit, in +this exact order (rewindable writes must precede every entry append, section 2): + +1. Reads: `generationKind.config.selectedTools` fresh (so a config change made while `afterTools` ran is not + lost); the snapshot is already in hand. +2. Fold `control` from the snapshot **in the assistant's call order**: `addTools` is additive (current list + followed by every requested name not already present, first occurrence wins, nothing dropped); `handoff` wins + over `terminate` (a handoff also ends the turn) and among several `handoff` messages the last in call order + wins. +3. Buffer the single `selectedTools` set, if `addTools` produced a change. This is the only rewindable write and + it comes before any entry. +4. For every tool outcome that is `orphaned` or `aborted` without a result entry, append a synthesized error + result, in call order, so the exchange is complete. +5. If `handoff`: append `pi.handoff{ head: "self", model: [user handoff message] }` **directly** through + `CoreTaskTx.entry` (not `write`), so the head exists before anything else placed in this batch. +6. If `handoff` or `terminate`: resolve the active group `done` (answer: assistant) and run the final-answer + boundary, whose follow-up placement and successor generation now come after the head and therefore start the + fresh context. Otherwise: post-tools boundary and `pi.generation{ inputs }` as the continuation with the group + plus placed steer. +7. Terminal snapshot. + +Reason for 5 before 6: a `write` would evaluate prospective busy after the successor generation was buffered and +queue the handoff behind it, so the next request would still see the old context. This direct control handoff +is not an inbox-ordered self-head write: queued steer/followUp items, even if admitted earlier, are placed after +the new head by step 6 and may start in its fresh context rather than being staled. + +Abort: `pi.post_tools` owns the active input group while it is live, so its fresh abort closure resolves every +ID in `input.inputs` as `unanswered/aborted` (writing the internal input-result state directly through core +authority; no public helper exists) and terminalizes `aborted: null`. Otherwise a marked post_tools would +strand placed inputs forever. Queued items are untouched here: task abort leaves them for a later idle send; +conversation abort has already withdrawn steer/followUp in its own mark batch. + +### 10.4 `pi.collapse` + +```ts +input: { reason: "threshold" | "manual" | "overflow"; through: Id; instructions?: string } +checkpoint: // `Base` is captured once, before `beforeCollapse`, and carried unchanged except `attempt` (retrying records the failed attempt; the next summarizing uses attempt + 1) + Base = { expectedHead: Id | null; instructions?: string; model: ModelRef; thinkingLevel: ThinkingLevel; retry: Stored; attempt: number } + | { phase: "summarizing" } & Base + | { phase: "retrying"; untilMs: number; lastError: string } & Base + | { phase: "prepared"; summary: string } & Base // candidate text is durable; only finalization remains +result: { summary: Id } +failure: { reason: "stale" | "declined" | "provider" | "no_model"; detail: string } +aborted: null +``` + +`through` is the last entry of a complete exchange (chosen by the creator, section 10.1 for automatic +collapse; `c.collapse` uses the same `keepRecent` rule). Execute: through one read-only `runtime.commit` +capture `Base` (`expectedHead` = current newest head, `generationKind.config` model/thinking/retry at this +position, `attempt: 1`) together with the entries to summarize; if `model` is unset, terminalize +`failed/no_model` immediately (before `beforeCollapse` and before any checkpoint), for manual, threshold and +overflow alike; run `beforeCollapse` (fold `first`) outside +the line on that snapshot. Then one commit: re-read `newestHead`; if it differs from `Base.expectedHead`, the +hook's decision was made against a superseded context → terminalize `failed/stale` (never pair old entries +with a new head). Otherwise: no handler answer → proceed with `input.instructions`; `{ decline: true }` → +`failed/declined` for every reason (manual, threshold, overflow), no summary, no fault; `{ instructions }` → +replaces `Base.instructions`; `{ summary }` → commit `prepared` with that text and skip the provider; else +commit `summarizing`. Call the summarizer outside the line. Provider error: classify with +`isRetryableAssistantError`; retryable, `retry.enabled` and `attempt <= retry.maxRetries` → commit +`retrying{ untilMs = now + retryDelayMs(retry, attempt), lastError }` (recording the attempt that failed), +sleep, commit `summarizing` with `attempt + 1`, call again; otherwise `failed/provider`. The +summarizer request is request-local and **tool-free**: every tool definition effective in the projected +historical request (the fold of `toolsAdded`/`toolsRemoved` across its `SystemMessage`s, which is authoritative +regardless of the current registry) is removed by a request-local `toolsRemoved` on the private copy, and no +new tools are offered; a summarizer response containing tool calls is treated as a provider +failure (`failed/provider`, "summarizer returned tool calls") and retried under the policy. On a successful response commit `prepared` +with the summary text. Closure: `newestHead` read first; if it differs from `expectedHead`, `failed/stale` +(someone else collapsed). Else append `pi.summary { data: {through}, model: [user summary], head: first entry +after through }` and return completed. Edits appended in between do not stale. Recover: `summarizing` → the +call may have happened; count it as that attempt's failure exactly like generation's `requesting` recovery: +if `retry.enabled` and `attempt <= retry.maxRetries`, commit `retrying{ untilMs: now + retryDelayMs(retry, +attempt), lastError: "interrupted" }`, sleep, then `summarizing` with `attempt + 1`; otherwise `failed/provider`. Never replay the same attempt number. +`retrying` → sleep out `untilMs`, then `summarizing` with `attempt + 1`; `prepared` → finalize only, no +provider call. One live collapse per conversation; a second creation returns the existing ID. Collapse +never blocks generation or input. Manual `c.collapse` returns the task ID; the caller reads its outcome. + +### 10.5 `pi.job` + +A job is a long-running external process owned by a task from its first effect. It is an ordinary kind: its +notices are passive `write`s, which `BaseTaskTx` provides, so it needs no core authority. Jobs are created by +hosts through `HostTx.task`, by other ordinary tasks through `BaseTaskTx.task`, and by tools through the gated +mediated task operation of section 11.1. + +```ts +input: JobInput (declared with ProcessHost below) +checkpoint: + | { phase: "waiting"; untilMs: number; occurrence: number } + | { phase: "spawning"; key: string; occurrence: number } // inflight: the host may have started it + | { phase: "running"; key: string; occurrence: number } +result: { exitCode: number; occurrences: number } +failure: { reason: "spawn" | "interrupted"; detail: string } +aborted: { killed: boolean } +output: JobOutput (declared with ProcessHost below; mutable, unlike the readonly host snapshots) +``` + +Jobs run through an injected **process host**, the only code that touches the OS. The host, not Pico, owns +whatever durability it can honestly provide: + +```ts +type JobInput = { readonly command: string; readonly args: readonly string[]; readonly cwd: string; + readonly env?: { readonly [name: string]: string }; + readonly notify: boolean; readonly rerun: boolean; readonly every?: number; readonly notBefore?: number }; +type ProcessSpec = Pick; +type ProcessOutput = { readonly stdout: string; readonly stderr: string; readonly droppedStdout: number; readonly droppedStderr: number }; // bounded tails; readonly host snapshot +type JobOutput = { -readonly [K in keyof ProcessOutput]: ProcessOutput[K] } & { exitCode?: number; occurrence: number }; // the task's mutable OutputState, mirrored from status() inside output.mutate +type ProcessStatus = + | ({ readonly status: "running" } & ProcessOutput) + | ({ readonly status: "exited"; readonly exitCode: number } & ProcessOutput) + | { readonly status: "unknown" }; // host has no durable record of this key +interface ProcessHost { // implemented by hosts; copy this verbatim + start(key: string, spec: ProcessSpec, ctx: Context): Promise; // idempotent per key: a second start for a key the host knows is a no-op. + // Rejects only when it knows no process started (ENOENT, EACCES, bad cwd); if the outcome is + // uncertain it must resolve and report through status(). A rejection is a domain outcome, not a fault. + status(key: string, ctx: Context): Promise; + kill(key: string, signal: "SIGTERM" | "SIGKILL", ctx: Context): Promise; // idempotent if key already exited or is unknown +} +``` + +Pico polls `status` (same backoff schedule as retries, capped at 1 s) and copies the bounded output into the +task output; it never holds a stream and never reads a PID. The bundled `nodeProcessHost()` keeps runs in process memory +only: after a restart every key is `unknown`. A host that redirects output to files and supervises the child +(or a remote executor) may return `running`/`exited` across restarts; that is the host's contract to keep, and +Pico does not specify how. The host is injected through `HarnessOpenOptions.processHost`; when absent, a job's +execute terminalizes `failed/spawn` ("no process host") before any checkpoint. Memory-storage tests use a fake +host. + +Execute, per occurrence `n`: if `notBefore`/`every` says wait, commit `waiting` and `sleep`. Commit +`spawning{ key: `${taskId}:${n}`, occurrence: n }` **before** any OS effect (the inflight checkpoint). Call +`host.start(key, ...)`, commit `running{ key, occurrence }`, then poll `status` until `exited`, mirroring +output into `output.mutate`. On exit: if `notify`, `tx.write` a `pi.notice` for this occurrence in the same +commit as the next checkpoint (`waiting` for a schedule, or the terminal snapshot for a one-shot); if `every`, +`output.replace` a fresh state for `n + 1` and loop under the same task ID (no backlog is inferred after +downtime); else return completed. + +`start` rejecting → the closure terminalizes `failed/spawn` with the host's message (and `write`s the notice +if `notify`); nothing escapes the task method. `status` rejecting (the host itself is broken or unreachable) +→ the closure terminalizes `failed/interrupted` with the host's message: the process outcome is unknown *and* +unknowable, so `rerun` does not apply (rerun is for `unknown`, where the host answered). A `kill` rejection +during fresh abort is the documented failing-abort case (7.3) and faults. `status` returning `unknown` at any point, during the ordinary +poll or on recover, means the process may or may not have started or finished and Pico cannot know: if +`rerun`, `start` again with the same key (a possible duplicate external effect, exactly the uncertainty the +recoverability disclaimer in 3.2 allows), else `failed/interrupted`. + +Recover: `waiting` → sleep until the stored `untilMs`, then continue that occurrence (commit `spawning`, ...). +`spawning` or `running` → `host.status(key)`; `running`/`exited` → continue as above; `unknown` → the rule +just stated. Abort, by checkpoint: none or `waiting` → no process exists, no host call, closure returns +`{ killed: false }`; `spawning`/`running` → `kill(key, "SIGTERM")`, `rt.sleep(now + 5000)`, then +`kill(key, "SIGKILL")`, and return `{ killed: true }` if both host calls resolve. Here `killed` means the host +accepted the cleanup sequence for a possibly started key, not proof that the process was still alive; `kill` +must be idempotent for an exited or unknown key. A rejection is the failing-abort fault described above. A +schedule ends wherever it is. Notices are passive writes: a notice about occurrence `n` is admitted with `n`'s +checkpoint and placed at the next boundary if the conversation is busy; `result` carries no entry ID because +placement may happen after the job terminalized. + +## 11. Tools and hooks + +### 11.1 Tools: normative behaviour, gated API + +The behaviour of the built-in `pi.tool` kind (10.2) and of every tool *result* is normative. The exact +programming interface an ordinary tool author writes against is **gated**: this is the one intentional gap in +this document. Nothing below the "Gated" heading may be implemented until its final shape is appended here. + +Normative (settled), independent of the gated shape: + +```ts +// what a tool declares, minus its execution signature +interface ToolDeclaration

    { + readonly name: string; readonly description: string; readonly parameters: P; + readonly replay?: "safe" | "unsafe"; // default unsafe + readonly output?: { maxBytes?: number; maxLines?: number; retain?: "head" | "tail" }; // default 64k / 200 / head +} +// what a tool produces (public, pi-ai typed content) +interface ToolResult { + content: readonly (TextContent | ImageContent)[]; + details?: D; // typed, for UIs; stored in entry data + isError?: boolean; + usage?: ToolUsage; + diagnostics?: readonly ToolDiagnostic[]; // about the call, rendered after the output + control?: ToolControl; +} +// durable payload pieces are exact type aliases (interfaces have no implicit index signature and would not satisfy JsonObject) +type ToolControl = { + readonly terminate?: true; // end the turn after this exchange + readonly handoff?: string; // end the turn and reset context with this message + readonly addTools?: readonly string[]; // additive: appended to selectedTools by post_tools before its continuation, so the next model request offers them; never removes +}; +type ToolDiagnostic = { readonly severity: "info" | "warn" | "error"; readonly message: string; readonly code?: string }; +``` + +Ordinary tool code never receives `commit`, `entry`, `write`, task lifecycle methods or any admission helper. +Reason: a tool is plugin code; the transcript and the task are Pico's. The `pi.tool` kind owns the tool's +task: it enforces `output` bounds on `content`, emits the truncation diagnostics it owns, stores the exact model +message in entry `model` and the rest (`ToolResultData`) in entry `data`, and streams the tool's live progress +as its own `TaskOutput` (10.2). `h.tools` registers `ToolDeclaration`s plus the gated execution +part. + +**Gated: the ordinary tool execution API.** Not settled, not to be inferred from the generic task runtime: +the `execute` signature, the facade a tool receives (previously sketched as a `ToolApi`; that sketch is +withdrawn), how the tool reaches the `pi.tool` task's `TaskOutput` for streaming progress, durable memos, the +keyed child-task and subagent operations, and the owned-descendant resolver. The settled requirements the +final shape must satisfy are: + +- It is a small `pi.tool`-mediated surface over the current invocation: the tool's identity is the stable + `pi.tool` task ID, its output is that task's `TaskOutput`, and durable strict-JSON memos are protected + state scoped to that task, retired with it. +- A keyed child-task operation, if included, uses one internal `pi.tool` transaction to create the child and + record its ID, kind, `abortWithTool: boolean` and optional shared-output ref in the complete parent + checkpoint before returning. Jobs remain tasks created this way; there is no `startShell` verb and no + adoption of an unfinished promise. +- A keyed subagent operation, if included, applies a `ConversationSeed` (section 8.1) in one internal + `pi.tool` transaction: `createConversation` (owner = the tool task), the seeded config/sticky/section + values, the internal `accept` of the first input through the fixed generation strategy, and the complete + parent checkpoint recording conversation ID, input ID, request key and `abortWithTool`. It then resolves the + existing restricted `TaskConversation` and waits outside the line. A subagent is an owned conversation, + never a subagent task or a spawn operation with its own lifecycle. +- A repeated creation key is either explicitly first-key-wins or compares a persisted canonical spec + (normalized parent position, input, request key, initial values, seed profile/version, `abortWithTool`); + never mutable current values. The choice is part of the gated decision. +- Returned task handles expose wait/abort, never commit. A local wait timeout is observation only: it returns + "timed out" without marking the child. The `pi.tool` terminal closure rereads the child on the line: a + terminal child yields the normal final result; a live child yields the continues-in-background result while + the child retains any shared output. No design may trust stale `Promise.race` state. +- Returned conversation handles are capped at the existing `TaskConversation`: scoped `send` plus the returned + input's result/wait/abort; no commit, direct entry, admission helpers, close, shutdown or unrelated state. + Any model-facing status/tail projection must be specified before it is added. +- `abortWithTool: false` means only that enclosing `pi.tool` abort does not mark that child; it is not a + generic detachment abstraction. Parent normal completion cancels neither. +- Later sibling tool calls need one narrow built-in resolver for strict owned descendants of their current + parent conversation, including ownership through terminal sibling tasks. Fork-only ancestry and unrelated + roots reject. It must not broaden `runtime.conversation` or `TaskConversation`. + +Until this decision is appended here, no ordinary tool can be executed: `pi.tool` (10.2) is implemented and +tested against an internal test executor that returns `ToolResult`s, and every built-in tool (including `bash` +and `subagent`) waits for the API. WP10 delivers `pi.tool` to that boundary and no further. + +### 11.2 Hooks + +Hooks are declared by the task kind that invokes them. There is no global hook interface. A kind declares +typed points; its implementation calls `runtime.hooks.run(kind.hooks.point, input, ctx)` at the moments it chooses; the +scheduler and kernel never interpret point names. Core kinds ship predefined points; ordinary kinds declare +their own the same way. + +```ts +type Fold = "collect" | "first" | "chain"; +interface HookPoint { + readonly fold: F; // how several handlers' outputs combine + readonly onThrow: "skip" | "abort"; // skip: report, drop this handler's output, continue + // abort: stop, kind receives { threw: error } + readonly [hookIn]?: In; readonly [hookOut]?: Out; // compile-only +} +declare function defineHookPoint(options: { readonly fold: "collect"; readonly onThrow: "skip" | "abort" }): HookPoint; +declare function defineHookPoint(options: { readonly fold: "first"; readonly onThrow: "skip" | "abort" }): HookPoint; +declare function defineHookPoint>(options: { readonly fold: "chain"; readonly onThrow: "skip" | "abort" }): HookPoint; // chain: object input, output is a patch of In +type HookPoints = Record>; + +type HookHandler

    = (input: HookIn

    , info: HookInfo, ctx: Context) => HookOut

    | void | Promise | void>; +interface HookInfo { readonly kind: string; readonly taskId: Id; readonly conversationId: Id } + +interface HookRunner { + run

    (point: P, input: HookIn

    , ctx: Context): Promise>; // point token from this kind's bundle +} +type HookResult

    = + P extends { fold: "collect" } ? { outputs: readonly HookOut

    []; threw?: unknown } : + P extends { fold: "first" } ? { output?: HookOut

    ; threw?: unknown } : + { output: HookIn

    /* chained */; threw?: unknown }; + +interface HarnessHookRegistry { + on[keyof NonNullable]>( + kind: K, point: P, handler: HookHandler

    , + ): () => void; // unregister; harness-wide has no scope option +} +interface ConversationHookRegistry { + on[keyof NonNullable]>( + kind: K, point: P, handler: HookHandler

    , options?: { readonly subtree?: boolean }, + ): () => void; // this conversation; subtree: also conversations it owns, recursively +} +// h.hooks: HarnessHookRegistry h.hooks.on(myKind, myKind.hooks.somePoint, handler) +// c.hooks: ConversationHookRegistry c.hooks.on(myKind, myKind.hooks.somePoint, handler, { subtree }) +``` + +Points are passed as tokens (`kind.hooks.somePoint`), not strings. Reason: plugin kinds may reuse point names, +and a token carries its owning kind, input/output types, fold and failure policy, so nothing is looked up by +name and nothing can collide. + +Fold semantics: `collect` runs every handler and returns all non-void outputs in order; `first` stops at the +first non-void output; `chain` is legal only for object `In` and requires `Out extends Partial`: handlers run +sequentially, each output is shallow-merged over the current input, a void return passes it through, and the +final current input is the result. The generic rule has no terminal predicate. One built-in point adds an +explicit short-circuit, stated on that point only: `beforeTool` stops the chain as soon as an output sets +`block`. Handler order: harness-wide in registration +order, then outermost conversation to innermost. Handlers run outside the line, receive and must forward +`ctx`, may take as long as they like (a human approval is a hook that waits), and may run again after a crash +because the kind re-enters the phase that ran them; external side effects need their own keys. The runner +awaits each handler's actual return, never a raced promise. Cancellation errors always propagate and unwind +the invocation; `onThrow` governs other errors only. Handlers hold no task, transaction or transcript +authority; they receive `HookInfo` for identity and use ordinary public handles for state. + +A decision a hook returns is revalidated in the commit that applies it (section 10); the hook itself commits +nothing. Hooks never run on the line, inside a closure, or after it; a kind that needs committed data for a hook +takes an immutable snapshot through a read-only `runtime.commit` first. Timing of every core point: +`systemInstructions` during preparation, before `prepared`; `beforeRequest` after `prepared`, before +`requesting`; `afterResponse` after every terminal provider message, before retry/failure classification and +before the generation closure; `onYield` after `afterResponse`, only on success and when no trigger is queued; +`beforeTool` after lookup and initial validation but before the `started` checkpoint; `started` is committed +immediately before invocation. `afterTool` runs after the tool returns, before the tool closure; `afterTools` after the read-only +snapshot, before the post_tools closure; `beforeCollapse` before the `summarizing` checkpoint. + +Core points (declared on the respective kind; `fold`/`onThrow` shown): + +| kind | point | input | output | fold / onThrow | +|---|---|---|---|---| +| generation | `systemInstructions` | `{ sections: SystemSectionDraft; config: Settings; tools: readonly ToolDeclaration[] }` | `{ tools?: ToolDeclaration[] }` (override only) | collect / skip; a skipped handler's draft edits are rolled back | +| generation | `beforeRequest` | `{ request: { messages: Message[] }; cutoff: Id }` | `{ request?: { messages: Message[] } }` | chain / skip | +| generation | `onYield` | `{ answer: AssistantMessage }` | `{ continue: string }` | first / skip | +| generation | `afterResponse` | `{ message: AssistantMessage; usage?: Usage; attempt: number }` (every terminal provider message: success, error, aborted, deferred fetch) | void | collect / skip | +| tool | `beforeTool` | `{ call: ToolCall; block?: { reason: string } }` | `{ call?: ToolCall; block?: { reason: string } }` | chain, stops on `block` / abort → block | +| tool | `afterTool` | `{ call: ToolCall; result: ToolResult }` | `{ result?: ToolResult }` | chain / skip | +| post_tools | `afterTools` | `{ assistant: Id; results: Id[] }` | void | collect / skip | +| collapse | `beforeCollapse` | `{ reason; through: Id; entries: Entry[] }` | `{ decline: true } \| { instructions?: string; summary?: string }` | first / skip | + +`beforeTool` is the one point with a short-circuit: an output that sets `block` ends the chain; the tool is not +invoked and the result is an error result with `block.reason`. Otherwise the final `call` is what runs, and only +its `arguments` may differ from the original: `id`, `name` and every other identity field (type, signature, +namespace, provider metadata) must equal the model's original call; a fold whose `call` changed any of them is +treated as `block` with reason "call identity changed" (a hook may narrow or normalize a call, never redirect it +to a different executor). The final `arguments` are then schema-validated exactly like the original call's: a +hook that rewrites them into something invalid produces the same synthetic `isError` result, never an invocation +with bad arguments. `onThrow: "abort"` there means a throwing handler sets `block` with the +error message. `beforeRequest`'s input is `{ request, cutoff }` and only `request` is patchable; `afterTool`'s is +`{ call, result }` and only `result` is patchable; `cutoff` and `call` are read-only context. + +This is the same derivation principle as configuration (section 2.1): a kind exposes two typed bundles, +`kind.config` and `kind.hooks`; `c.config` is assembled from the first, `c.hook` (section 13.1) from the +second. `c.hooks.on(kind, kind.hooks.point, handler, scope)` and `h.hooks.on(...)` are the generic forms for +any kind. Each point's metadata is owned by the kind that declares it; the flattened `c.hook.` methods +are wrappers and introduce no second schema. + +## 12. System prompt and tool loadout + +Pico sends pi-ai `{ messages }` only. System instructions and the tool loadout are `SystemMessage`s inside +`messages` at their historical positions. pi-ai owns native translation and best-effort cache preservation. +This depends on the pi-ai system-message API (`SystemMessage` with `toolsAdded`/`toolsRemoved`, messages-only +adapters, PR #9116), which is a hard dependency of WP9 and WP10: those packages wait for it and import its +types and adapters; Pico never ships a stand-in `SystemMessage` or its own provider-side translation. + +```ts +// pi-ai (PR #9116); reproduced here for reading only, imported in code. After the PR, pi-ai's `Message` union includes it, +// which `Entry.model: readonly Stored[]` and `SystemEntry extends Entry` rely on: the WP1 compile matrix passes only +// against a pi-ai that has landed it (a hard dependency, section 18). +interface SystemMessage { + role: "system"; content: string; toolsAdded?: Tool[]; toolsRemoved?: Tool[]; timestamp: number; +} +``` + +### 12.1 Sections + +```ts +interface SystemSection { readonly key: string; render(value: T): string } // default generic is the erased registry form +declare function defineSystemSection(definition: { readonly key: string; render(value: T): string }): SystemSection; +declare const systemSections: { readonly identity: SystemSection; readonly environment: SystemSection; readonly skills: SystemSection }; +// h.sections: MutableRegistry erased default generic; register rejects duplicate key; replace requires same key; remove keeps stored data +``` + +Tokens carry a stable key and a pure synchronous renderer. Only keys, payloads and rendered text are stored; +never renderers. Payloads are JSON; changing a registered payload type needs a compatible replacement. + +### 12.2 Draft + +```ts +interface SystemSectionDraft { + get(s: SystemSection): T | undefined; // owned copy + set(s: SystemSection, value: T): void; // existing key keeps position; new key appends + delete(s: { readonly key: string }): void; // explicit; every section token satisfies this; null is a valid payload + wrap(s: SystemSection, transform: (rendered: string) => string): void; // preparation-local +} +``` + +Preparation seeds one draft from the canonical section state (12.4). `systemInstructions` handlers run in +order (section 11.2) editing that draft. A handler that throws has its own mutations and wrappers discarded; +earlier handlers' changes stay. After all handlers: touched sections are rendered, wrappers apply in +registration order after rendering, and the result is frozen. The tool loadout defaults to +`generationKind.config.selectedTools` resolved against `h.tools` (unknown names are dropped and reported through +`onReport`, section 13); +handlers do not need to return tools. A handler that returns `tools` overrides the loadout for this +preparation with that complete list; the last such handler wins. + +Handlers must be replay-safe as a set: a handler that appends to a persisted seed accumulates unless an +earlier handler resets that section's base each time. Wrappers are not stored; untouched sections keep their +previously rendered text including old wrapper output. + +### 12.3 Managed entries + +```ts +type SectionRecord = + | { key: string; action: "set"; value: JsonValue; rendered: string } + | { key: string; action: "remove" }; +type SystemEntryData = { readonly baseline?: true; readonly sections: readonly SectionRecord[] }; +``` + +- **Baseline**: `baseline: true`, complete ordered state of every section. Its `SystemMessage.content` is every + section formatted as `## ${key}\n${rendered}`, in canonical order, joined by one blank line; `toolsAdded` is the + complete loadout; `timestamp` is the commit time. +- **Delta**: only changed sections (`set`) and explicit deletions (`remove`). Its `SystemMessage.content` states + each change: `The ${key} section now reads:\n${rendered}` for a set, `The ${key} section no longer applies.` + for a removal, joined by one blank line. `toolsRemoved` + lists previous complete definitions no longer wanted; `toolsAdded` lists new or changed complete + definitions. Removals apply before additions; same-name additions upsert. Tool definitions live only in + `SystemMessage` fields, never duplicated in `data`. +- A delta whose data changed but whose rendering and tools did not has `model: []` (metadata-only). +- Tool-only changes may have empty `content`. + +The frozen draft is diffed against the canonical state: unchanged data and rendering → no entry; else a +delta, or a baseline when 12.4 requires one. + +### 12.4 Canonical state, heads, forks, restarts + +Canonical section state = fold the fork-visible managed entries from the newest `baseline` forward through +its deltas. This uses the indexed `pi.system` kind scan and never depends on model heads or edits; a +collapse or reset does not erase section payloads. An optional prepared-state cache may shortcut the fold. + +A usable model baseline must follow the newest head. If the newest head is after the newest baseline, the +next preparation appends a fresh full baseline at the tail carrying omission edits for every retained +superseded managed entry (baselines and deltas between the head and the tail). This atomic supersession is +the only permitted edit of managed entries; any other omit/replace targeting them rejects. Already superseded +entries omitted by the current baseline do not cause another baseline. + +```text +10 user; 11 baseline; 20 assistant; 30 user; 31 delta; 35 notice; 40 assistant; 50 user +60 summary, head=30 +context at 60: [60, 30, 31, 35, 40, 50] +70 assistant; 80 user +81 baseline, edits:[{target:31, action:omit}] +context at 81: [60, 30, 35, 40, 50, 70, 80, 81] +``` + +Forks and owned conversations created with `parent` see the managed entries visible at their fork point. +Their first preparation compares desired state against that inherited canonical state and appends the delta +(or baseline, if a head intervened): for a host fork the desired state starts from the inherited sections; for +a task-owned child it starts empty, so every unseeded inherited section is removed and every inherited tool +not in the child's isolated loadout is in `toolsRemoved` (section 8.1). Inherited messages are never +rewritten. A conversation created without `parent` has no managed history; its first preparation appends a +full baseline from the seed plus handlers. Restart without a plugin keeps its payload and rendered text; re-registering a compatible +definition restores typed editing; only explicit `delete` removes a section. + +### 12.5 Preparation and staleness + +```text +on line: take the preparation snapshot S = { newestManaged: Id | null, + newestHead: Id | null, // decides baseline vs delta (12.4) + config: current generationKind.config values, + sectionsRev, toolsRev: registry revisions (h.sections, h.tools) } + and copy the canonical section fold and definitions +off line: run systemInstructions handlers on the draft; render; collect tools +on line, in the `prepared` commit: + re-take S'; if S' != S (any field) -> discard, repeat preparation (until one snapshot commits or the task is cancelled) + compute delta/baseline; if any -> buffer entry.append (managed; omission edits if baseline) + buffer checkpoint prepared{cutoff = newest visible entry after this append, system = entry id or null, ...} +``` + +Every mutable registry carries a monotonically increasing revision, bumped by `register`/`replace`/`remove`. +Reason for snapshotting config and registry revisions, not just the newest managed entry: a `c.config.set`, a +`replace` of a section renderer or a tool registration that commits while handlers run would otherwise pair the +managed entry at its historical position with instructions or a loadout computed from stale inputs. `newestHead` +is in the snapshot because a speculative collapse can append a summary head while handlers run; without it the +commit could append a delta against a baseline that now lies before the head, violating the fresh-baseline rule +of 12.4. A miss is ordinary churn, not a fault: preparation simply repeats and the next pass chooses baseline +versus delta against the new head; cancellation is the only exit. Whether an +offered tool's *executable* is still compatible when the model later calls it is a property of the gated +ordinary tool API (section 11.1); tool names in the checkpoint identify what was offered, nothing more. + +The request snapshot is captured after that commit and is immutable; a summary landing afterwards changes +the live context, not the request. A crash after the managed append but before the request preserves the +prepared instructions; the next preparation finds no change and appends nothing. + +`beforeRequest` runs after this commit on a private messages-only copy (section 10.1 step 2); the tool names +offered after transformation are recorded in the `requesting` checkpoint as the validation basis for the +model's calls. Transformed requests themselves are not stored. + +## 13. Harness + +```ts +interface HarnessOpenOptions = {}> { + readonly models: Models; // pi-ai; generationKind.config.model resolves against it + readonly tools?: readonly ToolDeclaration[]; // execution part gated (11.1) + readonly taskKinds?: readonly OrdinaryKind[]; + readonly entryKinds?: readonly EntryKind[]; + readonly sections?: readonly SystemSection[]; + readonly root?: RootSeed; // fresh root only; ignored on reopen + readonly rootValues?: readonly SeedWrite[]; // advanced alias for root.values + readonly onReport?: (report: Report, ctx: Context) => void; // non-fatal diagnostics (hook skipped, unknown tool name, unknown entry kind); off line; throws ignored + readonly processHost?: ProcessHost; // section 10.5; absent -> every pi.job fails { reason: "spawn", detail: "no process host" } at execute +} +type Report = + | { readonly type: "hook_skipped"; readonly kind: string; readonly point: string; readonly conversationId: Id; readonly error: unknown } + | { readonly type: "unknown_tool"; readonly conversationId: Id; readonly name: string } + | { readonly type: "unknown_entry_kind"; readonly kind: string }; +interface RootSeed> { + readonly config?: NoExtra, R>; // same shape and exactness as c.config.set + readonly values?: readonly SeedWrite[]; // any other conversation or session values (set or delete) + readonly sections?: readonly SectionSeed[]; // build with sectionSeed(section, value) +} +// Harness.open>(storage: Storage, options: HarnessOpenOptions, ctx); R is inferred from root.config +interface Harness { + root(ctx: Context): Promise; + conversation(id: Id, ctx: Context): Promise; + createConversation(seed: ConversationSeed, ctx: Context): Promise; // independent (no owner); may include first input; one commit + conversations(query: ConversationScan, ctx: Context): Promise>; // explicit query; `limit` is required by PageQuery + entries(query: EntryScan, ctx: Context): Promise>; // fork-aware, newest-first; pre-head history lives here + tasks(query: TaskScan, ctx: Context): Promise>; // hosts never touch Storage directly + commit(build: (tx: HostTx) => T | Promise, ctx: Context): Promise; + input(requestId: string, ctx: Context): Promise; + getEntry(id: Id, ctx: Context): Promise; + getEntry(kind: EntryKind, id: Id, ctx: Context): Promise; + getTask(id: Id, ctx: Context): Promise; + getTask(kind: K, id: Id, ctx: Context): Promise | undefined>; + value>(a: D): PublicValue, ResolvedValue>; // session scope; bound only + list(a: List): PublicList; + waitForIdle(ctx: Context): Promise; // all foreground tasks in the session; rejects Closed/Faulted + abortTask(id: Id, ctx: Context): Promise<"marked" | "terminal">; + resume(ctx: Context): Promise; close(ctx: Context): Promise; shutdown(ctx: Context): Promise; + inspect(ctx: Context): Promise; + readonly kinds: CoreTaskKinds; + readonly taskKinds: MutableRegistry; + readonly entryKinds: MutableRegistry; + readonly tools: MutableRegistry; + readonly sections: MutableRegistry; + readonly hooks: HarnessHookRegistry; + watchConversation(id: Id, options: ConversationWatchOptions, ctx: Context): Promise>; + watchSession(ctx: Context): Promise>; + watchSession(options: SessionWatchOptions, ctx: Context): Promise>; +} +interface ConversationHandle extends BoundPublicState { // value/list bind unbound definitions; a defaulted definition reads as T, others as T | undefined + readonly id: Id; + commit(build: (tx: ConversationTx) => T | Promise, ctx: Context): Promise; + send(input: SendInput, ctx: Context): Promise; + waitForIdle(ctx: Context): Promise; // this conversation's foreground set; rejects Closed/Faulted + abort(ctx: Context): Promise; + fork(options: ForkOptions, ctx: Context): Promise; // section 8.1; parent is this conversation + collapse(ctx: Context): Promise; collapse(options: { readonly instructions?: string }, ctx: Context): Promise; + reset(ctx: Context): Promise; reset(options: { readonly handoff?: string }, ctx: Context): Promise; // see below + readonly config: ConfigFacade; // section 2.1: model, thinkingLevel, selectedTools, profile, retry, steeringMode, followUpMode, threshold, keepRecent + readonly hook: HookFacade; // section 13.1: systemInstructions, beforeRequest, afterResponse, onYield, beforeTool, afterTool, afterTools, beforeCollapse + readonly hooks: ConversationHookRegistry; // generic: on(kind, kind.hooks.point, handler, { subtree }) + watch(ctx: Context): Promise>; + watch(options: ConversationWatchOptions, ctx: Context): Promise>; // === h.watchConversation(id, ...) +} +``` + +`reset` is a `write` of `pi.reset` or `pi.handoff{head: "self"}`: appended immediately when idle, queued when +busy. It resolves once the write is durably admitted (appended or queued), not when placed; placement is +observed through the watch (`entry` event with `head`). A `head: "self"` write can never go stale (its boundary +is itself). Cancelling `ctx` before admission cancels nothing durable; after admission the write stays. There is +no public handle for passive writes, and `InputHandle` is reserved for admitted user input. + +`fork({ abort: true })` aborts the source conversation's foreground set first ("go back"). `fork` inherits +rewindable state and managed system history through the parent walk; the seed fields override locally +(section 8.1). `h.createConversation(seed, ctx)` creates an independent conversation (no owner) and, unlike the +synchronous `tx.createConversation`, may carry the first input. + +Registries supply implementations, not selections: registering a tool does not offer it to the model; +`generationKind.config.selectedTools` does. `h.kinds` exposes the fixed core kinds as typed witnesses for +`config`/`hooks` derivation and `TaskOf` narrowing; possession of a witness confers no creation authority. + +Registries (`h.taskKinds`, `h.entryKinds`, `h.tools`, `h.sections`) are asynchronous and serialize on the line. +`register` rejects an existing name and any `pi.*` name; `replace` requires an existing ordinary name; +task-kind `replace`/`remove` reject while live instances exist; entry/section/tool removal keeps durable +records. In-flight invocations and preparations keep the definitions they started with; but any *new* creation +they attempt with the old token (`tx.task(oldKind, …)`, `tx.write(oldEntryKind, …)`) rejects `StaleDefinition` +(section 5.2), because the current registration is a different object. Hook registration +(`h.hooks.on`, `c.hooks.on`, `c.hook.*`) is different in kind: synchronous, process-local, returns an +unsubscribe function, touches no durable state and does not enter the line. + +Open order: + +1. Backend replay; initialize `nextId` after the highest committed ID; construct the disabled scheduler. +2. Install core kinds and built-in sections, then options. +3. Empty storage: create root conversation (ID 1) and apply `root` (`config` → the corresponding + `generationKind.config`/... definitions, `values` set/delete, `sections` → the root record's section-seed metadata) plus + `rootValues`, all bound to conversation 1, no duplicate addresses, in one commit. Nonempty: ignore both. +4. Load live tasks and the ownership ancestry they need. Do not load terminal history. Discover stored + entry-kind names through the kind index for `unknownEntryKinds`. +4a. Validate every live fixed built-in task (`pi.generation`, `pi.tool`, `pi.post_tools`, `pi.collapse` and + the ordinary-authority but fixed `pi.job`) against its fixed input/checkpoint schema. Any mismatch fails + open: built-in state that Pico itself cannot read is corruption, never an orphan. The five are always + installed, cannot be replaced, and never reach step 5. +5. Orphan reconciliation, one commit: every live task whose kind is not registered → `terminal/orphaned`, + scratch retired; every registered live foreground task in conversations it owns → marked. Queues kept. + Dependents see `orphaned` and decide. Registration never resurrects an orphaned task. +6. Return. Nothing runs until `resume`. + +Any failure in 1–5 (including 4a) closes the storage and rejects; there is no partially open Harness. + +### 13.1 Conversation config and hooks facades + +Both facades are assembled from the core kinds' `config` and `hooks` bundles. They add no schema and no +authority; every member is the generic operation with the kind or address bound. + +```ts +// config: one property per core config address, plus batched set/get (section 2.1). Usage: +// c.config.model PublicValue: get(ctx) | get(at, ctx), set, delete +// c.config.thinkingLevel PublicValue (defaulted: never undefined) +// c.config.selectedTools PublicValue +// c.config.steeringMode StickyPublicValue<"all" | "one-at-a-time", "all" | "one-at-a-time">: get(ctx), set, delete +// c.config.followUpMode same sticky queue-mode facade +// c.config.retry StickyPublicValue, Stored> +// c.config.threshold / keepRecent PublicValue +// c.config.set({ model, thinkingLevel, selectedTools, steeringMode, followUpMode, retry }, ctx) exact partial, one commit +// c.config.get(ctx) all current values, one read + +// hook: one method per built-in point, flattened across the fixed kinds exactly like config +interface CoreHooks { readonly generation: typeof generationHooks; readonly tool: typeof toolHooks; readonly postTools: typeof postToolsHooks; readonly collapse: typeof collapseHooks; readonly job: {} } +type CoreHookBundle = UnionToIntersection; +type HookFacade = { + readonly [N in keyof H]: (handler: HookHandler, options?: { readonly subtree?: boolean }) => () => void; +}; +type _AssertDisjointHooks = Assert>; // compile error on a duplicate flattened point name; re-asserted at install +type _HooksMatchKinds = Assert<{ [K in keyof CoreHooks]: NonNullable extends CoreHooks[K] ? true : false }[keyof CoreHooks]>; + +// the bundles the fixed kinds attach as `.hooks` (the points' In/Out types are the table in 11.2) +const generationHooks = { + systemInstructions: defineHookPoint<{ readonly sections: SystemSectionDraft; readonly config: Settings; readonly tools: readonly ToolDeclaration[] }, { readonly tools?: readonly ToolDeclaration[] }>({ fold: "collect", onThrow: "skip" }), + beforeRequest: defineHookPoint<{ readonly request: { readonly messages: readonly Message[] }; readonly cutoff: Id }, { readonly request?: { readonly messages: readonly Message[] } }>({ fold: "chain", onThrow: "skip" }), + onYield: defineHookPoint<{ readonly answer: AssistantMessage }, { readonly continue: string }>({ fold: "first", onThrow: "skip" }), + afterResponse: defineHookPoint<{ readonly message: AssistantMessage; readonly usage?: Usage; readonly attempt: number }>({ fold: "collect", onThrow: "skip" }), +} satisfies HookPoints; +const toolHooks = { + beforeTool: defineHookPoint<{ readonly call: ToolCall; readonly block?: { readonly reason: string } }, { readonly call?: ToolCall; readonly block?: { readonly reason: string } }>({ fold: "chain", onThrow: "abort" }), + afterTool: defineHookPoint<{ readonly call: ToolCall; readonly result: ToolResult }, { readonly result?: ToolResult }>({ fold: "chain", onThrow: "skip" }), +} satisfies HookPoints; +const postToolsHooks = { + afterTools: defineHookPoint<{ readonly assistant: Id; readonly results: readonly Id[] }>({ fold: "collect", onThrow: "skip" }), +} satisfies HookPoints; +const collapseHooks = { + beforeCollapse: defineHookPoint<{ readonly reason: "threshold" | "manual" | "overflow"; readonly through: Id; readonly entries: readonly Entry[] }, { readonly decline: true } | { readonly instructions?: string; readonly summary?: string }>({ fold: "first", onThrow: "skip" }), +} satisfies HookPoints; + +// Usage: +// c.hook.systemInstructions(h, { subtree: true }) === c.hooks.on(generationKind, generationKind.hooks.systemInstructions, h, { subtree: true }) +// c.hook.beforeTool(h) === c.hooks.on(toolKind, toolKind.hooks.beforeTool, h) +// c.hooks.on(myPluginKind, myPluginKind.hooks.myPoint, h) plugin kinds: generic registry, token-addressed +``` + +Singular `c.hook` is the flattened built-in convenience; plural `c.hooks` is the generic registry. Built-in +point names are unique across the fixed kinds by construction (`Assert>`) and re-checked when core kinds are installed. Plugin points are never flattened because plugin names can collide; they are +addressed by token. + +`c.watch(ctx)` / `c.watch(options, ctx)` is `h.watchConversation(c.id, options ?? {}, ctx)`. + +The Harness has neither facade. It keeps process-wide definition registries (`h.taskKinds`, `h.entryKinds`, +`h.tools`, `h.sections`), the typed witnesses `h.kinds`, and generic harness-wide `h.hooks.on(kind, +kind.hooks.point, ...)`. Configuration is conversation state, so it lives on the conversation. + +Advanced forms remain: `generationKind.config.model` tokens with `c.value(...)`, `c.hooks.on(kind, +kind.hooks.point, ...)`, `h.watchConversation(id, ...)`, `rootValues`. The primary surface is `root.config`, +`c.config.`, `c.hook.(...)`, `c.watch(...)`. + +### 13.2 Cloudflare Durable Objects + +One Durable Object hosts one Session, one Harness and one scheduler covering the root and every fork and +subagent. The Node implementation uses `AsyncLocalStorage` to reject line entry from inside any active Session +callback; a Cloudflare deployment therefore enables `nodejs_compat`. A request wake opens Pico, `send` durably +admits and ensures `resume`. An alarm or one logical +Cloudflare Task wake opens Pico and calls `resume`, which recovers all eligible work in all conversations. +Platform wake-up is not effect recovery: do not create one lane or scheduler per conversation, and do not use +platform replay as a second authority for provider/tool effects. If admission and the platform wake cannot +share a transaction, an adapter adds an ordered handoff or idempotent retry so acknowledged work always +causes another wake. Checkpoints and `recover` remain authoritative across invocation limits. + +## 14. Watch + +```ts +interface WatchOptions { readonly capacity?: number; readonly onError?: (error: unknown) => void } // default capacity 256 +interface ConversationWatchOptions extends WatchOptions { + readonly values?: readonly (Value | UnboundValue)[]; // plugin/application values; `pi.*` rejects (built-in config has view.config; internals are protected) + readonly lists?: readonly (List | UnboundList)[]; // plugin/application lists; same rule +} +interface SessionWatchOptions extends WatchOptions { readonly values?: readonly Value[]; readonly lists?: readonly List[] } // session scope only + +interface WatchedValue { readonly address: Value; readonly value?: JsonValue } +interface WatchedList { readonly address: List; readonly elements: readonly Element[] } // complete current list at capture; bound address +interface WatchedTaskOutput { readonly id: Id; readonly kind: string; readonly value: OutputState } +interface ConversationView { + readonly conversation: Conversation; + readonly entries: readonly Entry[]; // the active transcript in transcript order: every entry with id >= the newest head's boundary (the head entry sits at its own position) + readonly context: readonly Id[]; // model-visible IDs derived from `entries` (edits folded, omissions applied) + readonly tasks: readonly Task[]; readonly taskOutputs: readonly WatchedTaskOutput[]; + readonly inbox: readonly Element[]; + readonly config: ConfigSnapshot; // every built-in config value, always captured + readonly values: readonly WatchedValue[]; // only the explicitly requested plugin/application values + readonly lists: readonly WatchedList[]; // only the explicitly requested plugin/application lists + readonly readAt: Seq; +} +type ConfigSnapshot = ResolvedValues>; // same definitions as c.config; only `model` (no default) is `| undefined` +interface SessionView { readonly conversations: readonly Conversation[]; readonly values: readonly WatchedValue[]; readonly lists: readonly WatchedList[]; readonly readAt: Seq } +// Session-level usage totals and fault/report feeds are intentionally not part of v1 (not gated, not planned +// here). `HarnessOpenOptions.onReport` is the complete v1 non-fatal report channel; faults surface as `Faulted` +// rejections and `closed/session` watch deliveries. + +type ListOp = { type: "append"; element: Element } | { type: "remove"; id: Id } | { type: "clear" }; +type ConversationEvent = + | { type: "entry"; entry: Entry } | { type: "task_start"; task: Task } | { type: "task_update"; task: Task; previous: Task } + | { type: "task_end"; task: Task & { status: "terminal" } } | { type: "task_output"; id: Id; kind: string; delta: readonly Op[] } + | ConfigEvent // built-in config, typed by key + | { type: "value"; value: WatchedValue } // requested plugin/application values only + | { type: "list"; address: List; ops: readonly ListOp[] } // requested plugin/application lists; ops for one address in one commit, in write order + | { type: "inbox"; ops: readonly ListOp[] } | { type: "context"; ids: readonly Id[] }; +type ConfigEvent = { + [K in Exclude]: { + readonly type: "config"; + readonly key: K; + readonly value: ResolvedValue; // after a delete, the default, never undefined for defaulted members + readonly previous: ResolvedValue; + }; +}[Exclude]; // flat and discriminated on `key`: e.key === "model" narrows e.value to ModelRef | undefined +type SessionEvent = + | { type: "conversation"; conversation: Conversation; change: "created" } | { type: "value"; value: WatchedValue } + | { type: "list"; address: List; ops: readonly ListOp[] }; + +interface CommitEnvelope { readonly first: Seq; readonly last: Seq; readonly events: readonly E[] } +type WatchDelivery = ({ readonly type: "commit" } & CommitEnvelope) | { readonly type: "closed"; readonly reason: "overflow" | "session" }; // d.events, d.first, d.last +interface WatchHandle { readonly view: V; start(listener: (d: WatchDelivery) => void): void; unsubscribe(): void } +declare function applyConversationCommit(view: ConversationView, commit: CommitEnvelope): ConversationView; +declare function applySessionCommit(view: SessionView, commit: CommitEnvelope): SessionView; +``` + +Capture and subscribe happen in one line operation, so the view plus the stream after `readAt` is gapless +and duplicate-free. A conversation view holds the **active transcript** in transcript order: every fork-visible +entry whose ID is ≥ the newest head entry's boundary, through the current tail; the head entry itself sits at +its own chronological position (after the boundary), not first; with no head, the whole transcript. Placing +the summary before the retained tail is context projection (1.3), not the view. It also holds live tasks +directly in that conversation, every distinct output they reference, the current inbox, every built-in +configuration value as `config`, exactly the requested custom values as `values` and requested lists as +`lists` (session addresses or addresses bound to this conversation; others reject `ScopeViolation`; built-in +config and protected `pi.*` reject). A later source-conversation write emits nothing for an existing fork. + +`view.config` is derived from the same `CoreConfigBundle` as `c.config` (section 2.1): one property per +built-in definition (`model`, `thinkingLevel`, `selectedTools`, `profile`, `retry`, `steeringMode`, `followUpMode`, +`threshold`, `keepRecent`), current value or `undefined`, with the same default resolution the core tasks use. +A watch always captures these; `values` is for plugin and application addresses only, and a built-in config +definition listed there rejects (it has its own field and event; it is never silently folded into `config`), +as does any protected internal. Duplicate requested `values`/`lists` entries are deduplicated by address +identity (the tuple in section 2), keeping the position of the first request: each address appears once in +`view.values`/`view.lists` and produces one event per commit. Reason: every renderer shows the model and tool loadout; making +each client know the address list would duplicate the schema. A built-in config write emits a dedicated +flat `config` event: `{ type: "config", key, value, previous }`, typed by key. Reason: a generic `value` event with an address is mechanically sufficient but forces every UI to match +addresses back to names; the discriminated `key` gives `e.key === "model"` → `e.value: ModelRef | undefined` for free, +derived from the same bundle as `c.config`. A batched `c.config.set({...})` produces one +`config` event per changed key in one envelope; the reducer folds all of them into `view.config` before the +listener runs. Built-in definitions never appear as `value` events; requested plugin/application addresses +never appear as `config` events. Initial state is read from `view.config`. Protected state (`pi.inbox`, +`pi.input_result`, `pi.request`, `pi.output`, scratch, conversation-record seed metadata) is never exposed through `config` or +`values`; inbox and task outputs have their own fields. + +There is no `tail` count or other window option in v1. Reason: the head already defines what the model and +the user are working with; a numeric window would cut it arbitrarily. + +Entry reducer, exactly: + +```text +capture: entries = active transcript in transcript order (ids >= newest head's boundary; or all if no head) +entry event, no head: entries.push(entry) +entry event with head: boundary = entry.head (head:"self" was materialized to entry.id at append) + entries = entries.filter(e => e.id >= boundary) // drop everything before the retained boundary + entries.push(entry) // the new head entry itself comes last +``` + +So a summary with `head = firstRetained` keeps `firstRetained..current` and appends the summary; a reset or +handoff with `head = its own id` drops every prior active entry and appends itself. The truncation and the +append happen inside the fold of the same commit envelope, so no intermediate view shows both prefixes and no +separate truncate event exists: the `entry` event already carries `head`. The capture is the only time the +full active list is materialized; afterwards the view is maintained incrementally. `context` remains its own +full-ID event because edits and heads change the derived projection non-locally. Entries before the active +head are history, read through `h.entries({ conversationId, cursor, limit })`/`h.getEntry`, not through the live watch. + +Events of one commit arrive in one envelope, folded into `view` before the listener runs. A boundary commit +therefore shows inbox removal, entry append, result and successor task together, with no transient duplicate +or idle gap. Same-commit inbox append+remove emits no inbox event. A commit with no events after filtering is +not delivered and uses no capacity, while `view.readAt` still advances. A `context` event is emitted whenever +the derived context ID list changes. + +Reducers are pure and free of plugin kinds: `applyConversationCommit` needs no registry because every event +carries what it applies (`config` events carry `key`/`value` over the statically fixed `CoreConfigBundle`; `list` +events carry the bound address and ops). `task_start` on create; `task_update` on any non-terminal replacement; +`task_end` on terminal (always with outcome). `task_output` carries a decoded delta applied with +`applyImmutable()`; when the last referencing task ends in the same envelope, output events precede +`task_end` and the reducer then drops the output. `list` ops for one address within one commit are grouped +into one event in committed write order and folded onto `view.lists[address].elements` before the listener: +`append` pushes the element (stable ID), `remove` deletes by ID, `clear` empties. The inbox uses the same op +shape on its own field. No replay, cursor or acknowledgement protocol exists for lists; overflow closes the +watch like everything else. + +Overflow closes the watch with `overflow` after discarding queued deliveries; the consumer opens a new watch +and gets a fresh authoritative capture. No resume protocol. A listener throw closes only that watch and calls +`onError` off the line; an `onError` throw is ignored; the session never faults. Session close delivers +`closed/session` once. `unsubscribe` is idempotent and stops all callbacks. + +### 14.1 Plugins and remote clients + +Pico does not publish plugin state to UIs and has no plugin-config publication API. The integration pattern is +the existing Chord facet split, already used by `packages/coding-agent/examples/plugins/pi-example-plugin` +(`contract.ts`, `session.ts`, `tui.ts`) and the experimental services in +`packages/coding-agent/src/experimental/services/` (`models-provider.ts`, `transcript-provider.ts`) and +`src/experimental/mini/worker/`: + +- **contract**: `defineService(id)` with typed methods and `ReplicatedState` fields. The DTO is what the + plugin chooses to expose; it is not the plugin's Pico addresses. +- **session facet**: runs in the process that owns the Harness. It stores its durable state in its own Pico + definitions (`conversationValue`/`sessionValue`/lists) through `c.value`/`h.commit`, reconstructs a + presentation-safe `MutableReplicatedState` from those values on activation and reopen, and updates it from + Pico commits (its own `commit` results, or a watch on the values it cares about). It `env.provide`s the + service. +- **presentation facet** (TUI, web): `env.use(Service)`, subscribes to the replicated state, renders, calls + the typed methods. It never receives a Harness, an address, or a `ConversationView` of another plugin's + values. + +Pico is the durable authority; Chord `ReplicatedState` is a live snapshot/delta projection over RPC, never +storage. Built-in config stays special: `view.config` and the `config` event are automatic because every UI +needs them. Everything a plugin wants a UI to see goes through its own service DTO. This is guidance about how +to compose the existing pieces, not a Pico API; nothing in sections 1–14 changes for it. + +## 15. Backends + +```ts +MemoryStorage.create(): Storage +JsonlStorage.open(path: string, ctx): Promise +SqliteStorage.open(path: string, options: { readonly session: string }, ctx): Promise +``` + +A `Storage` is opened by its backend and handed to `Harness.open`. One Session owns a `Storage` while open; +that exclusivity and the `close` lifecycle are internal Session/backend concerns, not a separate public +concept. JSONL/SQLite hold a cross-process lock from open to close; two writers fail rather than race. Memory +has no lock but still rejects a second concurrent Session; the same `MemoryStorage` may be reopened by a later +Session after close for recovery tests. + +Persisted output deltas are decoded Chord `Op[]`; `Op` is strict JSON by construction (tuples of tag, path +segments and Chord `JsonValue` payloads), so it needs no codec. Replay validates every `Op` tuple shape before +applying. The JSONL file and the SQLite schema each carry their one existing format version; nothing is added +for `Op`. + +**JSONL**: one main file; one sidecar per live task scratch scope; one sidecar per live output with +post-creation deltas (task creation and the initial output base are in main). Record: `{ first: Seq, last: +Seq, maxId: Id, writes: Write[] }`, newline-terminated, `last - first + 1 === writes.length`. `maxId` is the +committed-ID high-water *after* this batch, written into every record of every file, main and sidecar alike. +Reason: sidecar appends (scratch list elements, output deltas) mint IDs; once that task or output retires its +sidecar is ignored, so the IDs would be lost if reopen only scanned retained writes. Because the retiring main +record carries the high-water forward, reopen takes `max(maxId)` over retained records only and is still +correct. fsync every record before +publication; fsync new files and their directories. Replay main first, derive live scratch/output IDs from +live tasks, replay only those sidecars. Retired sidecars are ignored even if malformed (a stale file cannot +resurrect state). Sequence ranges increase within a file and never overlap across retained files; gaps are +valid. Bytes after the last newline are a torn write: truncate, fsync, continue. A complete but malformed +line fails open. Sidecar unlink failure is harmless because the main record proves retirement. Reopen +initializes `nextId` to `max(maxId over retained records) + 1` and the applied `Seq` from the greatest retained +endpoint. + +```text +main seq 1..10 maxId 12 (task 12 created; its scratch sidecar opens) +scratch seq 11..15 maxId 40 (list elements 13..40 appended in the sidecar) +main seq 16 maxId 40 (task 12 terminal; sidecar retired, may be unlinked or left malformed) +reopen: retained = main only -> nextId = 41; no ID in 13..40 is ever reused +failed callback minted 41..43 without commit -> nothing persisted -> next open still starts at 41 +``` + +**SQLite**: tables for conversations, entries (facets as columns), tasks (current snapshot), values + +rewindable versions, list elements + clear markers, scratch, output deltas, commit boundaries, and one +`meta(max_id)` row updated in every commit transaction (the same high-water rule as JSONL). Index task +status/abort/conversation/kind/output and entry conversation/head/kind. `PRAGMA synchronous=FULL` or a +documented equivalent. Terminal `task.set` deletes scratch rows in the same transaction. Reopen loads live +tasks and named owner records, not terminal history. Schema version mismatch rejects; no migration in v1. + +Both run the same conformance suite as memory. + +## 16. State-indexed task authoring (`defineStateTask`) + +An optional public adapter for ordinary kinds with several durable phases. It compiles to an ordinary +`TaskKind`; the scheduler and kernel are unchanged. It makes one kind's checkpoint dispatch exhaustive at +compile time. + +```ts +// constraints abbreviated: always mean I,R,F,A extends JsonValue; C extends TaskCheckpoint; +// O extends OutputState; H extends HookPoints (section 3.3). Written out in full here because this block must compile literally. +type PhaseOf = C["phase"]; +type CheckpointAt> = Extract; +type PhasePayload> = Omit, "phase">; +type PhaseTask> = + Omit, "checkpoint"> & { readonly checkpoint: CheckpointAt }; +type InitialTask = + Omit, "checkpoint"> & { readonly checkpoint?: never }; + +type StateTransitionTx = Omit, "checkpoint">; +type StatePhaseTx> = StateTransitionTx & { checkpoint(value: CheckpointAt): void }; +type InitialStateRuntime = + Omit, "commit">; // no commit before the first checkpoint +type StateRuntimeFor, H extends HookPoints> = + Omit, "commit"> & { + commit(build: (tx: StatePhaseTx, current: PhaseTask) => V | Promise, ctx: Context): Promise; + }; + +interface PhaseTransition> { + readonly type: "transition"; readonly phase: P; + readonly commit: (tx: StateTransitionTx, current: RunningTask) => PhasePayload | Promise>; +} +interface StateTerminal { + readonly type: "terminal"; readonly closure: TerminalClosure; +} +type StateTransitions = + { [P in PhaseOf]: PhaseTransition }[PhaseOf]; // distributed: each member pairs one literal phase with exactly its payload +type StateResult = + StateTransitions | StateTerminal; + +interface StateActions { + transition

    , Actual extends PhasePayload>( + phase: P, + commit: (tx: StateTransitionTx, current: RunningTask) => NoExtra, Actual> | Promise, Actual>>, + ): PhaseTransition; + terminal(closure: TerminalClosure): StateTerminal; +} + +type PhaseHandler, H extends HookPoints> = + | { readonly role: "start"; readonly recover?: never; + run(task: PhaseTask, rt: StateRuntimeFor, actions: StateActions, ctx: Context): Promise> } + | { readonly role: "inflight"; + run(task: PhaseTask, rt: StateRuntimeFor, actions: StateActions, ctx: Context): Promise>; + recover(task: PhaseTask, rt: StateRuntimeFor, actions: StateActions, ctx: Context): Promise> }; +type PhaseMap = + { readonly [P in PhaseOf]: PhaseHandler }; + +type StateTaskDefinition = { + readonly kind: string; + readonly hooks?: H; // kind-owned hook points, as on defineTask + readonly config?: ConfigBundle; + readonly initial: { readonly role: "start"; + run(task: InitialTask, rt: InitialStateRuntime, actions: StateActions, ctx: Context): Promise> }; + readonly phases: PhaseMap; // handlers receive StateRuntimeFor<..., H> + abort(task: RunningTask, rt: AbortTaskRuntime, ctx: Context): Promise>; +} & TaskOutputDefinition; + +declare function defineStateTask(): + >( + definition: string extends PhaseOf ? never : ExactStateTaskDefinition, D>, + ) => D & TaskKind; // H and D["config"] inferred from the literal exactly as in defineTask; lifecycle unchanged +``` + +Compile contract: `C` is a finite union with literal `phase`s (broad `string` rejects); every phase has +exactly one handler and no extra key exists; each handler receives its complete narrowed checkpoint; `start` +has `run` only, `inflight` has `run` and `recover`; `initial` has no checkpoint and no `commit`; transition +payloads reject missing or visible extra fields for literals, variables and spreads; the returned +`StateResult` itself preserves the phase/payload correlation (`StateTransitions` distributes over the phases, +so a hand-built `{ type: "transition", phase: "a", commit: () => payloadForB }` is a compile error, not only +through `actions.transition`); `terminal` preserves the typed `TerminalClosure`. `ExactStateTaskDefinition`/`ExactPhaseMap`/`ExactPhaseHandler` are the +`NoExtra` helpers that enforce this; they are spelled in appendix A. + +Generated `execute`/`recover`: + +```text +execute: no checkpoint -> fence; initial.run + transition(P, commit) -> module-private commitStateTransition: on the line, check invocation and + expected source phase, run commit(tx, fresh current), buffer full task.set with {phase:P, ...payload}, + persist, return the applied PhaseTask

    = P extends HookPoint ? In : never; +type HookOut

    = P extends HookPoint ? Out : never; +declare const hookIn: unique symbol; declare const hookOut: unique symbol; + +// storage queries +interface PageQuery { readonly cursor?: Id; readonly limit: number } +interface Page { readonly items: readonly T[]; readonly next?: Id } +interface ConversationScan extends PageQuery { readonly parent?: Id; readonly owner?: Id } +interface EntryScan extends PageQuery { readonly conversationId: Id; readonly kind?: string; readonly through?: Id } +interface TaskScan extends PageQuery { readonly conversationIds?: readonly Id[]; readonly statuses?: readonly Task["status"][]; readonly kind?: string; readonly abort?: boolean; readonly outputId?: Id } + +// errors (all extend Error; constructor arguments shown) +declare class ReadAfterWrite extends Error {} +declare class ScratchRetired extends Error { readonly taskId: Id; constructor(taskId: Id) } +declare class OutputRetired extends Error { readonly outputId: Id; constructor(outputId: Id) } +declare class InvalidHistoryPosition extends Error { readonly at: Id; constructor(at: Id) } +declare class ConversationBusy extends Error {} +declare class InputNotFound extends Error {} +declare class TaskNotFound extends Error {} +declare class ScopeViolation extends Error {} +declare class NestedLineOperation extends Error {} +declare class Closed extends Error {} +declare class Faulted extends Error {} +declare class NothingToCollapse extends Error {} +declare class RequestKindMismatch extends Error {} +declare class StaleDefinition extends Error {} +``` + +## Appendix B. Changes from v1 (`pico-simple-handoff.md`) + +| v1 | v2 | why | +|---|---|---| +| gated packages (harness, provider, tool, hooks, collapse, jobs, sections, client) | all specified except the exact ordinary tool API (execution signature, facade, output access, memos, children, resolver), which stays gated with fixed requirements | user decision: describe the full harness; the tool API is the one allowed gap | +| host `HostTx.entry` with structural append validation | removed; hosts use `write` | queued writes only land at boundaries, so exchange splitting cannot happen; validation became assertions | +| `write` helper restricted to trusted built-ins | `write` on `BaseTaskTx` and `FinalTx` | user decision: passive queued writes are boundary-safe; ordinary kinds gain exactly that and nothing else | +| `pi.job` ordinary "unless a requirement proves otherwise" | stays ordinary | its notice is a `write` from the terminal closure, which ordinary kinds now have | +| queue batching rule "unsettled" | `steeringMode`/`followUpMode` sticky in `postToolsKind.config`, default `one-at-a-time` | matches current Agent/coding-agent defaults (the old "matching the lane harness" claim was wrong: the lane harness defaults to `all`) | +| watch `values` only | `values` and `lists`; `WatchedList` + `list` event with grouped append/remove/clear ops | user decision: direct watched-list support | +| `reset` returns `InputHandle` | `reset(): Promise` resolves on durable admission; passive writes have no public handle | `InputHandle` is for admitted user input only; matches the old normative API | +| defaults implied in prose | `default?` on the definition token, resolved by every reader | one place for a default | +| protected = four names (`pi.inbox`, `pi.input_result`, `pi.request`, `pi.output`) | unchanged; all `pi.*` reserved for built-ins; config tokens exported; section seeds live on the `conversation.create` record, not an address | restore the v1 rule; no new protected address | +| `generationKind.config` bundle + `c.config(kind)`/`c.settings` facades | `c.config.` per core address plus `c.config.set(patch)`, assembled from all core bundles; no `c.settings`, no `c.config(kind)`, no `c.core` | user decision: direct properties, one schema, plugin config stays in plugin bundles | +| `onYield` and `before_collapse` semantics in historical docs | specified in 10.1, 10.4, 9.5; continuation entry kind is `pi.user` | decision, not inherited | +| `defineStateTask` in section 5.1 | section 16, package 15 | unchanged contract, moved and condensed | +| section 14.4 system sections + usage-guide contract | section 12, unchanged design | user decision: the settled design stands, pi-ai PR will land | +| hook points as a fixed table on built-ins | declared per task kind via `defineHookPoint`; core kinds ship predefined points | user decision: hooks are kind-specific generically; ordinary kinds may declare their own | +| string point names (`system_instructions`, `before_tool`, ...) and a fixed hook table | camelCase `HookPoint` tokens on `kind.hooks` (`systemInstructions`, `beforeTool`, ...); `c.hook.` flattened across fixed kinds like `c.config`; plugins use `c.hooks.on(kind, kind.hooks.point, ...)` | user decision: derive like config, tokens avoid plugin name collisions | +| `systemInstructions` handler returns the loadout | loadout defaults to `selectedTools` resolved against `h.tools`; returned `tools` is an explicit override | common hook no longer needs `h.tools.select(...)` | +| `rootValues` only | `root: { config, values, sections }` primary, `rootValues` advanced alias | quick start does not import kind tokens | +| `WatchDelivery.commit.events` | `WatchDelivery` is the envelope: `d.events`, `d.first`, `d.last` | one less level | +| conversation watch `tail: 0..10000` window | removed; view holds the active transcript selected by the newest head; older entries via paged scans | user decision: the head defines the working set; no arbitrary window | +| built-in config visible in a watch only if listed in `values` | `view.config` always captured from `CoreConfigBundle`; `values` is custom-only; dedicated flat typed `config` event (`key`, `value`, `previous`) | user decision: renderers always need it; key-discriminated typing beats address matching | +| validation prose per write type | 5.2 split into validation (reject) and assertions (fault) | same behaviour; clearer which is a public contract | +| exclusion lists, repeated trace prose, undefined vocabulary | glossary, one statement per rule with reason | readability | diff --git a/packages/agent/docs/pico/pico-rendering.md b/packages/agent/docs/pico/pico-rendering.md new file mode 100644 index 000000000..a759177bd --- /dev/null +++ b/packages/agent/docs/pico/pico-rendering.md @@ -0,0 +1,305 @@ +# Pico rendering + +This document defines how a presentation renders a replicated Pico conversation. It covers local and +remote clients, initial attachment, live streaming, terminal handoff, renderer registration and stable +component identity. The harness model remains defined by `pico-v3.md`; the examples and client-facing +surface belong in `pico-usage-guide.md`. + +The target is the directness of coding-agent interactive mode and `experimental/mini`, without their +hard-coded event vocabulary or duplicate hydration logic. + +## 1. The presentation owns a replica, not agent state + +A presentation owns one `ConversationView`. It does not read storage, parse commits or hold live task, +provider, tool or model objects. + +Opening a watch atomically captures the view and registers its listener. The harness buffers changes +until delivery starts. Each committed change is folded into the view before listeners observe its +events. An in-process client uses the harness-owned view; a remote client applies the same exported +kind-free reducer before notifying its UI. + +Delivery over one WebSocket or SSE connection is ordered and gap-free. If delivery fails or a bounded +queue overflows, that watch closes. Reconnection opens a new watch and receives a fresh view. There are +no cursors, epochs, acknowledgements, deduplication, replay requests or in-place resnapshot protocol. + +The view is authoritative. Events identify the affected portion so a live renderer can avoid scanning +the whole view; they are not a second source of presentation state. + +## 2. Local view access + +The raw collections remain serializable. A local wrapper adds indexed and typed access: + +```ts +interface ConversationView { + readonly conversation: Conversation; + readonly entries: readonly Entry[]; + readonly tasks: readonly Task[]; + readonly previews: ReadonlyMap; + readonly inbox: readonly Element[]; + readonly values: ReadonlyMap; + readonly context: readonly Id[]; + readonly faulted: boolean; + readonly readAt: Id; + + entry(id: Id): Entry | undefined; + task(id: Id): Task | undefined; + + entriesOf(kind: K): readonly EntryOf[]; + tasksOf(kind: K): readonly TaskOf[]; + + preview>( + kind: TaskKind, taskId: Id, + ): P | undefined; +} +``` + +`preview(kind, taskId)` returns `undefined` when the task is absent, has another kind or has no live +preview. Built-in and plugin renderers use it without casts. `task(id)` is sufficient to classify a +`task_output` event because the reducer has already folded that event before the listener runs. + +A remote wire view may remain arrays and maps encoded as JSON. The wrapper builds its indexes locally; +those indexes are not another replicated representation. + +## 3. Two rendering paths + +Both paths use the same registered renderers and component keys. They differ only in how much of the +view they visit. + +### 3.1 Hydration + +Hydration runs on first attachment and after opening a replacement watch. It walks the current entries +and live tasks and upserts their blocks: + +```ts +function hydrate(view: ConversationView): void { + for (const entry of view.entries) entryRenderers.render(entry, view); + for (const task of view.tasks) taskRenderers.render(task, view); + chrome.sync(view); +} +``` + +This is the view-driven path used by Mini. Existing blocks are reconciled by stable key, so a fresh +view need not tear down unchanged terminal components. + +### 3.2 Live events + +Normal streaming uses targeted event handling, as interactive mode does: + +```ts +function handle(event: ConversationEvent, view: ConversationView): void { + switch (event.type) { + case "entry": + entryRenderers.render(event.entry, view); + break; + case "task_start": + case "task_update": + taskRenderers.render(event.task, view); + break; + case "task_output": { + const task = view.task(event.task); + if (task) taskRenderers.render(task, view); + break; + } + case "task_end": + taskRenderers.end(event.task, view); + break; + case "inbox": + case "value": + case "context": + chrome.sync(view, event); + break; + case "fault": + case "closed": + chrome.connection(event); + break; + } +} +``` + +A streamed token therefore updates one task renderer. It does not rebuild every transcript block or +run every layout. A client that does not need the optimization may ignore the event payload and call +`hydrate(view)` after every notification; it remains correct. + +The renderer callbacks, not a mandatory full-tree diff, are the shared path between hydration and live +updates. + +## 4. Renderer registry + +Entry and task renderers are registered by kind. Registration is typed by the kind object even though +the wire and internal registry use its string token: + +```ts +renderers.entry(noteKind, previous => ({ + render(entry, host, view) { /* entry is NoteEntry */ }, +})); + +renderers.task(downloadKind, previous => ({ + render(task, preview, host, view) { /* typed task and DownloadPreview */ }, + end(task, host, view) { /* terminal typed task */ }, +})); +``` + +The exact component type belongs to the presentation package, not Pico. The common contract is: + +- `render` upserts components through the host using stable keys; +- `end` handles terminal live-state cleanup and does not need a retired preview; +- renderers do not directly parent components in the transcript container; +- registration may replace or wrap `previous`; +- built-ins and plugins use the same API; +- removing renderer code does not affect the replicated view. + +An unregistered entry kind uses a generic renderer: model text when present, otherwise collapsed +kind/data. An unregistered task kind uses kind, status and a collapsed raw preview when present. +Unknown kinds must remain visible and must not break hydration. + +The heterogeneous registry may erase renderer types internally. That cast or validation boundary is +owned once by the registry; renderer authors do not cast task state, entry data or previews. + +## 5. Renderer host + +The host owns component instances and placement. Renderers only describe or update blocks through it. +At minimum it maintains: + +```text +entry id -> settled transcript block +live task id -> live task block or region +tool-call id -> tool component +inbox item id -> queued-input component +``` + +It provides keyed `get`, `upsert`, `move`/`adopt` and `remove` operations. A renderer can update an +existing component without knowing which container currently parents it. The host preserves expansion, +selection and other local UI state while adopting or moving a block. + +No persistent `tool task id -> tool-call id` map is required. Every tool-task state variant carries +its call: + +- on `task_output`, `view.task(event.task).state.call.id` identifies the component; +- on `task_end`, the event's terminal task carries the same call; +- tool-result entries use their indexed call key. + +A renderer may cache expensive presentation-only products such as parsed Markdown. Such caches are +never authoritative and may be discarded on hydration. + +## 6. Stable identity and terminal handoff + +Every rendered object has a stable semantic key: + +| Object | Key | +| --- | --- | +| transcript entry | entry ID | +| live generation | generation task ID | +| assistant content block | generation task ID + content index | +| tool component | tool-call ID | +| live tool preview lookup | tool task ID | +| inbox item | list-element/input ID | +| compaction, job or plugin task | task ID | + +A component may change ownership without changing its semantic identity. + +### Assistant handoff + +A generation preview renders a partial assistant message under the generation task ID. The final +assistant entry has `byTaskId` equal to that generation task. Its renderer adopts the existing live +component, applies the immutable final message and then associates the settled transcript position +with the entry ID. It does not append a duplicate assistant component. + +If no matching live component exists, as on initial attachment, the entry renderer creates the same +settled component directly. + +### Tool handoff + +Tool-call components are born while the assistant preview streams and are keyed immediately by the +call ID. A later tool-task start marks that same component running. Tool output updates it from the +typed preview. The terminal tool-result entry updates the same call-ID component with immutable final +output. No phase creates a second tool component. + +### Other tasks + +A task that has no terminal entry removes or finalizes its live block in `end`. A task kind that +requires a durable transcript representation writes that entry in the same commit in which it +settles. The view is already folded to the whole commit before listeners run, so no client-visible idle +or missing-result state exists between those events. + +## 7. Task provenance required by rendering + +Task creation records generic provenance just as entry creation does: + +```ts +interface Task { + // ... + readonly byTaskId?: Id; +} +``` + +`ConversationTx` fills `byTaskId` from the invocation whose commit created the task; it is absent for +tasks created outside a task invocation. Task kinds do not write it themselves. + +This replaces kind-specific provenance such as `JobState.origin`. A job created by a tool has +`byTaskId` equal to the tool task. A renderer follows that task and narrows it with `toolKind.is` to +select the originating tool renderer. Nested delegated work follows the chain as needed. + +## 8. Structured previews + +A preview is the complete current render model for one live task, not a text delta. + +The generation preview is the full partial `AssistantMessage` assembled from +`AssistantMessageFrame`s. It retains ordered content blocks, text, thinking, signatures, partial tool +arguments, provider metadata, usage and diagnostics. `task_output.ops` only transports changes; the +renderer reads the already-updated message through `view.preview(generationKind, task.id)`. + +A tool preview is the complete current `ToolOutputState`: text or structured content, plugin details, +images, usage, delegation, handoff, added tools, termination, diagnostics and truncation metadata. Tool +renderers own plugin-specific interpretation. The generic host does not flatten it to text. + +Terminal rendering uses immutable entries or terminal task state because settlement retires the live +preview. + +## 9. Layouts are optional presentation policy + +Grouping tool calls, collapsing a run until its final answer and similar structures are useful, but a +general composable layout/reconciliation pipeline is not required by the Pico watch or renderer +contract. + +A presentation may add layouts above the keyed renderer host. Layouts must preserve block keys and +must not make preview delivery depend on rebuilding the complete transcript. They should initially run +only for structural changes unless a particular layout explicitly depends on live preview content. + +The first coding-agent client should reach feature parity with direct keyed placement. Add a general +layout API only after at least two concrete layouts demonstrate that a shared abstraction is smaller +than their local implementations. + +## 10. Client-owned visible work + +A client operation that runs over time and should survive in the transcript is an ordinary task kind, +not a presentation-only event stream. For example, `!cmd` is a `pi.user_bash` task owned by the coding +agent or bash plugin. It shares process-output helpers with jobs, publishes a typed preview, and writes +its terminal entry in the same commit as settlement. + +The presentation starts it with a command and then handles its generic `task_start`, `task_output`, +`task_end` and `entry` events. Escape calls `abortTask(id)`. Plugins use exactly the same pattern for +client-initiated work. + +## 11. Mapping from interactive mode + +| Interactive-mode event | Pico rendering | +| --- | --- | +| `agent_start` / `turn_start` / assistant `message_start` | generation `task_start` | +| `message_update` | generation `task_output`; read typed assistant preview | +| `message_end` | assistant `entry`; adopt by `byTaskId` | +| `tool_execution_start` | tool `task_start`; identify component by call ID | +| `tool_execution_update` | tool `task_output`; read typed tool preview | +| `tool_execution_end` | tool `task_end` plus tool-result `entry` | +| `compaction_start` / `_end` | collapse `task_start` / `task_end`; summary `entry` | +| retry events | generation or collapse `task_update` | +| `agent_end` / `agent_settled` | no live foreground task after the commit | +| `queue_update` | `inbox` | +| thinking/model change | typed configuration `value` | +| custom entry | registered entry renderer or generic fallback | +| `bash_execution_update` | `pi.user_bash` `task_output` | + +The generic vocabulary changes dispatch, not component behavior. Compared with interactive mode, Pico +removes the second independently authored hydration implementation. Compared with Mini, it replaces +hard-coded operation fields with typed task kinds and replaceable renderers while retaining the same +small keyed-component model. diff --git a/packages/agent/docs/pico/pico-simple-blockers.md b/packages/agent/docs/pico/pico-simple-blockers.md new file mode 100644 index 000000000..bd3da1ae7 --- /dev/null +++ b/packages/agent/docs/pico/pico-simple-blockers.md @@ -0,0 +1,37 @@ +# Pico implementation state + +## Authority + +[`pico-simple-handoff.md`](pico-simple-handoff.md) is the sole normative implementation specification. +Read it completely before implementation. Other Pico documents and prototypes are historical inputs only. + +The original specification and WP1 received delegated review. Peer review then identified and resolved +TaskOutput contract inconsistencies in JSONL replay, mutation failure handling, watch reconnection, +kind replacement and generation partial state. + +## Git state + +- Branch: `pico` +- Branch base when created: `f3c672245`. +- WP1 implements compile-time declarations under `packages/agent/src/harness/pico/` with focused compile + coverage. +- Implement and commit one small work package at a time for user review. + +## Next action + +Review the WP1 commit, then begin WP2 only after explicit user approval. WP2 is the mutation algebra and +`MemoryStorage`; follow its prerequisites, acceptance criteria and one-commit boundary in the handoff. + +## Adopted core + +The harness owns `pending -> running -> terminal`. Task input is immutable typed JSON. A checkpoint is an +optional full replacement with `phase: string`. Execute/recover return typed terminal closures applied +atomically with outcome and scratch retirement. Cancellation is mark, revoke writes, signal, join, fresh +abort, then a restricted abort closure. + +`turn: true` is a trusted replaceable task-kind capability. Background is independent because speculative +manual collapse is background plus turn. Non-turn tasks use limited transactions and `accept`/`write`; +turn tasks may append model-affecting entries directly. Every missing live kind becomes orphaned at open. +Input groups belong to built-in generation/post_tools. V1 is job-first and excludes arbitrary unfinished +promise adoption. Shared TaskOutput storage/runtime/watch foundations are settled; concrete tool/job +payloads, bounded capture/spill policy and model projection remain gated. diff --git a/packages/agent/docs/pico/pico-simple-handoff.md b/packages/agent/docs/pico/pico-simple-handoff.md new file mode 100644 index 000000000..315b18f9d --- /dev/null +++ b/packages/agent/docs/pico/pico-simple-handoff.md @@ -0,0 +1,2654 @@ +# Pico v1 implementation specification + +This file is the sole normative implementation specification for the clean-room Pico harness under +`packages/agent/src/harness/pico/`. Other Pico documents and prototypes are historical inputs and test +inspiration only. An implementer must not need them to implement the work packages +marked ready here. + +The specification intentionally gates provider, built-in tool schemas, hook-scratch and renderer/client +integration where their APIs are not settled. Task output is part of the ready foundation; there is no +separate preview subsystem. A gated package must not be implemented by guessing. The storage, +entry/context, task, transaction, admission, scheduling, recovery and cancellation foundation is fully +specified here. + +## 1. Required behavior + +Pico stores sessions containing conversations, immutable transcript entries, durable tasks and scoped +state. One process owns a session at a time. A writable Harness owns one scheduler for the whole Session. +Once resumed, it considers eligible tasks in every conversation; foreground and background task effects both +run concurrently. Foreground affects idle and ordinary cancellation reach, never execution eligibility. +Every mutation and scheduler decision serializes on one commit line. Each conversation has at most one +active turn graph: the live tasks among the fixed `pi.generation`, `pi.tool` and `pi.post_tools` kinds. Other +tasks still run concurrently but never participate in that graph or block input admission. + +A task is one recoverable async operation: + +1. Creation durably stores immutable input and status `pending`. +2. Reservation durably changes status to `running` before `execute` performs effects. +3. The task may atomically replace a full typed checkpoint while it runs. +4. A `running` task encountered by a later process runs `recover`, including when no checkpoint exists. +5. `execute` or `recover` completes effects and cleanup, then returns a terminal closure. +6. The harness invokes the closure once on the line and atomically stores its buffered writes, terminal + outcome and scratch retirement. +7. Durable cancellation marks the task, revokes normal writes, signals and joins the old invocation, + then starts a fresh `abort` invocation from immutable input and the latest checkpoint. +8. `abort` completes cleanup, then returns a restricted closure whose writes, aborted outcome and scratch + retirement commit atomically. + +Recoverability does not preserve a JavaScript stack and does not imply exactly-once external effects. +A crash after an unkeyed external action but before recording its identity/result remains uncertain. + +Required exclusions: no imports from another harness implementation; no worker pool, polling scheduler, +lease, effect gate, per-conversation scheduler, stable or temporary serving attachment, public drive method, +replaceable core task kinds, public custom-turn task capability, kernel-interpreted global author lifecycle +graph, `patch`, `settle`, status epoch, workflow replay, generator DSL, one-plan-at-end restriction or +specialized core verbs such as `startShell`. +The optional per-kind state-indexed authoring adapter in section 5 is explicitly allowed; it compiles to an +ordinary `TaskKind` and changes no kernel lifecycle or scheduler behavior. + +## 2. Core data and identifiers + +All durable application payloads are strict JSON. + +```ts +type JsonPrimitive = null | boolean | number | string; +type JsonValue = JsonPrimitive | JsonObject | readonly JsonValue[]; +interface JsonObject { readonly [key: string]: JsonValue } + +type Id = number; +type Seq = number; +``` + +`Id` and `Seq` are distinct positive safe-integer domains. IDs are stable committed-object identity. Storage +synchronously mints process-local ascending IDs through `nextId()` while the Session FIFO line is held; +conversation, entry, task and list-append builders immediately carry the returned ID in their canonical +write. A callback or validation failure discards its writes but burns those IDs in the current open binding. +They become valid object identities only if the containing commit succeeds. Reopen derives the allocator +high-water from committed creation/list-append writes, so it may reuse IDs that were minted but never +committed by an earlier process; committed IDs are never reused. `commit` receives no separate allocator +value. IDs may be used for same-batch references before commit, but must not escape a failed callback. + +`Seq` is storage-assigned write order. Storage assigns one ascending sequence to each committed write and +returns the sequences in write order. Sequences order all main, scratch and shared writes but are not object +IDs; a creation's ID never has to equal its sequence. + +Local transaction and Storage APIs accept trusted typed values without defensive cloning or redundant +strict-JSON decoding. Durable values are readonly by contract; callers must not mutate values after passing +them to Pico. External wire/plugin boundaries validate before entering this layer. + +Use Chord's `Context` directly: + +```ts +import type { Context } from "@earendil-works/chord"; +``` + +Every asynchronous public, runtime, storage-adapter, provider, environment, hook and wait operation takes +a required final `ctx: Context`. Synchronous transaction builder methods take no context. + +## 3. Entries, conversations and context + +### 3.1 Stored records + +`Message`, `UserMessage`, `AssistantMessage`, `ToolResultMessage`, `TextContent`, `ImageContent` and +`ToolCall` are provider-neutral pi-ai types. Current pi-ai has no `SystemMessage`; typed `pi.system` +model projection is reserved for the gated messages-only integration package. + +```ts +type ContextEdit = + | { readonly target: Id; readonly action: "omit" } + | { readonly target: Id; readonly action: "replace"; readonly messages: readonly Message[] }; + +interface EntryIdentity { + readonly id: Id; + readonly conversationId: Id; + readonly kind: string; + readonly byTaskId?: Id; +} + +type EntryBase = EntryIdentity; +interface EntryData { readonly data: D } +interface ModelProjection { readonly model: readonly M[] } +interface ContextHead { readonly head: Id } +interface ContextEdits { readonly edits: readonly ContextEdit[] } + +type Entry = EntryBase & Partial; + +type EntryInput = + Omit & + (E extends ContextHead ? { readonly head: Id | "self" } : { readonly head?: never }); + +interface EntryKind { + readonly kind: string; + is(entry: Entry | undefined): entry is E; +} + +interface Conversation { + readonly id: Id; + readonly parent?: { readonly conversationId: Id; readonly at: Id }; + readonly owner?: Id; +} +``` + +Entries are append-only and never patched, reordered or renumbered. `data`, `model`, `head` and `edits` +are stored composable facets. No kind callback runs while reading or deriving context. Unknown entry kinds +retain all generic behavior because the facets are materialized. + +`Entry.byTaskId` is set by Pico when an invocation appends the entry and is absent for a host append. +Tasks do not have a generic `byTaskId`; a task kind stores required provenance in its typed input. + +Built-in kind strings are `pi.user`, `pi.assistant`, `pi.tool_result`, `pi.system`, `pi.notice`, +`pi.summary`, `pi.handoff` and `pi.reset`. Ready foundation witnesses are: + +```ts +type UserEntry = EntryBase & { readonly model: readonly [UserMessage] }; +type AssistantEntry = EntryBase & { readonly model: readonly [AssistantMessage] }; +type ToolResultEntry = EntryBase & EntryData & { readonly model: readonly [ToolResultMessage] }; +type NoticeEntry = EntryBase & Partial & { readonly model: readonly [UserMessage] }; +type SummaryEntry = EntryBase & EntryData<{ readonly through: Id }> & + { readonly model: readonly [UserMessage] } & ContextHead; +type HandoffEntry = EntryBase & { readonly model: readonly [UserMessage] } & ContextHead; +type ResetEntry = EntryBase & ContextHead; +``` + +Each user, assistant, tool-result, notice, summary and handoff entry contains exactly one corresponding +message. Reset has no model. `pi.system` is reserved but has no ready typed model witness. Stored entry-kind +names remain directly scannable from materialized entry facets; backends may index them so open need not +decode payload history. + +### 3.2 Direct append validation + +A direct append validates against committed state plus the builder's narrow pending-reference indexes: + +- The conversation exists, including one created earlier in the batch. +- `head: "self"` becomes the Storage-minted entry ID. +- A numeric head is a visible transcript entry at the append point. +- Let `P` be the newest visible prior head. A new head boundary cannot precede `P.head` in the logical + transcript. +- A head boundary cannot split a successful assistant/tool-result exchange. +- Every edit target is an earlier visible entry. An edit outside the selected range during later context + derivation is a no-op. +- Ordinary edits cannot target managed `pi.system` entries. The system-preparation operation is the only + authority that may append a complete fresh managed baseline and omission edits for superseded managed + entries in one batch. +- An assistant message is final-successful when `stopReason` is `"stop"`, `"length"` or `"toolUse"`. + `"pending"` and `"deferred"` are not appendable terminal assistant entries. `"error"` and `"aborted"` + may be stored for display but do not enter later model requests or create tool work. +- A successful assistant's tool-call IDs are unique. A tool-result entry's single message names one visible + call ID/name, has one result at most for that call, and belongs to the same conversation. + Fixed core task implementations remain responsible for creating the right tasks and input ownership. + +An exchange consists of one successful assistant entry containing tool calls and its tool-result entries. +A head may retain the assistant (boundary at or before it) or omit the complete exchange (boundary after +its last required result). It may not omit the assistant while retaining any result or retain only a +suffix of its result entries. While future results can still land in that conversation, no boundary after +an incomplete assistant is valid. A self-head may omit an entire incomplete exchange inherited through a +fork when the fork has no local result-producing task for it; source tasks/results are not inherited. + +### 3.3 Transcript versus model order + +The transcript preserves append chronology. Admission prevents ordinary model-visible writes from being +inserted into an active turn and relying on projection to repair arbitrary chronology. + +Projection performs only these deterministic transformations: + +1. Stored head placement. +2. Stored edit folding. +3. Tool-result placement in the assistant call order. +4. Request-local missing-result repair when a historical fork cuts a successful exchange before all + results. For each missing call, synthesize one `ToolResultMessage` with the original call ID + and name, `isError:true`, timestamp equal to the assistant timestamp, text + `"Tool result unavailable: history ends before this call completed."`, and details + `{ reason: "missing_after_fork" }`. The repair is request-local and is never stored. +5. Provider/model normalization in pi-ai. + +Speculative collapse is the deliberate exception. A summary head may append chronologically while another +turn runs. The head targets an old complete exchange boundary and projects before its retained tail, so it +cannot split the current exchange. + +### 3.4 Context derivation + +For a conversation and inclusive target entry `T`: + +```text +H = newest fork-visible entry at or before T with a stored head +from = transcript start when H is absent, otherwise H.head +range = fork-aware visible entries from `from` through T, inclusive +edits = for each target, the newest edit in range wins +entries = when H is absent: range + when H exists: H followed by range with every head entry removed +model = for each selected entry in order, omit/replace its projection using edits, + concatenate messages, then order/repair tool exchanges request-locally +``` + +An entry without `model` contributes no messages. A replacement changes the effective messages at the +target's position without changing its ID. A current-handle cache is disposable derived state. Historical +queries older than its cursor derive separately and never rewind the live cache. A request captures an +immutable selected-reference array and immutable effective replacements through a durable cutoff; later +entries, heads, edits and cache changes do not mutate it. + +### 3.5 Forks + +A fork creates a conversation with `parent = { conversationId, at }`. The target must be a visible entry +of the source; `"start"` means no inherited entries. Source entries keep their original IDs and owners. +A fork's logical transcript recursively includes each source prefix capped at the recorded fork point, +then its local entries. Later source changes are invisible. Tasks are never inherited. + +Any entry is a valid fork point. If it cuts a successful exchange, request projection supplies missing +results locally but creates or inherits no tasks. A fork link is history, not ownership. + +## 4. State and addresses + +```ts +type Scope = + | { readonly type: "session" } + | { readonly type: "conversation"; readonly conversationId: Id } + | { readonly type: "task"; readonly taskId: Id } + | { readonly type: "shared"; readonly id: Id }; + +declare const addressType: unique symbol; + +interface Address { + readonly scope: Scope; + readonly namespace: string; + readonly key?: string; + readonly kind: "value" | "list"; + readonly rewind: boolean; + readonly [addressType]?: T; +} + +interface Value extends Address { readonly kind: "value" } +interface List extends Address { readonly kind: "list" } +interface Element { readonly id: Id; readonly value: T } + +type StickyOptions = { readonly key?: string; readonly rewind?: false }; +type ConversationOptions = { readonly key?: string; readonly rewind: boolean }; +declare function defineValue( + scope: Extract, namespace: string, + options?: StickyOptions, +): Value; +declare function defineValue( + scope: Extract, namespace: string, + options: ConversationOptions, +): Value; +declare function defineList( + scope: Extract, namespace: string, + options?: StickyOptions, +): List; +declare function defineList( + scope: Extract, namespace: string, + options: ConversationOptions, +): List; +``` + +Addresses are fully bound. Session constructors require no ID; conversation constructors require a +conversation ID; task-scratch constructors require a task ID; shared constructors require a shared-scope +ID. Handles validate the bound ID and never silently replace it. Session, task and shared addresses are +non-rewindable; conversation addresses explicitly select `rewind:true` or `false`. Task scope is private +scratch retired with that task. Shared scope is capability-referenced and retires when a committed batch +leaves no live task reference. Ready packages compare addresses structurally by the tuple +`[scope type, scope id or null, kind, rewind, namespace, key or null]`. `key` participates in address identity +by selecting an optional member of a keyed family. Only property presence is non-identity: omitted `key` and +`key:undefined` are the same unkeyed singleton, while the empty string is a distinct key. Backends represent +the unkeyed slot directly, for example with a null sentinel. String wire encoding is deferred to client integration. + +Values use complete replacement and deletion. Lists use intrinsic append/remove/clear operations. For a +rewindable value, deletion records durable absence. For a rewindable list, remove and clear hide elements +only at and after their own write sequence, preserving earlier historical reads. Sticky state exposes current +contents; a backend may physically discard removed sticky elements. + +Historical `at` is an entry ID. Storage resolves it to that entry's creation sequence. Rewindable +conversation lookup chooses the newest local write at or before that sequence. If none exists and the +conversation has a parent, recurse into the parent capped at the earlier of the requested entry position and +`parent.at`. Lists combine inherited and local operations under the same caps. Scratch and sticky historical +reads reject. Forks inherit no scratch and inherit sticky conversation state only through an explicit copy +policy at creation. + +A rewindable conversation `getValue(..., at)` or `readList(..., at)` requires `at` to be a fork-visible entry +of that address's conversation; an unrelated/missing position rejects `InvalidHistoryPosition`. `undefined` +selects current state. + +Within one main transaction, every rewindable conversation value/list write must precede every entry +append. This makes state in the same `Write[]` visible at a fork through that entry while excluding later +state. Sticky/session state and task writes may occur anywhere. A violating builder throws and the +whole transaction rolls back. + +Namespaces beginning `pi.` are protected. Public/plugin constructors reject them unless created with an +internal built-in authority. Managed system records and input/result/receipt state cannot be written via generic entry, value or list +handles. + +Ready generic transactions reject shared addresses; the protected central task-output writer is their only +mutation authority. Host transactions may use any nonprotected session/conversation address. Normal and +abort task transactions may use session addresses and conversation addresses in the current task's conversation/owned subtree. A task +address is accepted only by that same task's `scratch` API; it is rejected by main transactions and by +another task. `TaskConversation.value/list` accepts session addresses and addresses bound to that handle's +conversation, and rejects every other conversation/task address. + +## 5. Task records and kinds + +```ts +interface TaskCheckpoint extends JsonObject { readonly phase: string } +type NoCheckpoint = never; + +declare const taskOutputType: unique symbol; // compile-only; durable identity is `kind` +interface TaskOutputKind { + readonly kind: string; + readonly [taskOutputType]?: O; +} +declare function defineTaskOutput(kind: string): TaskOutputKind; +interface TaskOutputSpec { + readonly kind: TaskOutputKind; + initial(input: I): O; +} +interface StoredTaskOutputRef { readonly id: Id; readonly kind: string } +interface TaskOutputRef extends StoredTaskOutputRef { + readonly [taskOutputType]?: O; +} +type TaskOutputField = [O] extends [never] + ? unknown + : { readonly output: TaskOutputRef }; +type TaskOutputDefinition = [O] extends [never] + ? { readonly output?: never } + : { readonly output: TaskOutputSpec }; + +type TaskOutcome = + | { readonly status: "completed"; readonly result: R } + | { readonly status: "failed"; readonly failure: F } + | { readonly status: "aborted"; readonly result: A } + | { readonly status: "orphaned" }; + +interface TaskBase { + readonly id: Id; + readonly conversationId: Id; + readonly kind: string; + readonly input: I; + readonly checkpoint?: C; + readonly after: readonly Id[]; + readonly background?: true; + readonly owns: readonly Id[]; + readonly output?: StoredTaskOutputRef; + readonly abort?: true; +} + +type Task = TaskBase & ( + | { readonly status: "pending" | "running"; readonly outcome?: never } + | { readonly status: "terminal"; readonly outcome: TaskOutcome } +); + +type RunningTask = + Omit, "output"> & TaskOutputField & + { readonly status: "running"; readonly outcome?: never }; + +type Completion = + | { readonly status: "completed"; readonly result: R } + | { readonly status: "failed"; readonly failure: F }; +``` + +Kind, conversation, input, dependencies, background and output reference are immutable after creation. A +checkpoint is optional; each checkpoint update constructs a full `task.set` snapshot containing the complete +checkpoint. Its `phase` is a kind-defined tag that the generic scheduler never interprets. Result, failure and +abort payloads are kind-specific strict JSON. Structurally identical checkpoint types are intentionally +assignable in TypeScript; kind attribution is supplied by the current runtime, not a nominal durable brand. +Differently shaped checkpoints, partial checkpoints and another kind's incompatible shape reject. + +A task kind has five durable payload types and a sixth task-output state type (`never` means no output). +Output state is a mutable object/array accepted by Chord `track`. `TaskOutputKind` contains only durable +string identity plus a compile-only output witness. `TaskOutputSpec.initial(input)` is synchronous and pure. +It runs only when task creation omits a shared ref; its returned object is transferred to the tracker and +must not be retained or mutated by the author. Its base commits atomically with task creation, and it never +runs when sharing. + +Ordinary extension tasks and Pico's fixed privileged core tasks use separate authoring factories. The +separate brands make their tokens non-interchangeable. `defineTask` is public. `defineCoreTask` is an internal +Pico authoring helper and is not a plugin extension point. + +```ts +declare const taskKindBrand: unique symbol; +interface TaskKindBase { + readonly kind: string; + readonly [taskKindBrand]: "ordinary"; +} +interface CoreTaskKindBase { + readonly kind: string; + readonly [taskKindBrand]: "core"; +} + +interface TaskKindMethods extends TaskKindBase { + execute(task: RunningTask, runtime: TaskRuntime, ctx: Context): + Promise>; + recover(task: RunningTask, runtime: TaskRuntime, ctx: Context): + Promise>; + abort(task: RunningTask, runtime: AbortTaskRuntime, ctx: Context): + Promise>; +} +interface CoreTaskKindMethods extends CoreTaskKindBase { + execute(task: RunningTask, runtime: CoreTaskRuntime, ctx: Context): + Promise>; + recover(task: RunningTask, runtime: CoreTaskRuntime, ctx: Context): + Promise>; + abort(task: RunningTask, runtime: CoreAbortTaskRuntime, ctx: Context): + Promise>; +} +type TaskKind = + TaskKindMethods & TaskOutputDefinition; +type CoreTaskKind = + CoreTaskKindMethods & TaskOutputDefinition; + +type NoExtra = + Actual & Record, never>; +type ExactJsonInput = + Expected extends readonly JsonValue[] ? Actual : + Expected extends JsonObject + ? Actual & Record, never> + : Actual; + +type KindDefinition = Omit; +type KindFactory = + >(definition: NoExtra, D>) => D & K; + +type TaskKindFactory = + KindFactory>; +type CoreTaskKindFactory = + KindFactory>; + +declare function defineTask(): + TaskKindFactory; +declare function defineCoreTask(): + CoreTaskKindFactory; +``` + +The factories preserve all payloads and output inference. `tx.task`, checkpoint methods and closures infer +from the token. `NoExtra` is also applied to the actual inferred task spec type, so visible extra top-level +fields in literals, variables and spreads reject. Deep structural exactness, casts and erased fields remain +normal TypeScript limits. Wire/plugin RPC boundaries validate strict JSON and optional schemas before calling +trusted local APIs. + +The fixed `pi.generation`, `pi.tool`, `pi.post_tools` and `pi.collapse` definitions use `defineCoreTask`. +`pi.job` uses ordinary `defineTask` unless a later concrete requirement proves it needs a narrow privileged +surface. Core definitions are installed internally and cannot be registered, replaced or removed. Ordinary +plugin definitions cannot use a protected `pi.*` name and never receive transcript/admission authority. +Generation, tool and post_tools are the fixed turn-task set; collapse has core transcript authority but is +not a turn task. The kernel identifies these roles only by their fixed registered slots/names and never +interprets their payload or checkpoint schemas. + +### 5.1 Optional state-indexed task authoring + +`defineStateTask` is an optional authoring adapter for kinds with several durable recovery phases. It +returns an ordinary `TaskKind`; `defineTask` remains available for simple ordinary kinds. Core kinds use the +separate internal `defineCoreTask` factory rather than this adapter in v1. The adapter makes one kind's +checkpoint dispatch exhaustive without adding author phases to stored `Task.status` or to the scheduler. + +```ts +type PhaseOf = C["phase"]; +type CheckpointAt> = + Extract; +type PhasePayload> = + Omit, "phase">; + +type PhaseTask> = + Omit, "checkpoint"> & { + readonly checkpoint: CheckpointAt; + }; + +type InitialTask = + Omit, "checkpoint"> & { + readonly checkpoint?: never; + }; + +type StateTransitionTx = + Omit, "checkpoint">; +type StatePhaseTx> = + StateTransitionTx & { + checkpoint(value: CheckpointAt): void; + }; +type InitialStateRuntime = + Omit, "commit">; +type StateRuntimeFor> = + Omit, "commit"> & { + commit( + build: (tx: StatePhaseTx, current: PhaseTask) => V | Promise, + ctx: Context, + ): Promise; + }; + +interface PhaseTransition> { + readonly type: "transition"; + readonly phase: P; + readonly commit: ( + tx: StateTransitionTx, + current: RunningTask, + ) => PhasePayload | Promise>; +} +interface StateTerminal { + readonly type: "terminal"; + readonly closure: TerminalClosure; +} +type StateResult = + | PhaseTransition> + | StateTerminal; + +type ExactPhasePayload, + Actual extends PhasePayload> = + NoExtra, Actual>; +interface StateActions { + transition

    , Actual extends PhasePayload>( + phase: P, + commit: ( + tx: StateTransitionTx, + current: RunningTask, + ) => ExactPhasePayload | Promise>, + ): PhaseTransition; + terminal( + closure: TerminalClosure, + ): StateTerminal; +} + +type PhaseHandler> = + | { + readonly role: "start"; + readonly recover?: never; + run( + task: PhaseTask, + runtime: StateRuntimeFor, + actions: StateActions, + ctx: Context, + ): Promise>; + } + | { + readonly role: "inflight"; + run( + task: PhaseTask, + runtime: StateRuntimeFor, + actions: StateActions, + ctx: Context, + ): Promise>; + recover( + task: PhaseTask, + runtime: StateRuntimeFor, + actions: StateActions, + ctx: Context, + ): Promise>; + }; + +type PhaseMap = { + readonly [P in PhaseOf]: PhaseHandler; +}; +type ExactPhaseHandler = + Actual extends { readonly role: infer Role } + ? Actual extends Extract + ? NoExtra, Actual> + : never + : never; +type ExactPhaseMap = + Actual & Record, never> & { + readonly [P in keyof Expected]: ExactPhaseHandler; + }; +type LiteralCheckpoint = + string extends PhaseOf ? never : C; + +type StateTaskDefinition = { + readonly kind: string; + readonly initial: { + readonly role: "start"; + run( + task: InitialTask, + runtime: InitialStateRuntime, + actions: StateActions, + ctx: Context, + ): Promise>; + }; + readonly phases: PhaseMap; + abort( + task: RunningTask, + runtime: AbortTaskRuntime, + ctx: Context, + ): Promise>; +} & TaskOutputDefinition; + +type ExactStateTaskDefinition = + NoExtra & { + readonly phases: ExactPhaseMap; + }; + +interface StateTaskFactory { + >( + definition: [LiteralCheckpoint] extends [never] + ? never + : ExactStateTaskDefinition, D>, + ): D & TaskKind; +} + +declare const commitStateTransition: unique symbol; +interface StateAdapterRuntime { + [commitStateTransition]

    >( + expectedSource: PhaseOf | undefined, + phase: P, + build: ( + tx: StateTransitionTx, + current: RunningTask, + ) => PhasePayload | Promise>, + ctx: Context, + ): Promise>; +} + +// Curried generics and output inference mirror defineTask. +declare function defineStateTask(): + StateTaskFactory; +``` + +`StateActions` is passed immediately before the final `ctx`; authors call `actions.transition(...)` and +`actions.terminal(...)`. The factory's outer and nested exact helpers capture actual phase handlers and +transition callback payloads. This declaration pattern was prototyped under the repository TypeScript +compiler with the positive and negative cases below; WP8A moves those cases into maintained compile tests. +Its compile contract is exact: + +- `C` is a finite discriminated union whose `phase` values are string literals; broad `string` rejects. +- Every `C["phase"]` has exactly one handler and no extra phase key exists. +- A handler's task contains the complete checkpoint variant for its phase. +- `initial` receives a running task with no checkpoint. Task creation remains checkpoint-free. +- A `start` phase has `run` and no `recover`; an `inflight` phase requires both. +- The target passed to `transition` determines the complete target payload. Missing or visible extra fields + reject for literals, variables and spreads. +- `terminal(closure)` preserves the existing typed `TerminalClosure`; terminal author phases do not exist. +- Output inference and fresh kind-level abort match `defineTask` exactly. + +The factory generates `TaskKind.execute` and `TaskKind.recover` around this internal loop: + +```text +scheduler reserves pending -> running and calls generated execute once + no checkpoint -> assert current unmarked invocation; initial.run + handler returns actions.transition(P, commit) + call the module-private commitStateTransition capability + on the line: validate expected source and invocation, run commit(tx, fresh current), + buffer a full task.set snapshot with the complete P checkpoint, + persist/apply, return applied PhaseTask

    + assert current unmarked invocation; dispatch P.run within the same outer TaskKind invocation + repeat until a handler returns terminal(closure) +generated execute returns closure to the existing terminal path + +reopen reserves the restored running task and calls generated recover once + no checkpoint -> fenced initial.run + start checkpoint P -> fenced P.run + inflight checkpoint P -> fenced P.recover + after any committed transition, entry becomes normal and the next fenced handler uses run +``` + +The scheduler never calls `execute` again for an internal transition. `commitStateTransition` is a +module-private symbol implemented by the task-kernel runtime and consumed only by the generated adapter; it +is not exported and grants no task-author capability. It performs one ordinary line transaction, then reads +the already-applied live-task projection, including checkpoint and `owns`. There is no second empty commit +and the adapter never fabricates a task from the stale handler argument. Transition callbacks run on the +line against fresh current state, may perform explicit committed reads before their first write and then +compose same-commit entries/tasks/state, and cannot perform effects, waits, hooks, sleeps or nested commits. + +`InitialStateRuntime` has no commit method. With no durable phase, allowing initial to commit child work would +let a crash rerun initial and duplicate them. Initial child/task/conversation creation therefore occurs in +its first returned transition callback, atomically with the first complete checkpoint. The next phase +resolves handles and waits only after that transition has committed. + +Checkpoint-backed handlers retain `runtime.commit` for durable bookkeeping that must precede a wait or +effect. The wrapped transaction may replace only its complete current-phase variant; phase changes use +returned transitions. A same-phase commit may atomically create a child task/conversation and record its +ID/policy in the current checkpoint, then the handler waits after the commit resolves. The handler's later +transition callback still receives fresh current state. + +`start` means the checkpoint does not represent an uncertain external effect, so reopen runs it normally. +`inflight` means an effect may have started, so reopen requires its explicit recovery path. This is a small +named state machine per task kind, not positional replay: only the latest complete checkpoint is durable. +There is no workflow history, deterministic replay requirement, generator, returned-plan journal, +one-commit-per-phase rule or global kernel lifecycle graph. + +Immediately before every internal handler call, the adapter synchronously validates the exact invocation +object, Context, session phase, durable mark and abort signal. If transition persistence wins first, the next +handler may be admitted and a later mark signals it normally. If the mark wins before the check, the next +handler is never invoked; the ordinary cancellation unwind returns from the outer execute/recover, which is +joined before fresh abort starts. This fence does not forcibly stop already-admitted third-party code; +handlers remain cooperatively cancellable. + +## 6. Mutation algebra and storage + +### 6.1 Mutations + +```ts +type NewTask = Omit & { + readonly status: "pending"; + readonly owns: readonly []; + readonly abort?: never; + readonly checkpoint?: never; + readonly outcome?: never; +}; + +type StateWrite = + | { readonly type: "value.set"; readonly address: Value; readonly value: JsonValue } + | { readonly type: "value.delete"; readonly address: Value } + | { readonly type: "list.append"; readonly address: List; readonly element: Element } + | { readonly type: "list.remove"; readonly address: List; readonly elementId: Id } + | { readonly type: "list.clear"; readonly address: List }; + +type Write = StateWrite | + { readonly type: "conversation.create"; readonly conversation: Conversation } | + { readonly type: "entry.append"; readonly entry: Entry } | + { readonly type: "task.create"; readonly task: NewTask } | + { readonly type: "task.set"; readonly task: Task }; +``` + +Each committed `Write` consumes one storage-assigned `Seq`; the returned sequences are contiguous, ascending +and positionally aligned with the submitted `Write[]`. `conversation.create`, `entry.append`, `task.create` and `list.append` carry IDs previously minted by +`Storage.nextId()`; Storage never equates them with sequences. +For a newly owned task output, the output ID defaults to the task ID; task creation and the protected shared +`pi.output` base append occur in the same `Write[]`. + +A `task.set` carries the complete replacement task snapshot. The Session/task kernel, not Storage, +constructs running, checkpointed, ownership, abort-marked and terminal snapshots. A terminal snapshot +atomically retires private scratch and, from the prospective complete task set, every shared output with no +live reference. The kind's closure materializes final output into its entry/outcome before retirement. A new +or same-commit task reference keeps the output live; terminal records do not. A retired output ID cannot be +resurrected. + +The Session validates scope and capability before persistence. Protected output writes retain private +per-append authorization while buffered; one `Write[]` may contain multiple authorized output appends. Each +append is one exact nonempty central-tracker flush. Scratch transactions contain only writes for their task. Storage remains mechanical: it atomically applies the canonical writes, maintains indexes and retirement, +assigns sequences, and derives/persists its committed ID high-water from creation/list-append IDs in the +canonical writes. It trusts the Session-supplied IDs and complete snapshots. It does not execute kind code, derive partial task transitions or validate application +payload schemas. Empty `Write[]` values are not passed to Storage, consume neither IDs nor sequences and +publish no event. An uncertain commit failure poisons the binding; the Session fail-stops and never retries +that callback on the same binding. + +### 6.2 Queries + +```ts +interface PageQuery { readonly cursor?: Id; readonly limit: number } +interface Page { readonly items: readonly T[]; readonly next?: Id } + +interface ConversationScan extends PageQuery { + readonly parent?: Id; + readonly owner?: Id; +} +interface EntryScan extends PageQuery { + readonly conversationId: Id; + readonly kind?: string; + readonly through?: Id; +} +interface TaskScan extends PageQuery { + readonly conversationIds?: readonly Id[]; + readonly statuses?: readonly ("pending" | "running" | "terminal")[]; + readonly kind?: string; + readonly abort?: boolean; + readonly outputId?: Id; +} +interface Storage { + nextId(): Id; + commit(writes: readonly Write[], ctx: Context): Promise; + getConversations(ids: readonly Id[], ctx: Context): Promise>; + scanConversations(query: ConversationScan, ctx: Context): Promise>; + getEntries(ids: readonly Id[], ctx: Context): Promise>; + scanEntries(query: EntryScan, ctx: Context): Promise>; + newestHead(conversationId: Id, at: Id, ctx: Context): Promise; + getTasks(ids: readonly Id[], ctx: Context): Promise>; + scanTasks(query: TaskScan, ctx: Context): Promise>; + getValue(address: Value, at: Id | undefined, ctx: Context): Promise; + readList(address: List, at: Id | undefined, ctx: Context): Promise[]>; +} +``` + +Scan rules: + +- `limit` is a positive integer. A returned `next` is the cursor for the next page. +- Conversation scans are ascending by ID; `cursor` is an exclusive lower bound. `parent` and `owner` are exact, + composable filters. +- Entry scans are fork-aware and newest-first. `through` is an inclusive fork-visible upper bound; `cursor` + is an exclusive upper bound, so the next page continues toward older entries. `kind` filters before limit. +- Task scans are ascending by ID; `cursor` is an exclusive lower bound. Tasks are never inherited. + `conversationIds`, `statuses`, `kind`, `abort` and `outputId` compose as exact filters. +- List reads return the complete logical list ascending by element ID. `at` selects the validated historical + entry position for rewindable conversation lists; `undefined` selects current state. There is no paged list + scan in v1. +- `newestHead` is the dedicated fork-aware descending limit-one operation through its required inclusive + bound. Latest value/head queries must not load and tail. +- Scan filters and fork caps apply before the limit and before payload decoding where the backend can avoid + it. Pages deliberately contain no storage-sequence snapshot. A complete list read is coherent at one + committed state and needs no multi-page snapshot protocol. + +Before Storage is called, the Session/kernel validates the complete task snapshots against committed state +and prior buffered references. It also proves invocation identity, normal-versus-abort authority, open +reconciliation authority, durable mark and Session phase. Storage trusts the resulting canonical writes and +applies them atomically. + +| write | structural current state | required complete snapshot | +|---|---|---| +| `task.create` | absent | pending, no checkpoint/outcome/abort, empty owns; ordinary/core token and output ref/base valid when declared | +| reservation `task.set` | pending | running; normal reservation requires terminal dependencies, marked abort reservation ignores them | +| progress `task.set` | running | running with complete checkpoint and/or complete immutable ownership list replacement | +| mark `task.set` | pending or running | same lifecycle state with `abort:true`; repeated mark helper is idempotent | +| completed/failed `task.set` | running and unmarked | terminal with matching outcome; scratch retirement follows snapshot | +| aborted `task.set` | running and marked | terminal with aborted outcome; scratch retirement follows snapshot | +| orphaned `task.set` | pending or running | terminal with orphaned outcome; scratch retirement follows snapshot | +| any replacement of terminal | terminal | reject | + +Task-output creation/reference validation is whole-batch: + +- A no-output kind stores no ref and cannot receive one. +- An output-capable task with no supplied ref stores `{ id: task.id, kind: spec.kind.kind }`; its one + protected initial list element is the same-batch `track(spec.initial(input)).flush()` and starts with one + `r` base. The initial value is transferred to the tracker; the author must not retain or mutate it. +- A supplied ref names an output with at least one live reference in committed tasks or the builder's private + pending-task index and matches the durable output-kind string. A task invocation may pass its own ref to a task it is + authorized to create; host code may share a live ref through its unrestricted task-creation authority. +- Several creations/terminalizations use the prospective post-commit live reference set. No ref may target a + retired output. +- Every protected `list.append` to `defineList({type:"shared",id}, "pi.output")` is an exact nonempty + central-tracker flush and has private buffered provenance from a current `writerTaskId` that references + that output. One commit may contain several independently authorized appends. + +Every owned `conversation.create` has one matching complete `task.set` snapshot in the same `Write[]` whose +`owns` includes the new conversation. Reject an unpaired side, ownership of an existing conversation, +duplicate ownership or mismatched owner IDs. Conversation ownership and the corresponding task `owns` +element are immutable after creation. + +List removal rejects an element ID that never existed or belongs to another list. It is idempotent only for +an existing same-list element already hidden/removed, and still consumes a write sequence. Clear is valid on +an empty list and consumes a sequence. Reading any task-scope address after its task is terminal rejects +`ScratchRetired` in all backends; it never returns an accidentally retained sidecar value. + +Explicit author-facing transaction reads run while the line is exclusively held and see committed state +only. They are allowed only before the first buffered write; any later explicit read rejects +`ReadAfterWrite`. There is no general transaction overlay and no read-your-writes. Semantic builder helpers +may continue after writes by consulting narrow private pending/reference indexes for their own same-callback +creations, request-key deduplication and structural validation. Those indexes are not exposed as readers; +authors carry values they just wrote. + +## 7. Transaction and capability surfaces + +Every runtime, host or conversation `commit` callback receives a synchronous write builder backed by one +canonical `Write[]`. Creation/list-append builders synchronously call `Storage.nextId()`, validate and buffer; +they never call `Storage.commit`. Explicit readers may await committed Storage only until the first write is buffered. After the +callback succeeds, the Session validates once and invokes `Storage.commit` exactly once. Public one-operation +mutators use the same transaction path. + +### 7.1 Common readers and state builders + +```ts +interface TxReaders { + getConversation(id: Id): Promise; + getEntry(id: Id): Promise; + getEntry(kind: EntryKind, id: Id): Promise; + getEntries(ids: readonly Id[]): Promise>; + getTask(id: Id): Promise; + getTask(kind: K, id: Id): Promise | undefined>; + getTasks(ids: readonly Id[]): Promise>; +} + +interface TxValue { + get(at?: Id): Promise; + set(value: T): void; + delete(): void; +} +interface TxList { + read(): Promise[]>; + read(at: Id): Promise[]>; + append(value: T): Id; + remove(id: Id): void; + clear(): void; +} +``` + +The aliases are: + +```ts +type AnyTaskKind = TaskKindBase; +type AnyCoreTaskKind = CoreTaskKindBase; +type AnyDefinedTaskKind = AnyTaskKind | AnyCoreTaskKind; +type PayloadsOf = + K extends TaskKind + ? { input: I; checkpoint: C; result: R; failure: F; aborted: A; output: O } + : K extends CoreTaskKind + ? { input: I; checkpoint: C; result: R; failure: F; aborted: A; output: O } + : never; +type InputOf = PayloadsOf["input"]; +type CheckpointOf = PayloadsOf["checkpoint"]; +type ResultOf = PayloadsOf["result"]; +type FailureOf = PayloadsOf["failure"]; +type AbortedOf = PayloadsOf["aborted"]; +type OutputOf = PayloadsOf["output"]; +type DistributiveOmit = T extends unknown ? Omit : never; +type TaskOf = + DistributiveOmit, CheckpointOf, ResultOf, FailureOf, AbortedOf>, "output"> & + TaskOutputField>; +``` + +The implementation may use distributive helper aliases to satisfy TypeScript variance without weakening +these public results. None may use `any`. Task-bound checkpoint methods exist only for the current kind. + +### 7.2 Internal admission-capable transaction + +```ts +interface Acceptance { readonly requestId?: string; readonly conversationId: Id; readonly inputId: Id } + +interface SendInput { + readonly requestId?: string; + readonly content: UserInput; + readonly whenBusy?: "followUp" | "steer" | "reject"; +} + +interface InputHandle { + readonly id: Id; + readonly conversationId: Id; + result(ctx: Context): Promise; + wait(ctx: Context): Promise; + abort(ctx: Context): Promise<"aborted" | "already_placed" | "not_found">; +} + +type TaskSpec = { + readonly conversationId?: Id; + readonly input: I; + readonly after?: readonly Id[]; + readonly background?: true; +} & ([O] extends [never] + ? { readonly output?: never } + : { readonly output?: TaskOutputRef }); + +interface ConversationCreateSpec { + readonly parent?: { readonly conversationId: Id; readonly at: Id }; +} + +interface TaskCreator { + task, OutputOf>>( + kind: K, + spec: NoExtra, OutputOf>, S> & { + readonly input: ExactJsonInput, S["input"]>; + }, + ): Id; +} + +interface TaskTxBase extends TxReaders { + checkpoint(value: C): void; + createConversation(spec: ConversationCreateSpec): Id; + value(address: Value): TxValue; + list(address: List): TxList; +} + +type BaseTaskTx = + TaskTxBase & TaskCreator; + +interface InternalAdmissionTx { + accept(conversationId: Id, options: InternalAcceptOptions): Promise; + queueInput(conversationId: Id, input: QueuedInput): Promise; + write(conversationId: Id, kind: EntryKind, input: EntryInput, requestId?: string): Promise; +} + +type CoreTaskTx = + TaskTxBase & TaskCreator & InternalAdmissionTx & { + entry(kind: EntryKind, conversationId: Id, input: EntryInput): Id; + } +``` + +Ordinary task authors receive `BaseTaskTx` and can instantiate only ordinary `TaskKind` tokens. Fixed +core definitions receive `CoreTaskTx`; its `task` method uses the same spelling but accepts either an +ordinary child token or one of Pico's fixed `CoreTaskKind` tokens. Thus generation can atomically create +tool and post_tools tasks, while plugin tasks cannot instantiate any core kind. `CoreTaskTx` also contains +direct entry and protected admission authority. A conversation-bound +internal variant omits explicit `conversationId`. In a task transaction, omitted task target +conversation means the current task's conversation. Explicit targets must be that conversation or a +conversation in its ownership subtree, including one created earlier in the batch. `createConversation` +automatically records `owner=current task` plus a complete replacement task snapshot containing the new +ownership; a host transaction creates an independent conversation unless an internal owner authority is supplied. Its optional parent must be the current +conversation or one in the current task's owned subtree; a task cannot inherit transcript/state from an +unrelated tree. A parent may be combined with ownership. +Data-only appends use the capability-restricted typed `entry` method with an `EntryKind`; there is no +parallel untyped `record` API. `accept`, `queueInput` and `write` are protected transaction helpers, not +ordinary public conversation methods. They remain available where built-ins must atomically compose child +creation, initial values, first input, first generation and a parent checkpoint. They are asynchronous because request-key lookup +may require a committed storage read; when invoked before any write they perform that read first, then synchronously buffer all +writes. After writes, they may use only their narrow private request/pending indexes and reject any path that +would require a new explicit read. They never reenter the line and may target a conversation created earlier +in the callback through its pending reference. To atomically create an owned child, copy state, accept its +first input, create its first generation and checkpoint the child/input IDs, the author performs every +explicit source read first and then buffers the complete composition. + +Direct entry authorization and admission helpers are distinct. Core code may buffer an authorized +model-visible append after validating placement. An ordinary runtime constructs only `BaseTaskTx`; a cast +cannot add the absent `entry` or protected admission methods. Every builder records private provenance for +validation; storage sees only the resulting mutation algebra. No new invocation-authority mode is needed: +the fixed dispatcher chooses the concrete runtime/builder implementation, while the existing invocation +identity continues only to fence stale writes and cancellation. + +### 7.3 Runtime and restricted conversation handle + +```ts +interface PublicValue { + get(at: Id | undefined, ctx: Context): Promise; + set(value: T, ctx: Context): Promise; + delete(ctx: Context): Promise; +} +interface PublicList { + read(ctx: Context): Promise[]>; + read(at: Id, ctx: Context): Promise[]>; + append(value: T, ctx: Context): Promise; + remove(id: Id, ctx: Context): Promise; + clear(ctx: Context): Promise; +} +interface ScratchTx { + value(address: Value): TxValue; + list(address: List): TxList; +} +interface ScratchReader { + value(address: Value): Pick, "get">; + list(address: List): Pick, "read">; +} +interface TaskOutput { + readonly ref: TaskOutputRef; + read(ctx: Context): Promise; + mutate(mutator: (state: O) => undefined, ctx: Context): Promise; + replace(value: O, ctx: Context): Promise; +} +interface ReadonlyTaskOutput { + readonly ref: TaskOutputRef; + read(ctx: Context): Promise; +} +type RuntimeOutput = [O] extends [never] + ? unknown : { readonly output: TaskOutput }; +type AbortRuntimeOutput = [O] extends [never] + ? unknown : { readonly output: ReadonlyTaskOutput }; +type FinalOutput = [O] extends [never] + ? unknown : { readonly output: O }; + +interface TaskConversation { + readonly id: Id; + send(input: SendInput, ctx: Context): Promise; + value(address: Value): PublicValue; + list(address: List): PublicList; +} + +interface AbortTaskConversation { + readonly id: Id; + value(address: Value): PublicValue; + list(address: List): PublicList; +} + +interface TaskRuntimeBase { + readonly taskId: Id; + scratch(build: (tx: ScratchTx) => T | Promise, ctx: Context): Promise; + conversation(id: Id, ctx: Context): Promise; + waitForTask(id: Id, ctx: Context): Promise; + abortTask(id: Id, ctx: Context): Promise<"marked" | "terminal">; + now(): number; + sleep(untilMs: number, ctx: Context): Promise; +} +interface CommitRuntime { + commit(build: (tx: Tx, current: RunningTask) => T | Promise, ctx: Context): Promise; +} +type TaskRuntime = + TaskRuntimeBase & CommitRuntime> & RuntimeOutput; +type CoreTaskRuntime = + TaskRuntimeBase & CommitRuntime> & RuntimeOutput; +``` + +`runtime.conversation`, `waitForTask` and `abortTask` accept only targets in the current task's ownership +tree: its own conversation, its owned descendants, their tasks, and the task's own dependencies. An +unrelated target rejects `ScopeViolation`. Host operations are unrestricted except for protected data and +normal lifecycle validation. This generic rule does not let one task resolve a conversation owned by a +sibling task. The future gated ordinary-tool descendant resolver required by section 14.3 is the narrow +exception; it will not broaden `runtime.conversation` or `TaskConversation`. + +`TaskConversation` has no raw commit, direct entry, close, shutdown, delete or host-wide registry methods. +An `InputHandle` returned to a task is likewise invocation-bound. Task operations capture the expected +invocation when the runtime/handle is constructed. Every supplied Context must contain that exact current +invocation object. An absent, foreign or stale identity +rejects; task methods never fall back to host authority. + +Exactly one authoritative Chord tracker is hydrated per live output ID. `read` snapshots it without exposing +a retained proxy. `mutate` invokes its callback synchronously on the commit line against the central proxy, flushes one +nonempty delta, and buffers its authorized append in the callback's canonical `Write[]`; the Session commits +that array once before resolving. Requiring an +`undefined` return rejects async callbacks at compile time. The callback cannot retain the proxy, call +another runtime method, or mutate after returning. Objects inserted through the proxy become tracker-owned; +the author must not retain and later mutate their raw references. If the callback throws, returns a thenable, +or causes tracker application to fail, nothing commits or publishes and the session fail-stops before +releasing the line. The poisoned in-memory tracker is never used +again; reopen reconstructs it from durable deltas. This avoids cloning the growing output before every delta. +`replace` uses the same tracker's `state` setter and commit path. Concurrent task writes serialize in global +commit order. Any persistence failure after tracker mutation also fail-stops. No observer sees uncommitted +tracker state. Runtime checks the ref's durable kind string, not JavaScript token +identity or structural TypeScript compatibility. + +### 7.4 Terminal and abort transactions + +```ts +type TerminalClosureFor = + (tx: Tx & FinalOutput, current: RunningTask) => + Completion | Promise>; +type FinalTx = BaseTaskTx & FinalOutput; +type CoreFinalTx = CoreTaskTx & FinalOutput; +type TerminalClosure = + TerminalClosureFor, I, C, R, F, O>; +type CoreTerminalClosure = + TerminalClosureFor, I, C, R, F, O>; + +interface AbortTx extends TxReaders { + checkpoint(value: C): void; + value(address: Value): TxValue; + list(address: List): TxList; + markTask(id: Id): Promise<"marked" | "terminal">; +} +interface CoreAbortTx extends AbortTx { + entry(kind: EntryKind, conversationId: Id, input: EntryInput): Id; + write(conversationId: Id, kind: EntryKind, input: EntryInput, requestId?: string): Promise; +} +type AbortClosureFor = + (tx: Tx & FinalOutput, current: RunningTask) => A | Promise; +type AbortFinalTx = AbortTx & FinalOutput; +type CoreAbortFinalTx = CoreAbortTx & FinalOutput; +type AbortClosure = + AbortClosureFor, I, C, A, O>; +type CoreAbortClosure = + AbortClosureFor, I, C, A, O>; + +interface AbortRuntimeBase { + readonly taskId: Id; + scratch(read: (scratch: ScratchReader) => T | Promise, ctx: Context): Promise; + conversation(id: Id, ctx: Context): Promise; + waitForTask(id: Id, ctx: Context): Promise; + abortTask(id: Id, ctx: Context): Promise<"marked" | "terminal">; + now(): number; +} +type AbortTaskRuntime = + AbortRuntimeBase & CommitRuntime> & AbortRuntimeOutput; +type CoreAbortTaskRuntime = + AbortRuntimeBase & CommitRuntime> & AbortRuntimeOutput; +``` + +Ordinary abort runtime exposes repeated restricted commits with `AbortTx`, plus scratch reads but no scratch +writes. It cannot create tasks/conversations, append entries, delete conversations, send or queue input. +Fixed core abort receives `CoreAbortTx`, which adds direct entry authority plus only the passive protected +`write` helper; it cannot create future work. This lets the core tool adapter atomically publish an aborted +tool result and lets core cleanup queue a boundary-safe passive write. Marking immediately revokes the normal invocation's +`TaskOutput` mutation authority before signaling its context. Fresh abort receives only +`ReadonlyTaskOutput`: `ref` and `read`, never `mutate`/`replace`. Its final closure receives a detached +`tx.output` snapshot captured on the line so it can persist final output-derived data in the same main batch +as terminalization and possible last-reference retirement. Checkpoint replacement is allowed during lengthy +fresh cleanup; a crash reruns abort from the newest checkpoint because the durable mark remains. + +Terminal closures run on the line after all effects and producer joins. They may await explicit committed +reads before buffering their first write and may then mint same-commit IDs, but may not perform external +effects, hooks, sleeps, waits or nested runtime operations. The kernel then buffers one full terminal +`task.set` snapshot with the returned outcome. A throwing closure persists nothing and faults the session. +Abort/close/fault may discard a normal closure without invoking it. Uncertain persistence fail-stops and +does not retry the closure on that handle. + +### 7.5 Host transactions and foundation handles + +Host transactions are not task invocations. Public host transactions may append model-affecting entries +directly only when no fixed turn task is live; otherwise direct append rejects. Protected built-in +transactions use the internal `write` helper when boundary-safe queuing is required. + +```ts +interface HostTx extends TxReaders { + task, OutputOf> & { readonly conversationId: Id }>( + kind: K, + spec: NoExtra, OutputOf> & { readonly conversationId: Id }, S> & { + readonly input: ExactJsonInput, S["input"]>; + }, + ): Id; + createConversation(spec: ConversationCreateSpec): Id; + value(address: Value): TxValue; + list(address: List): TxList; + entry(kind: EntryKind, conversationId: Id, input: EntryInput): Id; +} + +interface ConversationTx extends Omit { + task, OutputOf>, "conversationId">>( + kind: K, + spec: NoExtra, OutputOf>, "conversationId">, S> & { + readonly input: ExactJsonInput, S["input"]>; + }, + ): Id; + entry(kind: EntryKind, input: EntryInput): Id; +} + +interface InternalConversationAdmissionTx { + accept(options: InternalAcceptOptions): Promise; + queueInput(input: QueuedInput): Promise; + write(kind: EntryKind, input: EntryInput, requestId?: string): Promise; +} +type InternalHostTx = HostTx & InternalAdmissionTx; +type InternalConversationTx = ConversationTx & InternalConversationAdmissionTx; + +interface ConversationHandle { + readonly id: Id; + commit(build: (tx: ConversationTx) => T | Promise, ctx: Context): Promise; + send(input: SendInput, ctx: Context): Promise; + waitForIdle(ctx: Context): Promise<"idle" | "closed">; + abort(ctx: Context): Promise; + fork(options: { readonly at: Id | "start"; readonly abort?: boolean }, ctx: Context): Promise; + collapse(options: { readonly instructions?: string } | undefined, ctx: Context): Promise; + reset(options: { readonly handoff?: string } | undefined, ctx: Context): Promise; + value(address: Value): PublicValue; + list(address: List): PublicList; +} + +interface Harness { + root(ctx: Context): Promise; + conversation(id: Id, ctx: Context): Promise; + commit(build: (tx: HostTx) => T | Promise, ctx: Context): Promise; + input(requestId: string, ctx: Context): Promise; + getEntry(id: Id, ctx: Context): Promise; + getTask(id: Id, ctx: Context): Promise; + waitForIdle(ctx: Context): Promise<"idle" | "closed">; + abortTask(id: Id, ctx: Context): Promise<"marked" | "terminal">; + resume(ctx: Context): Promise; + close(ctx: Context): Promise; + shutdown(ctx: Context): Promise; +} +``` + +There is no partial public harness or feature-not-installed facade. Leaf packages are tested directly. +The single `Harness` is exported only when its required built-in dependencies are implemented. `send` +first commits or deduplicates durable admission, then non-abandoningly ensures scheduler activation, and only +then returns; it never waits for generation. Caller cancellation after admission cannot strand the accepted +input, and a request-key retry still returns its original handle. `waitForIdle`, task waits, input waits and +abort operations are observation/progress operations: they idempotently ensure the one session scheduler is +resumed, but a waiter never owns execution. + +## 8. Input admission and boundaries + +### 8.1 Records + +```ts +type UserInput = string | readonly (TextContent | ImageContent)[]; + +interface StoredEntryDraft { + readonly kind: string; + readonly data?: JsonValue; + readonly model?: readonly Message[]; + readonly head?: Id | "self"; + readonly edits?: readonly ContextEdit[]; +} + +type QueuedInput = + | { readonly mode: "steer" | "followUp"; readonly input: UserInput; readonly requestId?: string } + | { readonly mode: "write"; readonly entry: StoredEntryDraft; readonly requestId?: string }; + +type InputResult = + | { readonly status: "queued"; readonly requestId?: string } + | { readonly status: "placed"; readonly requestId?: string; readonly entry: Id } + | { readonly status: "done"; readonly requestId?: string; readonly entry: Id; readonly answer?: Id } + | { readonly status: "unanswered"; readonly requestId?: string; readonly entry?: Id; + readonly reason: "terminated" | "aborted" | "failed" | "stale"; readonly detail?: string }; + +type TerminalInputResult = Extract; + +type QueueMode = "all" | "one-at-a-time"; + +interface InternalAcceptOptions { + readonly input: UserInput; + readonly requestId?: string; + readonly whenBusy?: "followUp" | "steer" | "reject"; +} + +// Public facade: send(input, ctx) returns InputHandle after this internal operation commits. +``` + +The protected `pi.inbox` address is a conversation-scoped sticky (`rewind:false`) list bound to exactly one +conversation. It is not inherited by forks. It stores `QueuedInput`; each list-element ID is the `inputId`, +and ascending element IDs preserve Session admission order. Protected sticky values store `InputResult` +directly by input ID and `Acceptance` directly by session-wide request key. Result/receipt lookup is always a +point lookup in that protected state, never a transcript or inbox list scan. `Conversation.send` converts +`SendInput.content` to the internal accepted user input and returns an `InputHandle` bound to the accepted +input and its original conversation. + +Inbox reads expose one complete current materialized ordered collection. MemoryStorage retains that +collection directly; JSONL replays list operations once at open and then maintains it; SQLite reads all +current indexed rows in element-ID order and may physically delete rows on remove/clear. No backend replays +from the last clear on each inbox read. Historical +rewindable-list operation storage and fork-capped reads are unrelated to this sticky protected list. + +A request key names the first acceptance for the Session lifetime. Lookup occurs before conversation, busy +mode or payload comparison. A duplicate `send` returns a handle for the original acceptance and buffers +nothing, even when retried through another conversation with different content. The internal `accept`, +`queueInput` and `write` helpers obey the same first-key-wins rule. + +Only protected admission/boundary helpers write input results, using this transition table. Enqueue, +removal, result/receipt changes, entry placement and task creation selected by one admission/boundary decision +remain one main transaction: + +```text +absent -> queued +queued -> placed +queued write -> done +queued -> unanswered(aborted | stale) +placed -> done +placed -> unanswered(terminated | aborted | failed) +done | unanswered -> immutable +``` + +An idle accepted input may write `absent -> placed` in its one batch; an idle write may write +`absent -> done`. These are the only collapsed transitions and still create/remove the inbox element so +`inputId` is its list-element ID. + +Enqueue validation checks the stored entry draft as far as current state allows. At placement it is fully +revalidated. If a queued head no longer satisfies visibility, monotonicity or exchange-boundary rules, the +placement atomically removes it and writes `unanswered/stale`; ordinary staleness never faults the session. +Queued edits whose targets have moved outside context remain valid no-ops; protected managed-system edits +remain forbidden. + +### 8.2 Generation admission strategy + +The foundation does not guess a provider generation payload. Input placement receives one strategy: + +```ts +interface InternalBoundaryTx extends TxReaders { + task, OutputOf> & { readonly conversationId: Id }>( + kind: K, + spec: NoExtra, OutputOf> & { readonly conversationId: Id }, S> & { + readonly input: ExactJsonInput, S["input"]>; + }, + ): Id; +} +interface InternalGenerationAdmission { + readonly kind: K; + create( + tx: InternalBoundaryTx, + conversationId: Id, + inputs: readonly Id[], + ): Id | Promise; +} +``` + +This is a private admission-module/test seam, not a public Harness option. The callback runs inside the +current transaction and may perform committed transaction reads before buffering its one task creation; +it cannot append entries, mutate state, perform effects or enter the line. Validation requires exactly one new task in the fixed core generation slot in `conversationId` and requires +its returned ID to be that task. The expected token/name is `strategy.kind`; production wiring supplies the +fixed generation token. Accounting for exactly the supplied input IDs is the trusted +strategy's typed obligation; the erased kernel cannot inspect an arbitrary generation input to prove it. +WP5 tests supply a minimal `defineCoreTask` generation definition; the provider package supplies the fixed +production strategy. + +### 8.3 Busy and placement predicates + +`admissionBusy(conversation)` means at least one live task directly in that conversation has kind +`pi.generation`, `pi.tool` or `pi.post_tools`, regardless of foreground/background. The predicate is evaluated +against the prospective complete task set for a transaction. It is transcript-admission state, not scheduler +eligibility, idle observation or proof of input ownership. A speculative collapse has core transcript +authority but is not in the fixed turn-task set. Ordinary plugin tasks never make a conversation admission-busy. + +- `send` when `admissionBusy` is empty performs one idle admission transaction: select all older writes, and + all or the oldest steer/followUp according to their respective `QueueMode`; merge selected items in global + inbox order, then append the new input last; create one built-in generation owning those input IDs; mark + each placed; store any request receipt; commit. `whenBusy` is irrelevant. The successful main commit + kicks the scheduler. After that durable commit resolves, `send` non-abandoningly ensures the scheduler is + resumed before returning. +- `send` when `admissionBusy` is nonempty queues `followUp` by default, queues `steer` when requested, or + throws `ConversationBusy` and writes no request receipt when `reject` is requested. It creates no task at + admission; the harmless main-commit kick still occurs. +- Internal `write` appends and completes immediately when `admissionBusy` is empty; otherwise it queues. + +Boundary operations are transaction helpers, not nested commits: + +- **post-tools boundary:** select all writes and all or the oldest steer according to `steeringMode`, merge + selected items in global inbox order, and place them. Steering joins the active input group. FollowUp + remains queued. +- **final-answer boundary:** resolve the active group to the answer; select all writes plus all or the oldest + steer/followUp according to `steeringMode`/`followUpMode`; merge selected items in global inbox order, place + them into a new group and create its generation. +- **idle-turn boundary:** when a normal core terminal batch changes the prospective fixed turn-task set from + nonempty to empty and creates no fixed turn-task successor, Pico applies the same final-boundary queue-mode + selection and creates its new generation. Core code remains responsible for resolving its active input group; + the kernel never interprets a core payload or checkpoint. +- **abort boundary:** no automatic generation or queue drain. Conversation abort separately withdraws + queued steer/followUp; task abort preserves queues. A later idle send can consume preserved items. + +Canonical trace: + +```text +idle send A +TX[user A; generation G1(inputs:[A]); result A=placed; receipt] +kick -> G1 + +G1 calls X,Y +TX[assistant calls; tool X; tool Y; post_tools P1(after:[X,Y],inputs:[A]); + terminal G1] +kick -> X and Y concurrently + +busy send B(default followUp) -> TX[inbox B; result B=queued; receipt] +busy send S(steer) -> TX[inbox S; result S=queued; receipt] +busy internal write W -> TX[inbox W; result W=queued] + +X and Y finish in either order +TX[tool result X; terminal X] +TX[tool result Y; terminal Y] +model projection orders those result messages by the assistant's call order + +P1 boundary +TX[write W; user S; result S=placed; + generation G2(inputs:[A,S]); terminal P1] +kick -> G2 + +G2 final answer E2 +TX[assistant E2; A/S=done(answer:E2); user B; B=placed; + generation G3(inputs:[B]); terminal G2] +kick -> G3 +``` + +`steeringMode` and `followUpMode` are independent built-in `QueueMode` settings and default to +`"one-at-a-time"`, matching the old lane harness. `"all"` selects every queued item of that tag; +`"one-at-a-time"` selects only its oldest item. Selection never reorders the one inbox: selected tags are +merged in global element-ID order and unselected items keep their relative order. There is no queue-size or +drain-size bound in v1. An idle internal write appends and becomes `done` without creating work. Applications +that need speculative staged context for a later explicit turn +store it in their own sticky conversation list and fold it into that later `send`. No send creates a giant +turn task or immutable task-per-send dependency chain. + +Normal terminal finalization uses this exact order: + +```text +invoke returned closure and buffer its writes +obtain its completed/failed outcome +buffer the complete terminal task.set snapshot +compute prospective admissionBusy from committed live tasks plus the whole buffered Write[] +when eligible, append idle-boundary inbox/result/entry/generation mutations +validate and persist the complete batch +``` + +The terminal snapshot and automatic boundary are one atomic `Write[]` commit. Abort, orphan reconciliation, +close and fault never run automatic idle-boundary augmentation. WP5 implements the pure prospective planner; +WP6 integrates it with normal closure finalization. + +A speculative collapse never contributes to `admissionBusy`; an idle `send` may therefore create generation +work while collapse runs. If collapse completes while a generation/tool/post_tools task remains live, no idle +boundary runs; the fixed core owner processes the inbox later. Ordinary third-party tasks cannot own an +input group or participate in the turn-task set. Pico never infers or resolves an ordinary kind's private +payload as an input group. + +Input ownership for built-in turns is explicit `inputs: readonly Id[]` in generation/post_tools input or +checkpoint. Exactly one live built-in generation or post_tools owns each placed active group, and ownership +transfers in the same terminal batch. Generic scheduling never scans arbitrary checkpoints for ownership. + +### 8.4 Withdrawing input + +`InputHandle.abort()` serializes with placement. It first commits withdrawal when the queued item wins, then +non-abandoningly ensures the scheduler is resumed so admitted cleanup can progress. If the queued item wins, +Pico atomically removes only that item and writes terminal +`unanswered/aborted`, then returns `aborted`. If placement or any terminal result wins, it returns +`already_placed` and changes nothing. A missing input returns `not_found`. Later inbox items retain their +relative order; no task dependency is rewired. If a queued result exists but its inbox element is missing, +Pico faults the Session as corrupted state. + +### 8.5 Waiting for input + +`InputHandle.result()` is a point read and does not activate work. `InputHandle.wait()`: + +- validates that the accepted input still exists, otherwise rejects `InputNotFound`; +- idempotently resumes the session scheduler before observing terminal state; +- returns an existing `done` or `unanswered` result immediately; +- otherwise registers/checks/removes one cancellable point waiter on the line without polling; +- removes its signal listener and waiter exactly once on result, caller cancellation, close or fault; +- rejects caller cancellation, close and fault; shutdown eventually closes and rejects unresolved queued + waits rather than deleting their durable input; +- does not mark or cancel durable work when its caller cancels. + +The waiter observes durable state and owns no execution. Several handles or calls may wait for the same +result. There is no serving graph, blocker set or input-specific execution cycle detector. Generic cycle +validation remains limited to acyclic task dependencies, direct task self-wait, and waiting on a task whose +unresolved dependency path reaches the caller. Hidden plugin-promise cycles remain the author's +responsibility. + +## 9. Commit line and runtime authorization + +One FIFO async line serializes commits, calls to `Storage.nextId()`, reservations, marks, lifecycle +transitions, registry mutations, waiter registration/removal and watch capture. A transaction callback may +await committed storage reads while holding the line only before its first buffered write; a later explicit +read rejects `ReadAfterWrite`. It must not await effects, hooks, task/input/idle waits, sleeps or nested line +operations. Known Pico runtime/handle line reentry rejects `NestedLineOperation` before queueing rather than +silently deadlocking. Every intermediate, terminal and abort builder receives a fresh immutable current task read on +the line after all committed checkpoint/mark changes; it never receives only the method's older snapshot. + +Caller cancellation is checked while waiting to enter the line and again before the builder begins. Once +the builder is admitted, caller cancellation cannot abandon builder completion or persistence. Storage +receives an internal non-abandoning Context that preserves telemetry but not the caller's abort signal. Task +invocation identity, lifecycle and durable mark are still revalidated on the line before invoking its +builder. + +Commit sequence: + +```text +enter line +validate Session phase and caller/invocation authority +construct a capability-bound builder with an empty canonical Write[] +run callback; creation/list-append builders call Storage.nextId() synchronously on the line +explicit reads see committed state and must precede the first buffered write +on throw/rejection, discard Write[]; already-minted IDs remain burned in this binding +validate writes/order/references/capabilities using committed state plus private semantic indexes +if nonempty, call Storage.commit(Write[], internalCtx) exactly once +Storage derives/persists its committed ID high-water from created/list-appended objects in that Write[] +receive one storage-assigned Seq per write; returned created IDs become valid after success +apply the complete writes and sequences to all process indexes and views +unconditionally coalesce one session-scheduler kick for a successful main commit +resolve affected observation waiters +leave line +dispatch signals and listeners outside line; the scheduler drain dispatches its reservations outside line +resolve outer commit promise +``` + +Publication never precedes persistence. Every index observes a full batch before scheduling or idleness, +so terminal task + successor has no visible idle gap. Every successfully persisted main commit kicks after +application, even when its writes cannot affect eligibility; coalescing makes redundant kicks harmless. +Scratch and task-output-only commits need not kick because they cannot change task eligibility. + +Private invocation identity contains task ID, method (`execute`, `recover`, `abort`) and a unique object. +The scheduler installs that exact object in a derived Context and binds every task runtime/handle to it. Runtime +admission compares object identity with the invocation slot. Metadata/RPC transport never confers authority. + +Authorization table: + +| phase/method | normal commit | scratch write | terminal closure | abort restricted commit/closure | host admission | +|---|---:|---:|---:|---:|---:| +| open, execute/recover, unmarked | yes | yes | yes | no | yes | +| open, execute/recover, marked | reject before builder | reject before builder | discard without invoke | no | yes | +| open, abort invocation | no | read only | no | yes | yes | +| stopping | no | no | discard | existing/new abort only | reject | +| closing/faulted/closed | no | no | discard | no; close/fault signal it | reject | + +Internal safe admission/`write` authorization and raw direct-entry authorization are separate. Protected built-in +state and entry operations require internal helpers. Main and scratch writes from absent/foreign/stale task +identity reject; task-provided handles never treat them as host calls. + +## 10. Session scheduler, ownership and waits + +A writable Harness owns exactly one process-local scheduler for its Session. Process indexes after open are: + +- the complete live task map across every conversation; +- one invocation slot per running task ID; +- tasks by conversation plus fixed-turn-task and foreground subsets; +- reverse dependency edges; +- conversation parent/owner ancestry needed for capability and cancellation reach; +- point waiters for task, input and foreground-idle observation; +- scheduler enabled/dirty/runner state and Session phase. + +Open constructs these indexes but leaves the scheduler inert. `resume(ctx)` idempotently enables it until +close or fault, installs the drain runner if needed, records an initial coalesced kick, and returns without +waiting for any reservation, task callback or foreground idle. It always covers every conversation. `send`, `InputHandle.wait`, +`waitForTask`, foreground-idle waits, `collapse`, task/input/conversation abort operations, and any other +host/plugin task-creation facade that promises execution ensure resume. Read-only +inspection, view/watch capture, fork creation without input and configuration/state reads do not. + +Every successful main commit unconditionally calls the same coalescing kick after persistence and in-memory +application. A kick while disabled records no execution obligation because a later `resume` scans all +eligibility. While enabled, a kick sets dirty and ensures one drain runner. The drain repeatedly clears dirty, +reserves every currently eligible task on the Session line, and dispatches all reservations outside it. If a +commit races with drain exit, dirty causes another pass. There is no polling, per-conversation worker or lost +kick window. Scratch and task-output-only commits need not kick. + +Dependencies are immutable, acyclic IDs in the same ownership tree. They may reference tasks created +earlier in the same batch, including the current task from its terminal closure. A task becomes eligible +when every dependency is terminal, regardless of outcome. Missing from the complete live map means terminal +after existence was validated. A foreground task depending on endless background work intentionally keeps +foreground idle false. + +Reservation on the line considers all conversations: + +```text +marked live task -> fresh abort, ignoring dependencies +unmarked pending with terminal deps -> commit full running task.set snapshot, reserve execute +unmarked running loaded at open -> reserve recover +``` + +A pending task marked before reservation never becomes running for effects; the kernel writes a complete +running snapshot only as part of reserving fresh abort so `abort` receives `RunningTask`. A task created and +marked in one commit is likewise cleaned by abort without execute. Foreground and background tasks use the +same eligibility rules and both execute. `background:true` only excludes a task from foreground-idle and +ordinary foreground cancellation sets. + +There is at most one invocation per task ID in a process. Its slot remains held through method return, +producer/progress joins and terminal-closure acceptance or discard. The scheduler never awaits one task +before dispatching another task ID, so tasks in the root, forks and owned child conversations may run +concurrently. Callbacks, methods and signals never execute on the line. + +Ownership remains capability and cancellation structure, not execution structure. Foreground cancellation +reach starts with live foreground tasks directly in the requested conversation, recursively enters +conversations owned by those live foreground tasks, and repeats. A background or terminal owner breaks that +reach. Fork history grants neither ownership nor cancellation reach. Queued input is not a task. + +Conversation `waitForIdle` observes that foreground cancellation closure; Harness `waitForIdle` observes all +session foreground tasks. Both ensure resume, return immediately when already idle, or install one +cancellable point waiter. They do not wait for background work and do not alter scheduler scope. Cancelling a +wait removes only that waiter/listener and never cancels durable work. + +`waitForTask` ensures resume, rejects `TaskNotFound` for a missing target, returns a terminal task +immediately, or installs one cancellable point waiter for a live target. Direct self-wait and a wait on a +task whose unresolved dependency path reaches the caller reject. Dependency creation rejects cycles. No +waiter owns execution, and no serving-specific cycle or recursive execution scope exists. + +## 11. Cancellation, close, shutdown and fault + +### 11.1 Task and conversation abort + +`abortTask(id)` and abort-transaction `markTask(id)` reject `TaskNotFound` for a nonexistent ID, return +`terminal` for an existing terminal task, and return `marked` for either an unmarked or already-marked live +task. Each public abort operation first commits its mark or withdrawal, then non-abandoningly ensures the session +scheduler is resumed, so cleanup becomes eligible without changing execution scope. Repeated marks are idempotent and never cancel an already-running fresh +abort invocation. + +Conversation abort commits the mark/queue-withdrawal batch and records those task IDs as its tracked set. +Any mark issued by an abort invocation whose own task is in that tracked set is added transitively to the +same set. The operation observes until every tracked task is terminal and no tracked invocation remains. +Unrelated background work is excluded from the tracked set, although it continues executing normally. +Caller cancellation cannot abandon cleanup after the mark batch is admitted. + +The mark transaction: + +1. Compute the current foreground cancellation closure from the pre-commit live/ownership snapshot. +2. Mark every live task in it. +3. Remove queued steer/followUp in affected conversations and write terminal `unanswered/aborted` results. +4. Preserve queued write. + +When a mark wins: + +```text +commit a full task.set snapshot with abort:true +revoke execute/recover main and scratch writes immediately +leave line; signal old invocation +wait for method, effects and owned progress producers to return +on line discard any normal terminal closure and release old invocation +reserve fresh abort with new identity/controller/Context +abort may perform restricted checkpointed cleanup +abort returns closure +apply passive/direct core writes + full aborted terminal task.set snapshot + scratch retirement atomically +release abort invocation +``` + +Fresh built-in `pi.tool` cleanup reads its durable child records. It marks each child task with +`abortWithTool:true` and live foreground tasks in each owned child conversation with that flag, using atomic +mark-if-live; an already-terminal target counts as cleaned. Children with `abortWithTool:false` are excluded. +It never invokes public child conversation abort and therefore preserves child queues. Local deadlines are not durable marks: their +in-band errors remain kind-specific domain outcomes. Cancellation classification uses Pico's identity and +known reason, never error name alone. + +An uncooperative effect can delay abort indefinitely. Forced isolation is outside v1. + +### 11.2 Close + +Close is a nonpersistent line transition that stops admission and reservation. Earlier admitted line work +finishes; later operations reject except invocation completion. Outside the line, signal and join every +outstanding invocation, then close the backend binding. A durably terminal task has no outstanding invocation because terminal publication follows method return and producer joins. It writes no outcomes or marks. +Caller cancellation cannot abandon close. Repeated close calls share completion. + +### 11.3 Shutdown + +Shutdown enters `stopping` and commits full abort-marked snapshots for every live task in one batch. It +preserves every queued input/result. Its mark commit and cleanup path activate the session scheduler; shutdown +observes until live tasks and invocation slots are empty, then closes. Fresh abort handlers may commit; all normal work/admission rejects. A failing abort faults shutdown. +Caller cancellation cannot abandon an admitted shutdown. Explicit close may interrupt it; shutdown rejects +and durable marks recover later. Repeated shutdown calls share completion. + +### 11.4 Fault + +Unexpected persistence, invariant, capability, closure or task-contract errors fail-stop the session: +reject waiters immediately, stop admission/reservation, signal and join invocations, close storage, and +preserve durable unfinished tasks. Do not manufacture outcomes. Domain provider/tool failures must be +returned as typed outcomes by their kinds. + +## 12. Scratch + +Scratch uses the state vocabulary under `{type:"task", taskId}`. One scratch transaction writes one +live task only. It has value/list get/set/delete/append/remove/clear; reads are async, writes buffered. +Scratch is never rewindable, inherited or part of context. Task scope is private scratch. Shared scope is +separate observable ephemeral state: it may be referenced by several live tasks and is not retained merely +to preserve one task's unrelated scratch. + +Execute/recover may read and write its own scratch while unmarked. Abort may read but not write scratch; +it persists cleanup progress in the task checkpoint or permitted main state. A crash before terminal +outcome retains scratch. The terminal `task.set` snapshot retires it atomically. Every later scratch write rejects, even if a +sidecar unlink failed. + +A retry under one task ID resets attempt-specific live state through `runtime.output.replace` before new +output. Generation stores no second durable frame stream in task scratch: its task output is the sole +partial assistant representation. Chord persists compact append/replace/truncate deltas, not growing +whole-value copies. + +Harness-owned producers own every pending scratch promise, cancel on early exit, suppress only expected +mark/close rejection, report persistence faults and join before the invocation may finalize. Iterator exit +alone is not provider/process completion. Plugin code must await or catch every raw scratch write. + +## 13. Harness open and registries + +The final Harness has one construction API: + +```ts +interface CoreTaskKinds { + readonly generation: CoreTaskKindBase; + readonly tool: CoreTaskKindBase; + readonly postTools: CoreTaskKindBase; + readonly collapse: CoreTaskKindBase; + readonly job: TaskKindBase; +} +interface RootValueWrite { + readonly address: Value; + readonly value: T; +} +interface HarnessOpenOptions { + readonly taskKinds?: readonly TaskKindBase[]; + readonly entryKinds?: readonly EntryKind[]; + readonly rootValues?: readonly RootValueWrite[]; +} +interface OpenInspection { + readonly pending: readonly Task[]; + readonly running: readonly Task[]; + readonly orphaned: readonly Task[]; + readonly unknownEntryKinds: readonly string[]; +} +interface MutableRegistry { + get(kind: string, ctx: Context): Promise; + register(definition: T, ctx: Context): Promise; + replace(definition: T, ctx: Context): Promise; + remove(kind: string, ctx: Context): Promise; +} +interface Harness { + readonly kinds: CoreTaskKinds; + readonly entryKinds: MutableRegistry; + readonly taskKinds: MutableRegistry; + inspect(ctx: Context): Promise; +} +interface HarnessFactory { + open(binding: StorageBinding, options: HarnessOpenOptions, ctx: Context): Promise; +} +``` + +The two `Harness` declarations in this specification merge into one TypeScript interface. `register`, +`replace`, `remove` and `get` serialize on the line for ordinary plugin definitions. Register rejects an +existing name; replace requires an existing ordinary name. Replacing or removing an ordinary task kind +rejects while any live task of that kind exists; with no live instance, replacement affects only future +creations and terminal history remains unchanged. +The fixed core definitions are not members of the mutable registry and cannot be registered, replaced or +removed. Ordinary registration rejects every protected `pi.*` task name. + +Stable core names and roles are: + +```text +generation pi.generation defineCoreTask; turn task; transcript/admission authority +tool pi.tool defineCoreTask; turn task; transcript/admission authority +postTools pi.post_tools defineCoreTask; turn task; transcript/admission authority +collapse pi.collapse defineCoreTask; not a turn task; transcript/admission authority +job pi.job defineTask; not a turn task; ordinary task authority +``` + +Harness installs these exact production definitions. The internal admission strategy is bound to the fixed +generation token. `harness.kinds` exposes readonly typed witnesses for hooks, inspection and internal +composition; possession of a witness does not make it creatable through an ordinary transaction. + +Empty storage means its conversation scan is empty. The root conversation is the first committed object from +a fresh backend, has ID `1`, +has no parent/owner, and can never be deleted. Every `rootValues` address must be a value bound to conversation 1 +(or an authorized built-in address bound there); duplicate addresses reject. Root creation and all root +values commit atomically. Nonempty storage ignores `rootValues`. + +One Session exclusively owns a Storage instance while open; this ownership guard and backend binding +lifecycle sit above the minimal WP2 `Storage` interface, which has no `claim` or `close`. `MemoryStorage` +retains data and may be bound to a later Session after the first closes for recovery tests. JSONL/SQLite +bindings are not reusable and hold a cross-process session lock from backend open through binding close. +Initialization failure releases that binding before rejecting. Two writers fail rather than race. + +Open order: + +1. Finish backend replay/recovery, initialize `Storage.nextId()` from committed creation/list-append writes, + and construct the disabled scheduler. Kicks from later open-time main commits are harmless while disabled. +2. Install the fixed core entry/task definitions, then ordinary plugin options. +3. If storage is empty, create root conversation and apply `rootValues` in one commit. Reopen ignores + `rootValues`. +4. Load the complete live-task seed and required ownership ancestry. Discover stored entry-kind names via + the agreed scans; a backend may satisfy those scans from an index. +5. Validate every live fixed-core task against its installed fixed definition. +6. Reconcile every live ordinary task whose kind is absent in one commit of full task snapshots. +7. Finalize all indexes used by the disabled session scheduler, then return inspection/read handles without + starting work. Registry/open reconciliation is complete before `resume` can dispatch anything. + +Core slots `generation`, `tool`, `post_tools`, `collapse` and `job` always have their built-in implementation. +This ensures internal admission/boundary processing and unavailable-tool results remain available. Ordinary +task kinds may be removed only when no live instance exists. Entry and section definitions may be removed +without deleting durable records. In-flight ordinary invocations/preparations retain captured definitions; +later operations use the new registry. + +Missing-kind reconciliation uses one pre-reconciliation snapshot: + +- Find every pending/running task whose kind is absent, foreground or background. +- For each missing task's owned conversations, recursively mark every registered live foreground descendant + reachable through ownership. Preserve all queued input. +- Missing descendants are themselves orphaned rather than marked for code that does not exist. +- Terminalize each missing task as `orphaned`, retain input/checkpoint/history, remove live/dependency indexes + and retire scratch atomically. +- Dependents observe a terminal dependency and interpret `orphaned`. +- Keep ancestry links for capability and cancellation policy; after `resume`, registered marked descendants + are eligible even though the owner is now terminal. +- Report orphaned IDs from `inspect`. + +If validation, root creation, index loading or reconciliation fails, open closes/releases the supplied +storage binding before rejecting and returns no handle. Registration never resurrects orphaned tasks. + +## 14. Built-in contracts retained for later packages + +These contracts are normative behavior. Packages explicitly gated in section 18 wait for their unsettled +author APIs. + +### 14.1 Generation and post_tools + +Generation owns explicit input IDs and only minimal effect-recovery evidence. Its checkpoint may contain the +preparation/context cutoff, selected built-in model identity, attempt, retry wait, deferred handle and the +specific offered tool-name/version evidence needed for uncertain-response recovery. A request snapshot is +immutable, but it never copies generic plugin configuration or arbitrary conversation state. Built-in model, +thinking, system-prompt and tool selection remain fork-aware conversation state: a fork sees the built-in +configuration at its selected entry and may override it locally. A plugin needing exact recovery persists +its own compact key/version. `before_request` may transform a private messages-only request; actual offered +tools after transformation are the validation basis and must be recoverable only to the extent required by +that hook/effect contract. + +Assistant frames mutate the generation task output; scratch contains only unrelated private recovery data. +Retry and deferred polling loop cooperatively under one stable task ID. +`on_yield` receives the prospective assistant message/draft plus stable task ID, not an entry ID that does +not exist. The returned terminal closure atomically appends assistant/usage, resolves or transfers inputs, +and creates tools/post_tools/continuation. + +For calls, generation creates every tool task and one post_tools depending on them in its closure. Each tool +atomically appends exactly one result entry while terminalizing. Parallel completion therefore controls +transcript append chronology; request projection reorders tool-result messages into the assistant's call +order. Post_tools reads terminal outcomes in call order, handles completed/failed/aborted/orphaned, places the +post-tools boundary after every result exists, and either terminates/handoffs or creates a continuation while +transferring inputs atomically. + +Threshold or overflow creates collapse `C` and replacement generation `G` with `after:[C]`, carrying the +active inputs, in one closure. No invocation waits on collapse. `G` handles every terminal collapse outcome. + +Concrete transaction traces: + +- **Generation:** reservation is one full-snapshot commit. Preparation/checkpoint commits finish before the + provider effect, which runs outside the line; streamed output flushes use their own shared commits. After + the effect and producers finish, the terminal callback performs any committed reads first, then buffers + assistant/usage entries, tool and post_tools task creations, input transitions and the generation terminal + snapshot. The Session calls Storage once for that canonical `Write[]`. +- **post_tools:** reservation is one full-snapshot commit. It reads tool outcomes and their already-committed + result entries before writing. Its terminal callback buffers boundary placement, optional continuation + creation, input ownership transfer and its terminal snapshot; all persist in one Storage call. + +These atomic terminal commits do not absorb external effects. Every durable checkpoint-before-effect or +record-after-effect boundary remains a separate line transaction and Storage commit. + +### 14.2 Collapse + +Collapse captures reason, target prefix ending at a complete exchange, first retained boundary, newest head +and settings. It runs speculatively and may be background/manual or foreground/automatic. It checkpoints a +prepared candidate. Its terminal closure reads the current newest head on the line. A newer head returns +ordinary typed stale failure; intervening edits do not stale. Success appends summary head and completes in +one commit. Concretely, reservation commits a full running snapshot; summarization runs outside the line and +any prepared-candidate checkpoint is a separate commit. The terminal callback reads `newestHead` before its +first write, then buffers the summary append and terminal collapse snapshot in one `Write[]` and one Storage +call. Abort publishes no summary. One live collapse per conversation; simultaneous creation is +serialized and duplicate creation declines or reuses the existing ID according to the calling built-in. + +### 14.3 Tools, jobs and owned conversations + +Tool input includes immutable call and assistant entry ID. Missing implementation, invalid arguments, hook +block and ordinary throw become model-visible error results, not session faults. Recovery retries only with +a durable safe replay policy; otherwise it returns interrupted. Abort publishes an aborted tool result +through `CoreAbortTx` entry authority. A missing executable tool is handled by the built-in tool kind; it is +not a missing task kind. + +A concrete tool trace is: reserve with one full running-snapshot commit; commit any replay-policy checkpoint +before invoking the external tool; run that effect outside the line; then invoke the terminal closure. The +closure performs committed reads first and buffers its one tool-result append, final output-derived data and +full terminal task snapshot. Those writes are one canonical `Write[]` and one Storage call. Abort uses the +same one-commit final shape after its separately joined external cleanup. + +The fixed `pi.tool` `CoreTaskKind` owns `CoreTaskRuntime.commit` and its transaction capabilities. An +ordinary `ToolDefinition` receives neither. Its eventual adapter is a small `pi.tool`-mediated capability over the +current invocation's `TaskOutput`, durable scoped memos and keyed child creation; exact public names and +TypeScript types remain gated in section 18 and must not be inferred from generic task runtime. + +The later ordinary-tool bridge must preserve these requirements without exposing general commit: + +- Its invocation ID is the stable `pi.tool` task ID. Its output is the task's `TaskOutput`. Durable + strict-JSON memos are protected state scoped to that task and retire with it. +- A keyed child-task operation, if included, uses one internal parent-tool transaction to create the child + and write its ID, kind, `abortWithTool: boolean` and optional shared-output ref into the complete parent + checkpoint before returning. Jobs remain tasks. The exact task-spawn API and checkpoint schema are gated. +- A keyed subagent operation uses one internal parent-tool transaction to call `tx.createConversation`, + write the selected initial values, call the internal `tx.accept` helper to create the first generation, and record the child + conversation ID, input ID, request key and `abortWithTool: boolean` in the complete parent checkpoint. + After commit, the adapter resolves the existing restricted conversation handle and waits outside the + line. A subagent is an owned conversation, never a subagent task. +- `abortWithTool:false` means only that enclosing `pi.tool` abort does not mark that child. It is not a + generic detached-ownership abstraction. Parent normal completion cancels neither + value. +- A repeated creation key must either be explicitly first-key-wins or compare a persisted canonical spec. + Exact-match replay must persist normalized parent position, input, request key, initial values, protected + seed profile/version and `abortWithTool`; it must not compare mutable current values. The final choice and + record types remain gated. +- Returned task handles expose wait/abort, not commit. If local timeout is supported, wait accepts an + explicit timeout and returns `undefined` when observation times out without marking the child. Terminal, + timeout, caller cancellation, mark, close and fault races remove waiter/timer/listener state exactly once. + The exact handle type remains gated. +- Returned conversation handles reuse the existing restricted `TaskConversation` capabilities as the + ceiling: scoped `send` plus returned input result/wait/abort, and no raw commit/direct entry/internal + admission helpers/close/shutdown/unrelated state. Any model-facing status/tail projection must be specified before adding it. +- Later sibling tool calls need one narrow built-in resolver for strict owned descendants of their current + parent conversation, including ownership through terminal sibling tasks. Fork-only ancestry and unrelated + roots reject. This future resolver must not broaden generic `runtime.conversation`. + +The `pi.tool` adapter, not ordinary tool code, resolves timeout/background publication. An ordinary tool may +return a child ID after local observation timeout, but the `pi.tool` terminal closure rereads that child on +the commit line. A terminal child produces the normal final result; a live child produces the +continues-in-background result while its task retains any shared output. No design may trust stale +`Promise.race` state. + +Jobs are job-first: the durable job owns execution/output from the first effect. Ordinary tools will create +them through the gated mediated task operation, not a specialized `startShell` verb and not by adopting an +unfinished promise. A non-adoptable interrupted process is lost unless durable policy explicitly permits +rerun. Recurring schedules use one task ID and named checkpoint phases; no backlog is inferred after +downtime. Terminal notices are passive writes and work in either notification-before-completion order. + +#### Shared task output + +Task output is the only live-output mechanism; there is no separate preview subsystem. The exact declaration, +reference and facade types are in sections 5 and 7. A task kind declares `TaskOutputSpec`. Creation either +uses a compatible `TaskOutputRef` or synchronously computes a new base with `initial(input)`. Runtime +compatibility compares `TaskOutputKind.kind`, not token object identity or structural TypeScript compatibility. + +Output is a protected list at a fixed shared address: + +```ts +type TaskOutputDelta = readonly Op[]; +const taskOutputDeltas = defineList( + { type: "shared", id: taskOutputId }, + "pi.output", +); +``` + +Persist decoded Chord `Op[]`, not `WireOp[]`, in v1. The first list element is exactly the initial tracker +flush and starts with one `r` base. Every later element is exactly one nonempty flush from the single central +tracker, preserved as one atomic append. Replay folds the elements with `applyImmutable()` into a mutable +strict-JSON tree and then hydrates exactly one authoritative tracker. Tasks receive controlled +`TaskOutput`, never trackers, proxies, raw protected addresses, or independent mutable replicas. After +hydrating a tracker from replayed state, the kernel consumes its synthetic initial `flush()` locally to +establish the baseline; it does not persist that synthetic base. The first later mutation therefore remains +incremental unless the author explicitly calls `replace`. + +Several live tasks may reference and write the same task output. There is no mutable owner field or transfer +event. Retirement derives from prospective post-batch live `Task.output` references. The output remains while +at least one exists, retires atomically after the final reference terminalizes, and can never be resurrected. +Private task scratch remains independent. + +Tool timeout uses one stable task output: tool T creates O and background job B is created with T's +`TaskOutputRef`. B writes progress directly. T's terminal closure rereads B on the line. If B is live, T +materializes the immutable "continues as B" result and terminalizes while B retains O. If B already won, T +takes the normal completed path. B materializes final O before terminalizing as the last reference. Never +trust stale `Promise.race` state. Marking either task revokes that invocation's mutable output facade before +its signal; fresh abort can only read. + +Generation uses the same mechanism with an object wrapper around its `AssistantMessage`; frames mutate the +central task output and Chord emits structured deltas for text, thinking, signatures, tool calls/partial +arguments, metadata, usage and diagnostics. Generation materializes the immutable assistant entry before +its final output reference retires. Tool-specific APIs, if useful, are convenience facades over +`TaskOutput`, not a second storage or writer abstraction. + +Each protected output-list append mechanically emits one `task_output` event keyed by stable task-output ID +and carrying that exact decoded delta. `ConversationView.taskOutputs` folds deltas with `applyImmutable()`. +In-process fan-out gives each reducer immutable application rather than sharing a mutable decoded batch. If +wire path encoding is added later, each independently hydrated stream owns one encoder/decoder. + +### 14.4 System sections + +Pico sends pi-ai `{messages}` only; top-level `systemPrompt` and `tools` are absent. A typed +`SystemSection` has stable key and synchronous pure renderer. Preparation seeds one ordered draft from +durable payload/rendered state. Handlers run sequentially outer-to-inner and mutate get/set/delete/wrap; +failed skipped handler rolls back only its mutations. Missing contributors do not delete stored sections. + +Baseline records contain complete ordered state; deltas contain set/remove changes. Canonical data folding +is independent of model heads/edits. Payload-only change may have `model:[]`. Tool definitions exist only in +SystemMessage fields; removals precede additions and same-name additions upsert. A fresh post-head baseline +atomically omits retained superseded managed entries. Preparation captures section/registry snapshots, +retries managed-state staleness, and commits system entry plus request cutoff atomically. + +### 14.5 Hooks and external events + +Hooks are the preferred extension point inside built-in operations. They run outside the line, receive and +forward Context, may run again after crash, and return typed decisions that the built-in commits. A hook may +start and await ordinary durable side work while the built-in retains exchange ownership. Approval/question +policy and durable answer reuse belong to workspace/plugins. Exact namespaced hook scratch access remains a +gated API decision. + +A long-lived ordinary background listener injects user events through request-keyed `send`; a trusted +built-in may use the internal request-keyed `write` helper for passive transcript events. It may await the +returned `InputHandle` outside a transaction. It never mutates an already snapshotted provider request; +steering affects the next safe request boundary. + +## 15. Watch and backend contracts + +### 15.1 Watch + +```ts +interface WatchedValue { readonly address: Address; readonly value?: JsonValue } +interface WatchedTaskOutput { + readonly id: Id; + readonly kind: string; + readonly value: JsonObject | readonly JsonValue[]; +} +interface ConversationView { + readonly conversation: Conversation; + readonly tail: number; + readonly entries: readonly Entry[]; + readonly context: readonly Id[]; + readonly tasks: readonly Task[]; + readonly taskOutputs: readonly WatchedTaskOutput[]; + readonly inbox: readonly Element[]; + readonly values: readonly WatchedValue[]; + readonly readAt: Seq; +} +interface SessionView { + readonly conversations: readonly Conversation[]; + readonly values: readonly WatchedValue[]; + readonly readAt: Seq; +} +type InboxOp = + | { readonly type: "append"; readonly item: Element } + | { readonly type: "remove"; readonly id: Id } + | { readonly type: "clear" }; +type ConversationEvent = + | { readonly type: "entry"; readonly entry: Entry } + | { readonly type: "task_start"; readonly task: Task } + | { readonly type: "task_update"; readonly task: Task; readonly previous: Task } + | { readonly type: "task_end"; readonly task: Task & { readonly status: "terminal" } } + | { readonly type: "task_output"; readonly id: Id; readonly kind: string; + readonly delta: readonly Op[] } + | { readonly type: "value"; readonly value: WatchedValue } + | { readonly type: "inbox"; readonly ops: readonly InboxOp[] } + | { readonly type: "context"; readonly ids: readonly Id[] }; +type SessionEvent = + | { readonly type: "conversation"; readonly conversation: Conversation; readonly change: "created" } + | { readonly type: "value"; readonly value: WatchedValue }; +interface CommitEnvelope { + readonly first: Seq; + readonly last: Seq; + readonly events: readonly E[]; +} +type WatchDelivery = + | { readonly type: "commit"; readonly commit: CommitEnvelope } + | { readonly type: "closed"; readonly reason: "overflow" | "session" }; +interface WatchOptions { + readonly capacity?: number; + readonly onError?: (error: unknown) => void; +} +interface ConversationWatchOptions extends WatchOptions { + readonly tail: number; + readonly values?: readonly Address[]; +} +interface SessionWatchOptions extends WatchOptions { + readonly values?: readonly Address[]; +} +interface WatchHandle { + readonly view: V; + start(listener: (delivery: WatchDelivery) => void): void; + unsubscribe(): void; +} +declare function applyConversationCommit( + view: ConversationView, + commit: CommitEnvelope, +): ConversationView; +declare function applySessionCommit( + view: SessionView, + commit: CommitEnvelope, +): SessionView; +interface WatchService { + watchConversation( + conversationId: Id, + options: ConversationWatchOptions, + ctx: Context, + ): Promise>; + watchSession( + options: SessionWatchOptions, + ctx: Context, + ): Promise>; +} +``` + +`tail` is an integer from 0 through 10,000. A conversation capture contains its last `tail` logical +fork-visible entries, live tasks directly belonging to that conversation, every distinct task output they +reference, current inbox and exactly the requested values. Conversation watch values may be session addresses or addresses bound to that +conversation; foreign-conversation/task addresses reject. A session watch accepts session addresses only. +A later source-conversation write emits no event for an existing fork. The pure reducer uses `view.tail` to +retain only the newest logical transcript entries. A session capture contains all current conversations and +exactly its requested session values. Rendering uses settled entries, live tasks, task outputs, inbox and +selected values from this one view. A boundary commit removes an inbox element, appends its entry, updates +the protected input result and creates any successor task atomically; its visible entry/inbox/task changes +arrive in one envelope, so clients reconcile stable IDs without a transient duplicate or idle gap. + +Capture and subscription registration occur in one line operation. Default capacity is 256 complete commit +envelopes; an explicit capacity must be a positive integer. `start` is synchronous and may be called once. +It delivers commits after the captured storage `Seq` in `readAt`, then live commits, in sequence order. The +view is folded through a whole envelope before its listener runs. Events from one commit are never split. +Same-commit inbox append/remove cancels and emits no inbox event. A commit producing no events after this +subscription's filters is not delivered and consumes no buffer capacity, while the in-process +`handle.view.readAt` still advances. Emit a full `context` event whenever the derived context-ID array +changes, including ordinary entry appends. + +When capacity would be exceeded, discard queued commits and close that watch with reason `overflow`; it +never pauses or maintains a hidden advancing view. The consumer reconnects by opening a new watch, whose +atomic capture is the new authoritative state and sequence barrier. There is no resnapshot/resumption API. +`unsubscribe` is idempotent, discards buffered deliveries and prevents future callbacks. If a listener +throws, close only that watch without another listener call and invoke `options.onError` outside the line; +never fault the session. An error thrown by `onError` is ignored. Session close delivers `closed/session` once +unless unsubscribed. + +`task_start` is `task.create` at pending. Every `task.set` replacement is derived against its previous full +snapshot: reservation and checkpoint/abort/ownership changes emit `task_update`; a terminal replacement emits +`task_end` and always includes an outcome. Each protected shared append +emits one `task_output`; the reducer applies its delta with `applyImmutable()`. Envelope order places output +before a same-batch final `task_end`, after which the reducer removes an output with no remaining live task +reference. Sharing requires no separate event. Reducers are kind-free and pure. Conversation values are only the explicitly requested addresses; session values are explicitly +configured by the session watch caller in the final Harness API. + +### 15.2 Backend construction and durability + +```ts +interface StorageBinding { + readonly storage: Storage; + close(ctx: Context): Promise; +} +interface MemoryStorageFactory { create(): StorageBinding } +interface JsonlStorageFactory { open(path: string, ctx: Context): Promise } +interface SqliteStorageFactory { + open(path: string, options: { readonly session: string }, ctx: Context): Promise; +} +``` + +JSONL/SQLite `open` acquires the cross-process session lock and finishes replay before returning the +exclusive binding. Binding `close` releases resources and the lock and is idempotent. Memory has no +cross-process lock but its binding still enforces one open Session at a time. Binding lifecycle is outside +the minimal mechanical `Storage` interface. + +Every JSONL main, live-scratch and live-shared commit is fsynced before in-memory publication or commit +resolution. Creating the main file, a sidecar or a containing directory requires fsync of the created file +and each affected parent directory before publication. Sidecar unlink need not be directory-durable because +the fsynced main record proving retirement is authoritative. No weaker durability mode exists in v1. + +JSONL uses one main file, one file per live task scope, and an optional file per live shared task output that +has post-creation deltas, all sharing the sequence. Task creation and its initial output base exist only in +the atomic main record; creating an output does not require a shared sidecar. Each newline-terminated record +is: + +```ts +interface JsonlRecord { + readonly first: Seq; + readonly last: Seq; + readonly writes: readonly Write[]; +} +``` + +`last - first + 1` equals `writes.length`. Replay initializes the next-ID allocator after the greatest +conversation, entry, task or list-element ID carried by retained canonical writes. Creation and element IDs +are validated as Storage-minted identities and need not match any sequence. +Replay main first, including each live output's initial protected base append. From its final live tasks, +derive the live scratch IDs and distinct live task-output refs; then replay only their existing sidecars as +later deltas. A missing shared sidecar means no post-creation write. Retired scratch/shared files are ignored +even if malformed, so a stale file cannot resurrect state. Every retained shared record contains only +protected shared-address writes, and replay folds it after the base already recovered from main. Retained sequence ranges increase within each file and never overlap +across retained files; gaps are valid. Validate file scope, task/address references, object IDs, output refs +and lifecycle/scope structure. Reopen derives the next-ID allocator from all complete retained +creation/list-append writes; the Session's last applied `Seq` is the maximum complete retained endpoint +including clear/remove. + +A torn suffix is bytes after the final newline. Truncate those bytes and fsync before append. A +newline-terminated record that is malformed JSON or structurally invalid fails open. Persist and fsync the +record before applying it to memory. + +SQLite uses indexed conversations, full immutable entry facets, current tasks, values plus rewindable +versions, list elements/clear markers, task scratch, shared task-output refs/deltas and commit boundaries. Index +task status/abort/conversation/kind/output ID and entry conversation/head/kind. Terminal transaction deletes scratch rows atomically. Reopen +loads live tasks and named owner records, not terminal history into residency. Version mismatch rejects. +Use durable SQLite transactions with `PRAGMA synchronous=FULL` or an explicitly documented equivalent at +least as strong. Journal mode is backend-private if locking, atomic scratch retirement and conformance hold. +There is no migration API until the gated migration package is specified. + +### 15.3 Cloudflare Durable Object host guidance + +Pico scheduling and platform wake-up are separate. One Durable Object should naturally host one Pico +Session, one open Harness and its one scheduler, covering the root and every fork/owned conversation. +Cloudflare alarms or one logical Cloudflare Task may wake an evicted process; they do not become a second +effect-recovery authority and must not create one lane or scheduler per conversation. + +```text +request wake -> open Pico -> send() durably admits -> send() ensures resume -> scheduler runs +alarm wake -> open Pico -> resume() -> all eligible tasks in all conversations run/recover +``` + +The merged Cloudflare Agents lane example uses explicit per-lane admission/execution passes because that +lane runtime requires them; Pico must not copy that shape. If Pico admission and the platform wake cannot share +one transaction, an adapter needs a small ordered handoff or idempotent retry rule so acknowledged durable +work can always wake another process. Cloudflare's invocation limit still bounds one uninterrupted external +effect; Pico checkpoints and `recover()` remain authoritative across wakes. + +## 16. Complete foundation race matrix + +Every race is tested in both orders with fake clocks/effects and storage barriers: + +- Persistence paused after construction: readers see old complete state. +- Commit callback throws after creating objects: no Storage commit or writes; minted IDs remain burned in + that open binding but may be reused after reopen because they were never committed. +- Explicit read before first write succeeds against committed state; explicit read after a buffered write + rejects `ReadAfterWrite` without persistence. +- State-only writes advance `Seq` without calling `nextId`; a later creation proves object ID and write + sequence are independent. Reopen initializes the next-ID allocator from that committed creation. +- Send durable but reply lost: request-key retry and lookup return the original `InputHandle`. +- Duplicate request key in one transaction or through another conversation: session-wide first receipt wins + without payload comparison and no second inbox item/task is created. +- Idle send atomically places the input, records result/receipt and creates exactly one generation. +- Busy send atomically queues followUp by default, records result/receipt and creates no task. +- Main plus scratch in one batch: reject all. +- Effect returns versus abort mark: one invocation; fresh abort only after old return. +- Post-mark main/scratch builder: reject before callback. +- Normal closure versus mark/close/fault: commit before mark wins; otherwise closure is not invoked. +- Closure throws: no closure writes/outcome; session faults. +- Terminal persistence uncertain: fail-stop; closure not retried on handle. +- Pending task marked before reservation: execute never runs; fresh abort does. +- Repeated abort mark while abort runs: abort Context remains active. +- Close versus commit: admitted earlier commit finishes; later mutation rejects; completion messages admitted. +- Shutdown versus send: send commits before the mark batch or rejects; queues are preserved. +- Shutdown crash before child cleanup: reopen plus resume runs fresh aborts and preserves child queues. +- Broken abort handler: idle waits/shutdown reject; invocations join; Session closes with the task durable. +- Open invokes no task callback; one resume dispatches all eligible restored tasks in every conversation. +- Resume versus main commit in both orders: no eligible task is missed and no task is invoked twice. +- Main commit while a drain exits: dirty/kick handoff cannot lose work; many commits coalesce safely. +- Send, input wait and abort on an inert open each ensure one session-wide resume. +- Tasks in different conversations run concurrently; foreground and background both run; background + continues after foreground idle. +- Task abort preserves queued future input unless the explicit conversation policy withdraws it. +- Generation terminalization atomically creates all tool tasks and their one post_tools dependent. +- Each tool terminalization atomically appends its one result entry; parallel completion may determine + transcript order, while model projection restores assistant call order. Post_tools starts once after every + tool is terminal and atomically places its boundary and creates its continuation. +- Final-answer terminalization atomically resolves the active input group and creates the selected follow-up + successor generation. +- Cancelled task/input/idle waiter removes only that waiter/listener and never affects execution. +- Dependency terminal versus dependent reservation: dependent starts once. +- Direct self-wait and dependency cycles reject before durable deadlock. +- Background collapse plus idle send: generation may start concurrently because collapse is not in the fixed turn-task set. +- Collapse concurrent with generation plus listener send: generation boundaries retain inbox authority. +- Collapse abort with queued input: no successor; queue remains for a later idle send. +- FollowUp send versus final answer: queued then consumed, or consumed by the boundary transaction; never + lost or duplicated. Steering versus post_tools is tested in both orders. +- Queued write versus post_tools preserves the complete assistant/tool-result block. +- Queued head versus newer head: placement writes unanswered/stale, not session fault. +- Parallel tool completion either order: post_tools starts once and projects call order. +- Abort versus post_tools closure: no unmarked successor and every active input resolves once. +- Withdraw versus place input: one terminal result. +- Several inputs in one group: all done results name the same answer. +- Overflow chain: collapse and replacement atomic; no invocation waits on collapse. +- Competing summary head: stale failure; intervening edits do not stale. +- Watch capture versus commit: base includes commit or stream delivers it, never gap/duplicate. +- One boundary envelope contains inbox removal, entry placement, result update and successor creation without + a transient duplicate or idle gap. +- A fork sees selected historical/config state, inherits no tasks or inbox, and executes independently after + its first send. +- Watch overflow/listener throw: that watch closes; a newly opened watch captures one fresh authoritative + base without a resumption protocol. +- JSONL main through Seq 100/live scratch through Seq 150: reopen at applied Seq 150 and initialize + `nextId()` after the greatest created/list-appended ID in all retained complete records. +- Terminal Seq 151/unlink failure: ignore scratch even malformed; applied sequence remains 151 and IDs are + never reused. +- JSONL torn final suffix versus malformed complete line: truncate only former. +- 100,000 terminal children/two live tasks: open reads live seed and required ancestry only. +- Missing owner kind with registered descendants: owner orphaned and scratch retired; descendants marked, + queues retained, later session resume runs their fresh abort. +- Ordinary task-kind replace/remove versus a live instance: reject without registry change; after every + instance is terminal, replacement affects only future creations. +- Ordinary runtime/transaction has no direct-entry or core-task-creation surface; casts add no implementation. +- New task plus initial task-output base: both persist or neither; `initial` is not called for a supplied ref. +- Two tasks mutate one task output: central tracker commits deltas in line order without a lost update; a + permitted shared `Write[]` with multiple authorized appends preserves every append atomically. +- Mark versus task-output mutation: commit before mark wins; otherwise mutation rejects before callback. +- Last reference terminal versus same-batch child sharing: prospective references retain output; otherwise + final materialization and retirement are atomic. +- Task-output delta versus watch capture/overflow: base or exact ordered delta, never gap/duplicate/mutable + alias. +- Shared sidecar after terminal/unlink failure: reopen ignores it; no resurrection. +- State adapter live transitions through several phases: scheduler calls generated `execute` once; each next + handler receives the authoritative post-commit checkpoint/owns projection. +- Initial child creation versus crash: no transition commit means initial reruns with no child; committed + child plus first checkpoint dispatches the next phase without duplication. +- Crash after an inflight checkpoint versus a start checkpoint: generated `recover` dispatches required + inflight recovery versus normal start run, once per restored invocation. +- Same-phase bookkeeping commit versus abort: child/reference and complete parent checkpoint both commit + before the wait or neither does; post-mark commit rejects. +- State transition versus abort mark: transition-first may admit the next fenced handler; mark-first prevents + that handler call and unwinds the outer invocation before fresh abort. +- Known runtime/handle line reentry rejects `NestedLineOperation` before queueing. + +## 17. Ready implementation packages + +Every package is a separate commit. Before each commit run its focused test and `npm run check`; do not run +the full credential-sensitive suite. Each package may use an internal kernel with supplied test kinds and +memory storage; it need not expose incomplete production Harness behavior. + +### WP1 — Core types and kind witnesses + +Prerequisites: none. + +Deliver: +- strict JSON, distinct `Id`/`Seq`, entry, conversation, task/checkpoint/outcome and address types, including + background mode and non-rewindable shared scope identity; +- `EntryKind`, ordinary `TaskKind`, fixed-core `CoreTaskKind`, `TaskOutputKind`, typed/erased output refs and + payload extractors; +- ordinary versus core runtime/transaction/output/closure declaration types, with method bodies deferred; +- no imports from existing harness implementation. + +Accept: +- compile tests for exact ordinary/core task input, full checkpoint with phase, kind-specific outcomes and + no-output/owned-output/compatible-shared-output surfaces; +- ordinary normal/final/abort transactions have no direct entry or core creation; core variants do; +- `CoreTaskTx.task` accepts ordinary and core children through the same method name; +- incompatible checkpoint shape rejects; structurally identical shape is documented assignable; +- Pico declarations introduce and explicitly spell no `any`; imported pi-ai types are exempt; +- compile-only focused test and repository check green. + +### WP2 — Mutation algebra and MemoryStorage + +Prerequisites: WP1. + +Deliver: +- exact canonical `Write` algebra and the minimal Storage interface, with no claim/close/kind-name methods; +- mechanical application of canonical writes and full `task.create`/`task.set` records; +- synchronous `Storage.nextId()`, storage-assigned write sequences, committed next-ID recovery from canonical + creation/list-append writes, and empty-write behavior; +- conversations, entries, complete task snapshots and terminal scratch/shared retirement; +- current and rewindable values/lists including tombstones/clear/remove; +- ascending conversation scans, fork-aware newest-first entry scans, dedicated newest head, ascending task + scans, complete ordered list reads and the exact cursor/filter rules in section 6.2. + +Accept: +- hand-built `Write[]` conformance for every query and write; +- returned `Seq[]` aligns one-for-one with writes and demonstrably differs from Session object IDs; +- rejected storage commit leaves durable state and internal sequence unchanged while already-minted IDs stay + burned in the open binding; +- a terminal task snapshot makes scratch unreadable atomically and retires unreferenced shared output; +- fork/history/deep-cap tests; scans have no `readAt`; complete list reads are coherent at one state; callers + honor the readonly durable-value contract; +- focused test and repository check green. + +### WP3 — Commit line and raw transaction kernel + +Prerequisites: WP2. + +Deliver: +- FIFO async Session line serializing calls to synchronous `Storage.nextId()`; +- synchronous buffering into one canonical `Write[]`; explicit async committed reads only before first write; +- process-local ID burning on failed callbacks and the rewindable-before-entry rule; +- ordinary/core/abort transaction construction and capability separation; +- narrow pending/reference indexes for semantic helpers, with no general overlay or read-your-writes; +- atomic task-output base creation, compatible reference admission and protected namespace/kind checks; +- no public send/InputHandle/runtime handles yet; admission helpers remain internal. + +Accept: +- concurrent callbacks serialize; an allowed async read holds the line; read after first write rejects + `ReadAfterWrite`; +- creation/list-append builders call only synchronous `Storage.nextId`; callback failure burns those IDs in + the binding, and callback success invokes `Storage.commit` exactly once; +- reopen may reuse never-committed IDs but starts after every committed created/list-appended ID; object IDs + and returned storage sequences are distinct; +- same-callback references validate only through private semantic indexes and are not author-readable; +- an ordinary transaction cannot instantiate a core token; casting adds no direct-entry implementation; + typed data-only host entry works; +- managed/protected generic writes reject; +- no callback/signal dispatch on line. + +### WP4 — Entries, context and forks + +Prerequisites: WP3. + +Deliver: +- typed entry builders and built-in entry witnesses; +- direct append structural validation, head/self/edit/exchange rules; +- exact fork-aware context derivation and disposable cache/snapshot rules; +- missing-result and call-order normalization adapter boundary using faux messages. + +Accept: +- facet combinations, unknown kinds, heads monotonic and exchange-safe; +- summary-under-running-turn trace; repeated heads; edit winner/no-op/protected targets; +- arbitrary fork cutoff and incomplete exchange repair without tasks; +- error/aborted assistant exclusion; immutable request snapshot. + +### WP5 — Input/admission kernel + +Prerequisites: WP4. + +Deliver: +- queued/input-result/acceptance schemas and protected addresses, including conversation-scoped sticky inbox + identity/current materialization and direct protected result/receipt point lookup; +- fixed turn-task classification by the live `pi.generation`/`pi.tool`/`pi.post_tools` set directly in each + conversation, evaluated prospectively over complete batches; no persisted busy flag or payload inspection; +- protected core-task creation unavailable to ordinary task/host transactions; +- internal async in-transaction accept/queue/write with committed pre-write lookup and private pending-key + dedupe, moved behind the protected admission capability; +- `SendInput`/`InputHandle` declarations and an internal/test send facade; final public Harness export remains + gated, and scheduler-backed wait/activation integration belongs to WP7; +- safe enqueue and placement revalidation; +- post-tools/final/idle boundary transaction helpers; +- pure prospective fixed-turn-set idle-boundary planner, without provider execution or payload inspection. + +Accept: +- complete followUp/steer/write mode table; idle send same-batch append/remove; busy send queues without task + creation; duplicate keys including same batch and cross-conversation retry; +- inbox reads one complete current materialized ordered collection without per-read log replay; forks inherit + no inbox; result/receipt reads perform no list scan; +- independent `steeringMode`/`followUpMode` all-versus-oldest selection, default one-at-a-time, preserves + global inbox order and unselected relative order; no v1 queue/drain bound; +- ordinary tasks never affect admission busy; core task creation is accepted only from protected admission or + `CoreTaskTx`, and generation may instantiate fixed tool/post_tools children through `tx.task`; +- create conversation + internal accept through the private fixed-generation strategy + parent checkpoint atomically; +- speculative collapse does not make admission busy; concurrent collapse/generation traces; +- stale queued head terminal result; no fault; +- queued ownership never inferred from arbitrary checkpoints. + +### WP6 — Terminal-closure task kernel and scratch + +Prerequisites: WP5. + +Deliver: +- pending `task.create` and durable full-snapshot running reservation; +- one central Chord tracker per live task output plus controlled read/mutate/replace facade; +- ordinary/core capability-specific execute/recover runtimes and normal terminal closures; +- invocation identity/slot; atomic completed/failed outcome plus scratch retirement; +- scratch transactions and normal closure seal/disposition behavior; +- focused scheduler fixtures using typed test kinds while activation remains deferred to WP7. + +Accept: +- running intent precedes effect; reopen chooses recover including no checkpoint; +- repeated recovery from newer checkpoint under stable ID; +- one closure outcome, same-commit Storage-minted result ID, throwing closure rollback/fault; +- normal closure commits exactly once when still authorized; WP8 adds mark/close/fault discard races; +- scratch crash retention/terminal retirement/post-retirement rejection; +- shared output concurrent writes, final closure read/materialization and reference-derived retirement. + +### WP7 — Session scheduler, dependencies and waits + +Prerequisites: WP6. + +Deliver: +- disabled-scheduler bootstrap plus loading of the complete live-task/dependency seed needed by an internal + reopen/test factory; final binding lifecycle and Harness integration remain in WP8; +- one inert-until-resumed scheduler over that complete Session live/dependency index; +- idempotent `resume`, unconditional coalesced post-main-commit kick and no-lost-dirty drain handoff; +- foreground-idle accounting while foreground and background tasks both execute; +- task/input/idle point waiters that observe only and ensure resume when progress-seeking; +- task dependency-cycle and direct/dependency self-wait checks, with no execution ownership scopes. + +Accept: +- open invokes nothing; resume dispatches every eligible task across all conversations; +- dependencies mean terminal and wake once; terminal/dependency commits kick successors; +- distinct task IDs, including root/fork tasks, run concurrently; one task ID has one invocation; +- resume/main-commit and commit/drain-exit races lose no kick and duplicate no invocation; +- many kicks coalesce while all eligible tasks reserve; background continues after foreground idle; +- input result/missing/terminal-fast-path/caller-cancel and task/idle waiter cleanup have no leak; +- send and input wait on an inert reopen resume the Session; abort activation and close/fault/shutdown waiter + behavior are tested in WP8. + +### WP8 — Cancellation and lifecycle + +Prerequisites: WP7. + +Deliver: +- mark/revoke (including mutable task output)/signal/join/fresh-abort sequence and abort runtime construction; +- restricted repeated ordinary/core abort commits and mark/close/fault normal-closure discard; +- conversation abort operation, completion wait and queue policy; +- close, shutdown and fault phase machines with shared repeated completions; +- integration of the WP7 disabled-scheduler/live-task bootstrap into final Harness initialization, including + exclusive Storage binding, required ownership ancestry, task-registry capture and missing-kind orphan + reconciliation; any loading/validation/reconciliation failure closes the binding before rejecting. + +Accept: +- every cancellation/lifecycle row in section 16 applicable to memory; +- fresh abort checkpoint/reopen with read-only task output; repeated mark does not cancel abort; +- core passive abort writes but no future work; core tool direct abort result; ordinary abort has neither; +- close writes no outcomes; shutdown marks all and preserves queues; fault preserves unfinished state; +- task/input/conversation abort on an inert Harness commits its durable request, resumes the Session and + completes the specified cleanup without changing scheduler scope; +- initialization starts nothing; every missing live ordinary kind orphans, retires scratch and marks + descendants; ordinary task replace/remove rejects while that kind has a live instance; +- failed reconciliation yields no usable Session; no history scan beyond live seed/required ancestry. + +### WP8A — State-indexed task authoring adapter + +Prerequisites: WP8. + +Deliver: +- exact `defineStateTask` declarations, action constructors and compile tests from section 5.1; +- generated ordinary `TaskKind.execute`/`recover` dispatch loop with no scheduler changes; +- module-private transition commit bridge returning the authoritative applied task projection; +- initial/start/inflight role dispatch, same-phase-only checkpoint-backed commits, and a cancellation fence + before every internal handler call; +- no built-in generation/tool/provider implementation. + +Accept: +- broad-string checkpoints reject; phase maps and handlers reject missing/extra literal, variable and spread + fields; each handler receives its narrowed complete checkpoint variant; +- inflight requires recover; start forbids it; initial has no checkpoint and no commit; +- transition payloads reject missing/extra literal, variable, spread and async-return fields while preserving + output/terminal-closure typing; +- initial child creation and first checkpoint are one transition batch in crash tests; checkpoint-backed + same-phase child creation commits before wait; direct intermediate phase change rejects; +- one scheduler execute across multiple live transitions; reopen dispatch matrix for absent/start/inflight; +- transition result includes checkpoint/owns from the applied projection with no fabricated task or empty + follow-up commit; +- transition-versus-mark passes in both orders: a winning mark prevents the next handler call; +- no workflow history, positional replay, generator, one-plan-at-end rule or specialized effect verbs. + +### WP9 — Watch foundation + +Prerequisites: WP8. + +Deliver the kind-free commit-derived view/event reducer, including `taskOutputs`/`task_output`, atomic +capture/subscription, bounded overflow closure and reconnect-by-new-capture. There is no lag/resnapshot or +preview subsystem. + +Accept the watch races and task lifecycle event meanings in sections 15–16. + +### WP10 — JSONL backend + +Prerequisites: WP8; WP9 only for cross-backend watch tests. + +Deliver the backend and all recovery rules in section 15. Run the same storage/admission/task conformance +stream as memory, including real process-kill recovery for running/checkpoint/scratch/shared-output/terminal +cases and no-resurrection sidecar cleanup. + +### WP11 — SQLite backend + +Prerequisites: WP8; WP9 only for cross-backend watch tests. + +Deliver indexed persistence/residency, shared task-output persistence/retirement and conformance in section 15. Update the session-backend package only +through a reviewed adapter boundary; no import from existing harness runtime/session implementations. + +## 18. Gated packages and explicit decisions + +Do not implement these until their listed decision is settled and appended to this specification: + +- **Harness open/public handles/fixed core wiring:** wire the single final `Harness` only when the required + built-in kinds are implemented. It owns root/rootValues, public ordinary registries, fixed core kinds, the + one session scheduler, `resume`, `send`/`InputHandle`, idle waits and the production generation-admission + strategy. There is no temporary foundation harness and no public accept/queue/drive composition. +- **Provider generation/system integration:** verify landed pi-ai messages-only behavior and define complete + generation input/checkpoint/result/failure/abort schemas, retry/usage dedupe and deferred recovery table. +- **Tool/post_tools:** define the complete built-in payload/checkpoint/outcome schemas, bounded/spill policy + and model-visible projection on settled `TaskOutput`. Specify and review the small ordinary-tool mediated + memo/task/conversation capabilities in section 14.3; `ToolDefinition` code never receives task commit + authority. Job-first only; no arbitrary promise adoption. +- **Hooks:** define task identity and namespaced scratch capability, then confirm hook points/typed decisions. +- **Collapse provider implementation:** define summarizer request/checkpoint/result schemas and retry budgets; + the context/head/chain foundation above is ready. +- **Jobs/subagents:** define exact job task payloads and terminal notification protocol on the settled output + API. A subagent is an owned conversation, not a subagent task. Keyed creation, cancellable task waits and + later sibling-tool resolution remain part of the gated ordinary-tool capability design. + Ownership/admission/wait foundations are ready. +- **Typed system sections:** define the concrete draft/persistence API and preparation staleness retry code + after pi-ai verification; section 14.4 fixes required semantics. +- **Client/rendering integration:** expose renderer projections over task-output watch state; delivery + coalescing remains deferred. +- **Conversation deletion, runtime schema bundle, renderer registry/layouts and migration:** later milestones, + not foundation. Root deletion is always forbidden; descendant/fork-reference deletion policy must be + specified before adding a delete API. + +## 19. Repository and process rules + +Implementation location: `packages/agent/src/harness/pico/`. Do not import from +`packages/agent/src/harness/runtime`, `packages/agent/src/harness/session`, `agent-harness.ts`, Pico, Pico2 +or DOM. Read reusable leaf implementations before copying, then own the copy under Pico. Allowed package +boundaries include Chord and pi-ai. + +Use erasable TypeScript syntax. Pico declarations must not introduce or explicitly spell `any`; imported +pi-ai declarations are exempt and their durable values receive strict-JSON boundary validation. Check +external API types in `node_modules`; do not guess. Test +with package-specific Vitest commands and `npm run check`; never run the credential-sensitive full suite. +Commit only files changed by the current work package, using explicit paths. One reviewed commit per work +package; do not begin the next until the user has reviewed the previous commit. diff --git a/packages/agent/docs/pico/pico-usage-guide.md b/packages/agent/docs/pico/pico-usage-guide.md new file mode 100644 index 000000000..dbde696b9 --- /dev/null +++ b/packages/agent/docs/pico/pico-usage-guide.md @@ -0,0 +1,1572 @@ +# pico + +Durable agent harness for pi: one session file, any number of conversations, every piece of work +recorded as a task that survives a crash, and a view any UI can render. + +**Note**: this guide is about using the harness. `pico-simple-handoff.md` is the sole normative +implementation specification and the reference for why things are the way they are. Provider/system, hook, ordinary-tool, job and subagent facades remain gated +where marked; their examples show intended behavior, not permission to guess unsettled APIs. Required task output, +storage and cancellation guarantees are retained; the separate steps proposal has not been adopted. + +## Table of Contents + +- [The Mental Model](#the-mental-model) +- [Calls and Cancellation](#calls-and-cancellation) +- [Installation](#installation) +- [Quick Start](#quick-start) +- [Sessions and Conversations](#sessions-and-conversations) + - [Opening](#opening) + - [What Happens on Reopen](#what-happens-on-reopen) + - [Closing](#closing) + - [Cloudflare Durable Objects](#cloudflare-durable-objects) +- [Configuration](#configuration) + - [Settings Exported Alongside Kinds](#settings-exported-alongside-kinds) + - [Reading and Writing](#reading-and-writing) + - [What Is Rewindable](#what-is-rewindable) +- [System Prompt and Tool Loadout](#system-prompt-and-tool-loadout) + - [How It Works](#how-it-works) + - [Answering the Hook](#answering-the-hook) + - [Changing the Loadout](#changing-the-loadout) + - [Sections from Plugins](#sections-from-plugins) + - [Subagents Have Their Own](#subagents-have-their-own) + - [Compaction, Forks and Restarts](#compaction-forks-and-restarts) +- [Sending Input](#sending-input) + - [send and InputHandle](#send-and-inputhandle) + - [Input While Busy](#input-while-busy) + - [Aborting](#aborting) +- [Watching](#watching) + - [The View](#the-view) + - [Events](#events) + - [Rendering](#rendering) + - [Remote Clients](#remote-clients) + - [Session Watch](#session-watch) +- [Forks](#forks) +- [Compaction and Reset](#compaction-and-reset) +- [Subagents](#subagents) +- [Jobs and Schedules](#jobs-and-schedules) +- [Plugin State](#plugin-state) + - [Values and Lists](#values-and-lists) + - [Atomic Commits](#atomic-commits) +- [Writing Tools](#writing-tools) + - [The Sink](#the-sink) + - [Diagnostics](#diagnostics) + - [Long-Running Tools](#long-running-tools) +- [Hooks](#hooks) +- [Writing Kinds](#writing-kinds) + - [Entry Kinds](#entry-kinds) + - [Task Kinds](#task-kinds) +- [Recovery](#recovery) +- [Storage Backends](#storage-backends) + +## The Mental Model + +A **session** is one storage file (or one row set in SQLite). It holds **conversations**. A +conversation has three things: + +- a **transcript**: an append-only list of immutable **entries**. User messages, assistant messages, + tool results, summaries, system instructions, and anything a plugin wants to record. Entries are + never edited or reordered. +- **tasks**: the units of work. Generating a response is a task, running a tool is a task, so is + compacting, running a background process, or anything a plugin wants done. A task has immutable input, + pending/running/terminal lifecycle, optional complete checkpoints and optional task output. Those durable + boundaries make a crash recoverable. +- **state**: keyed **values** and **lists**, for the model in use, plan mode, a game board, whatever + a plugin needs to remember. + +What the model sees, the **context**, is not stored as a list. Each entry may store `model` +messages, a **head** boundary, and **edits** that omit or replace earlier model messages. The harness +prepends the newest head, reads forward from its stored boundary, folds retained edits, then performs +request-local tool and provider normalization. Compaction appends a head; tool-result pruning +appends an edit. Forks, compaction and reset never mutate old entries. + +Object identity and write order are separate: + +```typescript +type Id = number; // Storage-minted stable identity for committed conversations, entries, tasks and list elements +type Seq = number; // storage-assigned committed write order +``` + +Creation and list-append builders synchronously call `Storage.nextId()` while holding the Session line. A +failed callback discards its writes but burns those IDs in the current open binding. They become valid object +identities only after commit succeeds. Reopen initializes after the greatest committed creation/list-append +ID and may therefore reuse IDs that were minted but never committed by an earlier process; committed IDs are +never reused. `commit` receives no allocator argument. Each write receives a `Seq`. Historical entry-ID +lookup resolves that entry's sequence internally. Watch barriers and commit envelopes use `Seq`, never +object IDs. + +All main writes happen through **commits**: a closure that runs on the Session's single mutation line, where +everything inside it lands together or not at all. Tasks run concurrently; mutations never do. + +Opening is inert. `resume()` activates one scheduler for every eligible task in every conversation. It stays +active while the Harness is open, and every successful main commit coalesces a scheduler kick. Foreground and +background tasks both execute; foreground only controls idle and ordinary abort reach. A UI **watches** a +conversation and gets one authoritative **view** it can render directly. + +## Calls and cancellation + +Every asynchronous harness, conversation and task-runtime operation takes a required final `Call`. +`Call` is a type alias of Chord `Context`: it carries an abort signal, telemetry parent and, for task +code, a private typed invocation identity. It is not model context. No casts or admission helpers +are needed. Pure accessors, synchronous registrations and methods inside a transaction take no Call. + +```typescript +import type { Call } from '@earendil-works/pi-agent'; +import { BACKGROUND_CONTEXT, withCancel } from '@earendil-works/chord/context'; + +const call: Call = BACKGROUND_CONTEXT; // host call, without cancellation +const { context: waitingCall, cancel } = withCancel(call); +const input = await conversation.send({ content: 'Inspect the parser' }, call); +const waiting = input.wait(waitingCall); +cancel(); // removes this waiter; does not abort durable work +await waiting; // rejects with cancellation +``` + +Tasks receive `(task, runtime: TaskRuntime, call: Call)` and forward `call`. Tools receive +`(toolCallId, params, out, runtime: ToolRuntime, call: Call)`. Environment/provider/hook operations +interpret the signal; custom handlers must cooperate. Derive a Call for nested telemetry or a tighter +deadline and pass it onward. The scheduler does not inherit a waiter caller's signal into task execution. + +The line reads task identity through a private `createContextKey`; Chord returns the +correct type from `call.value(key)`. Derived calls preserve that exact object. Stale task writes +reject. No invocation identity is serialized over RPC; trusted host bindings supply it locally. +Deliberately using an unrelated host Call or ignoring cancellation is an in-process escape, not +something a facade or type can prevent. + +## Installation + +```bash +npm install @earendil-works/pi-agent +``` + +## Quick Start + +```typescript +import { Harness, JsonlStorage, systemSections, type Call } from '@earendil-works/pi-agent'; +import { BACKGROUND_CONTEXT } from '@earendil-works/chord/context'; +import { readTool, writeTool, bashTool } from '@earendil-works/pi-agent/tools'; +import { generationKind } from '@earendil-works/pi-agent/kinds'; +import { builtinModels } from '@earendil-works/pi-ai/providers/all'; + +const call: Call = BACKGROUND_CONTEXT; + +// One file per Session. Reopening restores it inert; send/wait/resume activates recovery. +const storage = await JsonlStorage.open('./session.jsonl'); + +// The built-in kinds (generation, tool, post_tools, collapse, job; the entry kinds) and the +// subagent and job tools are registered by open. You add models and the tools you want. Nothing runs yet. +const h = await Harness.open(storage, { + models: builtinModels(), + tools: [readTool, writeTool, bashTool], + // Address/value pairs, applied only when creating the root; reopening preserves stored settings. + rootValues: [ + [generationKind.config.model, { provider: 'anthropic', modelId: 'claude-opus-5' }], + [generationKind.config.thinking, 'high'], + [generationKind.config.selectedTools, ['read', 'write', 'bash']], + ], +}, call); + +const c = await h.root(call); + +// Configuration is durable state; this hook edits the prepared section payloads. +// The harness stores changed payloads and rendered system messages before each request. +c.hooks.on(generationKind, 'system_instructions', ({ sections, config }, call) => { + sections.set(systemSections.identity, 'You are a careful engineer working in this repository.'); + sections.set(systemSections.environment, { cwd: process.cwd() }); + return { tools: h.tools.select(config.selectedTools) }; // complete tool definitions +}, { subtree: true }); + +// Watch the conversation. `view` is a plain object a UI renders from; subscribe whenever you like. +// The view is complete as of capture and commit envelopes follow from its Seq barrier. +const w = await h.watch(c.id, { tail: 100 }, call); +w.start(delivery => { + if (delivery.type !== 'commit') return; + for (const event of delivery.commit.events) { + if (event.type === 'task_output') { + const output = w.view.taskOutputs.find(value => value.id === event.id); + process.stdout.write(renderTaskOutput(output)); + } + if (event.type === 'entry') console.log(`\n[entry ${event.entry.id} ${event.entry.kind}]`); + } +}); + +// Admission is durable and resumes the whole Session; waiting observes this input's explicit result. +const input = await c.send({ + requestId: 'cli-1', + content: 'Inspect the parser and list the public API', +}, call); +const result = await input.wait(call); +const answer = result.status === 'done' && result.answer + ? await h.getEntry(result.answer, call) + : undefined; +console.log(answer?.model?.[0]?.content); + +w.unsubscribe(); +await h.close(call); // cancels nothing durable; reopen restores here, then send/wait/resume continues +``` + +Run it, kill it in the middle, run it again: the second run recovers whatever was in flight +(publishing a partial answer, rerunning or reporting an interrupted tool) and continues. That is the +whole point. + +Snippets below assume `h`, `c` and `call` set up like this. Optional options before Call are passed +as `undefined` when unused. A task or hook always forwards its supplied Call, not this host root. + +## Sessions and Conversations + +### Opening + +`Harness.open(storage, options, call)` takes the storage backend and everything that defines behaviour: + +| option | what it is | +|---|---| +| `models` | a pi-ai `Models` collection | +| `tools` | what the model may call, besides the built-in `subagent` and `job` tools; a `ToolRegistry` or an array | +| `kinds` | ordinary plugin entry and task kinds, added beside the fixed built-ins; protected `pi.*` names reject | +| `rootValues` | explicit initial root configuration, applied only in the fresh root's creation commit | +| `sections` | initial custom typed section definitions, in addition to built-ins | + +Registries supply implementations, not selections. Registering read/write/edit/bash does not +select them automatically. Supply initial model/thinking/selectedTools through `rootValues`, or +configure the fresh conversation before generation. On reopen, `rootValues` is ignored: durable +configuration wins. Children inherit the values selected by their spawn policy; forks inherit +rewindable configuration at the fork point. Missing required generation configuration is an error. + +The built-in kinds are fixed and installed by `open` itself because `send`, safe-boundary admission and +`collapse` cannot work without them. They cannot be registered, replaced or removed. Their readonly +witnesses remain available through `h.kinds` for hooks and inspection: + +```typescript +h.kinds.generation // config: model, thinking, selectedTools, ...; hooks: system_instructions, before_request, on_yield +h.kinds.tool // hooks: before_tool, after_tool +h.kinds.collapse // hooks: before_collapse +h.kinds.job +``` + +Plugins extend these operations through their typed hooks and may register ordinary durable task kinds for +side work. They cannot supply an alternative generation/tool/post_tools graph or obtain the core transcript +transaction surface. + +Open checks the recorded kind strings without scanning the transcript and reports live work, but +starts nothing. An unregistered historical entry kind is reported, not rejected: its stored +`model`, `head` and `edits` still build context, while its typed data and custom renderer are +unavailable. Open reconciles missing live task kinds before returning: + +```typescript +const { pending, running, orphaned, unknownEntryKinds } = await h.inspect(call); +// pending: tasks whose effects never began +// running: tasks the prior process left in flight; recover() handles them after resume +// orphaned: live tasks whose kind was missing and was terminalized during open +// unknownEntryKinds: stored entries that retain generic facets but lack typed plugin narrowing +``` + +### What Happens on Reopen + +Nothing runs during open. Resume once to start or recover every eligible foreground and background task in +every conversation. Activation is idempotent and returns after initial dispatch scheduling, not after idle: + +```typescript +const h = await Harness.open(storage, opts, call); +await h.resume(call); +``` + +`send`, `InputHandle.wait`, `collapse`, task/input abort and foreground-idle waits also ensure this +session-wide resume. +Use explicit `resume()` when restored work must progress and no new progress-seeking command will be issued. +A read, inspection, fork without input, value lookup or watch capture does not activate recovery. + +### Closing + +```typescript +await h.close(call); // cancel in-process work, write nothing; everything resumes on the next open +await h.shutdown(call); // mark live tasks, wait for abort cleanup, close; queued input survives +``` + +`close` is the normal exit: it stops admission on the commit line, then signals and joins owned +invocations outside it, without writing task outcomes. Earlier line operations have already finished; +later mutations reject. Close waits for actual invocation completion, regardless of caller cancellation. +An uncooperative task can delay it indefinitely. + +`shutdown` closes normal admission and atomically marks live tasks only. Queued input and its queued +result records remain stored, including in idle conversations. Fresh abort handlers resolve their +already-running input groups and finish cleanup. Built-in child cleanup marks tasks only, never drains +child queues, including after a crash and reopen. Shutdown waits for both live tasks and running calls +to disappear before closing. Preserved queues create no work merely from reopen or resume; a later explicit +idle send consumes queued writes/follow-ups/steering according to the normal boundary policy. + +Once admitted, caller cancellation does not abandon shutdown. Repeated lifecycle calls share +completion; explicit close interrupting shutdown makes shutdown reject. Task calls cannot invoke +host lifecycle methods. + +### Cloudflare Durable Objects + +Use one Durable Object per Pico Session: one open Harness, one session scheduler, and any number of root, +forked and owned conversations. A request that calls `send` activates the scheduler. An alarm or one logical +Cloudflare Task wake should open Pico and call `resume`, which recovers all eligible Session work. + +Platform wake-up is not Pico effect recovery. Do not create one Cloudflare lane/task per conversation, and +do not use Cloudflare replay as a second authority for provider/tool effects. If durable Pico admission and +the platform wake cannot share a transaction, add an ordered handoff or idempotent retry so acknowledged +work can always cause another wake. Pico checkpoints and `recover()` remain authoritative across process +and platform invocation limits. + +## Configuration + +### Settings Exported Alongside Kinds + +There is no generic settings object and generic `TaskKind` does not capture configuration. Built-in/product +packages export typed address bundles alongside their kinds, so each address is still spelled once: + +```typescript +generationKind.config +// { +// model: conversationValue('pi.model', { rewind: true }), +// thinking: conversationValue('pi.thinking', { rewind: true }), +// selectedTools: conversationValue('pi.tools.selected', { rewind: true }), +// profile: conversationValue('pi.prompt.profile', { rewind: true }), +// budgetMs: conversationValue('pi.tool.budget', { rewind: false }), +// } +``` + +A UI can list built-in configuration from these exported bundles. A plugin defines and exports its own +addresses the same way; those values remain conversation state and are never copied wholesale into a task. + +### Reading and Writing + +```typescript +// all of a kind's values, one batched read +const { model, thinking } = await c.config(generationKind).get(call); + +// some of them, one commit +await c.config(generationKind).set({ thinking: 'low' }, call); + +// `settings` is config(generationKind), because that is what every UI touches +await c.settings.set({ model: { provider: 'openai', modelId: 'gpt-5.6' } }, call); + +// one value, one point read, by its declared address +const tools = await c.value(generationKind.config.selectedTools).get(call); +``` + +A change is a normal commit: it appears in the watch stream as a `value` event, and the next +generation picks it up. Nothing is cached in memory that could disagree with storage. + +### What Is Rewindable + +A value is either **rewindable** (its history is kept, a fork at an entry sees the value in force +there) or **sticky** (current only; the UI's present, not the conversation's history). Model, +thinking and selected tools are rewindable: forking at yesterday's answer gets yesterday's model. +Something like `ui.expanded` is sticky. + +Children created by `spawn` don't inherit history; they are initialized explicitly (see +[Subagents](#subagents)). Generation tasks do not copy this generic configuration into their durable input. +They carry only minimal built-in effect-recovery evidence such as input IDs, context cutoff, selected model +identity, attempt/deferred IDs and offered tool-name/version evidence when required. A plugin needing exact +recovery persists its own compact key/version. Fork configuration remains built-in fork-aware conversation +state, so a fork sees the values in force at its selected entry and local overrides do not affect the parent. + +## System Prompt and Tool Loadout + +### How It Works + +Pico sends pi-ai only `{ messages }`: no parallel top-level `systemPrompt` or `tools`. Managed system +entries contain the exact rendered pi-ai messages, including complete tool additions/removals. Pi-ai +owns native/fallback translation and best-effort cache preservation. + +Configuration changes remain ordinary durable value writes. System entries separately record the +instructions prepared for requests—not proof of delivery. Host files, discovery caches, callbacks and +renderer functions are not stored. Section JSON payloads and final rendered text are stored, so a +missing plugin cannot make historical requests depend on its renderer. + +The target pi-ai system-message API and messages-only adapter behavior are integration prerequisites +([#9116](https://github.com/earendil-works/pi/pull/9116), with coding-agent integration in +[#9117](https://github.com/earendil-works/pi/pull/9117)). They were open when reviewed; this guide describes +the intended contract, not a claim that those PRs already implement the agreed adapter behavior. + +### Answering the Hook + +A typed section token names the payload and its append-time renderer: + +```typescript +interface SystemSection { + readonly key: string; + render(value: T): string; +} + +const rulesSection = defineSystemSection({ + key: 'myplugin.rules', + render: rules => rules.map(rule => `- ${rule}`).join('\n'), +}); +await h.sections.register(rulesSection, call); +``` + +Tokens use stable string keys in storage. Payloads must be JSON-representable; changing a registered +payload type requires a compatible replacement or migration. The typed token supplies normal get/set +inference without casts in plugin code. Built-ins export tokens through `systemSections`, such as +identity, environment and skills; their values come from the host, not from the registry. + +Each generation seeds one private ordered section draft from the last durable prepared state. Handlers +run sequentially, harness-wide first and innermost conversation last, editing that same draft: + +```typescript +c.hooks.on(generationKind, 'system_instructions', ({ sections, config }, call) => { + sections.set(systemSections.identity, 'You are a coding assistant.'); + sections.set(systemSections.skills, skillsCache.current); // complete typed skill data + sections.set(rulesSection, ['Run relevant tests.']); // authoritative base for later transforms + return { tools: h.tools.select(config.selectedTools) }; // complete selected JSON definitions +}, { subtree: true }); +``` + +The draft provides: + +```typescript +interface SystemSectionDraft { + get(section: SystemSection): T | undefined; + set(section: SystemSection, value: T): void; + delete(section: string | { readonly key: string }): void; + wrap(section: SystemSection, transform: (text: string) => string): void; +} +``` + +`get` returns an owned copy; use `set` to change the draft. Existing keys retain their position; new +keys append. `delete` is explicit—null remains a valid payload. Registered compatible definitions are +required for typed get/set/wrap; deletion can use a stable key even if its definition is unavailable. +No ordering configuration exists, and reordering alone emits no update. + +After handlers finish, touched sections render outside the line. Wrappers apply in registration order +after rendering. The frozen result is diffed against stored payloads and rendered text. Changed data +with unchanged rendering produces a metadata-only system entry (`model: []`); changed rendering with +unchanged data still produces a system-message update. No historical read runs these functions. + +### Sections from Plugins + +A later handler can modify a built-in section's structured payload, not parse its prose: + +```typescript +c.hooks.on(generationKind, 'system_instructions', ({ sections }, call) => { + const skills = sections.get(systemSections.skills); // typed skill array | undefined + sections.set(systemSections.skills, + (skills ?? []).filter(skill => skill.name !== 'deploy')); +}); +``` + +Appending to the same key is ordinary typed get-and-set: + +```typescript +c.hooks.on(generationKind, 'system_instructions', ({ sections }, call) => { + sections.set(rulesSection, [ + ...(sections.get(rulesSection) ?? []), + 'Check migration safety.', + ]); + sections.wrap(rulesSection, text => `Repository policy:\n${text}`); +}); +``` + +This example relies on the earlier host handler resetting `rulesSection` to its authoritative base +on every preparation. Without that reset, repeatedly appending to a persisted seed accumulates text. +The whole transformation chain must produce the same result when applied again, or start from a +refreshed base; individually idempotent handlers are not sufficient when they interact. + +Wrappers are preparation-local. Untouched sections keep their stored rendered text, including old +wrapper output when the contributing plugin disappears. Explicit set/wrap or a renderer replacement +recomputes it. Refreshing the base deliberately rebuilds wrappers from currently installed handlers; +we do not promise to preserve missing wrappers across that refresh. + +Skills discovery can remain in the hook owner's closure or a host service. Watch local changes or poll +a remote source at a bounded interval; hook calls read the cached snapshot. Failed refresh is not +removal: retain the last successful snapshot. The base hook explicitly deletes its section when the +source really disappears. If a handler fails and is skipped, discard its draft mutations/wrappers, +not earlier handlers' changes; a half-finished refresh must not remove instructions. No discovery callback or private cache state is stored with sections. + +### Changing the Loadout + +Configuration is durably persisted when changed: + +```typescript +await c.settings.set({ selectedTools: ['read', 'grep'] }, call); +``` + +The next preparation renders the final draft and compares it with previous prepared state. A transcript +might look like this (`readDefinition` etc. mean complete JSON definitions, not executable functions): + +```text +100 user +110 system baseline: identity + rules; add read/write +120 assistant +125 config write: selectedTools=read/grep (durable state, not a transcript entry) +130 user +140 system delta: changed rules; remove write; add grep +150 assistant +``` + +Stored baseline: + +```typescript +const baseline: SystemEntry = { + id: 110, conversationId: 1, kind: 'system', + data: { baseline: true, sections: [ + { key: 'pi.identity', action: 'set', + value: 'You are a coding assistant.', rendered: 'You are a coding assistant.' }, + { key: 'myplugin.rules', action: 'set', + value: ['Run relevant tests.'], rendered: '- Run relevant tests.' }, + ] }, + model: [{ + role: 'system', + content: '## pi.identity\nYou are a coding assistant.\n\n' + + '## myplugin.rules\n- Run relevant tests.', + toolsAdded: [readDefinition, writeDefinition], timestamp: 1000, + }], +}; +``` + +Stored change after a plugin modifies rules: + +```typescript +const change: SystemEntry = { + id: 140, conversationId: 1, kind: 'system', + data: { sections: [{ key: 'myplugin.rules', action: 'set', + value: ['Run relevant tests.', 'Check migration safety.'], + rendered: '- Run relevant tests.\n- Check migration safety.', + }] }, + model: [{ + role: 'system', + content: 'The myplugin.rules section now reads:\n' + + '- Run relevant tests.\n- Check migration safety.', + toolsRemoved: [writeDefinition], toolsAdded: [grepDefinition], timestamp: 2000, + }], +}; +``` + +Tool definitions live only in SystemMessage fields, not duplicated in section data. Compare definitions +structurally by name; a changed schema/description is a complete `toolsAdded` upsert. Removal includes +the previous complete stored definition. Apply removals before additions. Tool-only changes may have +empty instruction content. Hooks supplying tools replace the complete desired loadout; the last +supplied list wins, and no list means no desired tools. + +Explicit section deletion stores `{ key, action: 'remove' }` and a system message saying that section +no longer applies. Omitting a hook or unregistering its definition is not deletion. + +Pico sends: + +```typescript +const request = { + messages: [user100, ...baseline.model, assistant120, user130, ...change.model], +}; // no top-level systemPrompt or tools +``` + +For unsupported provider/model combinations, pi-ai translates system messages to ``-bracketed +user messages at their historical positions and derives any bulk wire tool declarations it needs. +Pico never hoists the baseline or flattens changes into a rewritten top-level prompt. Cache preservation +is best-effort; a fallback user message does not have native system priority. + +### Subagents Have Their Own + +The exact spawn helper is gated; this example fixes the required configuration semantics. Children +explicitly choose their durable configuration. Subtree handlers supply defaults, and inner +hooks can replace built-in payloads or add sections: + +```typescript +const childId = await c.spawn({ prompt: 'Audit the tests', + values: { inherit: [generationKind.config.model], + set: [[generationKind.config.selectedTools, ['read', 'grep']]] }, +}, call); +const child = await h.conversation(childId, call); +child.hooks.on(generationKind, 'system_instructions', ({ sections }, call) => { + sections.set(systemSections.identity, AUDITOR_IDENTITY); +}); +``` + +Definitions can be supplied initially through `Harness.open(..., { sections: [...] }, call)` or changed +later through `h.sections.register/replace/remove`. Registration is mutable process state, serialized +on the line; an in-flight preparation retains its definition snapshot. `register` rejects duplicate +keys; `replace` is explicit and compatible; `remove` unregisters code without erasing stored sections. +Entry/task registries follow the parallel `h.entryKinds`/`h.taskKinds` API. Replacing or removing an +ordinary task kind rejects while live tasks of that kind exist; after they are terminal, replacement affects +future tasks only. At open, every live task with a missing kind is terminalized as +`orphaned`, regardless of foreground/background; registered descendants selected by cleanup policy are +marked. Later registration never resurrects terminal tasks. + +### Compaction, Forks and Restarts + +Canonical section state is reconstructed from fork-visible managed system entries back to the most +recent baseline, then folded forward. Model heads and projection omissions do not erase these section +payloads. This uses existing indexed kind scans, with an optional prepared-state cache. A fresh baseline +checkpoints the whole state; no extra full-state value or token/renderer serialization is needed. + +Consequently, restarting without a plugin retains its JSON payload and rendered text—even if compaction +removed its original baseline from model context. Untouched unknown sections also appear in the next +fresh baseline. Re-registering a compatible definition restores typed editing; explicit deletion is +how the host removes an abandoned section. + +Every generation checkpoints `requestThrough`, an inclusive transcript cutoff. It captures canonical +section state and definitions before running hooks/renderers outside the line. Preparation then checks +on the line that no managed section write changed its seed; if one did, repeat preparation. A head-only +change does not stale the section data, but may require a baseline rather than a delta. + +One line operation commits the system entry and inflight intent/cutoff, catches the live context cache +up, and captures an immutable array of effective entry references after the full batch is durable. +Later cache updates do not mutate that array or its replacement projections. Request-local transforms +copy what they modify. Reconstructing an older cutoff reads storage without rewinding the live cache. + +```text +prepare through 51 → requestThrough=51; capture request snapshot +60 summary lands → live cache changes, request snapshot does not +70 answer lands → answer to the already prepared request +``` + +A usable model baseline must follow the newest head entry. Otherwise the next preparation appends a +full baseline carrying ordinary omission edits for superseded retained managed system entries: + +```text +10 user; 11 baseline; 20 assistant; 30 user; 31 managed delta; 35 job notice; 40 assistant; 50 user +60 summary, head=30 +context at 60: [60 summary, 30 user, 31 delta, 35 notice, 40 assistant, 50 user] +70 assistant +80 user +81 system baseline, edits:[{ target:31, action:omit }] +context at 81: [60 summary, 30 user, 35 notice, 40 assistant, 50 user, 70 assistant, 80 user, 81 baseline] +``` + +The baseline stays at its appended tail position. Its edits omit managed baselines/deltas, not unrelated +notifications with role=system. This atomic baseline supersession is the only permitted edit of managed +system projections; arbitrary omit/replace edits targeting them reject. Change their instructions through +the section draft instead. Before preparation, old retained deltas remain visible. Generic head writers +and context projection need no system-specific callbacks or hidden filtering. + +Repeated heads use the same mechanism. Fold effective tool declarations after planned omissions; add +all desired tools and explicitly remove unwanted declarations that remain in other system messages. +`baseline:true` is pico metadata, not a reset command understood by pi-ai. + +Already superseded entries omitted by the current baseline must not cause repeated baselines. + +A crash after configuration changes preserves them. A crash after system append but before request +preserves prepared instructions; unchanged data/rendering yields no duplicate delta. Forks inherit +only their visible section history and rewindable config. Current host-source or renderer changes can +produce a new prepared update, but never re-render old model messages. + +```typescript +// Fork the earlier loadout example at 120: baseline 110 selected read/write, before the grep change. +const b = await c.fork({ at: 120 }, call); +await b.settings.set({ selectedTools: ['read'] }, call); +const forkInput = await b.send({ content: '...' }, call); // toolsRemoved=[write]; 110 stays visible +await forkInput.wait(call); +``` + +`before_request` may transform a private request copy, which must remain messages-only. These changes +do not mutate stored section state. The transcript is not an exact audit of arbitrary transformed +requests without optional separate capture. Tool-call validation uses the actual offered definitions +after transformation, plus normal implementation and permission checks. + +## Sending Input + +### send and InputHandle + +`send` is the application-facing admission operation. It first durably accepts or deduplicates content, then +non-abandoningly ensures the one Session scheduler is resumed, and returns an `InputHandle`; it does not wait +for a model answer. Caller cancellation after admission cannot strand the accepted input. + +```typescript +const input = await c.send({ + requestId: 'req-42', + content: 'Inspect the parser', +}, call); + +const result = await input.wait(call); // done | unanswered +const answer = result.status === 'done' && result.answer + ? await h.getEntry(assistantKind, result.answer, call) + : undefined; +``` + +`input.id` is the stable ID of the protected inbox element allocated for acceptance, even when idle +admission appends and removes that element in one commit. `pi.inbox` is a conversation-scoped sticky, +non-rewindable list and is not inherited by forks. Ascending element IDs preserve Session admission order. +Generation and `post_tools` carry input IDs explicitly. Results are direct protected values keyed by input ID, +and request receipts are direct protected values keyed by the Session-wide request key; neither lookup scans +the transcript or inbox. `input.result(call)` is a point read and does not activate work. +`input.wait(call)` ensures resume, returns an existing terminal result immediately or installs one cancellable +observation waiter. Cancelling that Call removes only that waiter; it neither aborts the input nor changes +scheduler scope. Several waiters may join the same result. + +Inbox reads return one complete current materialized ordered collection. Memory retains that collection +in process, JSONL replays operations once at open and then maintains it, and SQLite reads all current indexed +rows in element-ID order and may physically remove them on remove/clear. No inbox read replays history from +the last clear; rewindable list history is unrelated. An +admission or boundary decision commits its inbox removal together with result/receipt changes, entry +placement and successor task creation in one main transaction. + +A result moves `queued -> placed -> done | unanswered`. Idle send may collapse absent directly to placed. +`done` names the answering entry when one was owed; an internal passive write is done with no answer. +Terminal results never change. + +A request key is Session-wide and first-key-wins for the Session lifetime. Lookup happens before conversation, +busy mode or payload comparison. A retry through any conversation returns a handle bound to the original +conversation/input and writes nothing: + +```typescript +const first = await c.send({ requestId: 'req-42', content }, call); +const same = await anotherConversation.send({ requestId: 'req-42', content: 'different' }, call); +console.assert(first.id === same.id && first.conversationId === same.conversationId); +const existing = await h.input('req-42', call); +``` + +Use `c.waitForIdle(call)` to observe that conversation's foreground cancellation closure, or +`h.waitForIdle(call)` for all foreground tasks in the Session. These waits ensure resume but do not wait for +background work. Background jobs/collapse still execute after foreground idle. + +### Input While Busy + +A conversation is admission-busy while a live generation/tool/`post_tools` turn task owns transcript +progress. A speculative background collapse has model-entry authority but is not admission-busy. + +```typescript +const follow = await c.send({ content: 'Then write the tests' }, call); // default while busy +const steer = await c.send({ + content: 'Focus on the tokenizer first', + whenBusy: 'steer', +}, call); +``` + +| mode | lands | asks for | +|---|---|---| +| `steer` | next complete post_tools boundary, or final boundary | joins the active input group when safe | +| `followUp` | after the final assistant answer | starts the successor input group | +| `write` | next safe boundary | passive transcript content; no generation | + +`followUp` is the busy-send default. While idle, `whenBusy` is irrelevant: send starts one current input +group. `whenBusy:'reject'` throws without recording a request receipt. `steer` never mutates a provider +request already snapshotted or in flight. `write` is a protected agent/plugin admission mode rather than a +common public conversation method; built-ins compose it through internal transaction helpers when it must +land atomically with other state. An idle write appends and becomes done with no task. Applications needing +speculative staged context for a later explicit turn can keep it in their own sticky conversation list and +fold it into that later `send`; Pico does not standardize that application policy. + +Safe chronology is preserved in full: + +```text +send A while idle +TX[ user A; generation G1(inputs:[A]); result A=placed ] +kick -> G1 + +G1 returns calls X,Y +TX[ assistant calls; tool X; tool Y; + post_tools P1(after:[X,Y],inputs:[A]); terminal G1 ] +kick -> X and Y concurrently + +send B while busy -> TX[followUp B; result B=queued] +send S with steer -> TX[steer S; result S=queued] +internal write W -> TX[write W; result W=queued] + +X and Y finish in either order +TX[tool result X; terminal X] +TX[tool result Y; terminal Y] +model projection restores assistant call order + +P1 terminal boundary +TX[write W; user steer S; generation G2(inputs:[A,S]); terminal P1] +kick -> G2 + +G2 final answer E2 +TX[assistant E2; A/S=done(answer:E2); user followUp B; B=placed; + generation G3(inputs:[B]); terminal G2] +kick -> G3 +``` + +`steeringMode` and `followUpMode` independently use `"all" | "one-at-a-time"` and default to +`"one-at-a-time"`, matching the old lane harness. At each boundary, `"all"` selects every queued item of +that tag and `"one-at-a-time"` selects only its oldest item. All writes are selected. Selected tags are +merged in global inbox order; unselected items retain their relative order. Pico imposes no queue-size or +drain-size bound in v1. No giant turn task and no task-per-send dependency chain exists. A logical turn remains generation -> parallel tools -> post_tools -> continuation generation, with input +ownership transferred atomically at each stage. Each tool appends its own result entry atomically with +terminalization. Transcript chronology may reflect parallel completion; model projection restores the +assistant's call order before the continuation request. + +The inbox exposes complete entry drafts to the authoritative conversation view. Any queued input can be +withdrawn until placement wins: + +```typescript +const status = await follow.abort(call); // aborted | already_placed | not_found +``` + +Withdrawal and placement serialize. If withdrawal wins, only that element is removed and becomes +`unanswered/aborted`; later items retain order. If placement wins, abort returns `already_placed`. No task +dependency is rewired. + +### Aborting + +```typescript +await c.abort(call); // active foreground graph in this cancellation ownership closure +await h.abortTask(id, call); // one task, including a background job or schedule +``` + +Both operations first commit their durable mark/withdrawal, then non-abandoningly ensure the Session scheduler +is resumed so fresh cleanup can progress. An abort mark is a durable request, not terminal settlement: + +```text +100 generation execute invocation A running +110 abort=true commits; A loses main/scratch/output write authority + line releases; A's signal fires; provider exits; A returns + scheduler releases A and reserves fresh abort invocation B +120 B commits display/result cleanup + full terminal aborted snapshot atomically +``` + +If normal settlement wins first, the task is terminal. If the mark wins, normal writes reject, the old +invocation is joined, and only fresh `abort()` performs durable cleanup. Cancellation does not undo external +effects. + +Conversation abort marks its foreground cancellation closure, withdraws queued `steer`/`followUp` as +`unanswered/aborted`, and preserves `write`. Task abort preserves all queued future input unless a specific +built-in cleanup policy says otherwise. Abort terminalization never runs a normal final-answer +boundary, so it cannot accidentally start a queued successor. Shutdown marks all live tasks but preserves all +queued items/results. + +Tool and generation kinds, not waiters or the scheduler, own their durable outcomes and input-result updates. +They recover only committed checkpoint/scratch evidence; missing final usage is unknown, not zero. Runtime +scratch writes reject after cancellation too: await or catch them. Harness producers drain their own pending +writes and never recreate retired scratch. + +## Watching + +Tool-facing convenience APIs remain provisional alongside the sink. Durable task output and per-commit watch +delivery are normative; delivery coalescing is deferred. + +### The View + +A UI never reads storage or parses commits. It watches a conversation and gets a **view**: a plain +JSON object the harness keeps current, plus typed events that say what changed. + +```typescript +const w = await h.watch(c.id, { tail: 100, values: [myPlugin.config.mode] }, call); +w.view // ConversationView, captured atomically with the subscription +w.start(listener); +w.unsubscribe(); +// After overflow/failure, open a new watch and replace the client view atomically. +``` + +```typescript +interface ConversationView { + conversation: Conversation; + tail: number; + entries: Entry[]; // newest logical fork-visible tail + context: Id[]; // model-visible entry IDs + tasks: Task[]; // live tasks directly in this conversation + taskOutputs: WatchedTaskOutput[]; // shared live generation/tool/job/collapse output + inbox: Element[]; // queued followUp/steer/write + values: WatchedValue[]; // exactly the requested values + readAt: Seq; // coherent storage sequence barrier +} +``` + +Task output is where streaming lives. A generation output contains its partial assistant message; tool and +job outputs contain their current output state. The same output ID may be referenced by more than one live +task during durable handoff. Entries, tasks, task outputs, inbox and values are captured and updated as one +authoritative replicated view. + +### Events + +```typescript +type InboxOp = + | { type: 'append'; item: Element } + | { type: 'remove'; id: Id } + | { type: 'clear' }; + +type ConversationEvent = + | { type: 'entry'; entry: Entry } + | { type: 'task_start'; task: Task } + | { type: 'task_update'; task: Task; previous: Task } + | { type: 'task_end'; task: Task } + | { type: 'task_output'; id: Id; kind: string; delta: Op[] } + | { type: 'value'; value: WatchedValue } + | { type: 'inbox'; ops: InboxOp[] } + | { type: 'context'; ids: Id[] }; + +interface CommitEnvelope { + first: Seq; + last: Seq; + events: ConversationEvent[]; +} +``` + +The view is authoritative and the event is a wake-up: when the listener runs, `w.view` has already +been folded. Deliveries are whole commit envelopes ordered after `view.readAt`. Inbox operations are +combined per commit; an idle send's append and immediate remove emits no inbox event. A renderer may ignore the event payload entirely and be correct. +Because every piece +of work is a task of a known kind, four task events cover what used to need a name per case: + +| you want to know | look at | +|---|---| +| currently in a turn | `view.tasks` contains a fixed generation/tool/post_tools task after folding the whole commit; a continuation generation is not a new logical turn | +| a retry is scheduled | `task_update`, checkpoint phase `retry_wait`, attempt and `notBefore` | +| the response is streaming | `task_output` on the generation's output ID; `view.taskOutputs` | +| a tool is running / its output | `task_start` / `task_output` / `task_end` on the tool task | +| compaction started / ended | `task_start` / `task_end` where `collapseKind.is(task)` | +| the model changed | `value` with `addr === generationKind.config.model` | +| queued input changed | `inbox` | + +Events of one commit arrive together, in order, so a finished task and its successor never show +as an idle gap. + +### Rendering + +A renderer is a function of the view, diffing against what it last drew. Settled entries are keyed +by id and only appended; live things are keyed by task id, and a tool block keeps its key when the +task settles and its result entry appears, so nothing is torn down and rebuilt: + +```typescript +function render(view: ConversationView) { + transcript.sync(view.entries); + const outputById = new Map(view.taskOutputs.map(output => [output.id, output.value])); + const valueByAddress = new Map(view.values.map(value => [value.address, value.value])); + + const gen = view.tasks.find(t => generationKind.is(t)); + streaming.set(gen?.output ? outputById.get(gen.output.id) as AssistantMessage : undefined); + status.set( + gen?.checkpoint?.phase === 'retry_wait' ? `retrying (attempt ${gen.checkpoint.attempt})` : + gen?.checkpoint?.phase === 'deferred' ? 'waiting for provider' : + view.tasks.some(collapseKind.is) ? 'compacting…' : undefined); + + for (const t of view.tasks.filter(toolKind.is)) + toolBlocks.upsert(t.id, { call: t.input.call, phase: t.checkpoint?.phase ?? t.status, + output: t.output ? outputById.get(t.output.id) as ToolOutputState : undefined }); + + for (const t of view.tasks.filter(jobKind.is)) + jobBlocks.upsert(t.id, { tool: t.input.origin?.tool ?? 'job', + output: t.output ? outputById.get(t.output.id) as ToolOutputState : undefined }); + + queue.set(view.inbox.map(i => i.value)); + working.set(view.tasks.some(t => !t.background)); + statusLine.set({ model: valueByAddress.get(generationKind.config.model) }); +} +``` + +Tool components are registered once per tool name and fed one shape, `ToolOutputState`, whether it +comes from a live tool task's output, a settled `tool_result` entry, or a job the tool started +(`input.origin.tool` says which component). + +### Remote Clients + +The view is plain JSON and every event is proportional to its change, so a process without a +Harness (mini's TUI, a phone) runs the same fold over complete commit envelopes. +`applyConversationCommit(view, commit)` is exported and needs no kinds; task-output ops are Chord delta ops, +which the client applies with the same +module. Keep the original atomic watch subscribed. Do not snapshot, unsubscribe and invent a separate stream +cursor; reconnect after overflow with a fresh atomic watch and replace the client view. + +```typescript +// worker // UI process +const w = await h.watch(c.id, { tail: 100 }, call); on('view', m => { view = m.view; render(view); }); +send({ type: 'view', view: w.view }); +w.start(delivery => send({ type: 'delivery', delivery })); + on('delivery', m => { + if (m.delivery.type === 'commit') + view = applyConversationCommit(view, m.delivery.commit); + render(view); + }); +``` + +### Session Watch + +What isn't one conversation's: the conversation list, session values, usage totals, faults, and +reports (a hook threw, a task kind misbehaved and was stopped). + +```typescript +const sw = await h.watch(call); +sw.view.conversations; sw.view.values; sw.view.readAt; +sw.start(delivery => { + if (delivery.type !== 'commit') return; + for (const event of delivery.commit.events) + if (event.type === 'conversation') tree.refresh(); +}); +``` + +## Forks + +A fork is a new conversation whose transcript starts as a shared prefix of the source. Nothing is +copied and nothing in the source is deleted. It carries the context, canonical prepared instructions +and rewindable values visible at that entry. Its next preparation may append changes from current +host sources; it does not rewrite inherited messages. + +```typescript +const alt = await c.fork({ at: answer.id }, call); // source work keeps running independently +const altInput = await alt.send({ content: 'Try a different implementation' }, call); +await altInput.wait(call); + +const back = await c.fork({ at: earlier.id, abort: true }, call); // "go back": aborts the source's foreground first +``` + +Any transcript entry is a valid fork point, including an assistant with unanswered tool calls or one +of several results. The request projection supplies missing results for a successful incomplete +exchange without inheriting or executing the source tasks. Which conversation a UI treats as +"current" is the UI's business; the harness only has conversations. + +```typescript +const all = await h.conversations(undefined, call); +const independent = all.items.filter(x => x.owner === undefined); // root and forks; same Session scheduler +const children = await h.conversations({ parent: c.id }, call); // forks of c; owned children use ownedFrom +``` + +## Compaction and Reset + +Compaction appends a summary entry with its model message and first retained entry ID stored on the +entry. The context becomes the summary followed by the transcript from that boundary. Manual speculative +compaction is background; threshold/overflow compaction is foreground. Either executes automatically, and +speculative compaction may run while the model keeps working: ordinary entries landing meanwhile +remain after the prepared boundary. Only a competing head makes the summary stale; edit entries do +not. + +```typescript +const collapseId = await c.collapse(undefined, call); +const collapseId = await c.collapse({ instructions: 'keep the API decisions verbatim' }, call); +``` + +Threshold and overflow compaction happen inside the generation; nothing to call. Reset starts the +context over, with or without a handoff message: + +```typescript +await c.reset({ handoff: 'Continue from here: we settled on a recursive-descent parser.' }, call); +await c.reset(undefined, call); // /clear +``` + +The transcript keeps everything either way; only the context changes. + +## Subagents + +**Gated:** exact model-tool commands and owned-conversation helper types below are illustrative until the +ordinary-tool capability design is settled. A subagent is a conversation. There is no separate object to talk to: the model gets one tool with +a `command` argument, and the API gets a conversation handle. + +```typescript +// the model calls: +subagent({ command: 'run', prompt: 'Audit the tests', context: 'fresh', tools: ['read', 'grep'] }) // waits for the answer +subagent({ command: 'spawn', prompt: 'Profile the build' }) // returns the child's id +subagent({ command: 'send', id: 88, text: 'also check CI' }) +subagent({ command: 'status', id: 88 }) +subagent({ command: 'wait', id: 88 }) +subagent({ command: 'stop', id: 88 }) +``` + +```typescript +// the API +const childId = await c.spawn({ prompt: 'Profile the build', context: 'fresh', + values: { inherit: [generationKind.config.model] } }, call); +const child = await h.conversation(childId, call); +const childInput = await child.send({ content: 'also check CI' }, call); +await childInput.wait(call); +await child.abort(call); +``` + +`run` keeps the calling tool in flight while it waits for the child's `InputHandle`, so ownership makes the +child part of the parent's foreground cancellation reach. `spawn` settles the tool at once; the child is +excluded from that cleanup policy, runs while the parent goes on, and only `child.abort()` (or `stop`) ends +it. The one Session scheduler executes either child automatically, concurrently with unrelated +conversations. Both survive restart like any other durable work. + +## Jobs and Schedules + +**Gated:** exact job payloads and convenience methods remain unsettled; the scheduler/task-output behavior +here is required. A job is a background task that runs a process: durable, recoverable, killable, with its task output +streamed through the authoritative view. The model normally gets one from `bash` (asked to background, or run +past its budget) and controls it through the `job` tool: + +```typescript +job({ command: 'wait', id: 91, budgetMs: 30_000 }) +job({ command: 'status', id: 91 }) +job({ command: 'stop', id: 91 }) +job({ command: 'list' }) +``` + +From the API a job is a task; a schedule is a job with `every`: + +```typescript +const dev = await c.commit(tx => tx.task(jobKind, { background: true, + input: { cmd: 'npm run dev', cwd } }), call); + +const nightly = await c.commit(tx => tx.task(jobKind, { background: true, + input: { cmd: 'npm test', cwd, every: 24 * 3600_000, notBefore: tonightAt(2) } }), call); + +await h.abortTask(nightly, call); // ends the schedule wherever it is +``` + +A schedule is one running task updating its kind-defined named checkpoint phases across occurrences; there +is no trail of run IDs to chase. A job whose starting call returned early appends a `notice` entry when it finishes, so the +model learns of it on its next turn without polling. Live output stays in shared task-output state; the job +materializes terminal data into its outcome/notice before the final output reference retires. + +## Plugin State + +### Values and Lists + +Declare an address once; read and write through handles or inside commits. The address carries the +scope (session or conversation), the rewind policy and the payload type. + +```typescript +const planMode = conversationValue('plan.mode', { rewind: true }); +const moves = conversationList('game.moves', { rewind: true }); +const expanded = conversationValue('ui.expanded', { rewind: false }); +const name = sessionValue('pi.session.name'); + +await c.value(planMode).set(true, call); +const on = await c.value(planMode).get(call); +const then = await c.value(planMode).get(entryId, call); // as of an entry: rewindable only + +const elementId = await c.list(moves).append({ x: 1, y: 2 }, call); +const currentMoves = await c.list(moves).read(call); +const movesAtEntry = await c.list(moves).read(entryId, call); // rewindable only +await c.list(moves).remove(elementId, call); +await c.list(moves).clear(call); + +await h.value(name).set('parser work', call); +``` + +Rewindable state is what makes plugins survive forks: a fork before the move doesn't see the move, +a fork before plan mode was turned on doesn't have it on. Nothing to register, nothing to re-derive. + +### Atomic Commits + +Anything that must land together goes in one commit: a closure on the session's write line. Reads +inside it are asynchronous and see committed state; await them in an async builder. Writes are +synchronous and ids are final when returned; a throw discards everything. Awaiting storage reads does +not release the line. Never await external effects, task/input/idle waits or another commit inside a builder. +Known nested Pico line operations reject before queueing instead of deadlocking. + +```typescript +const entry = await c.commit(tx => { + tx.value(planMode).set(false); // rewindable state first + const id = tx.entry(myPlugin.noteKind, { data: { text: 'plan accepted' } }); + tx.value(expanded).set(true); // sticky state may follow + tx.task(myPlugin.reminderKind, { background: true, // tasks anywhere + input: { about: id, at: Date.now() + 3600_000 } }); + return id; +}, call); +``` + +### Appending Entries + +`tx.entry` appends immediately and returns the entry id. Use it for entries that are part of what +your task is doing, and for entries the model never sees (`data` only, no `model`). + +For model-visible passive content written while a turn may be busy, trusted built-ins use the protected +transaction `write` helper. It appends immediately when admission is idle and otherwise queues for the next +complete post_tools or final-answer boundary. It is intentionally not a public conversation method; +application user input goes through `send`. That is not a style preference: appending a +model-visible entry between an assistant's tool calls and their results changes the prefix the next +request replays, which providers reject and which invalidates Anthropic thinking signatures. +`tx.entry` rejects that one case rather than corrupting the next request, and the error points here. + +Only the fixed generation, tool, post_tools and collapse definitions receive the internal +`CoreTaskRuntime`, whose commits provide direct model-entry and protected admission authority. Ordinary +`defineTask` kinds receive `TaskRuntime`/`BaseTaskTx` without those methods. Generation, tool and post_tools +are the fixed turn-task set used for admission; speculative collapse is privileged but outside that set, so +it does not block an idle send. + +Two more things when writing entries directly: appending a `user` entry is not the same as asking +for an answer (use `send`), and a head entry may only narrow context, +never widen it, and may not split an exchange. + +### Write Order + +One rule: rewindable conversation value/list writes must precede entries in the same commit. That +makes state written alongside an entry visible to a fork at that entry while excluding later +commits. Session state and sticky conversation state may appear anywhere because forks never +reconstruct their history; they may therefore reference a new entry id. Tasks may also appear +anywhere. A violating builder call throws before anything is persisted. + +## Writing Tools + +**Gated:** exact sink and ordinary-tool runtime names below are illustrative, not implementation +instructions. Durable task output and the required streaming, bounded capture, diagnostics, typed details, +usage, control outcomes and job/child cleanup semantics are normative. Do not implement this package until +the final mediated tool API is appended to the normative handoff. + +### The Sink + +A tool's `execute` returns nothing. Everything it produces goes through a sink, so output streams +to the UI as it happens, is bounded once in one place, and settles into a `ToolOutputState` that +the transcript, the model and every renderer share. Failure is a throw; the harness sets `isError`. + +```typescript +import { Type, type Tool } from '@earendil-works/pi-agent'; + +export const countLinesTool: Tool<{ i: string; path: string; pattern?: string }, { lines: number; matching: number }> = { + name: 'count_lines', + description: 'Count lines in a file, optionally only those matching a pattern', + parameters: Type.Object({ + i: Type.String({ description: 'What you are trying to find out' }), // intent: streams first, shows in the UI + path: Type.String(), + pattern: Type.Optional(Type.String()), + }), + output: { maxBytes: 64_000, maxLines: 200, retain: 'head' }, // the sink enforces this + replay: 'safe', // read-only: may be rerun after a crash + + async execute(toolCallId, params, out, runtime, call) { + const lines = getOrThrow(await runtime.env.readTextLines(params.path, {}, call)); // ExecutionEnv: FileSystem & Shell + const re = params.pattern ? new RegExp(params.pattern) : undefined; + let matching = 0; + for (const [i, line] of lines.entries()) { + if (re && !re.test(line)) continue; + matching++; + out.write(`${i + 1}: ${line}\n`); // bounded by `output`; the sink truncates and diags + } + if (matching > 200) out.diag('warn', `showing 200 of ${matching} matching lines`, 'cap'); + out.details.lines = lines.length; // typed, for UIs + out.details.matching = matching; + }, +}; +``` + +The sink: + +| call | effect | +|---|---| +| `write(text)` / `replace(text)` / `image(img)` | the content the model reads | +| `details` | the tool's typed object for UIs; mutate it | +| `usage(u)` | accumulates | +| `addTools(names)` | change the loadout from the next turn on | +| `terminate(true)` | stop the turn after this exchange, whether or not the call failed | +| `handoff(message)` | ask for a context reset after the exchange (what `new_context` does) | +| `delegate(jobId)` | this call's work continues as that job | +| `diag(severity, message, code?)` | commentary about the call, kept out of the data | + +A tool never touches the transcript, its siblings or the inbox. Ordinary tool code receives no task or host +commit authority. The built-in `pi.tool` adapter may eventually expose narrow keyed memo, child-task and +owned-conversation operations. Those mediated operations atomically update the parent checkpoint internally; +exact names and types remain gated. + +### Diagnostics + +Anything that is *about* the call rather than its output goes through `diag`: truncation, a spilled +file, a corrected path, "the file changed on disk since you read it". The harness emits the ones it +owns (the sink itself reports truncation and spill); a tool adds only what it alone knows. +Tool settlement puts the output first and the commentary after it and stores that exact message in +entry `model`. Non-message details, usage, control flags, diagnostics and truncation metadata become +entry `data`. A UI combines the two and renders diagnostics as callouts by severity: + +```text +...last matching line + +[warn] stopped at 500 matches + +``` + +### Long-Running Tools + +A process-running tool is job-first; Pico never adopts an arbitrary unfinished tool promise. The required +trace is: + +```text +pi.tool task T owns task output O +internal TX[create background job B sharing O; checkpoint T with B + abortWithTool policy] +kick -> Session scheduler runs B automatically +ordinary tool observes B for its local budget +adapter terminal closure rereads B on the line + B terminal -> materialize normal result from O + B live -> materialize continues-as-B result; terminalize T while B retains O +B eventually materializes final O, terminalizes its last ref, and emits any passive notice atomically +``` + +Never trust stale timeout or `Promise.race` state. Creation records cleanup policy before returning a child +handle. Fresh tool abort reads that durable record and atomically marks a non-detached live job; an already +terminal job counts as cleaned. The scheduler already runs the job, so waits only observe. The same shared +task output lets the UI keep one `bash` component across tool-to-job handoff without copying a second live +output stream. Exact ordinary-tool handle methods remain gated. + +## Hooks + +**Gated:** exact hook names, task identity and namespaced scratch capability remain unsettled. The examples +show required semantics, not a final API. Hooks belong to the kind that runs them. A kind declares its points and their types; you register +handlers per kind and point, harness-wide or scoped to a conversation. Handlers run outside the +write line and their decisions are re-validated inside a commit, so they may take as long as they +like (a human approval is a hook that waits). + +```typescript +// policy: everywhere +h.hooks.on(toolKind, 'before_tool', async ({ toolName, args, conversationId }, call) => { + const conversation = await h.conversation(conversationId, call); + if (toolName === 'write' && await conversation?.value(planMode).get(call)) + return { block: { reason: 'plan mode: no edits' } }; + if (toolName === 'bash' && !(await ui.approve(args, { signal: call.abortSignal }))) + return { block: { reason: 'denied by user' } }; + return { args }; // may rewrite arguments +}); + +h.hooks.on(toolKind, 'after_tool', async ({ toolCallId, output }, call) => { metrics.record(output.usage); }); + +h.hooks.on(generationKind, 'before_request', async ({ request }, call) => ({ request: withTracing(request) })); + +h.hooks.on(generationKind, 'on_yield', async ({ answer, conversationId }, call) => { + if (await goalIncomplete(conversationId, call)) return { continue: 'The goal is not met yet; continue.' }; +}); + +h.hooks.on(collapseKind, 'before_collapse', async ({ reason, entries }, call) => { + if (reason === 'manual' && entries.length < 10) return { decline: true }; +}); + +// instructions: per conversation (see System Prompt and Tool Loadout) +c.hooks.on(generationKind, 'system_instructions', handler, { subtree: true }); +``` + +Points and their fail behaviour: + +| kind | point | returns | on throw | +|---|---|---|---| +| generation | `system_instructions` | edits section draft; optional complete tools | reported, skipped | +| generation | `before_request` | a transformed request | reported, skipped | +| generation | `after_response` | nothing | reported | +| generation | `on_yield` | `{ continue?: string }` | reported, skipped | +| tool | `before_tool` | `{ args? }` or `{ block }` | **blocks the tool** | +| tool | `after_tool` | nothing | reported | +| collapse | `before_collapse` | `{ decline? \| instructions? \| summary? }` | reported, skipped | + +These failure policies exclude cancellation control errors, which propagate to unwind the invocation. +A handler receives the active Call as its final argument and must forward it to waits/effects. The +harness awaits its actual return, not an abandoned raced promise. A handler may run again after a +crash, so its external side effects need their own idempotence. + +Approval/question policy, presentation and durable answer reuse belong to workspace/plugins, using +existing scratch or scoped values as appropriate. Pico does not add a separate memo subsystem. How +hooks receive task identity and authorized scratch access is still an integration question; the +examples do not establish a new hook-payload API. + +## Writing Kinds + +Kinds are how you extend the harness with new behaviour rather than new state. An entry kind is a +typed name for an immutable transcript shape; a task kind says how a kind of work runs. + +### Entry Kinds + +Entry facets combine. `data` is optional kind-specific JSON for logic and custom UI rendering; +`model` is an optional stored `Message[]`; `head` and `edits` are optional stored context controls. +No facet is derived while reading. + +```typescript +interface NoteData { text: string; pinned?: boolean } +type NoteEntry = EntryBase & EntryData; + +export const noteKind = defineEntryKind('myplugin.note'); + +await c.commit(tx => tx.entry(noteKind, { + data: { text: 'Parser plan accepted', pinned: false }, +}), call); // transcript/UI only; the model sees nothing +``` + +A plugin that wants both typed data and a model message keeps the append-time conversion in an +ordinary helper: + +```typescript +type PinnedEntry = EntryBase & EntryData & ModelProjection; +export const pinnedKind = defineEntryKind('myplugin.pinned'); + +function appendPinned(tx: ConversationTx, data: NoteData, timestamp: number): Id { + return tx.entry(pinnedKind, { + data, + model: [{ role: 'user', content: `${data.text}`, timestamp }], + }); +} +``` + +Heads and edits are supplied the same way: + +```typescript +type WindowEntry = EntryBase & EntryData<{ retainFrom: Id }> & ContextHead; +export const windowKind = defineEntryKind('myplugin.window'); + +tx.entry(windowKind, { + data: { retainFrom }, + head: retainFrom, // first retained entry, inclusive +}); + +type ToolResultEditEntry = EntryBase & ContextEdits; +export const toolResultEditKind = defineEntryKind('myplugin.tool_result_edit'); + +tx.entry(toolResultEditKind, { + edits: replacement === undefined + ? [{ target, action: 'omit' }] + : [{ target, action: 'replace', messages: replacement }], +}); +``` + +`head: 'self'` in an append draft stores the new entry id, which is how reset and handoff discard +everything earlier. Commit validation requires a stored head boundary not to move before the +previous visible boundary. Edits apply in transcript order; the newest edit per target wins. Append +another head or edit when context should change; no entry-kind callback runs during context reads. + +Reads are typed by the kind, or untyped and narrowed: + +```typescript +const note = await h.getEntry(noteKind, id, call); // NoteEntry | undefined (also undefined for another kind) +const any = await h.getEntry(id, call); // Entry | undefined +if (noteKind.is(any)) any.data.pinned; +``` + +A missing plugin removes that narrowing and its custom renderer, but not context behavior: the +entry's model messages, head and edits are stored independently of the kind. + +### Task Kinds + +An ordinary task kind declares immutable input, a complete optional checkpoint union, +completed/failed/aborted payloads, optional task-output state and three methods. The generic scheduler +interprets only pending/running/terminal, dependencies, background, abort mark and checkpoint presence. A +checkpoint's `phase` belongs to the kind; it is not a global workflow step. + +Here is a reminder that fires once: + +```typescript +type ReminderInput = { readonly about: Id; readonly at: number }; +type ReminderCheckpoint = { readonly phase: 'firing'; readonly effectKey: string }; + +export const reminderKind = defineTask< + ReminderInput, + ReminderCheckpoint, + { readonly fired: boolean }, + { readonly message: string }, + { readonly cleaned: boolean } +>()({ + kind: 'myplugin.reminder', + + async execute(task, runtime, call) { + if (task.input.at > runtime.now()) await runtime.sleep(task.input.at, call); + await runtime.commit(tx => { + tx.checkpoint({ phase: 'firing', effectKey: `reminder:${task.id}` }); + }, call); // durable intent before the external effect + + await notifications.fire(task.input.about, `reminder:${task.id}`, call); + return () => ({ status: 'completed', result: { fired: true } }); + }, + + async recover(task, runtime, call) { + if (!task.checkpoint) return reminderKind.execute(task, runtime, call); + const fired = await notifications.recover(task.checkpoint.effectKey, call); + return () => ({ status: 'completed', result: { fired } }); + }, + + async abort() { + return () => ({ cleaned: true }); + }, +}); +``` + +The scheduler reserves pending as running before `execute`, and reserves a restored running task for +`recover`, including one with no checkpoint. Methods perform effects outside the line and return a terminal +closure. The Harness invokes that closure once on the line and atomically commits its buffered writes, full +terminal task snapshot and scratch/output retirement. A throwing closure commits nothing and faults the +Session. + +Pico's fixed privileged definitions use its internal `defineCoreTask` factory instead. Their callbacks +receive `CoreTaskRuntime`, so `runtime.commit` supplies `CoreTaskTx`. That transaction uses the same +`tx.task(...)` spelling but accepts both ordinary and fixed core tokens: + +```typescript +const generationKind = defineCoreTask()({ + kind: 'pi.generation', + + async execute(task, runtime, call) { + // ...provider effect... + return (tx, current) => { + const toolTaskIds = completedAssistant.toolCalls.map(call => + tx.task(toolKind, { input: makeToolInput(current, call) })); + tx.task(postToolsKind, { + input: makePostToolsInput(current), + after: toolTaskIds, + }); + return { status: 'completed', result: generationResult }; + }; + }, + + recover, + abort, +}); +``` + +An ordinary `BaseTaskTx.task(...)` accepts only ordinary `defineTask` tokens. `defineCoreTask` is not a +plugin API; fixed core definitions cannot be registered, replaced or removed. The fixed dispatcher supplies +the core runtime implementation. The kernel does not inspect core input/checkpoint schemas. + +Use `defineStateTask` when one ordinary kind has several named durable recovery phases. It compiles to the same +ordinary `TaskKind`; the scheduler still invokes one outer execute/recover operation and never interprets +those phases. Do not create a giant turn task: built-in conversation work remains generation -> parallel +individual tools -> post_tools -> continuation generation, plus independent collapse and job tasks. + +Rules for task authors: + +1. Commit a complete checkpoint before an uncertain external effect. Recovery uses immutable input and the + latest committed checkpoint; it does not recover a JavaScript stack. +2. Effects, hooks, sleeps and waits happen outside transaction callbacks. Inside `runtime.commit`, perform + explicit committed reads before the first buffered write and use only the supplied `tx` afterwards. +3. Return a terminal closure only after effects and producers finish. Cancellation may discard an uncommitted + normal closure; fresh `abort` owns cleanup. +4. Prefer immutable `after` dependencies for task prerequisites. Task waits only observe and ensure Session + resume. Direct self-wait and dependency cycles reject; never race away unfinished work. +5. Keep generic configuration in scoped values, not task snapshots. Persist only compact evidence required + to recover this kind's effect. + +The one scheduler keeps the complete Session live/dependency index on the mutation line and invokes task +methods outside it. Every successful main commit kicks it; scratch/task-output-only commits need not. It +reserves all eligible tasks in every conversation, runs distinct IDs concurrently, and allows at most one +invocation per task ID. Foreground and background both run. Foreground controls idle and ordinary abort +reach only. + +A kind may declare task output with `TaskOutputSpec.initial(input)`. Runtime mutates the one authoritative +Chord-tracked output; each nonempty delta persists and appears as `task_output`. Shared refs let a tool hand +output to a background job without copying it. The last live reference materializes final data and retires +the output atomically. Task output is not scratch and there is no separate live-output subsystem. + +An unexpected task-contract error faults the Session: observation waits reject, admission stops, running +invocations are signalled and joined, and the binding closes. Domain tool/provider failures must be returned +as typed outcomes. Supply every required ordinary kind during open; a missing live kind is already orphaned +before `inspect` returns and later registration cannot resurrect it. Fixed core kinds are never replaced. + +## Recovery + +There is nothing to write. After a crash: + +```typescript +const h = await Harness.open(storage, opts, call); +await h.resume(call); +``` + +| what was running | what happens | +|---|---| +| a generation, streaming | its task output contains the partial assistant state; recovery publishes it, then retries within budget or fails | +| a generation, between retries | it sleeps out the remaining backoff and tries again | +| a tool with `replay: 'safe'` | runs again | +| any other tool | an "interrupted" error result; the turn continues | +| a `run` subagent tool | finds its child and waits on the existing input result again | +| a job | reruns if it said so, otherwise records `lost` | +| a scheduled job | sleeps until its time | + +Successors are written in the same commit as the settlement that decides them, so a crash never +leaves "the tools finished but nobody started the next turn": post_tools already exists and is +startable. + +## Storage Backends + +| backend | loads | good for | +|---|---|---| +| `MemoryStorage` | everything | tests, ephemeral sessions | +| `JsonlStorage` | main log plus live scratch/task-output sidecars | local Sessions; the default | +| `SqliteStorage` | only what queries return | long sessions, many sessions in one file, servers | + +All three answer the same queries with the same results. Entries are immutable and read whole; indexed +queries select the relevant entries before decoding. Values are stored whole on every set. Lists store +append/remove/clear operations and return one complete coherent logical list in ascending element-ID order; +v1 has no paged list scan. There is no automatic value diffing or Chord storage codec. +Chord deltas are used for task-output/watch delivery. Streaming task output appends compact operations, not +raw provider events with repeated growing partial snapshots. + +JSONL reopens by replaying complete main commits, then sidecars for surviving live scratch and shared task +outputs. Per-file `Seq` gaps are expected. Reopen initializes `Storage.nextId()` after the greatest +conversation, entry, task or list-element ID in retained complete canonical writes, and separately restores +the last applied sequence from the greatest retained complete record endpoint. Retired sidecars are +ignored even if unlink failed; their later main settlement already proves retirement. +Only unterminated final lines are discarded and removed before further appends; malformed complete +main/live-scratch batches fail open. + +```text +main through Seq 100; live scratch through Seq 150 -> reopen applied Seq 150 and initializes nextId +main settles at Seq 151; unlink fails -> ignore retired scratch; applied Seq remains 151 +``` + +The conformance suite compares one mutation stream across all backends. A repeated whole-value set +can write quadratic bytes when the value grows; incremental lists avoid that without hidden encoding. + +```typescript +const storage = await SqliteStorage.open('./sessions.db', { session: 'parser-work' }); +``` diff --git a/packages/agent/docs/pico/pico-usage-v2.md b/packages/agent/docs/pico/pico-usage-v2.md new file mode 100644 index 000000000..1e834b2ef --- /dev/null +++ b/packages/agent/docs/pico/pico-usage-v2.md @@ -0,0 +1,773 @@ +# pico + +Durable agent harness. One session file, any number of conversations, every piece of work is a task that +survives a crash, and a view a UI renders directly. + +This guide shows how to use it. `pico-handoff-v2.md` is the implementation specification. + +- [Mental model](#mental-model) +- [Quick start](#quick-start) +- [Opening, closing, recovery](#opening-closing-recovery) +- [Sending input](#sending-input) +- [Configuration](#configuration) +- [System prompt](#system-prompt) +- [Watching](#watching) +- [Forks, compaction, reset](#forks-compaction-reset) +- [Plugin state](#plugin-state) +- [Writing tools](#writing-tools) +- [Hooks](#hooks) +- [Writing entry kinds](#writing-entry-kinds) +- [Writing task kinds](#writing-task-kinds) +- [Subagents and jobs](#subagents-and-jobs) +- [Storage backends](#storage-backends) + +## Mental model + +A **session** holds **conversations**. A conversation has: + +- a **transcript**: append-only immutable **entries** (user, assistant, tool result, summary, notice, ...); +- **tasks**: units of work (generate, run a tool, compact, run a process). Each survives a crash; +- **state**: keyed values and lists (model in use, plan mode, whatever a plugin needs). + +The model **context** is derived from entries, not stored: the newest head entry says where context starts, +edits omit or replace earlier messages. Compaction appends a summary with a head; nothing is rewritten. + +All writes go through **commits**: one callback, one atomic batch. Tasks run concurrently, commits serialize. + +Every async call takes a Chord `Context` last (`ctx`). It carries cancellation and telemetry. Cancelling a +wait removes the waiter; it never cancels durable work. + +## Quick start + +```typescript +import { Harness, JsonlStorage, systemSections } from '@earendil-works/pi-agent/pico'; +import { builtinModels } from '@earendil-works/pi-ai/providers/all'; +import { BACKGROUND_CONTEXT as ctx } from '@earendil-works/chord/context'; + +const storage = await JsonlStorage.open('./session.jsonl', ctx); +const h = await Harness.open(storage, { + models: builtinModels(), + // no `tools`: the ordinary tool API is not settled yet (see Writing tools); this is a tool-free agent + root: { // applied once, when the session file is new + config: { + model: { provider: 'anthropic', modelId: 'claude-opus-5' }, + thinkingLevel: 'high', + }, + }, +}, ctx); + +const c = await h.root(ctx); + +// System prompt: edit typed sections before each request. Stored when it changes, replayed after a restart. +// Registered on this conversation only: subagents get their own identity (see Subagents). +c.hook.systemInstructions(({ sections }) => { + sections.set(systemSections.identity, 'You are a careful engineer working in this repository.'); + sections.set(systemSections.environment, { cwd: process.cwd() }); +}); + +// Watch: `w.view` is always current; each delivery is one commit's events. +const w = await c.watch(ctx); +w.start((d) => { + if (d.type !== 'commit') return; + for (const e of d.events) { + if (e.type === 'task_output') process.stdout.write(renderStreaming(w.view.taskOutputs)); + if (e.type === 'entry') console.log(`\n[${e.entry.kind} ${e.entry.id}]`); + if (e.type === 'config' && e.key === 'model') statusLine.set(e.value); // typed: ModelRef | undefined + } +}); + +// Send: durable, returns at once. Wait for this input's answer. +const input = await c.send({ requestId: 'cli-1', content: 'List the public API of the parser' }, ctx); +const result = await input.wait(ctx); +if (result.status === 'done' && result.answer) console.log(result.answer.message); + +// Change settings any time; the next request uses them. +await c.config.model.set({ provider: 'openai', modelId: 'gpt-5.6' }, ctx); + +w.unsubscribe(); +await h.close(ctx); +``` + +Kill it mid-run and run it again: the second run recovers the in-flight generation (or job) and continues; +tools will recover the same way once their API lands. + +## Opening, closing, recovery + +```typescript +const h = await Harness.open(storage, options, ctx); // inert: nothing runs yet +await h.resume(ctx); // start/recover every eligible task +``` + +`send`, `wait`, `waitForIdle`, `collapse` and abort operations call `resume` for you. Reads, inspection and +watch do not. After a crash, `open` + `resume` is the whole recovery procedure: + +| was running | on resume | +|---|---| +| generation | partial output is restored; request retried or failed within budget | +| tool with `replay: 'safe'` | runs again (once the tool API lands) | +| other tool | "interrupted" error result; the turn continues (once the tool API lands) | +| tool waiting on a child | waits again on the same child (once the mediated child API lands) | +| job | asks the process host for the run's status; if the host still knows it, continues; if not, reruns under the same key if `rerun` (possibly duplicating the effect), else `interrupted`. `nodeProcessHost()` forgets runs on restart | + +```typescript +const { pending, running, orphaned, unknownEntryKinds } = await h.inspect(ctx); +``` + +`orphaned` are live tasks whose kind was not registered at open; they were terminalized. Register all +kinds before opening. + +```typescript +await h.close(ctx); // stop, write nothing; everything resumes next open +await h.shutdown(ctx); // abort every live task cleanly, keep queued input, then close +``` + +Options: + +| option | | +|---|---| +| `models` | pi-ai models; `generationKind.config.model` is resolved against it | +| `tools` | tool declarations the model may call; not usable yet, the execution API is gated (see Writing tools) | +| `taskKinds`, `entryKinds`, `sections` | plugin kinds and system section definitions | +| `root.config` | initial built-in config (`model`, `thinkingLevel`, `selectedTools`, ...) for a fresh session; ignored on reopen | +| `root.values`, `root.sections`, `rootValues` | advanced: arbitrary initial values / section payloads for the fresh root | +| `processHost` | how `pi.job` runs processes; `nodeProcessHost()` for local jobs (in-memory: forgets runs on restart); omitted means every job fails with reason `spawn` | + +Registries supply implementations, not selections: registering `bash` does not select it. Select through +`c.config.selectedTools`. `h.tools`, `h.taskKinds`, `h.entryKinds`, `h.sections` have +`register`/`replace`/`remove` for changes after open; replacing or removing a task kind with live tasks +rejects. Hooks are registered on `h.hooks` or `c.hooks` (below), not in open options. + +## Sending input + +```typescript +const input = await c.send({ content: 'Inspect the parser', requestId: 'req-1' }, ctx); +input.id; // stable input id +await input.result(ctx); // queued | placed | done | unanswered, no waiting +const r = await input.wait(ctx); // done | unanswered +await input.abort(ctx); // aborted | already_placed | not_found + +if (r.status === 'done') { + r.input; // UserEntry: the placed user entry + r.answer?.entry; // AssistantEntry, when an answer was owed + r.answer?.message; // the stored AssistantMessage (entry.model[0]); text, thinking, tool calls, usage +} else { + r.reason; // terminated | aborted | failed | stale + r.input; // present if the input had been placed before it went unanswered +} +``` + +Outcomes come with their entries attached; there is nothing to look up afterwards. `message` is the message as +stored, not the current context projection, so a later edit or compaction does not change what `wait` +returned. + +`requestId` makes `send` idempotent for the session: retrying returns the original handle and writes nothing. +`h.input(requestId, ctx)` finds it later. + +While the conversation is busy (a generation or tool is live), input is queued: + +```typescript +await c.send({ content: 'then write tests' }, ctx); // followUp: after the answer +await c.send({ content: 'focus on the tokenizer', whenBusy: 'steer' }, ctx); // steer: at the next tool boundary +await c.send({ content: '...', whenBusy: 'reject' }, ctx); // throws ConversationBusy +``` + +Steer joins the current turn after the running tools finish. FollowUp starts a new turn after the final +answer. Both survive restarts. Passive content (a notice the model should see next time) uses `write`: + +```typescript +await c.commit((tx) => tx.write(noticeKind, { + model: [{ role: 'user', content: 'CI finished: 3 failures', timestamp: Date.now() }], +}), ctx); +``` + +`write` appends immediately if idle, otherwise queues for the next boundary. It never asks for an answer. + +There is no "queue this for the next run" mode. If you accumulate context that should not trigger a turn yet +(observations, partial instructions), keep it in your own value or list and fold it into a single `send` when +you do want an answer; use `write` only when a passive transcript entry is the point. + +```typescript +await c.waitForIdle(ctx); // this conversation's foreground work +await h.waitForIdle(ctx); // whole session +await c.abort(ctx); // abort this conversation's foreground cancellation set (turn, foreground tasks, owned subagents); withdraws queued steer/followUp +await h.abortTask(id, ctx); // one task; queued input is kept +``` + +## Configuration + +Built-in configuration is a set of typed properties on `c.config`, one per value, plus a batched setter: + +```typescript +await c.config.model.set({ provider: 'openai', modelId: 'gpt-5.6' }, ctx); +await c.config.thinkingLevel.set('high', ctx); +await c.config.selectedTools.set(['read', 'grep'], ctx); +await c.config.followUpMode.set('all', ctx); + +const model = await c.config.model.get(ctx); // current (or the definition's default) +const then = await c.config.model.get(entryId, ctx); // as of an entry (rewindable values only) +const mode = await c.config.followUpMode.get(ctx); // sticky: current only + +// several at once, one commit; keys and value types are exact +await c.config.set({ model: { provider: 'openai', modelId: 'gpt-5.6' }, thinkingLevel: 'low', followUpMode: 'all' }, ctx); +const all = await c.config.get(ctx); +``` + +| property | type | default | rewindable | +|---|---|---|---| +| `model` | `ModelRef` (`{ provider, modelId }`) | none: generation fails `no_model` | yes | +| `thinkingLevel` | `ThinkingLevel` (`off` … `xhigh`, `max`) | `'off'` | yes | +| `selectedTools` | `string[]` | `[]` | yes | +| `profile` | `string` | `'default'` | yes | +| `retry` | pi-ai `RetryPolicy` | `{ enabled: true, maxRetries: 3, baseDelayMs: 2000, maxAgentDelayMs: 60000 }` | no | +| `steeringMode`, `followUpMode` | `'all' \| 'one-at-a-time'` | `'one-at-a-time'` | no | +| `threshold` | `number` (context tokens; 0 = never) | `0` | yes | +| `keepRecent` | `number` (tokens kept after compaction) | `20000` | yes | + +Defaults live on the definitions and are what `get`, `view.config` and `config` events return when nothing is +set; `delete` restores them. The types say so: `c.config.followUpMode.get(ctx)` is `'all' | 'one-at-a-time'`, +never `undefined`; only `model` (no default) can be `undefined`. + +These are the same definitions the built-in tasks read (`generationKind.config.model` and friends). Each kind +exports a typed bundle of reusable conversation-value definitions; `c.config` is assembled from all of them, +so there is no second schema, and a name clash between kinds or with `set`/`get` is a compile error. The +definitions carry no conversation ID; the handle binds them, and you can pass them anywhere a conversation +value is accepted (`c.value(generationKind.config.model)`, `root.values`, child seeds). Plugin configuration is +not in `c.config`; plugins declare their own values and use `c.value(token)`. + +Rewindable means a fork at yesterday's answer gets yesterday's model. Queue modes are sticky: they describe +how you want the present handled, not history. The next generation reads current values; nothing is cached. + +## System prompt + +Pico sends pi-ai `{ messages }` only. Instructions and the tool loadout are system messages inside the +transcript at the position they took effect, so the model's cached prefix survives a change and a fork sees +exactly what its source saw. + +Sections are typed tokens with a renderer. A `systemInstructions` hook edits a draft before each request: + +```typescript +import { defineSystemSection, systemSections } from '@earendil-works/pi-agent/pico'; +import { generationKind } from '@earendil-works/pi-agent/pico/kinds'; + +const rules = defineSystemSection({ + key: 'myplugin.rules', + render: (r) => r.map((x) => `- ${x}`).join('\n'), +}); +await h.sections.register(rules, ctx); + +// host, harness-wide: sets the authoritative base every time, so it runs first +h.hooks.on(generationKind, generationKind.hooks.systemInstructions, ({ sections }) => { + sections.set(systemSections.identity, 'You are a coding assistant.'); + sections.set(systemSections.skills, skillsCache.current); + sections.set(rules, ['Run relevant tests.']); +}); + +// plugin, on this conversation: runs after the host and modifies typed payloads, never parses prose +c.hook.systemInstructions(({ sections }) => { + sections.set(rules, [...(sections.get(rules) ?? []), 'Check migration safety.']); + sections.wrap(rules, (text) => `Repository policy:\n${text}`); +}); + +// rare: override the loadout for this request instead of c.config.selectedTools +c.hook.systemInstructions(({ tools }) => ({ tools: tools.filter((t) => t.name !== 'bash') })); +``` + +Handlers run harness-wide first, then outer conversation to inner, on one draft; the plugin above therefore +sees the host's base. Registration is synchronous and process-local (it returns an unsubscribe function); +`h.sections.register` is asynchronous because the section registry is shared state. `get` returns a copy; +`set` keeps an existing key's position; `delete` is explicit (null is a value); `wrap` transforms rendered +text and is not stored. A throwing handler loses only its own edits. The tool loadout is +`c.config.selectedTools` resolved against `h.tools`; a handler that returns `tools` overrides it for that +request. + +After the hooks, the draft is compared to what was last stored. No change: nothing is written. A change: +one `pi.system` entry with the changed sections' payload and rendered text, and a system message with +`toolsRemoved`/`toolsAdded`. After a compaction or reset, the next request gets a fresh complete baseline. +A plugin that disappears leaves its rendered text in place until someone deletes the section. + +Because handlers rerun on every preparation, a handler that appends to what it reads accumulates unless an +earlier handler resets the base. Make the chain idempotent as a set. + +## Watching + +```typescript +const w = await c.watch({ values: [planMode], lists: [moves] }, ctx); // or c.watch(ctx); === h.watchConversation(c.id, ...) +w.view; // ConversationView, captured atomically with the subscription +w.start((d) => { // one delivery per commit: { type: 'commit', first, last, events } | { type: 'closed', reason } + if (d.type === 'commit') for (const e of d.events) { /* w.view is already updated */ } +}); +w.unsubscribe(); +``` + +```typescript +// abbreviated sketch; the exact readonly declarations are in the handoff, section 14 +interface ConversationView { + conversation: Conversation; + entries: Entry[]; // the active transcript in order: everything from the newest head's boundary through now + context: Id[]; // entry ids the model currently sees, derived from entries + tasks: Task[]; // live tasks in this conversation + taskOutputs: { id: Id; kind: string; value: OutputState }[]; // streaming assistant text, tool progress + inbox: Element[]; + config: { model?, thinkingLevel, selectedTools, profile, retry, steeringMode, followUpMode, threshold, keepRecent }; // defaults resolved: every key except model is present and never undefined + values: { address: Value; value?: JsonValue }[]; // only what you asked for in { values: [...] } + lists: { address: List; elements: Element[] }[]; // only what you asked for in { lists: [...] } + readAt: Seq; +} +``` + +A requested list is captured whole and then kept current by `list` events (`append`/`remove`/`clear` ops in +commit order, already folded when the listener runs). Built-in config cannot be requested as a value; it is +always in `view.config`. + +`view.config` has the same properties as `c.config`, so a status line reads `view.config.model` and a settings +panel binds to it directly; nothing needs to be requested. `values` is for your own state (`planMode` above). + +Built-in config changes arrive as their own event, typed by key: + +```typescript +w.start((d) => { + if (d.type !== 'commit') return; + for (const e of d.events) { + if (e.type === 'config') { + // e.key narrows e.value / e.previous + if (e.key === 'model') statusLine.setModel(e.value); // ModelRef | undefined + if (e.key === 'selectedTools') toolbar.set(e.value); // readonly string[]: defaulted, never undefined + } + if (e.type === 'value') plugins.update(e.value.address, e.value.value); // your requested values + if (e.type === 'list') board.render(w.view.lists); // ops already folded + } +}); +``` + +`c.config.set({ model, followUpMode })` yields two `config` events in the same delivery; `w.view.config` already +reflects both when the listener runs. Read the initial state from `w.view.config`, then follow events. + +The listener is a wake-up: `w.view` is already updated when it runs. A renderer that only looks at the view is +correct. + +`entries` is what the conversation is currently working with, in transcript order, not a fixed number of +recent entries. It is captured once and then maintained incrementally: an ordinary entry appends; an entry that +carries a `head` (summary, reset, handoff) first drops the entries before its retained boundary and then +appends itself, all within the same delivery. A summary therefore sits after the entries it retains (the model +sees it first; that is context projection, not the view). A reset or handoff points at itself, so it clears the +list and becomes the first entry. The older transcript is still in storage and readable through `h.entries({ conversationId: c.id, limit: 50 }, ctx)` (newest first, paged) and `h.getEntry`, it just is +not part of the live view. There is no window size to tune. + +| you want | look at | +|---|---| +| in a turn | `view.tasks` has a `pi.generation`/`pi.tool`/`pi.post_tools` | +| streaming text | `view.taskOutputs` entry for the generation's `output.id` | +| a tool running | `pi.tool` task; its `output` for progress | +| retrying | generation `checkpoint.phase === 'retrying'` | +| current model / tools / thinking | `view.config.model`, `.selectedTools`, `.thinkingLevel`; changes: `config` event with `key`/`value`/`previous` | +| compacting | `pi.collapse` task present | +| queued input | `view.inbox` | + +Overflow (listener too slow, 256 commits buffered) closes the watch with `{ type: 'closed', reason: +'overflow' }`. Open a new one; its view is the fresh truth. + +Remote UIs: ship `w.view` once, then every delivery. `applyConversationCommit(view, commit)` is exported and +needs no kinds. + +```typescript +const sw = await h.watchSession({ values: [sessionName] }, ctx); // conversation list, session values +``` + +### Plugin state in a UI + +Requested Pico values and lists (as in the watch above) are for the process that owns the Harness: the plugin's +session-side facet, or a host renderer in the same process. Pico does not push plugin state to remote UIs. A +plugin that has something to show elsewhere follows the Chord facet pattern +used by `coding-agent/examples/plugins/pi-example-plugin` and the experimental `models-provider` / +`transcript-provider` services: + +```typescript +// contract.ts: what the UI may see and call +export interface PlanService { + readonly state: ReplicatedState<{ active: boolean; steps: string[] }>; + toggle(input: { active: boolean }, ctx: Context): Promise; +} +export const PlanService = defineService('myplugin.plan'); + +// session.ts: runs beside the Harness; Pico is the durable truth, the replicated state is a projection +import { BACKGROUND_CONTEXT } from '@earendil-works/chord/context'; +export default defineFacet({ + id: 'myplugin/session', + setup(env) { + const state = env.replicatedState({ active: false, steps: [] }); + env.provide(PlanService, { + state, + async toggle({ active }, ctx) { + await c.value(planMode).set(active, ctx); // durable + state.state.active = active; state.publish(ctx); // live + }, + }); + env.onActivate(async () => { // reopen: rebuild the projection from Pico + state.state.active = (await c.value(planMode).get(BACKGROUND_CONTEXT)) ?? false; + state.publish(BACKGROUND_CONTEXT); + }); + }, +}); + +// tui.ts: never sees the Harness or addresses +export default defineFacet({ + id: 'myplugin/tui', + setup(env) { + const plan = env.use(PlanService); + env.onActivate(() => env.own(plan.state.subscribe((value) => badge.set(value.active)))); // (value, context, delivery) => void + }, +}); +``` + +Built-in config is the exception and is automatic (`view.config`); everything else a UI should see is a +service DTO the plugin defines. + +## Forks, compaction, reset + +```typescript +const alt = await c.fork({ at: entryId }, ctx); // shares history through entryId; independent after +const alt2 = await c.fork({ at: 'start' }, ctx); // empty transcript, same session +const back = await c.fork({ at: earlier, abort: true }, ctx); // "go back": abort the source's turn first +await alt.send({ content: 'try recursive descent instead' }, ctx); +``` + +Forks inherit rewindable state (model, tools) and the system messages visible at the fork point. They inherit +no tasks and no queued input. Nothing in the source changes. + +```typescript +const taskId = await c.collapse({ instructions: 'keep API decisions verbatim' }, ctx); +await h.getTask(taskId, ctx); +``` + +Manual compaction runs in the background while the model keeps working; it declines with `NothingToCollapse` +if everything already fits in `keepRecent`. Threshold compaction is created by generation automatically when the +context estimate crosses `threshold` and runs in the foreground (aborting the conversation aborts it too). Both append a `pi.summary` entry with a head; the transcript keeps everything. + +```typescript +await c.reset({ handoff: 'We settled on recursive descent; continue from the lexer.' }, ctx); +await c.reset(ctx); // /clear +``` + +Reset appends a `pi.handoff` or `pi.reset` entry with `head: self`. If the conversation is busy it is queued for +the next boundary; either way `reset` resolves once the request is durable, and the entry shows up in the watch +when it lands. + +## Plugin state + +```typescript +import { conversationValue, conversationList, sessionValue } from '@earendil-works/pi-agent/pico'; + +type Move = { readonly x: number; readonly y: number }; +const planMode = conversationValue('plan.mode', { rewind: true }); +const moves = conversationList('game.moves', { rewind: true }); +const expanded = conversationValue('ui.expanded', { rewind: false }); +const name = sessionValue('app.name'); + +await c.value(planMode).set(true, ctx); // c binds it to its own conversation +await c.value(planMode).get(ctx); +await c.value(planMode).get(entryId, ctx); // rewindable only +const id = await c.list(moves).append({ x: 1, y: 2 }, ctx); +await c.list(moves).read(entryId, ctx); +await c.list(moves).remove(id, ctx); +await h.value(name).set('parser work', ctx); + +planMode.bind(otherConversationId) // explicit Value when you need one +``` + +A conversation address is declared once and bound when used. It carries rewind policy and payload type. +Your definitions work everywhere: `c.value`, commits, `root.values`, child seeds, watch `values`. Pico's +built-in config tokens (`generationKind.config.model` and friends) are exported and work the same way. Pico's +internals (inbox, input results, request receipts, task output) are not addressable through any +generic handle; they surface only as `view.inbox`, `InputHandle` and `view.taskOutputs`, and you cannot define +values in those namespaces. + +Anything that must land together goes in one commit (assumes `noteKind` and `reminderKind` are registered, see +below; a kind object that is not the registered one is rejected as `StaleDefinition`): + +```typescript +const id = await c.commit(async (tx) => { + const current = await tx.value(planMode).get(); // reads first + tx.value(planMode).set(!current); // rewindable state before entries + const note = await tx.write(noteKind, { data: { text: 'toggled' } }); + tx.task(reminderKind, { background: true, input: { about: note, at: Date.now() + 3600_000 } }); + return note; +}, ctx); +``` + +Rules inside a commit: reads before the first write (`ReadAfterWrite` otherwise); rewindable writes before +entry appends; no awaiting anything but `tx` reads; a throw discards everything. + +## Writing tools + +**The ordinary tool API is not settled.** This is the one deliberate gap in Pico v1: what a tool's `execute` +receives and how it streams progress is still being decided, so nothing in this section is a compile contract. +What *is* settled: + +- A tool declares `name`, `description`, `parameters` (TypeBox schema), `replay: 'safe' | 'unsafe'` (may the + call be rerun after a crash) and `output` bounds (`maxBytes`, `maxLines`, `retain`). +- A tool returns a `ToolResult`: `content` (text/images the model reads), typed `details` for UIs, `isError`, + `usage`, `diagnostics` (commentary about the call, rendered after the output), and optional `control`. +- `control`: `{ terminate: true }` ends the turn after this exchange; `{ handoff: message }` ends it and resets + context with that message (what `new_context` does); `{ addTools: [...] }` adds tools to `selectedTools` so + the very next model request (normally the continuation right after this tool round) already offers them. +- Pico enforces the output bounds, reports truncation, stores the exact model message in the result entry and + the rest in its data, and streams the tool's live progress as the tool task's output (`view.taskOutputs`). +- A throw becomes an `isError` result; it never faults the session. Tools cannot touch the transcript, the task + or the inbox, and get no commit authority. +- Not yet available: durable per-call memos, starting a background job from a tool, starting a subagent from a + tool. Requirements are fixed (keyed and idempotent across a crash, cleanup policy recorded before the tool + returns, a wait timeout never cancels the child, a job is a task and a subagent is a conversation) but the + API shape is not. Until it is, the built-in `bash` cannot background and there is no `subagent` tool. + +Illustration only (the `execute` signature and the `tool` facade are the gated part): + +```typescript +// ILLUSTRATIVE, not the final API +const params = Type.Object({ path: Type.String(), pattern: Type.Optional(Type.String()) }); + +export const countLines = { + name: 'count_lines', + description: 'Count lines in a file', + parameters: params, + replay: 'safe', + output: { maxBytes: 64_000, maxLines: 200, retain: 'head' }, + async execute(args /* Static */, tool /* gated facade */, ctx) { + const lines = (await fs.readFile(args.path, 'utf8')).split('\n'); + const matching = args.pattern ? lines.filter((l) => l.includes(args.pattern!)) : lines; + return { + content: [{ type: 'text', text: matching.join('\n') }], + details: { lines: lines.length }, + diagnostics: matching.length > 200 ? [{ severity: 'warn', message: `showing 200 of ${matching.length}` }] : [], + }; + }, +}; +``` + +## Hooks + +Hooks belong to the task kind that runs them; each kind declares its own typed points. The built-in points are +flattened onto `c.hook`, one method each: `systemInstructions`, `beforeRequest`, `afterResponse`, `onYield`, +`beforeTool`, `afterTool`, `afterTools`, `beforeCollapse`. Each is the generic +`c.hooks.on(kind, kind.hooks.point, handler, { subtree })` with kind and point bound; `h.hooks.on(...)` +registers harness-wide. Plugin kinds use the generic form with their own point tokens. Input, output and +failure behaviour come from the point: + +```typescript +import { toolKind } from '@earendil-works/pi-agent/pico/kinds'; // only for the generic form + +// conversation-scoped, named +c.hook.beforeTool(async ({ call }, { conversationId }, ctx) => { // a `block` from an earlier handler already stopped the chain + if (call.name === 'write' && await c.config.profile.get(ctx) === 'plan') + return { block: { reason: 'plan mode: no edits' } }; + if (call.name === 'bash' && !(await ui.approve(call.arguments, ctx))) + return { block: { reason: 'denied by user' } }; + return { call: { ...call, arguments: normalize(call.arguments) } }; // may rewrite arguments +}, { subtree: true }); // shared policy: applies to subagents this conversation owns too + +c.hook.afterTool(async ({ call, result }) => { metrics.record(call.name); }); +c.hook.beforeRequest(async ({ request }) => ({ request: withTracing(request) })); +c.hook.onYield(async ({ answer }, { conversationId }, ctx) => { + if (await goalIncomplete(conversationId, ctx)) return { continue: 'The goal is not met yet; continue.' }; +}); +c.hook.beforeCollapse(async ({ reason, entries }) => { + if (reason === 'manual' && entries.length < 10) return { decline: true }; +}); + +// harness-wide, generic +h.hooks.on(toolKind, toolKind.hooks.beforeTool, handler); // same as c.hook.beforeTool, for every conversation +h.hooks.on(myKind, myKind.hooks.myPoint, handler); // your own kinds' points, by token +``` + +| kind | point | returns | several handlers | on throw | +|---|---|---|---|---| +| generation | `systemInstructions` | draft edits; `{ tools? }` | all run | its edits dropped, skipped | +| generation | `beforeRequest` | `{ request? }` | chained: each output is shallow-merged into the input the next handler sees | skipped | +| generation | `onYield` | `{ continue: string }` or nothing | first non-void answer wins | skipped | +| generation | `afterResponse` | nothing | all run | reported | +| tool | `beforeTool` | `{ call? }` or `{ block }` | chained (patch); `block` stops the chain | **blocks the tool** | +| tool | `afterTool` | `{ result? }` | chained (patch) | skipped | +| post_tools | `afterTools` | nothing | all run | reported | +| collapse | `beforeCollapse` | `{ decline }` or `{ instructions?, summary? }` | first answer wins | skipped | + +Hooks run outside the commit line and may take as long as they like (a human approval is a hook that waits). +They may run again after a crash; key external side effects. Cancellation always propagates. `onYield`'s +`continue` appends a user entry and starts another generation for the same input; if the user typed something +meanwhile, that wins and the yield decision is dropped. + +Your own task kinds declare points the same way (see below); `h.hooks.on(myKind, myKind.hooks.myPoint, ...)` then +type-checks against your declaration. + +## Writing entry kinds + +```typescript +import { defineEntry, type EntryBase, type EntryData, type ModelProjection } from '@earendil-works/pi-agent/pico'; + +type NoteEntry = EntryBase & EntryData<{ text: string }>; +export const noteKind = defineEntry('myplugin.note'); +await h.entryKinds.register(noteKind, ctx); // or pass it in Harness.open({ entryKinds: [noteKind] }) + +// Data-only: the model sees nothing. `write` resolves with its admission/input id, not the later entry id; +// the committed entry appears in the watch. +await c.commit((tx) => tx.write(noteKind, { data: { text: 'plan accepted' } }), ctx); + +type PinnedEntry = EntryBase & EntryData<{ text: string }> & ModelProjection; +export const pinnedKind = defineEntry('myplugin.pinned'); +await h.entryKinds.register(pinnedKind, ctx); +const text = 'Always run the tests before claiming a fix works.'; +await c.commit((tx) => tx.write(pinnedKind, { + data: { text }, + model: [{ role: 'user', content: `${text}`, timestamp: Date.now() }], +}), ctx); + +// later, from a watch event or an entry scan: +const entry = await h.getEntry(entryId, ctx); +if (noteKind.is(entry)) console.log(entry.data.text); +``` + +An entry may also carry `head` (context starts here) and `edits` (omit/replace earlier messages). A missing +plugin loses typed access, not context behaviour: facets are stored. + +## Writing task kinds + +```typescript +import { defineTask } from '@earendil-works/pi-agent/pico'; + +type Input = { readonly about: Id; readonly at: number }; +type Checkpoint = { readonly phase: 'firing'; readonly key: string }; + +export const reminderKind = defineTask()({ + kind: 'myplugin.reminder', + + async execute(task, rt, ctx) { + if (task.input.at > rt.now()) await rt.sleep(task.input.at, ctx); + await rt.commit((tx) => tx.checkpoint({ phase: 'firing', key: `reminder:${task.id}` }), ctx); + await notifications.fire(task.input.about, `reminder:${task.id}`, ctx); + return () => ({ status: 'completed', result: { fired: true } }); + }, + + async recover(task, rt, ctx) { + if (!task.checkpoint) return reminderKind.execute(task, rt, ctx); // crashed before the checkpoint: nothing external happened + // the checkpoint proves we intended to fire; the call is keyed, so firing again is safe if it did not land + if (!(await notifications.check(task.checkpoint.key, ctx))) { + await notifications.fire(task.input.about, task.checkpoint.key, ctx); + } + return () => ({ status: 'completed', result: { fired: true } }); + }, + + async abort() { + return () => null; + }, +}); + +await h.taskKinds.register(reminderKind, ctx); // fine for a fresh session. Once tasks of this kind can exist across restarts, + // pass it in Harness.open({ taskKinds: [reminderKind] }) on every open: open orphans + // live tasks whose kind is not registered *at open*, and nothing resurrects them +const id = await c.commit((tx) => tx.task(reminderKind, { background: true, input: { about: entryId, at: Date.now() + 3600_000 } }), ctx); +const task = await h.getTask(id, ctx); +``` + +The contract: + +1. Pico marks the task `running` before calling `execute`. Commit a checkpoint before any external effect you + would need to reconcile after a crash. +2. `execute`/`recover` do their effects, then return a closure. Pico runs the closure on the commit line; its + writes, the outcome and scratch cleanup commit atomically. Do not perform effects in the closure. +3. `recover` is called instead of `execute` when a running task is found at open. Read `task.checkpoint` and + decide. +4. On abort, the old invocation is signalled through `ctx` and its later writes rejected. `abort` then runs + as a fresh call for cleanup and returns its own closure. +5. `after: [ids]` runs the task after those are terminal (any outcome). `rt.waitForTask(id, ctx)` observes. +6. Configuration lives in scoped values, not in task input. Task input is what recovery needs. +7. A task may `tx.write(...)` a passive entry (a notice) in any commit or in its terminal closure. It lands + now if the conversation is idle, at the next boundary if busy. Tasks never append directly and never ask + for an answer; that is `send`. + +A kind may declare hook points. It decides when to run them; handlers registered on `h.hooks`/`c.hooks` for +this kind and point are invoked outside the line: + +```typescript +import { defineHookPoint } from '@earendil-works/pi-agent/pico'; + +export const deployKind = defineTask()({ + kind: 'myplugin.deploy', + hooks: { + beforeDeploy: defineHookPoint<{ target: string }, { approve: false; reason: string } | { approve: true }>({ + fold: 'first', onThrow: 'abort', + }), + }, + async execute(task, rt, ctx) { + const decision = await rt.hooks.run(deployKind.hooks.beforeDeploy, { target: task.input.target }, ctx); + if (decision.threw || decision.output?.approve === false) + return () => ({ status: 'failed', failure: { message: 'deployment declined' } }); + ... + }, + ... +}); + +await h.taskKinds.register(deployKind, ctx); +h.hooks.on(deployKind, deployKind.hooks.beforeDeploy, async ({ target }, info, ctx) => + (await ui.approve(`deploy to ${target}?`, ctx)) ? { approve: true } : { approve: false, reason: 'denied' }); +``` + +`fold` says how several handlers combine (`collect` all outputs, `first` non-void wins, `chain` shallow-merges +each handler's patch into the input the next handler sees). `onThrow` says whether a throwing handler is skipped +or stops the point. + +Kinds declaring `output` get `rt.output` (read/mutate/replace) whose deltas stream to watchers: + +```typescript +// abbreviated: recover/abort omitted +type Progress = { pct: number; log: string[] }; +const progressKind = defineTaskOutput('myplugin.progress'); +defineTask<{ readonly steps: number }, never, { readonly done: true }, { readonly message: string }, null, Progress>()({ + kind: 'myplugin.long', + output: { kind: progressKind, initial: () => ({ pct: 0, log: [] }) }, + async execute(task, rt, ctx) { + await rt.output.mutate((o) => { o.pct = 50; o.log.push('halfway'); }, ctx); + return () => ({ status: 'completed', result: { done: true } }); + }, + // ... +}); +``` + +## Subagents and jobs + +A subagent is a conversation owned by the task that created it. There is no subagent task and no subagent +object; the model will get one tool, the API gets a conversation handle. The tool-side operation is not yet +specified (see Writing tools). What is settled: a child never inherits the parent's `c.config` implicitly. Its +creator seeds `model`, `selectedTools`, `profile` and any other definitions it wants, plus initial system +section values, in the same commit that creates the child and accepts its first input, so the first +generation already sees them. A child created without a first input is inert; register its hooks and set its +config, then `send`. From the host a subagent is just another conversation: `h.conversation(id, +ctx)`, `h.conversations({ owner, limit: 100 }, ctx)`. Its own `systemInstructions` hooks (registered on the child +handle or inherited via `subtree`) give it its own identity. Aborting the owning tool call aborts it unless +the (gated) child API created it with `abortWithTool: false`. + +A job is a process: + +```typescript +// Harness.open(storage, { ..., processHost: nodeProcessHost() }, ctx) -- without a host, jobs fail with reason 'spawn' +const id = await c.commit((tx) => tx.task(jobKind, { + background: true, + input: { command: 'npm', args: ['run', 'dev'], cwd, notify: false, rerun: true }, +}), ctx); +await h.abortTask(id, ctx); // SIGTERM, then SIGKILL +``` + +`nodeProcessHost()` keeps runs in memory only: after a restart it no longer knows them, so a job either +reruns (`rerun: true`, same key, possibly repeating the effect) or fails `interrupted`. A supervising host can +do better; that is the host's contract, not Pico's. + +Its stdout/stderr tail (and how many bytes were dropped) is in its task output; `notify: true` appends a `pi.notice` when it finishes (per occurrence for a schedule) so the +model sees it on its next request (which may be the continuation of the current turn). + +## Storage backends + +```typescript +MemoryStorage.create() // tests +await JsonlStorage.open('./session.jsonl', ctx) // local; one file plus live sidecars +await SqliteStorage.open('./sessions.db', { session: 'parser' }, ctx) // many sessions, servers +``` + +Same behaviour everywhere. JSONL and SQLite take a cross-process lock; a second writer fails rather than +racing. diff --git a/packages/agent/docs/pico/pico-v3.md b/packages/agent/docs/pico/pico-v3.md new file mode 100644 index 000000000..b1a57f7fd --- /dev/null +++ b/packages/agent/docs/pico/pico-v3.md @@ -0,0 +1,2577 @@ +# pico v3 + +A design for the agent harness: how a session is stored, how work is scheduled, and how the +built-in agent behaviour is composed from that. Code in this document illustrates the model; it is +not a package export. In snippets, `call` is the current `Call` (Chord Context, §6.5), required on +asynchronous public/runtime operations; `runtime` is the task/tool capability object. Transaction +builders do not take another Call. + +**Design status:** the tool/sink/preview interfaces and related examples remain provisional and need +simplification before implementation. They preserve the capabilities to cover, not a final authoring +surface. Storage, atomic publication, cancellation ownership and bounded-output requirements remain +constraints on that redesign. The separate steps/returned-plans proposal has not been adopted. + +## 1. Goals + +**Few concepts, replaceable behaviour.** A session is conversations, immutable entries, durable +tasks, and scoped state. Agent behaviour (generate, run tools, compact, delegate) is written as +task kinds. The scheduler understands task lifecycle, dependencies and cancellation, nothing +about prompts or summaries. Storage understands objects and atomic batches, nothing about tasks. +Replace a task kind and you change the behaviour; the scheduler and storage stay. + +**One writer per session.** Execution is concurrent; writing is not. Every mutation, including +working-state writes, goes through one owner that runs commits one at a time. A commit reads +committed state, constructs its writes and their ids, commits them as one batch, then publishes. +Different sessions can have different owners and share a SQLite file. After a crash a new owner +resumes from durable state; it must never run alongside the old one. + +**Durable work, explicit recovery.** Accepted work survives a crash, including work that hasn't +started. Opening a session starts nothing. A task records its intent before doing anything external +and commits the outcome together with whatever should happen next. A crash in the middle leaves +uncertainty, not proof that nothing happened; each task kind decides how to recover: retry, +adopt, or report interruption. Nothing promises exactly-once external effects. + +**Long sessions without a residency layer.** Old entries and finished tasks are queryable but never +retained because they were once seen. Routine execution reads the current context and the live +tasks; it never scans accumulated history. What memory a backend uses is a backend choice: memory +and JSONL keep everything loaded; SQLite keeps only what queries return. + +**Measure.** Same workload on each harness and backend: CPU, process memory, bytes on disk +including sidecars. Small writes, indexed reads, batched lookups. A faster prototype is a reason to +look, not proof that its shortcuts belong here. + +## 2. Conversations + +A session holds conversations. A conversation has three things: + +- a **transcript**: an append-only list of immutable entries (messages, tool results, summaries, + plugin-defined entries), which is what happened; +- **tasks**: the durable units of work that act on it, such as a generation or a tool call, each + with a status that changes as it runs (§5); +- **state**: keyed values and lists that plugins and the harness read and write, such as the model + in use or a plan-mode flag (§4). + +What the model sees, the context, is derived from the transcript by a fixed rule. Tasks and state +are never model input by themselves. + +### 2.1 Entries and their kinds + +```ts +type Id = number; // a session sequence number; one alias so the representation can change + +type ContextEdit = + | { readonly target: Id; readonly action: "omit" } + | { readonly target: Id; readonly action: "replace"; readonly messages: readonly Message[] }; + +interface EntryIdentity { + readonly id: Id; // its sequence number + readonly conversationId: Id; + readonly kind: string; + readonly byTaskId?: Id; // which task wrote it +} + +interface EntryBase extends EntryIdentity { + readonly key?: string; // optional indexed key, e.g. the tool call a result answers +} + +interface EntryData { + readonly data: Data; // kind-specific durable data; never model input by itself +} + +interface ModelProjection { + readonly model: readonly Model[]; // provider-neutral messages materialized at append +} + +interface ContextHead { + readonly head: Id; // first retained transcript entry, inclusive +} + +interface ContextEdits { + readonly edits: readonly ContextEdit[]; +} + +type Entry = EntryBase & Partial< + EntryData & ModelProjection & ContextHead & ContextEdits +>; + +type UserEntry = EntryBase & ModelProjection; +type AssistantEntry = EntryBase & ModelProjection; + +interface SummaryData { readonly summarizedThrough: Id } +type SummaryEntry = EntryBase & EntryData & ModelProjection & ContextHead; +``` + +The facets combine. A user entry may be only `EntryBase & ModelProjection`; a tool +result may also carry non-message `EntryData` for its renderer; a summary has data, +a model projection and a head; a reset needs only a head. Entries append and are never patched, +reordered or renumbered. Ids increase in transcript order but need not be consecutive. + +`model` is the exact provider-neutral message projection chosen by the writer. It is not the final +provider request: context selection, edit folding, tool-exchange normalization and provider +conversion still happen per request. `data` is optional arbitrary kind-specific JSON for typed +logic and rendering. Nothing implicitly derives one from the other, so storing both never requires +duplicating a message. + +An entry kind is only a name and type witness: + +```ts +interface EntryKind { + readonly kind: string; + is(entry: Entry): entry is E; +} +``` + +A plugin normally keeps append-time construction in a typed helper. It may compute `model`, `head` +and `edits` from its data before calling `Tx.entry`, but no such computation is registered on the +kind or run while reading. Asynchronous inputs are prepared before entering the commit line. + +A stored `head` is the inclusive first retained entry id. Presence of the column marks and indexes +the entry as a head. `Tx.entry` accepts `"self"` for reset and handoff drafts and stores the minted +entry id. Every new head must satisfy: + +```text +newHead.head >= previousVisibleHead.head +``` + +The commit line validates this directly from stored entries. A head cannot move context backwards. +Its boundary must also not split an exchange, whoever appends it: the same rule compaction follows +(§2.3), applied to plugin heads so a head can never leave tool results without their call. +Older heads are controls, not retained context entries: the newest head replaces them and carries +any predecessor summary information it still needs. + +Stored `edits` omit or replace the model projection of earlier visible entries. Arbitrary edits may +not target managed `system` entries: they could make model instructions disagree with canonical +prepared section state (§8.2), so the commit rejects. The sole exception is generation preparation +appending a complete fresh baseline with omission edits for superseded managed baselines/deltas. +Those controls and their replacement instructions commit together (§8.2). Edits are folded in +transcript order; the newest applicable edit for a target wins. The target entry is never changed, +and an edit whose target is outside the selected range is a no-op. Retained edit entries apply again +on later turns. Tool-result pruning and shortening are edits. Dynamic policies such as "always keep +only the last N exchanges" are unsupported; code appends a new head or edit when context should +change. There is no `compose` callback or read-time entry-kind behavior. + +The built-ins are `user`, `assistant`, `tool_result`, `system`, `notice`, `summary`, `handoff` and +`reset`. Their writers materialize model messages when they append. `reset` stores no model; +`handoff` stores its message and starts a context; managed `system` entries store the exact pi-ai +`SystemMessage` prepared at their transcript position. They record canonical prepared instructions, +not proof that a provider received them. Pi-ai owns provider translation and best-effort cache +preservation; immutable entries alone do not guarantee an unchanged wire prefix: + +```ts +type SectionChange = + | { readonly key: string; readonly action: "set"; readonly value: JsonValue; readonly rendered: string } + | { readonly key: string; readonly action: "remove" }; + +interface SystemData { + readonly baseline?: true; // complete prepared section state (§8.2) + readonly sections?: readonly SectionChange[]; +} + +type SystemEntry = EntryBase & EntryData & ModelProjection + & Partial; // fresh baseline may omit superseded managed entries +``` + +A baseline stores the complete ordered section state as set records. A delta stores changed sets +and explicit removals. Payloads are JSON; null is a valid payload, not deletion. `rendered` is the +final section text, so a missing renderer never prevents replay or construction of a fresh baseline. +Diff by key, payload and rendered text; reordering alone emits nothing. A payload-only change still +persists a section-state delta but needs no instruction message (`model: []`). Rendered changes and +removals produce materialized system messages. Never parse prose to reconstruct section state. + +Complete tool definitions are stored only in the SystemMessage `toolsAdded`/`toolsRemoved` fields, +not duplicated in `data`. Added definitions are upserts by name, including same-name schema changes; +removals contain the previous complete definitions. Apply removals before additions. Section and tool +names are unique within their respective change lists; structural comparison ignores incidental JSON +object-key order. Plugin-defined entries omit `model` when the model should not see them. The kind +types reads and writes; stored objects are trusted after wire validation (§7.3), and a kind that changes +its data shape ships a migration. + +```ts +type ToolResultEntry = EntryBase & EntryData & ModelProjection; +const toolResultKind = defineEntryKind("tool_result"); + +tx.entry(toolResultKind, conversationId, { data: resultData, model: [message], key: callId }); +const e = await harness.getEntry(summaryKind, id, call); // SummaryEntry | undefined +const any = await harness.getEntry(id, call); // Entry | undefined +if (summaryKind.is(any)) any.data.summarizedThrough; // narrowed +``` + +Without a kind, a read returns the untyped entry. A typed read of another kind returns `undefined`, +the same as a missing entry; batched reads omit it. The session records the distinct kind strings +without scanning entries. Open reports an unregistered entry kind but does not reject: stored +`model`, `head` and `edits` preserve context, while only kind-specific typing and rendering are +unavailable. + +### 2.2 Context is derived, not stored + +The generation derives context from the transcript: + +```text +H = newest entry with stored `head` visible at the request/fork target +from = no H → transcript start; otherwise H.head +range = fork-aware transcript entries from `from` through target, inclusive +controls= fold stored `edits` in transcript order +context = no H → eligible range entries + H exists → H, then eligible range entries excluding every head + omitted targets disappear; replaced targets keep their id and use replacement messages +messages= concatenate stored `model` arrays; normalize tool exchanges for the selected model +``` + +An entry without `model` contributes no messages. An edit entry may independently have a model +projection. Existing tool results are placed with their calls in call order. For a successful +assistant cut before all its results, request-local pi-ai transformation supplies missing results; +results beyond the fork cutoff and source tasks are never inherited. Error/aborted assistant +outputs are excluded from subsequent requests and never create tool tasks/results. + +No context list is stored, so nothing can drift from the transcript. A conversation handle keeps an +optional process-local cache of the current derived entries, candidate membership and winning edits. +`contextEntries(through)` loads it once, catches it up with entries after its cursor, slices it from +the stored boundary when a new head arrives and folds stored edits. Generation tasks receive that +handle; `ConversationView` exposes the resulting context ids. A historical target older than the +cache is derived separately and does not rewind it. Each request receives an immutable snapshot. + +On a cold handle, cost is the fork-aware candidate range plus edit folding and fork depth, not +necessarily the final projected-message count. Memory and JSONL answer from their in-memory indexes; +SQLite performs indexed head/range reads. Warm handles process only appended entries. The cache is a +convenience owned by the handle and may be garbage-collected with it; the transcript is the truth. +Generation preparation persists `state.requestThrough` and captures its effective context snapshot +on the line after the complete batch is durable (§8.2). The snapshot copies the selected reference +array, not immutable entry bodies. Later cache updates never mutate that array or its effective +replacement projections; request-local normalization/hooks copy what they change. Historical cutoff +reconstruction does not rewind the live cache, and request-only references are released after use. + +### 2.3 Compaction and reset + +**Compaction** appends one summary with a stored model projection and head boundary: + +```text +transcript [10 user, 20 asst, 30 user, 40 asst] context [10, 20, 30, 40] +append 50 summary { model:[summary], head:30 } +scan [30 user, 40 asst, 50 summary] +context [50 summary, 30 user, 40 asst] +``` + +The summary is newer than 30 and 40 but projects before them. The summarized prefix must end on a +complete exchange, lie inside the current context and include the previous head's information. +Managed system entries retained across a head are explicitly superseded by the next prepared baseline +(§8.2); this does not split the retained tool exchange or add hidden filtering to generic projection. +A summary prepared against an old context +may still land later: ordinary entries appended while it runs are at or after its prepared retained +boundary and remain in the range. Only a competing head invalidates it; intervening context edits +do not (§8.4). + +**Reset** appends a `handoff` head with `model` or a `reset` head without it. Both pass +`head: "self"` to `Tx.entry`, which stores the new entry id as the boundary. The transcript keeps +everything. Reset by itself neither cancels tasks nor requests a response. + +### 2.4 Forks share history, not future changes + +A fork is a new conversation whose transcript starts as a prefix of its source, shared by +reference: entries 10 and 20 below keep their ids and their owning conversation; nothing is copied. + +```text +A: 10 ─ 20 ─ 30 ─ 40 ─ 50 ─ 70 + └──────────── 90 ─ 100 B, forked at 20 +B transcript: [10, 20, 90, 100] +``` + +Because context is derived from the transcript, B's context is what A's context was at the fork +point: the newest head among `[10, 20, ...]` is the one that existed then, and none that A added +later can be seen. Later appends, compaction and resets in A cannot affect B. Live tasks are not inherited; +continuing B is new work. + +### 2.5 History and ownership are different relationships + +```ts +interface Conversation { + readonly id: Id; + readonly parent?: { readonly conversationId: Id; readonly at: Id }; // forked from, at entry + readonly owner?: Id; // task that created it +} +``` + +```text +history: A ── fork at P ──> B B: parent = A at P, no owner; independent +ownership: A ── task T ── owns ──> C C: owner = T; a child, e.g. a subagent's conversation +``` + +A fork never enters its source's drive or cancellation scope. An owned conversation does, through +the task that owns it. A child can start with fresh or inherited context; that is separate from +which state it is initialized with (§4.3). + +## 3. Identity and commits + +### 3.1 Ids are minted while building the commit + +One session-wide sequence orders every mutation. A creation's sequence number is the object's id. + +```ts +// last committed: 99 +const entryId = await conv.commit(tx => { + tx.value(planMode).set(true); // 100 + const id = tx.entry(userKind, { model: [message] }); // 101 + tx.task(generationKind, { state: { status: "pending", input: id } }); // 102 + return id; +}, call); +// one batch [100–102] persisted and published; entryId === 101 here +``` + +A commit can reference ids it just minted. It cannot hand them to an external service or return +them to a caller before the batch is durable. A rejected commit consumes nothing: the next one +starts at 100 again. Numbers are positive safe integers; no uuids, no reserved ranges. + +### 3.2 Rewindable conversation state before entries + +A commit can write rewindable conversation state and an entry together (a tool that turns on plan +mode and writes its result), and a fork at that entry must include the state. It must not include +state committed later: a `/model` change made after an answer belongs to a fork made after it, not +to a fork at the answer. So the boundary is exactly the entry's commit, and the way to know it +without a field, a stamp or a lookup is write order: **in one commit, rewindable conversation value +and list writes come before entries.** Sequence order is call order, so the state is numbered below +the entries, and a fork at entry X selects the fork-visible transcript and rewindable conversation +history through X: + +```ts +// tool settlement, one commit +await runtime.commit(tx => { + tx.value(planMode).set(true); // 300 + const result = tx.entry(toolResultKind, { data: resultData, model: [message] }); // 301 + tx.settle(task, "done", { call: task.state.call, assistant: task.state.assistant, output, result }); // 302 +}, call); + +await conv.value(model).set("claude-opus-5", call); // 303, a later commit + +const b = await conv.fork({ at: 301 }, call); // has the result and plan mode; does not have the new model +``` + +The builder throws if a rewindable conversation value/list write follows an entry; the plan fails +before persistence and consumes no ids. Session state and sticky conversation state may appear +anywhere because forks do not reconstruct their history, so they may reference ids minted earlier +in the same commit. Tasks also go anywhere; they are not inherited by forks and usually come after +the entries they name. Historical reads (§4) take an entry as their position; "before the first +entry" is the empty position. Any transcript entry is a valid fork point, including an assistant call +or one of several tool results. Projection repairs the resulting successful incomplete exchange for +the request only; it never inherits or executes the source tasks. + +### 3.3 Failure before and after admission + +Validation failure: no writes, no ids consumed, the session keeps working. Uncertain persistence: +stop using this handle, reopen, recover the last complete batch and its `lastSeq`, continue +from that. No observer ever sees half a commit. An effect must not treat an uncertain write as +permission to redo the same external action with new ids. + +### 3.4 Request keys name acceptances + +Every accepted input has an `inputId`, the id of its `pi.inbox` list element (§8.1). A caller may +also supply an opaque request id. Acceptance atomically stores a session value from that key to the +new identity: + +```text +400 append pi.inbox item inputId = 400 +401 inputResult[400] = queued or placed +402 request["req-42"] = { conversationId, inputId:400 } +``` + +`harness.acceptance(requestId, call)` returns that receipt, for a caller whose accept response may +have been lost. A request key names one acceptance for the life of the session: a second `accept` +or `queueInput` with the same key writes nothing and returns the stored `inputId`, whatever its +payload or mode. Nothing is compared, so no content or digest is kept for that purpose; a client +that reuses a key for different input gets the first acceptance back, and `result(inputId, call)` +says what became of it. Without a request id, no mapping is written. The mapping is session state, +so it may follow and reference the newly minted inbox element. + +## 4. State + +State is addressed independently of the transcript. Its writes share the sequence, so one +position (an entry, §3.2) selects context and rewindable state together. + +### 4.1 Scopes + +| Scope | Rewind | Lifetime | +|---|---|---| +| Session | Sticky: one current value | The session | +| Conversation | Rewindable or sticky, chosen per address | The conversation and its fork history | +| Scratch | Never rewound, never inherited | One live execution, until settlement | + +```ts +const sessionName = sessionValue("pi.session.name"); +const plan = conversationValue("plugin.plan", { rewind: true }); +const expanded = conversationValue("ui.expanded", { rewind: false }); +const moves = conversationList("game.moves", { rewind: true }); +``` + +```ts +type Scope = { session: true } | { conversation: number } | { scratch: number /* task id */ }; +interface Address { readonly scope: Scope; readonly namespace: string; readonly key?: string; readonly rewind: boolean } +interface Value extends Address { readonly __value?: T } // phantom: the payload type +interface List extends Address { readonly __list?: T } +``` + +An address binds namespace, key, scope and rewind policy, and carries its payload type. The +constructors above make `Value`/`List` for a scope; a conversation handle binds +`conversation: id` itself, so plugin code writes `conversationValue("plugin.plan")` and never the id. +Values are strict JSON; reads return copies. A conversation address cannot be used as a session address. +Storage persists scope and policy; nothing depends on the address object surviving a restart. + +### 4.2 Vocabulary + +```text +value: tx.value(addr).get(), .set(v), .delete() +list: tx.list(addr).append(v) → element id, .remove(id), .clear(), .read(cursor, limit) +``` + +A rewindable history reads as of an entry (§3.2): a list appended at 40 and 50, cleared at 60 and +appended at 70 reads `[A, B]` at entry 55, `[]` at 65, `[C]` at 75. A remove hides one element the +same way a clear hides all of them. Clear hides; it doesn't erase +what an earlier fork needs. Deleting a rewindable value records absence. Sticky state exposes only +its current contents, so a removed sticky list element may be discarded physically. + +### 4.3 Initialization is explicit; forks inherit + +A new child conversation (a subagent) chooses its initial state independently of its context: + +```ts +spawn({ prompt, context: "inherit", + values: { inherit: [generationKind.config.model, generationKind.config.thinking], + set: [[generationKind.config.selectedTools, ["read", "grep"]]] } }) +``` + +One commit creates the child, copies the selected current values, applies overrides, appends the +user prompt, creates the first generation; that generation writes the child's first `system` +entry from `system_instructions` (§8.2). Unselected values are absent in the child. Lists are not +copied unless a policy says so. + +A historical fork inherits differently: all rewindable history as of the fork position, no sticky +state unless selected, the same session state, no scratch. + +### 4.4 Capped-source lookup + +A fork stores no copies. Reading a rewindable address in fork B, forked from A at an entry: + +```text +read(B, addr, at): // at = an entry id + local version ≤ at? set → value; delete → absent + no parent? absent + read(B.parent.conversationId, addr, min(at, B.parent.at)) +``` + +Lists combine inherited ranges with local appends the same way, honouring clears at every level. +Cost is fork depth, never unrelated history. A child initialized by copying (§4.3) has an owner but no `parent`, so lookup stops there. + +### 4.5 Structured plugin state + +Plugins with structured state build it on values and lists: a checkpoint value `{ through, state }` +plus a delta list; hydrate by reading the checkpoint visible at a position and the deltas after it. +The plugin owns its delta vocabulary and reducer; storage runs nothing. + +### 4.6 Scratch + +Scratch is a task's working state while it runs: streamed frames, a checkpoint, a memo. One scope +per task, any number of addresses in it, durable across a restart, gone when the task settles. + +```ts +import { AssistantMessageFrameEncoder, type AssistantMessageFrame } from "@earendil-works/pi-ai"; + +const frames = scratchList("frames"); +const checkpoint = scratchValue("checkpoint"); + +// a new attempt starts clean; then each frame is one small scratch commit +await runtime.scratch(sc => { sc.list(frames).clear(); }, call); +const { context: streamCall, cancel } = withCancel(call); +const stream = runtime.models.stream(model, request, streamCall); +const encoder = new AssistantMessageFrameEncoder(); +try { + for await (const event of stream) { + const frame = encoder.encode(event); + if (frame !== undefined) { + await runtime.scratch(sc => { sc.list(frames).append(frame); }, streamCall); + } + } +} finally { + cancel(); // stop this producer on any early exit, not only a task abort + await stream.result(); // join it; iterator exit alone is not provider completion +} + +// after a restart, recover reads what made it to disk +const partial = await runtime.scratch(sc => sc.list(frames).read(), call); + +// settlement: the result becomes an entry; settle retires the scope in the same commit +await runtime.commit(tx => { + const id = tx.entry(assistantKind, { model: [assemble(partial)] }); + tx.settle(task, "done", { inputs: task.state.inputs, assistant: id }); +}, call); +``` + +Persist compact frames, not raw provider events with a growing `partial` snapshot on every token. +The encoder stores content deltas and necessary start/end metadata; terminal outcome/usage is recorded +separately. Tool/job scratch similarly appends explicit output operations or bounded checkpoints. +These are application records in ordinary lists, not a Chord storage codec. + +Three rules: + +- A scratch commit writes one task's scratch and nothing else; a main commit never writes scratch. + On JSONL that makes every commit one file (§7.4). +- `settle` deletes the task's scratch in the same commit. Crash before settle: task and scratch are + both there for recovery. After: both gone. +- A task stops writing scratch before it settles. A frame that arrives late can't recreate what + settle deleted. + +A retry is the same task (§5.3), so an attempt clears the scratch before it starts streaming; +what is on disk is always the current attempt's. Scratch is never inherited, never rewound, never in the transcript; a task that wants to keep +something writes an entry or a value at settlement. + +## 5. Tasks + +A task is durable work belonging to one conversation. Its kind supplies behaviour; its record +supplies the inputs, progress and outcome that must survive a process. + +### 5.1 The record + +```ts +type TaskRole = "start" | "inflight" | "terminal"; + +type TaskStateBase = JsonObject & { readonly status: string }; // state is strict JSON with a status +type TaskRoles = Readonly>; + +interface Task { + readonly id: Id; + readonly conversationId: Id; + readonly kind: string; + readonly role: TaskRole; // materialized from the kind's roles map on every write + readonly state: State; // a tagged union; the status is its discriminant + readonly after: readonly Id[]; // dependencies, fixed at creation + readonly background?: true; // fixed at creation; absent = foreground (§5.6) + readonly turn?: true; // materialized from the kind; this task drives a turn (§5.8) + readonly owns?: readonly Id[]; // conversations this task created (§5.7) + readonly abort?: true; // durable cancellation mark (§6.3) +} +``` + +**State is a tagged union over status.** A kind declares one variant per status, carrying exactly +the fields that exist in it, so nothing is an optional that a reader has to guess about and a +dependent narrows instead of probing: + +```ts +type ToolStates = + | { status: "planned"; call: ToolCall; assistant: Id } + | { status: "running"; call: ToolCall; assistant: Id; args: JsonObject; replay: "safe" | "never"; + jobId?: Id; cancelJobOnAbort?: boolean } + | { status: "done"; call: ToolCall; assistant: Id; output: ToolOutputState; result: Id } + | { status: "aborted"; call: ToolCall; assistant: Id; output: ToolOutputState; result: Id }; +``` + +The status lives only in `state.status`. Storage extracts it into an indexed column when it writes +the record, so a scan never decodes state, and `role` is on the record because a reader of an +unregistered kind cannot compute it from the kind's map. `Tx` derives the role on every write. +Fields common to every variant must have the same type in each. + +A kind maps its statuses to three roles: + +```text +start the driver may call execute generation: pending, retry_wait, deferred +inflight intent was committed; call recover generation: streaming; job: running +terminal done; never patched again generation: done, failed, aborted +``` + +`Tx.task`, `patch` and `settle` derive the role from the registered kind's status map and persist it +on every write. Storage selects live rows using the role index without inspecting task state or +running kind code. Loading their full records may decode state; terminal history is not decoded. +Callers cannot edit the role independently. + +**`orphaned` is derived, not declared.** A kind never writes it and never lists it. The harness adds +one terminal variant to every state union, carrying the fields common to all of the kind's own +variants: + +```ts +type CommonPayload = Pick>; +type Orphaned = CommonPayload & { readonly status: "orphaned" }; +type ToolState = ToolStates | Orphaned; // orphaned carries only call and assistant +``` + +`keyof S` selects the keys present in every union member; `Pick` keeps their types and optionality. +This is the intersection of field names, not a TypeScript intersection of the variants (which would +intersect incompatible status literals into `never`). Kind definition rejects inconsistent types or +optionality for common fields, and rejects a user-declared `orphaned` status. A field optional in every +variant remains optional on orphaned; a variant-specific field is not exposed. + +It is written in one situation only: at open, for a live foreground task whose kind is not +registered in this process (§6.4). The stored record keeps whatever fields it had; the type +promises only the common ones, which is all a dependent may rely on, because the kind never ran its +cleanup. Background tasks of unregistered kinds are parked instead, and resume through `recover` +when their kind returns. This is the only case in which the harness settles a task it does not own. + +### 5.2 Kinds + +```ts +declare const taskType: unique symbol; // compiler-only witness; never populated or serialized +interface TaskType> { + readonly [taskType]?: { readonly state: (state: S) => S; readonly roles: R }; +} +type TypedTask> = Task & TaskType; +interface TaskDefinition> extends TaskType { + readonly kind: string; + readonly roles: R; +} + +interface TaskKind = TaskRoles> extends TaskDefinition { + readonly initialStatus: StatesWithRole["status"]; // helper in §9.2 + is(task: Task | undefined): task is ReadTask; + readonly turn?: true; // this kind drives a turn (§5.8) + readonly hooks?: HookSpecs; // the points this kind runs (§8.7) + readonly config?: Config; // the values this kind reads (below) + preview?: { // what UIs see while the task runs (§9.4) + init(scratch: ScratchReader): Preview; // built once, on attach or reopen, from durable scratch + }; // afterwards the kind mutates runtime.preview.state in place + execute(task: TypedTask, runtime: TaskRuntime, call: Call): Promise; + recover(task: TypedTask, runtime: TaskRuntime, call: Call): Promise; + abort(task: TypedTask, runtime: TaskRuntime, call: Call): Promise; +} + +type ReadTask> = + TypedTask, R & { readonly orphaned: "terminal" }>; +type TaskState = K extends TaskDefinition ? S | Orphaned : never; +``` + +`Task` remains the raw stored record. `TypedTask` adds only a compile-time witness for the complete +state union and literal role map, so narrowing `task.state` does not lose the other transition targets. +Typed reads include `orphaned`; execute/recover/abort receive only the kind's declared variants. +The witness is not a runtime capability: invocation identity and current durable state still decide +write authority on the line. + +Use `defineTaskKind()({ ... })`: the first call binds the declared union (and optional hook, +config and preview types); the second infers the literal roles from the definition. It checks an exact +role-map entry for each declared status, a start-role initial status and common-field compatibility. +Its return type retains `TaskDefinition`, including the compiler-only state witness; +otherwise kind-witnessed reads could infer statuses from roles but lose their payload types. +Do not widen the result to a generic `TaskKind` or its roles to `TaskRoles` when using +typed mutations: that would discard the information needed to distinguish patch from settle. + +A kind that reads config declares it, so the values it depends on are on the kind and nowhere +else, typed, and usable by callers that initialize a conversation: + +```ts +type ConfigSpec = Record>; + +const generationKind = defineTaskKind()({ + kind: "generation", + config: { + model: conversationValue("pi.model", { rewind: true }), + thinking: conversationValue("pi.thinking", { rewind: true }), + selectedTools: conversationValue("pi.tools.selected", { rewind: true }), + profile: conversationValue("pi.prompt.profile", { rewind: true }), + budgetMs: conversationValue("pi.tool.budget", { rewind: false }), + }, + ... +}); + +await c.config(generationKind).set({ model: "claude-opus-5" }, call); // or c.settings.set(...) for the generation +const cfg = await runtime.config(generationKind, call); // { model, thinking, ... } typed, read on the line +await c.spawn({ prompt, values: { inherit: [generationKind.config.model], + set: [[generationKind.config.selectedTools, ["read", "grep"]]] } }, call); +``` + +`runtime.config(kind, call)` reads every declared value in one batched read. A UI can list what a +conversation is configured with by walking the registered kinds' `config`. Values a plugin defines +for itself are declared on the plugin's own kind the same way. + +`TaskRuntime` is what an execution gets from the driver: + +```ts +interface TaskRuntime { + readonly taskId: Id; + commit(plan: (tx: ConversationTx) => T | Promise, call: Call): Promise; + scratch(plan: (sc: ScratchTx) => T | Promise, call: Call): Promise; + readonly preview: Tracker; // flushed after successful scratch commits + now(): number; // injected clock, not Date.now() in task code + sleep(untilMs: number, call: Call): Promise; + config(kind: { readonly kind: string; readonly config?: C }, call: Call): Promise>; + conversation(id: Id, call: Call): Promise; + abortTask(id: Id, call: Call): Promise; + waitForTask(id: Id, options: { budgetMs?: number } | undefined, call: Call): Promise; + getTask(id: Id, call: Call): Promise; // typed/batched reads also available + getEntry(id: Id, call: Call): Promise; + value(addr) / list(addr) // session state handles; async methods take Call + readonly models: ModelRegistry; // stream/deferred operations take Call + readonly tools: ToolRegistry; + hooks(kind: { readonly kind: string; readonly hooks?: HookSpecs }): HookRunner; // run(point, input, call) + readonly env: ExecutionEnv; // existing Context-final methods; no wrappers +} +``` + +The registry maps kind names to kinds; a replacement must understand the persisted statuses and +state of its live tasks. Open reads the live tasks anyway (`inspect`), so an unregistered live kind +is handled there rather than rejecting the session (§5.1, §6.4); terminal tasks of a forgotten kind +are only a problem when read typed. + +Typed access uses the kind as witness. Public/runtime reads are asynchronous and require Call; +transaction reads use the transaction view without another Call: + +```ts +// public/runtime +getTask>( + kind: TaskDefinition, id: Id, call: Call): Promise | undefined>; +getTasks>( + kind: TaskDefinition, ids: readonly Id[], call: Call): Promise>>; +// inside Tx: same asynchronous typed reads, but no additional Call +getTask>( + kind: TaskDefinition, id: Id): Promise | undefined>; +getTasks>( + kind: TaskDefinition, ids: readonly Id[]): Promise>>; +``` + +The getter returns `ReadTask` without casts, or `undefined` when the task is missing or of another +kind; without the kind it returns the untyped task. `kind.is(task)` narrows to the same read type. +A bare numeric id supplies no state/role witness for typed mutation; read it with its kind first. + +### 5.3 What an execution may do + +Four rules, and they are the whole contract: + +1. Commit an inflight status before any external effect. A crash before that point re-runs + execute from the top; a crash after it goes through recover. +2. You may block on the world: a provider stream, a process, a human inside a hook, a conversation + you created. The driver runs executions concurrently; a blocked one holds up nothing. +3. Make a durable status transition or settle before returning, except when cancellation/close + unwinds the invocation. The driver checks a status epoch (§6.1), not final-status equality. +4. Prefer fixed `after` dependencies for prerequisite work. Child conversations use `drive(call)`; + separately running jobs may use the bounded `waitForTask` API (§8.3). Known self/dependency waits + reject (§6.2); never abandon an unfinished effect to release an invocation slot. + +Timing is the task's business: a retry stores `notBefore` in its state and sleeps in its own +execute (`runtime.sleep`, which observes Call cancellation). Not everything on the signal throws: +pi-ai may return `stopReason: "aborted"`. The kind handles its domain outcome normally; if a durable +abort mark won, its attempted commit rejects and the driver runs the fresh abort handler (§6.3). +A recurring job sets its next `notBefore` and returns in a start status. The scheduler has no timers. + +A task is one logical operation, not one attempt. Its declared status graph may contain cycles: a +generation revisits `streaming` across retries and deferred polls, and a schedule loops from +`planned` through `running` back to `planned` under one stable id. Recovery uses only the current +status, state, role and scratch; it does not reconstruct the path taken. Returning to the starting +status after intermediate committed transitions is valid. Returning without any transition is a +contract violation; transitions are not a generic proof of progress or a diagnosis of bad cycles. + +### 5.4 Dependencies mean terminal, not successful + +`after` lists tasks that must be terminal before this one starts. Terminal, not done: a task whose +dependency failed or was aborted still starts and decides for itself what that means. Dependencies +are set at creation, reference existing tasks in the same ownership tree, form no cycles, and are +never edited. A task may create a successor whose `after` includes itself in its settlement commit; +it cannot add new dependencies to itself. A dependency outside an attached subtree may require +another explicit drive; attaching one scope never implicitly executes a sibling scope. + +A foreground task may depend on a background one, and it is live while it waits, so its conversation +stays busy and `drive` does not resolve until the dependency settles. That is what the overflow +chain wants (`G' after: [C]`). It is a trap for work that may never settle: a foreground task +depending on a recurring schedule keeps its conversation busy forever. Depend on work that ends, or +wait inside the task's own execute instead. + +### 5.5 post_tools joins an exchange + +The generation's settlement publishes the exchange atomically: + +```ts +await runtime.commit(tx => { + const assistant = tx.entry(assistantKind, { model: [message] }); // calls [A, B] + const tools = message.calls.map(call => + tx.task(toolKind, { state: { status: "planned", call, assistant } })); + tx.task(postToolsKind, { + after: tools, + state: { status: "waiting", assistant, tools, inputs: task.state.inputs }, + }); + tx.settle(task, "done", { inputs: task.state.inputs, assistant }); +}, call); +``` + +Each tool executes and settles itself: hooks, execution, result entry, terminal variant. It does not +look at siblings, queues or context. post_tools starts when both are terminal, reads them with the +typed getter, narrows on their status, and decides what happens next: + +```ts +async execute(task, runtime, call) { + await runtime.commit(async tx => { + const tools = await tx.getTasks(toolKind, task.state.tools); // Map> + const outcomes: ToolOutputState[] = []; + for (const t of tools.values()) { + switch (t.state.status) { + case "done": case "aborted": outcomes.push(t.state.output); break; // both wrote their result entry + case "orphaned": // kind unregistered at open (§5.1) + tx.entry(toolResultKind, unavailableResult(t.state.call)); break; + default: throw new Error(`tool ${t.id} still live in post_tools`); + } + } + if (outcomes.some(o => o.terminate)) { + for (const inputId of task.state.inputs) { + const r = await tx.value(inputResult(inputId)).get(); + if (r?.status !== "placed") throw new Error(`Invalid active input ${inputId}`); + tx.value(inputResult(inputId)).set({ status: "unanswered", requestId: r.requestId, entry: r.entry, + reason: "terminated" }); + } + return tx.settle(task, "stopped", { assistant: task.state.assistant, tools: task.state.tools }); // no successor + } + const added = outcomes.flatMap(o => o.addedTools ?? []); + const selectedTools = added.length ? [...current, ...added] : current; + if (added.length) tx.value(generationKind.config.selectedTools).set(selectedTools); + const handoff = outcomes.find(o => o.handoff); + if (handoff) tx.entry(handoffKind, { + data: { text: handoff.handoff }, model: [handoffMessage(handoff.handoff)], head: "self", + }); + const inputs = await landPostToolsInbox(tx, task.conversationId, task.state.inputs); // writes + steer (§8.1) + tx.task(generationKind, { state: { status: "pending", ...nextGeneration(task, { selectedTools, inputs }) } }); + tx.settle(task, "done", { assistant: task.state.assistant, tools: task.state.tools, inputs }); + }, call); +} +``` + +`inputResult(id)` above is the sticky conversation value address for that input (§8.1), not a +driver operation. The abort handler resolves the same owned inputs as `unanswered` with reason +`aborted` and settles in one commit. The runtime rejects a marked execute commit before its closure runs; no branch is needed +in the normal settlement. Inbox-placement helpers above implement exactly the mode table in §8.1. + +Sequential tools are the same mechanism with `B after: [A]`. + +```text +crash before generation settles → generation live; recover +crash after it settles → assistant, tools and post_tools exist; nothing to repair +crash after the last tool → post_tools exists and is startable; nothing to infer +``` + +### 5.6 Foreground and background + +A task is foreground unless created with `background: true`; the flag is fixed at creation. Generations, tools, post_tools and automatic collapses are +foreground; jobs, schedules, spawned subagents and manual collapses are not. The +foreground set of a conversation is its live foreground tasks, plus, through any live foreground +task that owns a conversation, that conversation's foreground set. Abort and "is this conversation +busy" are defined on that set. Detached work is cancelled by its own handle. + +### 5.7 Ownership + +A task that creates a conversation adds it to `owns`; the conversation records `owner`. Both are +written in the creation commit. A task may own several (a fan-out tool driving three children); +a conversation has one owner. Ownership defines drive scope and cancellation reach. Fork +provenance is a different link (§2.5) and creates no ownership. + +### 5.8 Turn tasks and appending entries + +A kind that drives a turn declares `turn: true`; `Tx` materializes it onto the task like `role`, and +storage indexes it. The built-in generation, tool, post_tools and collapse kinds declare it; a +plugin kind that drives its own turn declares it too. That gives one predicate, computed from the +index without decoding state or knowing any kind name: + +```text +inTurn(conversation) = live tasks in it with turn: true +``` + +It exists because of one hazard. An entry with a `model` appended between an assistant's tool calls +and their results changes the prefix the next request replays: providers that validate message order +reject it, and Anthropic thinking signatures are invalidated. The lane harness holds the same +invariant by deferring custom messages while streaming; pico holds it with `inTurn`. + +`tx.entry` appends immediately and returns the id, which is what a turn task needs (the generation's +assistant entry, a tool's result, a head with `"self"`). It rejects in exactly one case: + +```text +reject if the entry has a `model` + and the committing task is not turn: true, or there is no committing task + and inTurn(entry's conversation) is not empty +``` + +So an entry without a `model` is never blocked, a turn task writing its own exchange records is +never blocked, and nothing is blocked in a conversation with no live turn task. The rejection names +`tx.write`. + +`tx.write(kind, draft)` is the door for model-visible entries from outside a turn: it appends +immediately when `inTurn` is empty, and otherwise queues as the `write` mode (§8.1), landing at the +next post_tools or final-answer boundary. It returns an `inputId`, not an entry id, because the entry +may not exist yet. `ConversationHandle.write` is the same wrapped in a commit. + +`inTurn` is also the honest definition of "busy" for a UI: a foreground subagent tool is in it +because it is a tool, a background job is not. + +## 6. Scheduling and cancellation + +### 6.1 The commit line is the scheduler + +Every driver decision runs on the same serialized line as commits. Only task methods, signal +listeners, watcher callbacks and telemetry callbacks run outside it. There is no polling loop, +`Wake`, effect-admission gate, worker pool or second scheduler. + +Process state consists of: + +| State | Purpose | +|---|---| +| `live: Map` | current non-terminal tasks only | +| `running: Map` | one execute/recover/abort call per task id | +| `byConversation`, `dependents` | live-task membership and reverse live dependency edges | +| ownership links | immutable ancestry needed by live tasks, outstanding calls and attached roots | +| attached roots | stable conversation ids or one session-wide flag | +| waiters | temporary drive/task waiters, indexed by their scope or target | +| phase | open, stopping, closing, faulted, closed | + +Open seeds the complete live map before building readiness indexes, but starts no task. After a +successful batch, the line applies every task change to these indexes before scheduling or testing +idle. A settlement removes the task and its dependency edges; creating a successor in the same +batch never exposes an idle gap. `statusEpoch` advances on each actual committed status change, +including intermediate changes in a batch whose final status equals its initial status. State-only +patches, abort marks, same-status patches and failed commits do not advance it. + +Normal execution does not scan storage after every task returns: + +```text +task commit on line: + persist whole batch + update live/dependency/conversation indexes + reconsider changed tasks, affected dependents and waiters + reserve eligible calls in running + publish complete state; leave line + dispatch calls/signals/listeners outside line + +invocation completion on line: + validate the invocation object matches running[taskId] + inspect that task in live (absence means it already settled) + check outcome and statusEpoch; remove its running slot + reconsider that task and affected waiters + reserve eligible calls; leave line + dispatch outside line +``` + +Only the current invocation may change its owned task's state/status or settle it. Host cancellation +may set the abort mark, but host or sibling status writes cannot mask a broken invocation's progress +guard. Task code receives a snapshot; later authority is checked against the line's live record and +invocation identity, not that snapshot. No extra storage read is needed for this check. + +A task can be reserved only when it is served, live and not already in `running`: + +```text +marked → abort, irrespective of after +unmarked, role=start → execute when every dependency is terminal +unmarked, role=inflight → recover +``` + +Existing dependency ids are validated on creation. Once the complete live map is seeded, a valid +dependency absent from it is terminal. Dependency settlement wakes only its live dependents. +New attachments may inspect the current live map once; ordinary completions do not scan all tasks. +An outstanding invocation retains its slot even after it commits terminal status, until its method +actually returns. Different task ids execute concurrently. Retaining the slot is the abort join: +there is no separate invocation that waits for, or overwrites, its current owner. + +A normally returning execute/recover must have settled or advanced its captured `statusEpoch`. +`planned → running → planned` is valid; returning without any committed status transition is not. +Cancellation/close unwind is the exception. An abort handler must settle before returning. The driver +does not diagnose arbitrary changing-but-buggy cycles. + +Unexpected task-contract errors fail-stop the session, rather than poisoning individual tasks and +leaving their dependents waiting forever. Reject all pending drives and shutdown with the fault +immediately; stop admission and claims, signal owned calls outside the line, join them and close. +Do not manufacture terminal outcomes. The report includes task id, kind, method and error. Reopen +starts nothing: the host can inspect and mark a task before driving, or replace a broken kind with +one that understands and settles its stored state. Ordinary provider/tool failures are domain +outcomes handled by their kinds, not driver faults. + +### 6.2 Attachments and waiters + +`conversation.drive(call)` attaches one stable conversation id; `harness.drive(call)` attaches the +session. Each call registers an independent temporary waiter. Repeating a drive never retains another +scope object. Resolving or cancelling a waiter removes it and its signal listener, but keeps the +attachment, so background and recurring work continue. Session attachment subsumes serving filters; +separate conversation waiters still retain their own idle predicate. + +Serving follows ownership, never fork provenance. A conversation attachment serves all its owned +descendants, even when their owner task has settled. Discover membership starting from conversations +with live tasks, walking `conversation.owner → owner task's conversationId` upward. Batch named +ancestor reads through `getConversations` and `getTasks` and cache only the extracted immutable links; +never enumerate historical children or retain terminal owner payloads. +Owner liveness always comes from `live`, not the ancestry cache. Release unused ancestry when no +live task, outstanding invocation or explicit attachment needs it. + +```text +100,000 completed children; 2 live tasks +→ open reads 2 live tasks and only their required ancestry +→ later task completions use current indexes +→ no traversal of the 100,000 historical children +``` + +Idleness is decided on the line against current indexes: + +- Conversation: no direct live foreground task in that conversation. Reaching foreground descendants + requires a live foreground owner in the root, so this is equivalent to an empty foreground set. +- Session: no live foreground task anywhere, including foreground tasks in detached children. + +Both resolve `"idle"`, or `"closed"` when close wins. Session faults reject, not resolve idle. +A full-quiescence wait, if exposed separately, may never complete with a recurring task. + +Drive uses `call.abortSignal` to cancel only the actual registered waiter, not durable work. Register +and check cancellation on the line; a signal listener enqueues removal. Every outcome removes the +waiter and listener exactly once. Do not race an uncancellable drive promise with an outer promise. +Cancelling `prompt` after acceptance also leaves its accepted input durable. + +Task-bearing calls reject known self-waits: a foreground task cannot drive its own foreground scope, +nor wait for a task whose unresolved dependency path leads back to it. The latter also catches a +background task T driving a foreground D with `after:[T]`. Apply the checks before prompt acceptance +and reject new work that would introduce such an indexed wait cycle. A foreground ancestor reached +through a settled/background owner is not automatically the caller's foreground scope. Arbitrary +cycles hidden in plugin promises remain the task author's responsibility. Dependencies elsewhere +in an ownership tree may require another explicit attachment; never silently widen a drive scope. + +### 6.3 Cancellation is a request, then an abort invocation + +```text +abortTask(id, call) mark one live task; reject terminal +conversation.abort(call) + one commit: mark its current foreground ownership closure; + withdraw queued steer/followUp in affected conversations; keep write/nextRun +``` + +`abort: true` is a durable request, not terminal status. When that mark commits, the current +execute/recover invocation loses mutation authority immediately. Its later main and scratch writes +reject `TaskCancelled` on the line, before running their builders. This is normal control flow, not +a kind bug. The signal is dispatched after leaving the line; standard effects and waits observe it +cooperatively. No new admission gate is required. An effect may start in the interval before signal +delivery and then be cancelled; cancellation neither rolls it back nor promises exactly-once I/O. + +```text +100 task streaming; execute invocation A owns it +110 abort=true commits; A's mutation authority revoked + line releases; A's signal fires + stream returns/throws; execute finishes local finally cleanup and returns + A's completion enters line; remove A, reserve fresh abort invocation B + line releases; kind.abort(task, runtime, callB) runs +120 partial/error outcome and unanswered(aborted) input results, if the kind needs them +121 task terminal aborted; scratch retired in the same commit + abort method returns; remove B +``` + +Normal execute/recover settlement needs no mark check and no `tx.origin` branch. If its settlement +wins first, the task is already terminal and a later abort rejects. If the mark wins first, the whole +normal settlement rejects and the driver runs `abort`. A provider's in-band `stopReason: "aborted"` +is not itself a durable mark: without a mark, the kind must commit its failure/retry outcome; with +a mark, its attempted domain commit rejects `TaskCancelled` and cleanup runs in the fresh invocation. +An invocation cancelled only by a task-local deadline may still commit its domain result. + +The driver recognizes its own `TaskCancelled` and the invocation's known cancellation reason, +including supported cancellation errors correlated with the mark/close. An arbitrary exception does +not become harmless merely because a signal fired. Hooks propagate cancellation rather than treating +it as an ordinary fail-open/fail-closed decision. Expected close/stale callback rejections are handled +at their boundary; uncertain persistence still faults the session. + +`abort()` receives a new identity, telemetry span and initially active controller. Repeated marks +never cancel an already-running abort handler. Only close/fault cancels it. Abort ignores `after`, +may perform cleanup effects and write outcomes, but cannot create successor tasks/conversations or +queue future work via accept/nextRun. A target with no owned call and no attached scope remains +marked until driven; marking it does not implicitly attach unrelated work. + +Fresh cleanup can use only durable records. A tool that creates a cancellable background job records +the job id and its cancellation policy on its own task in that same commit. Its abort handler uses +those references, never locals from execute or a post-mark catch that tries another mutation. Child +conversation ownership is already durable. Built-in cleanup marks only the child's live foreground +tasks; it does not call the public conversation abort operation that also withdraws queued inputs. +This task-only cleanup policy applies on every abort invocation, including after crash/reopen, so +shutdown cannot accidentally discard preserved queues during recovery. Cleanup's cancellation request treats an already-terminal +target as finished, using an atomic mark-if-live operation internally or handling that specific public +terminal result; a read-then-mark does not close the race. + +Scratch before the mark is authoritative. Harness-owned sinks/progress bridges own and handle their +pending scratch promises, drop late cancelled output and drain callbacks before invocation completion. +Task code must await/catch its own scratch writes; no `void runtime.scratch(...)` without error handling. +Do not turn rejected scratch into a successful no-op or race away an unfinished tool/hook. If code +ignores the signal and never returns or reaches a rejecting harness operation, abort/close can wait +forever. An isolate kill boundary is future work; RPC cancellation alone is not forced termination. + +### 6.4 Open, close, shutdown, delete + +**Open** reports unregistered entry kinds, seeds live/ownership indexes, exposes inspection/queries, +and starts nothing. A live task whose kind is not registered cannot run: a background one is parked +(not started, not recovered, not counted for idleness) and resumes through `recover` when its kind +returns; a foreground one would keep its conversation busy forever, so open settles it in the +derived `orphaned` variant (§5.1) and reports it. `inspect` returns those separately. Unknown entries preserve context through stored facets. + +**Close** runs one nonpersistent line operation that enters `closing` and stops admission and claims. +All earlier line operations have already finished. Later storage operations reject; internal invocation +completion messages remain admitted. Outside the line, signal and join every owned call, including +terminal tasks whose methods are still unwinding, then close storage. No new abort marks or outcomes +are written. Never hold the line while awaiting a task that may itself be awaiting a line operation. + +**Shutdown** runs one line operation: enter `stopping` and commit one batch marking all live tasks. +Queued inputs and their queued result records remain untouched, including in idle conversations. +There is no inbox sweep or special storage query. After this batch, only abort cleanup mutations are +admitted; an abort handler already running remains authorized. Cancelled execute/recover mutations +reject. Abort handlers resolve their already-running input groups, but built-in child cleanup only +marks tasks and never drains queues (§6.3), including after restart. Serve cancellation session-wide; +once both live tasks and running invocation slots are empty, close. A failed abort handler faults +and rejects shutdown rather than leaving an eternal drain. Preserved queues create no work by +inference on reopen/drive; a later acceptance places eligible queued input at the defined boundary. + +Repeated lifecycle calls share completion. Task-token-bearing close/shutdown/session-wide drive reject +rather than waiting on their own invocation. Close joins regardless of its caller's signal; once +shutdown is admitted, caller cancellation cannot abandon marking, joining or closing. Explicit close +may interrupt shutdown; shutdown then rejects and remaining marked tasks recover later. Lifecycle +entry points are exceptions to ordinary admission so repeated close works while already closing. + +**Delete** rejects while either live tasks or outstanding invocation slots remain in the subtree, +then rejects new work. Entries inherited by independent forks are never erased. + +### 6.5 Call, telemetry and storage version + +`Call` is an alias of Chord `Context`, not a second context implementation. The driver installs a +private typed identity alongside its signal; span/budget derivation preserves it without casts: + +```ts +import type { Context } from "@earendil-works/chord"; +import { createContextKey, withAbortSignal, withContextValue } from "@earendil-works/chord/context"; + +export type Call = Context; + +interface Invocation { + readonly taskId: Id; + readonly method: "execute" | "recover" | "abort"; + readonly controller: AbortController; + readonly initialStatusEpoch: number; +} + +const invocationKey = createContextKey("pico.invocation"); // private + +// parentWithoutCallerCancellation retains the host telemetry parent, not the drive waiter's signal. +const call: Call = withContextValue(invocationKey, invocation, withAbortSignal( + invocation.controller.signal, parentWithoutCallerCancellation, +)); +const identity = call.value(invocationKey); // Invocation | undefined, no assertion +``` + +The line compares object identity with `running.get(identity.taskId)`. `TaskRuntime` is bound to its +expected invocation and rejects absent/foreign/stale identities; public host calls may have no token +while the session is open. Returned ordinary conversation handles also receive Call on every async +operation, so they preserve attribution without a task-specific facade. Capturing a public handle and +deliberately supplying `BACKGROUND_CONTEXT` remains an in-process cooperative escape, not isolation. + +Every async Harness/Conversation/TaskRuntime method takes a required final `call: Call`, including +reads, waits and mutations. Options before it are passed as `undefined` when unused. Pure accessors, +synchronous registration/disposal and operations inside Tx do not require a Call. `TaskRuntime` and +`ToolRuntime` name capabilities; `call` names cancellation/telemetry/identity; model context keeps its +existing meaning. There is no `TaskContext`, `CallContext`, admission helper or signal-injecting facade. + +The driver starts one telemetry span per execute/recover/abort invocation, independent of the drive +caller's cancellation. Commits use the supplied Call's current telemetry parent; nested provider/tool/ +hook spans derive and forward a Call. Existing Context-taking env, provider and hook boundaries +interpret the signal rather than merely carrying it. Node operations without interruptible APIs may +only check before/after; hooks and plugin code must cooperate. Commit admission uses lifecycle, mark, +method and identity, not a blanket signal check: cancellation never abandons admitted persistence. + +RPC transports cancellation and selected telemetry metadata, never the invocation object. A trusted +host-side binding reattaches the local identity for remote task calls. A cancelled RPC waiter does not +prove remote code stopped; no slot is released until completion or a real kill boundary confirms it. + +Telemetry schema names/attributes are a separate implementation package; the host supplies the tracer +and no spans are stored in session data. Storage metadata carries a version; mismatches reject and a +backend exposes migration before open. None of these mechanisms require a renewable session lease. + +## 7. Storage + +One interface: point reads, batched reads, bounded indexed scans, one atomic commit. No journal +reader, no second transaction layer, no residency API. + +### 7.1 What is stored + +Conversations (`parent` for forks, `owner` for owned children), entries, tasks +(with indexed role and scheduling fields), values and their rewindable versions, list elements and +clear markers, scratch scopes. Context is not stored (§2.2). + +### 7.2 Queries, from the callers' side + +| Caller | Reads | Never | +|---|---|---| +| Driver open/attach | complete live-task seed and named owner records; then committed-batch index updates | repeated full scans after calls, historical child traversal | +| Shutdown | current live-task index | inbox sweep, historical conversation traversal | +| Context | newest head at a target; fork-aware range from its returned boundary | unrelated transcript history | +| post_tools | tool tasks by id | sibling scans | +| UI | a transcript page before/after an id, with a limit | the whole conversation | +| Validation | named task/conversation records and full entries by id | unrelated records | +| Fork | transcript and rewindable conversation history ≤ the entry id | today's state filtered | +| State consumer | latest version by index; a bounded list range | a replay of unrelated records | +| Reopen (SQLite) | live tasks and the conversations they need | every historical transition | + +"Latest" is an indexed descending query with limit 1, never load-and-take-last. Filters apply +before limits and decoding. Entries are immutable and read whole; there is no separate header or +projection API. Named owner-task lookups use existing batched `getTasks`, even for terminal owners. + +```ts +interface Page { readonly items: readonly T[]; readonly next?: Id; readonly readAt: Id } + +interface Cursor { readonly after?: Id; readonly before?: Id; readonly limit: number } +interface ConversationQuery extends Cursor { readonly parent?: Id; readonly ownedFrom?: Id } +interface EntryQuery extends Cursor { readonly conversationId: Id; readonly kind?: string; readonly key?: string; + readonly from?: Id; readonly through?: Id } // inclusive logical range +interface TaskQuery extends Cursor { readonly conversationIds?: readonly Id[]; readonly live?: boolean; + readonly role?: TaskRole; readonly kind?: string; readonly abort?: boolean } +interface ListQuery extends Cursor { readonly at?: Id } // at = an entry id (§3.2) +interface ValueQuery { readonly scope: Scope; readonly namespace: string; readonly after?: string; readonly limit: number } // keys, ordered + +interface Version { readonly seq: Id; readonly value: T } +interface Element { readonly id: Id; readonly value: T } + +interface Storage { + readonly lastSeq: Id; // last committed sequence + commit(batch: CommitBatch): Promise<{ first: number; last: number }>; + + getConversations(ids: readonly Id[]): Promise>; + scanConversations(q: ConversationQuery): Promise>; + + getEntries(ids: readonly Id[]): Promise>; + scanEntries(q: EntryQuery): Promise>; // full entries, fork-aware + newestHead(conversationId: Id, at: Id): Promise; // full entry, target-capped, fork-aware + + getTasks(ids: readonly Id[]): Promise>; + scanTasks(q: TaskQuery): Promise>; // omitted conversationIds: entire session + + getValue(addr: Value, at?: Id): Promise | undefined>; // at = an entry id + scanValues(q: ValueQuery): Promise }>>; + readList(addr: List, q: ListQuery): Promise>>; + + close(): Promise; +} +``` + +`Value`, `List`, `Address` and `Scope` (session / conversation / scratch) are the address types +of §4.1; +`Conversation`, `Entry`, `Task` and `TaskRole` are the records of §§2 and 5; `CommitBatch` is §7.3. + +Pages carry a cursor and the sequence they were read at. Transcript reads are logical conversation +reads: they combine each fork's capped source prefix with local entries, carrying every ancestor +cutoff recursively. `from` and `through` are inclusive entry positions in that logical transcript; +source entries after a fork point never appear. `newestHead(conversationId, at)` applies the same +cutoffs and returns the newest visible entry with `head` at or before `at`. Rewindable conversation +state follows the same ancestor cutoffs. Session state stays current; sticky conversation state is +copied from current state only when the fork explicitly selects it. Tasks are never inherited. +Historical reads on sticky or scratch addresses reject. Coherent multi-read operations run on the +line; a cursor alone doesn't freeze mutable state. + +### 7.3 Batches + +```ts +type StateWrite = + | { type: "value.set"; addr: Address; value: JsonValue } + | { type: "value.delete"; addr: Address } + | { type: "list.append"; addr: Address; value: JsonValue } + | { type: "list.remove"; addr: Address; element: number } + | { type: "list.clear"; addr: Address }; + +type MainWrite = + | StateWrite + | { type: "createConversation"; conversation: Conversation } + | { type: "deleteConversation"; id: number } + | { type: "entry"; entry: Entry } + | { type: "task"; task: Task } + | { type: "patch"; id: Id; role: TaskRole; state: TaskStateBase; abort?: true } + | { type: "settle"; id: Id; role: "terminal"; state: TaskStateBase }; // also retires scratch + +type CommitBatch = + | { readonly kind: "main"; readonly writes: readonly MainWrite[] } + | { readonly kind: "scratch"; readonly task: number; readonly writes: readonly StateWrite[] }; +``` + +A scratch batch can only carry state writes, by type; `ScratchTx` (§4.6) only exposes those. Write +*i* gets sequence `lastSeq + 1 + i`. + +The Tx read `lastSeq` when it started and numbered its writes +from there; the line guarantees nothing landed in between, so storage numbering from its own +`lastSeq + 1` yields the same ids. Records that carry an id (`entry`, `task`, a list element) let +storage check that for free. The harness validates against +committed state plus earlier writes in the batch (kinds validate status transitions; the harness +enforces ownership, dependencies, exchange rules and admission; nobody validates payload shapes: +stored objects are trusted, and schema validation belongs at wire boundaries). For an entry with +`head`, validation requires the stored id to be visible and at or after the previous fork-visible +head's stored boundary. Stored edit targets must be earlier visible entries; managed-system targets +are restricted to baseline supersession (§2.1). Input-result transitions +are validated, terminal results cannot change, and consuming/cancelling an inbox element writes its +result in the same batch. No entry-kind code runs. Storage checks scope, sequences and structure, +prepares only touched data, persists atomically, publishes. A commit with no writes writes nothing. + +### 7.4 Memory and JSONL + +The memory backend holds current maps (conversations, tasks, values), ordered entries per +conversation, version histories, and lookup indexes (ids, addresses, live-task membership, parent +and owner links). Indexes reference the same immutable payloads. JSONL is the memory backend plus +one line per batch: + +```json +{"first":100,"writes":[ + {"type":"value.set","conversationId":1,"namespace":"plugin.plan","value":true}, + {"type":"entry","conversationId":1,"kind":"user","model":[{"role":"user","content":"Inspect","timestamp":0}]}, + {"type":"task","conversationId":1,"kind":"generation","state":{"status":"pending"}}]} +``` + +Write all bytes, then publish in-memory changes, never before. Durability against process crash is +required; fsync policy is a backend option. + +**Whole values, ordinary list operations.** Every `value.set` stores the complete supplied value, +just like the lane harness. There is no automatic diff, Chord encoding or hidden replay chain for a +value. Lists store append/remove/clear operations. Repeatedly setting a growing whole value can write +quadratic bytes; use compact appended frames/output operations or bounded checkpoints for streaming. +Chord deltas are used only for preview/watch delivery (§9.4), not storage serialization. + +**Scratch** goes to `session.scratch/`, one file per task, with the same plain batch format. +Retirement in the main file is authoritative; unlink happens afterwards. Task ids are never reused. +A retry keeps its task and clears its scratch through a persisted list operation, not by deleting +the only record of its latest sequence. + +**Recovery across files.** Main and scratch commits share one session sequence, so gaps inside any +one file are normal: + +```text +main: 100 create generation +scratch: 101–150 frame appends +crash before settlement +reopen: main says generation live; replay its scratch; lastSeq=150; next write=151 + +alternatively: +main: 151 settle generation, retire scratch +crash before unlink +reopen: generation terminal; ignore old scratch; lastSeq=151 +``` + +On open: + +1. Replay complete main batches, reconstructing task liveness and scratch retirement. +2. Replay scratch files only for surviving live tasks. Ignore retired scratch entirely, even if + malformed; it cannot affect state or the sequence high-water mark. +3. Validate increasing, nonoverlapping retained batch ranges and task/scope references. Per-file gaps + are valid; do not require the main log to contain discarded scratch positions. +4. Set `lastSeq` to the maximum complete batch endpoint across main and surviving scratch files, + including scratch clear/remove writes. A retired scratch file's sequences are already below its + later main settlement, so deleting it loses no high-water information. +5. Discard only unterminated final lines in replayed files and physically remove those torn suffixes + before appending again. Any malformed complete main or surviving scratch batch makes open fail. + +No new index, journal or cross-file transaction is required: each commit still writes exactly one file. + +### 7.5 SQLite + +Tables: conversations, entries, tasks (current records including terminal), values and +value_versions, list_elements and clear_markers, scratch, commit_boundaries, kinds (one row per +distinct entry kind), session_metadata (storage version, §6.5). An entry row has nullable JSON +columns for `data`, `model` and `edits`, plus a nullable integer `head`; storage understands those +core fields but never a plugin's data shape. Keys start with the session id. Indexes follow the query +table: entries by (conversation, id), (conversation, kind, key, id) and a partial index +(conversation, id) where head is not null; tasks by (role, id) and (conversation, role, id) with a +partial index on non-terminal roles; value_versions by (conversation, namespace, key, seq); +conversations by parent and by owner. One commit is one transaction: number the writes from +`lastSeq + 1`, insert rows, record the boundary, advance `lastSeq`. No full rewrite of anything per +append; no shutdown-specific inbox index. + +Values are stored whole, current and historical: a point read must not become a replay chain, and +disk is cheaper than that. Scratch is rows in the `scratch` table keyed by (task, address), written +in the task's own small transactions, and retired by `DELETE ... WHERE task = ?` inside the +settlement transaction: atomic with the result and the terminal status, nothing to unlink. Reopen +reads live tasks and named records they reference (including terminal owners), not accumulated +history. Once ownership links are extracted, terminal owner payloads need not remain resident. + +## 8. Built-in flows + +Task kinds, not scheduler special cases. Braces group one commit. Every +attempt records its usage; every settlement retires its scratch. Hooks and external calls run +outside the line, and their decisions are re-validated inside it. + +### 8.1 Accepting input + +Queued input is a conversation-scoped sticky list, `pi.inbox`. Its list element id is the stable +`inputId`; its value carries the complete entry draft, so a UI can render queued text or images +without another read: + +```ts +type UserInput = string | readonly (TextContent | ImageContent)[]; + +type QueuedInput = + | { readonly mode: "steer" | "followUp" | "nextRun"; readonly input: UserInput; readonly requestId?: string } + | { readonly mode: "write"; readonly kind: EntryKind; readonly entry: EntryInput; readonly requestId?: string }; + +type InputResult = + | { readonly status: "queued"; readonly requestId?: string } + | { readonly status: "placed"; readonly requestId?: string; readonly entry: Id } + | { readonly status: "done"; readonly requestId?: string; readonly entry: Id; readonly answer?: Id } + | { readonly status: "unanswered"; readonly requestId?: string; readonly entry?: Id; + readonly reason: "terminated" | "aborted" | "failed"; readonly detail?: string }; +``` + +The three modes that ask for a turn carry user content; a `write` carries its own entry kind and +the same `EntryInput` (§9.2) `Tx.entry` takes, and asks for nothing. `inputResult(id)` denotes the sticky conversation Value address with +namespace `pi.inputResult` and key `String(id)`, typed as `InputResult`. It is ordinary stored +state, not a driver callback. + +`accept({ input, requestId?, whenBusy? }, call)` is "the user hit enter": when the conversation is +idle it places the entry and creates a generation in one commit; when it is busy it queues, +`followUp` by default, `steer` or `reject` if the caller says so. Either way it returns an +`inputId`, and `result(inputId, call)` says which happened. `queueInput(input, call)` is the +explicit form for a specific mode, and `abortInput(inputId, call)` withdraws a queued one. + +| Mode | Placement | Effect on the answer group | +|---|---|---| +| write | next post_tools or final boundary | none; its result is `done` with no `answer` (§5.8) | +| steer | next post_tools or final boundary | joins the active group at post_tools; starts the next group after a final answer | +| followUp | final-answer boundary | starts the next group | +| nextRun | next explicit idle `accept` | joins the group started by that acceptance | + +A generation and its `post_tools` carry `inputs: Id[]`. Generation-with-calls creates the tools and +exactly one `post_tools` with the same group in its settlement commit. `post_tools` places writes, +places selected steering in admission order, extends the group with those steer ids, creates the +continuation generation and settles. At a final answer, the generation resolves every current +input to that answer, places writes, then places selected steer/followUp items into a new group and +creates its generation. An `on_yield` continuation retains the current group. No tail scan +attributes results. + +Placement appends the entry, removes the list element and writes `placed` in one commit; a `write` +is placed and `done` in that same commit, so `placed` is never observable for it. Only the +generation and `post_tools` transition a group: a final answer records `done` with the answer's +entry id; the harness giving up records `unanswered` with reason `failed`; a tool's `terminate` +records `terminated`; an abort of the run, or `abortInput` on a queued item, records `aborted`. +Foreground abort withdraws queued steer and followUp while preserving write and nextRun. Failure +and terminate may place safe writes but do not consume queued inputs that require a successor. +Exactly one live generation or `post_tools` owns an active group, and ownership transfers in the +same commit that settles the previous owner. Input groups/results are built-in harness behavior, +not payload fields the generic driver infers from arbitrary plugin tasks. These built-in kinds are +registered at open; a compatible replacement retains their input-result responsibilities. A missing +tool implementation is handled by the surviving tool task kind, not by orphaning the input owner. + +The list is stored as append/remove/clear operations, not as a rewritten array. Inbox watch events +carry the same operations; an idle same-commit append/remove emits none. Memory and SQLite may +discard a removed sticky element; JSONL retains its append record, so any payload later copied into +a transcript entry appears twice on disk, including an idle acceptance. Once an unplaced item is +withdrawn, its draft is no longer queryable; `inputResult` retains only its terminal status and +optional request id. + +### 8.2 Generation + +```text +create: capture inputs, the kind's declared config (model, thinking, selected tools, profile; + §5.2, rewindable values), provider options and retry policy, so a retry uses what the + attempt used +execute: if a collapse is needed by threshold: { create collapse C; create generation G' after:[C]; settle } + desired = seed typed section draft from durable prepared values/rendered text; + run system_instructions handlers, then touched renderers/wrappers outside line; + capture complete desired tool definitions; no discovery/cache callbacks in storage + { inspect current head and effective managed system entries; + if no valid current-epoch baseline: append full baseline plus stored omission edits; + else if prepared state differs from desired: append materialized section/tool delta; + capture requestThrough = exact resulting transcript tip; status streaming } + project at requestThrough, not the later tail; before_request with Call on a request copy + validate messages-only mode and capture actually offered tools; stream; frames to scratch +outcome: + calls { assistant; tools; post_tools carrying inputs; settle done } + final { assistant; resolve every current input; place next-group inbox items and successor if any; + settle done } + deferred { status deferred, handle } → execute again: poll with sleeps until final + retryable { status retry_wait, attempt+1, notBefore } → execute again: sleep, then stream + overflow { settle failed(overflow); create collapse C; C's settlement creates G' } + failure { retain partial/error outcome for display if useful; resolve inputs unanswered(failed); + settle failed; no tools/results } + aborted provider returns in-band: attempt domain failure/retry commit; + durable mark present → TaskCancelled; execute unwinds; driver calls abort below +``` + +**Durable configuration, prepared instructions.** Config changes persist immediately as ordinary +scoped value writes: model/thinking/selected tools/profile rewind with forks, while each other address +keeps its declared policy. Host files and catalogue implementations are not stored as configuration. +The `system_instructions` hook uses the generation's captured config to produce desired sections and +complete selected tool definitions. Its answer is prepared outside the line. A system entry records +canonical instructions prepared for a request, not proof of delivery or an external acknowledgement. + +**Typed section definitions and one mutable draft.** A token identifies a JSON payload type and an +append-time renderer. It is not an entry kind, discovery callback or read-time projection hook: + +```ts +interface SystemSection { + readonly key: string; + render(value: T): string; +} + +interface SystemSectionDraft { + get(section: SystemSection): T | undefined; // owned copy; call set to change the draft + set(section: SystemSection, value: T): void; + delete(section: string | { readonly key: string }): void; + wrap(section: SystemSection, transform: (text: string) => string): void; +} + +interface SkillInfo { name: string; description: string } +const skillsSection = defineSystemSection({ + key: "pi.skills", + render: skills => skills.map(skill => `- ${skill.name}: ${skill.description}`).join("\n"), +}); +``` + +Built-in definitions are exported through `systemSections`: identity carries a string, environment +carries structured data such as `{ cwd: string }`, and skills carries typed skill descriptions. +Plugins may define others. Payloads must be JSON-representable. Typed getters use the registered definition's +payload contract; plugins need no casts. Registration rejects duplicate keys; replacing a definition +must preserve the persisted payload shape or migrate it. Payload shape validation belongs at wire +boundaries, as for other typed stored values, not to arbitrary renderer casts. + +Every preparation seeds a private ordered draft from durable section values and rendered text. Hooks +edit that same draft in registration order, harness-wide first and innermost scope last. Under the +skip-failed-handler policy, discard that handler's mutations/wrappers while retaining earlier changes; +a failed refresh must not publish a partial deletion. `set` replaces +an existing payload in place; a new key appends. `delete` explicitly removes a key (even by string when +its renderer is unavailable). `get`/`set`/`wrap` require a registered compatible definition. Thus a later +plugin can filter or append typed skills by get-and-set, rather than parsing rendered prose. Hooks +return only optional complete desired `tools`; the last supplied list wins, omission preserves an +earlier handler's list, and no supplied list means no desired tools. + +After hooks, render touched sections outside the line. `wrap` composes text transforms in registration +order after the section renderer, without mutating the shared token. Wrappers live only in this draft; +only their final text is stored. Untouched seeded sections retain their stored rendered text, including +when their contributor or renderer is absent. An explicit set/wrap or compatible renderer replacement +recomputes that text. Refreshing a base intentionally rebuilds its wrappers; preserving a missing +wrapper across such a refresh is not promised. Freeze/discard the draft after preparation; later leaked +callbacks cannot alter the committed snapshot. + +**No accidental accumulation.** Host hooks should set complete authoritative base payloads each time +before transformation plugins run. Otherwise the composed transformation chain must reach a fixed +point when applied to its own previous output; each handler being individually idempotent is not enough. +Blindly appending to a seeded array/text each preparation would accumulate duplicates. Registration +order provides composition, not automatic deduplication of arbitrary payloads. + +Diff final JSON payloads and rendered text by key, ignoring incidental object-key order. Persist changed +payloads even if rendering is identical; those changes can be data-only system entries (`model: []`). +Changed rendering with identical payload also emits an instruction update, including renderer/wrapper +changes. Baselines and simultaneous changes use draft order; pure reordering is a no-op. There is no +separate order option, order-change message or generic source-renderer framework. + +**Durable seed, independent of model retention.** Reconstruct canonical section state through the +fork-visible target by reading managed `system` entries back to the latest baseline, then folding their +section records forward. Use existing indexed kind scans and an optional per-handle prepared-state +cache. A fresh baseline checkpoints the complete state. This fold is independent of model heads and +projection edits: an omitted historical model message does not erase its persisted section payload. +That is how an unavailable plugin's instructions survive a compaction that dropped its old baseline. +No extra full-state value, blob store or storage API is required. + +Separately inspect the effective model projection to decide whether it has a usable current-epoch +baseline. Arbitrary edits targeting managed system entries reject (§2.1); changing instructions uses +the section draft instead. Supersession omissions written by the current baseline do not trigger +another rebaseline. A fresh baseline uses the canonical payload/rendered seed plus this preparation's +changes, even when the previous model baseline fell outside the retained range. + +Hook-private discovery/cache state stays in closures or host services, not in section records. +Refresh skills through filesystem events or bounded polling, not remote scans per hook call. A failed +refresh is not a deletion: retain the last successful snapshot; seed first use before claiming a base. +Missing hook registration also is not deletion; use the draft's explicit delete operation. + +Fold effective pi-ai system-message tool fields from all projected messages, including declarations +outside managed entries. Diff complete definitions structurally by name. Additions and same-name +replacements emit `toolsAdded`; removals emit the previous stored definition in `toolsRemoved`. +Tools-only changes may use empty content. The adapter, not pico, renders compatibility notices. + +```text +100 user +110 system baseline: identity + plan_mode; toolsAdded=[read, write] +120 assistant +125 config commit: plan_mode=true; selectedTools=[read, grep] (not a transcript entry) +130 user +140 system delta: replace plan_mode; toolsRemoved=[write]; toolsAdded=[grep] +150 assistant +``` + +A crash between 125 and 140 preserves the config; the next generation prepares the missing change. +A crash after 140 but before the provider call also preserves that entry. An unchanged hook answer +produces no duplicate delta. Neither event proves the provider saw the instructions. A fork uses its +visible prefix and inherited rewindable config; changed host sources on restart may cause a new delta, +but cannot rewrite old messages. + +**Head changes and fresh baselines.** A current-epoch baseline is a managed baseline after the newest +visible head entry in transcript order (not after its retained boundary). With none, append a complete +baseline carrying ordinary omission edits for superseded retained managed system entries. Include old +baselines as well as deltas, but never unrelated system notices. Store the omissions on that same +baseline entry, not on the head, so arbitrary head writers remain generic and a fork cannot land +between replacement instructions and their omission controls. + +```text +10 user; 11 baseline; 20 assistant; 30 user; 31 system delta; 35 job notice; 40 assistant; 50 user +60 summary, head=30 +context at 60: [60 summary, 30 user, 31 delta, 35 notice, 40 assistant, 50 user] +70 assistant +80 user +81 system { baseline:true, full sections, full desired toolsAdded, + edits:[{ target:31, action:omit }] } +context at 81: [60 summary, 30 user, 35 notice, 40 assistant, 50 user, 70 assistant, 80 user, 81 baseline] +``` + +The baseline stays at its appended position after the retained tail. Before preparation, old retained +system entries remain visible; generic context derivation never secretly suppresses them. Every +request goes through preparation first. Repeated heads use the same rule. Reset/handoff often leave +no old managed entries inside the retained range, so the new baseline needs no omission targets. +Compute the remaining tool fold after planned omissions: a baseline adds the complete desired tool +set and explicitly removes unwanted declarations that remain from other effective system messages. +`baseline:true` is pico metadata, not a magic reset understood by pi-ai. + +**Frozen request cutoff and cache.** The generation stores the following request-specific field: + +```ts +// Request fields on the streaming variant; other generation variants/fields are omitted. +interface GenerationStreamingState { + readonly status: "streaming"; + readonly inputs: readonly Id[]; + readonly requestThrough: Id; // inclusive transcript entry cutoff, required after preparation +} +``` + +Capture the durable section-state cursor and registered definitions before running hooks/renderers +outside the line. Preparation retains that definition snapshot; later registrations affect subsequent +preparations. On the line, verify that no concurrent managed section-state write changed the seed; +if one did, rerun preparation outside the line. A head-only change does not stale the section seed. +Then inspect the current head and effective model state, choose baseline versus delta, and persist +the new system entry if needed and the generation's inflight intent with `requestThrough`. +The cutoff is the resulting transcript tip, even when no system entry was needed. Deferred variants +retain the cutoff for their prepared request; an unprepared pending variant has none. After persistence, +while still on the line, catch up the handle's context cache and capture a separate immutable array +of the selected entry references and effective replacement projections. This is the request snapshot; +entry bodies are immutable and can be shared. Never hand out mutable cache bookkeeping or mutate a +published replacement. Request-local normalization/hooks copy the objects they modify. + +```text +prepare through 51 → requestThrough=51, snapshot captured +head 60 lands → live cache advances; request snapshot stays unchanged +request streams → uses snapshot through 51 +next preparation → sees head 60; writes a fresh baseline if required +``` + +If a head lands while the hook is running, preparation sees the new head and chooses a baseline. +If it lands after preparation, the captured request is unaffected. Historical/recovery reads through +`requestThrough` derive independently from storage when the live cache has advanced; they do not rewind +it. Release request-only references after use; no registry of historical cache versions is retained. +The cutoff identifies canonical transcript input, not arbitrary request-local hook changes. + +**Pi-ai owns provider translation.** Pico passes only `{ messages }`, with neither top-level +`systemPrompt` nor `tools`, even when there are no selected tools. Both fields absent selects the +agreed pi-ai system-message mode; explicit empty strings/arrays are not absence. Adapters translate +the stored message history for their provider/model, preserving cache prefixes where possible. +Unsupported mid-conversation instruction/tool changes fall back to ``-bracketed user messages +at their historical positions and adapter-derived bulk wire tool declarations. Pico never hoists a +baseline, flattens historical changes into a rewritten prompt, or sends a parallel top-level tool list. +Fallback user messages do not have native system priority; cache reuse is best-effort, not universal. + +The pi-ai `SystemMessage` shape comes from [PR #9116](https://github.com/earendil-works/pi/pull/9116); +[PR #9117](https://github.com/earendil-works/pi/pull/9117) covers coding-agent prompt/tool deltas. These +were open when this contract was reviewed. Messages-only mode and its adapter fixtures are an external +integration requirement, not a claim that the current main branch already implements them. + +**Request-local overrides.** `before_request` receives a private request copy and may transform it. +Validate afterward that it remains messages-only. Such transformations neither mutate stored entries +nor alter the canonical section fold for the next generation. Exact transformed-request audit requires +separate optional capture; the transcript alone records only canonical preparation. Validate returned +tool calls against the definitions actually offered after transformation, plus implementation and +permission checks—not today's catalogue or the untransformed tool set. + +One task lives through retries and deferral; retries share a budget carried in state. Overflow +does not keep the generation alive: it settles and hands the retry to the collapse's settlement, +carrying the attempt count, so nothing live ever waits on the collapse. + +```text +recover streaming: frames in scratch → retain/publish the partial for display; settle without tools/results + no frames → retry within budget, else fail +recover deferred: execute again; the handle is in state +abort: fresh invocation after execute/recover returned: + read committed scratch; optionally retain partial for display, excluded from requests; + in one commit resolve owned inputs unanswered(aborted), record known usage, settle aborted; + no tool tasks/results; settlement retires scratch +``` + +Usage is recorded for failed, deferred and discarded attempts too; a missing report is unknown cost, +not zero. A report received only after the cancellation cutoff may not have reached scratch and is +therefore unknown to fresh abort cleanup. The record is a session list, `pi.usage`, one element per +attempt (conversation, task, model, tokens, cost), appended in the settlement commit together with an update of the session +value `pi.usage.totals`, so stats are a point read and never a fold. Tools and jobs append the same +way through the sink's `usage`. Persistence and invariant failures are not provider errors: they fault the session. + +### 8.3 Tools and post_tools + +```text +tool execute: + resolve the tool in the registry (the loadout may have changed since the turn started); + validate the model's arguments against its schema + before_tool → allow(args) | block(reason, terminate?) (a human approval waits inside the hook) + block / invalid / unknown tool → own error result, no invocation + { status running; effective args; replay policy } + invoke tool(callId, params, out, runtime, call); the sink owns its scratch promises (§9.3) + after_tool outside the line + { result entry from the folded sink; usage; settle done with result id and ToolOutputState } +``` + +The tool never touches context, siblings or queues. A throwing `before_tool` blocks the tool. An +ordinary tool throw is an error result, not a cancellation. A missing executable tool is also an +ordinary unavailable-tool result: the built-in `tool` kind remains registered and `post_tools` can +continue or finish the input group. This differs from a genuinely missing task kind (§6.4). + +```text +recover: replay only if the stored policy allows; else an interrupted result from the checkpoint +abort: previous invocation already returned; read durable scratch and cleanup references; + mark owned child foreground tasks/non-detached jobs, without touching child queues; + write aborted error result; settle +``` + +post_tools is the code in §5.5: read the tool tasks, stop on terminate, write the handoff if one +was requested, place writes and steering, carry the extended input group into the next generation, +and settle. Its abort resolves its input group as `unanswered` with reason `aborted` and settles +with no successor. + +**Blocking budget.** A tool call may not block a turn indefinitely. Tools that run processes or +child conversations create a job (or a conversation) first and wait on it with the budget; if the +budget runs out, the call settles now with what it has, `out.delegate(job)` and a diag, and the +work continues as the job: + +```ts +const job = await runtime.commit(async tx => { + const task = await tx.getTask(toolKind, runtime.taskId); + if (task?.state.status !== "running") throw new Error("Expected a running tool"); + const { status, ...payload } = task.state; + const id = tx.task(jobKind, { background: true, + state: { status: "planned", cmd, cwd, origin: { tool: "bash", task: runtime.taskId, callId: toolCallId } } }); + tx.patch(task, status, { ...payload, jobId: id, cancelJobOnAbort: true }); // complete running payload + return id; +}, call); +const done = await runtime.waitForTask(job, { budgetMs: runtime.budgetMs }, call); +const output = await runtime.jobOutput(job, call); +out.replace(output.text); out.capture(output.truncation); +if (!done) { + out.delegate(job); + out.diag("info", `still running as job ${job}; use job wait / status / stop`, "budget"); +} +``` + +The budget is the generation's config (`budgetMs`, default a few minutes); the tool kind passes it +as `runtime.budgetMs`. `waitForTask` observes Call cancellation and unregisters its waiter. A marked +tool cannot mutate from its execute catch; its fresh abort handler reads the stored job reference and +cancels non-detached work. Explicit backgrounding records a different cancellation policy. Successful +delegation settlement atomically records that the job is detached and requests its completion notice; +if the job has already finished, that transaction places the notice itself. Use a separate sticky +notification record rather than patching the state of a running job from its former owner. + +**Remaining budget integration question:** adopting arbitrary non-delegating tool work after timeout +requires an explicit transfer of effect and sink ownership before the source invocation releases its +slot. Racing and abandoning the tool promise is not a correct implementation. Preserve this capability +for a separate ownership design; crash recovery remains `lost`. The job-first path above needs no +promise adoption and is the initial implementation path. + +`new_context` is a tool that calls `out.handoff(message)`; the reset happens here, after the exchange +is complete, never inside the tool, because a head landing mid-exchange would cut it in half. Conflicting handoffs reject rather than pick an order. + +A final answer's continuation is decided in the generation's settlement. `on_yield` may request an +explicit continuation that keeps the current input group. Otherwise the answer resolves that group, +then eligible steer/followUp items are placed into a new group and its generation. An owned +conversation's final answer still resolves its own inputs; its owner tool observes that result while +driving it (§8.5). An idle conversation whose tail is a user entry creates no work by inference. + +### 8.4 Collapse + +```text +create: capture the prefix to replace (ending on a complete exchange), its first retained entry, + the newest head id, context and settings; one live collapse per conversation +execute: { status summarizing } + before_collapse (may decline or supply the summary); call the summarizer; candidate to scratch + { if no head newer than the captured one: + append summary { model:[summary], head:firstRetained }; settle done + else: settle failed(stale) } +``` + +Entries that land while it runs remain at or after the prepared retained boundary, so they stay in +context without anyone doing anything. Context edits that land meanwhile affect active projection +but do not invalidate or recompute the summary. A summary may land while a generation streams, +since the stream already projected its context and the replaced prefix is old. Automatic collapses +(threshold, overflow) are foreground; manual ones are background. A collapse never creates a +generation except through the settlement chain in §8.2. Abort records usage and settles without +publishing. Prefix pruning/windowing appends another entry with a stored head; pruning or replacing +individual retained entries uses edit entries (§2.1). Dynamic `compose` policies are unsupported. + +### 8.5 Subagents + +A subagent is a conversation. There is no subagent task, and there is one tool, `subagent`, whose +argument is a command: + +```ts +type SubagentCommand = + | { command: "run"; prompt: string; context?: "fresh" | "inherit"; tools?: string[] } // foreground + | { command: "spawn"; prompt: string; context?: "fresh" | "inherit"; tools?: string[] } // background + | { command: "send"; id: Id; text: string } + | { command: "status"; id: Id } + | { command: "wait"; id: Id } + | { command: "stop"; id: Id }; +``` + +```text +run: { create child conversation (this task owns it); initial values; accept prompt → inputId; first generation } + await child.drive(call) // cancels this waiter, not the child + child.result(inputId, call); { result entry; settle } + recover: the child exists; drive it again with the fresh Call + abort: mark child's foreground tasks only, preserve its queues; write own error result and settle + +spawn: the same creation commit; settle at once with the child's id +send: child.accept({ input: text }, call) → queued if the child is busy +status: the child's tail and live tasks +wait: await child.drive(call); child.result(lastInputId, call); settle (recover: drive again) +stop: child.abort(call) +``` + +`id` is whatever `spawn` returned, passed back by the model; the tool resolves it with +`runtime.conversation(id, call)`. There is no separate subagent registry to rebuild after a restart. +Between `spawn` and `wait`, whatever drives the parent's tree drives the child. A spawned child is owned by a task that +is already terminal, so it is not in the parent's foreground set and aborting the parent leaves it +alone; `stop` is the way to cancel it. Inherited context for a child is the parent's context at the +last complete exchange before the launching one. + +### 8.6 Jobs and schedules + +A job is a background task that runs a process. Its state: + +```ts +type JobInput = { + cmd: string; cwd: string; limits?: ShellOutputLimits; + origin?: { tool: string; task: Id; callId: string }; // originating call, for UIs + every?: number; notBefore?: number; rerun?: "safe"; // recurrence and recovery policy +}; +type JobState = JobInput & ( + | { status: "planned" } + | { status: "running"; startedAt: number } + | { status: "exited"; output: ToolOutputState; exitCode: number } + | { status: "killed"; output: ToolOutputState } + | { status: "lost" } +); +``` + +Starting one is creating the task; a tool does it in its execute (§8.3), a UI in a commit: + +```ts +const jobId = await conv.commit(tx => + tx.task(jobKind, { background: true, state: { status: "planned", cmd: "npm test", cwd, every: 6 * 3600_000 } }), call); +``` + +```text +states: planned { cmd, cwd, every?, notBefore? } | running { …, startedAt } | + exited { …, output, exitCode } | killed { …, output } | lost { … } +roles: planned=start; running=inflight; exited/killed/lost=terminal +execute: sleep until notBefore with Call; { status running; startedAt = runtime.now() } + env.exec(command, options, call), with bounded capture and spill + harness progress bridge owns scratch writes, handles cancellation and drains callbacks + after process and progress bridge return: + { if notification requested: append notice and consume notification request; + if recurring: status planned, notBefore = runtime.now() + every; + otherwise: settle exited with bounded output and exitCode } + a post-mark write rejects TaskCancelled; no in-band killed settlement +recover: safe rerun policy → execute again with the fresh Call + otherwise { consume pending notice request; append lost notice if requested; settle lost } +abort: previous invocation and its process/progress callbacks have returned + read committed output from scratch; + { consume pending notice request; append killed notice if requested; settle killed } +``` + +The progress bridge is harness-owned sink plumbing (§6.3), not a plugin-created detached promise. +It observes every scratch result, suppresses only expected cancellation/close rejections, reports +persistence faults, and is joined with process execution. Notification requests are current sticky +state keyed by job id; consuming them and publishing the notice is atomic on every terminal path, +including exited, lost and killed. If delegation follows settlement, delegation publishes that same +outcome instead. The two orderings must not change notification behavior. + +`runtime.env` is the harness's `ExecutionEnv` (`FileSystem & Shell`); `exec` runs the command, captures +output through `onUpdate` (here into scratch, so a watcher can show it live) and returns when it +exits or the signal fires. `running` is inflight, so a restart goes through `recover`; since the +shell doesn't hand out an adoptable process, recover reruns if the job declared that safe and +otherwise settles `lost`. A recurring job is the same task looping `planned → running → planned`; +`abortTask(id)` ends it wherever it is. Output stays in scratch while it runs and in the terminal state +after, where `jobOutput`, the `job` tool and UIs read it; there is no result entry, since a call that +waited already carries the output. A job whose call returned early appends a `notice` entry with a stored `SystemMessage` when it +finishes, so the next generation learns of it without +polling. Interval and catch-up policy are in state, +so a restart never launches a backlog. + +### 8.7 Hooks + +Hooks belong to kinds. A kind declares its points and their types; plugins register handlers per +kind and point; the harness only routes: + +```ts +type HookPoints = Record; + +interface GenerationHooks extends HookPoints { + system_instructions: { input: { conversationId: number; config: GenerationConfig; sections: SystemSectionDraft }; + output: void | { tools?: readonly Tool[] } }; // sections mutate one shared preparation draft + // handlers run sequentially; new keys append; set replaces; delete is explicit + before_request: { input: { request: ProviderRequest }; output: { request?: ProviderRequest } }; + after_response: { input: { response: AssistantMessage; usage: Usage }; output: void }; + on_yield: { input: { answer: AssistantEntry }; output: { continue?: string } }; +} +interface ToolHooks extends HookPoints { + before_tool: { input: { toolCallId: string; toolName: string; args: JsonObject }; + output: { args?: JsonObject; block?: { reason: string; terminate?: boolean } } }; + after_tool: { input: { toolCallId: string; output: ToolOutputState }; output: void }; +} +interface CollapseHooks extends HookPoints { + before_collapse: { input: { reason: "manual" | "threshold" | "overflow"; entries: readonly Entry[] }; + output: { decline?: boolean; instructions?: string; summary?: string } }; +} + +const generationKind = defineTaskKind()({ ..., hooks: { before_request: {}, after_response: {}, on_yield: { failClosed: false } } }); +const toolKind = defineTaskKind()({ ..., hooks: { before_tool: { failClosed: true }, after_tool: {} } }); + +// a plugin registers a handler +harness.hooks.on(toolKind, "before_tool", async ({ toolName, args }, call) => { + if (toolName === "bash" && !(await ui.approve(args, { signal: call.abortSignal }))) return { block: { reason: "denied" } }; // may wait for a human +}); + +// the kind runs it +const decision = await runtime.hooks(toolKind).run("before_tool", { toolCallId, toolName, args }, call); +``` + +A handler can be scoped to a conversation instead of the whole harness; for `system_instructions` +that is the normal case, since a subagent is a different agent: + +```ts +root.hooks.on(generationKind, "system_instructions", hostInstructions, { subtree: true }); // root and what it owns +child.hooks.on(generationKind, "system_instructions", investigatorInstructions); // this child only; overrides for it +``` + +Scoped handlers run after harness-wide ones for the same point, innermost conversation last, so a +child's sections win by key. Registration is process state, not durable; a presentation registers +when it attaches. + +Handlers run in registration order, each seeing the merged output so far; the kind decides what +to do with the result. A handler runs outside the line and its decision is applied inside a commit +whose runtime validates current invocation authority. Handlers receive a required final Call, +forward it to waits/effects, and must return before their parent task invocation releases ownership. +Cancellation control errors bypass ordinary hook failure policies. Handlers may run again after a +crash, so their external side effects need their own idempotence. `failClosed` says what a throw means: for `before_tool` +a throw blocks the tool; for the others a throw is reported and skipped. `before_tool` may wait as +long as it likes (a human approval is a hook that waits); the task is still `planned` meanwhile +and a crash simply asks again. A plugin kind declares its own points the same way, and a plugin +that replaces a built-in kind keeps its hook names so existing handlers keep working. Commit +listeners are not hooks: they observe, never decide, and never await a commit on the same line. + +Approval/question workflows and durable answer reuse belong to the workspace/plugin layer. Pico +supplies task identity and existing storage capabilities, not a separate memo or approval subsystem. +The exact hook access to task id and authorized scratch remains an integration question; no extra +hook-payload fields or answer-replay protocol are established here. + +## 9. API + +### 9.1 Handles + +```ts +interface ConversationHandle { + readonly id: Id; + snapshot(call: Call): Promise; + accept(options: { input: UserInput; requestId?: string; + whenBusy?: "followUp" | "steer" | "reject" }, call: Call): Promise<{ inputId: Id }>; + prompt(options: { input: UserInput; requestId?: string; + whenBusy?: "followUp" | "steer" | "reject" }, call: Call): Promise; + result(inputId: Id, call: Call): Promise; + drive(call: Call): Promise<"idle" | "closed">; + queueInput(input: QueuedInput, call: Call): Promise<{ inputId: Id }>; // §8.1 + write(kind: EntryKind, input: EntryInput, call: Call): Promise<{ inputId: Id }>; // §5.8 + abortInput(inputId: Id, call: Call): Promise<"aborted" | "not_found">; + abort(call: Call): Promise; + collapse(options: { instructions?: string } | undefined, call: Call): Promise; + reset(options: { handoff?: string } | undefined, call: Call): Promise; + fork(options: { at: Id | "start"; abort?: boolean; values? }, call: Call): Promise; + spawn(options, call: Call): Promise; + value(addr) / list(addr) // async reads/writes take a final Call + config(kind: { readonly kind: string; readonly config?: C }): { get(call: Call): Promise>; set(partial: Partial>, call: Call): Promise }; + readonly hooks: { on(kind, point, handler, o?: { subtree?: boolean }): () => void }; // scoped to this conversation (§8.7) + readonly settings: ConfigHandle; // sugar: config(generationKind) + commit(plan: (tx: ConversationTx) => T | Promise, call: Call): Promise; +} + +interface Harness { + readonly entryKinds: EntryKindRegistry; // mutable registration (§9.6) + readonly taskKinds: TaskKindRegistry; + readonly sections: SystemSectionRegistry; + root(call: Call): Promise; + conversation(id: Id, call: Call): Promise; + conversations(query, call: Call): Promise>; + acceptance(requestId: string, call: Call): Promise<{ requestId: string; conversationId: Id; inputId: Id } | undefined>; + inspect(call: Call): Promise<{ start: Task[]; inflight: Task[]; orphaned: Task[]; parked: Task[] }>; + drive(call: Call): Promise<"idle" | "closed">; + getEntry(id, call); getEntry(kind, id, call); getEntries(ids, call); getEntries(kind, ids, call); + entries(conversationId, page, call); + getTask(id, call); getTask(kind, id, call); getTasks(ids, call); getTasks(kind, ids, call); + abortTask(id: Id, call: Call): Promise; + value(addr) / list(addr) // session state handles; async methods take Call + commit(plan: (tx: Tx) => T | Promise, call: Call): Promise; + watch(conversationId: Id, options: { tail: number; values?: Address[]; raw?: boolean }, call: Call): Promise>; + watch(call: Call): Promise>; + deleteConversation(id: Id, call: Call): Promise; + shutdown(call: Call): Promise; + close(call: Call): Promise; +} +``` + +`Call` is defined in §6.5. `call.abortSignal` cancels the drive caller's wait, not the work. +`fork({ at, abort: true }, call)` marks the source's foreground set in the same commit that creates the fork, for "go back to that point"; which +conversation a UI treats as current is the UI's business. `prompt` returns the answer entry only +when its input's result is `done` with an `answer`; `unanswered`, or input still `placed` when the +drive returned, gives `undefined`. `result(inputId, call)` is one sticky-value point read, never a +transcript scan. After an uncertain remote response, `acceptance(requestId, call)` recovers the +conversation and input identity, or the caller simply retries with the same request key and gets +the same `inputId` back (§3.4). + +### 9.2 Commits + +```ts +const entryId = await harness.commit(tx => { + tx.value(planMode).set(true); // state first (§3.2) + const id = tx.entry(noteKind, conversationId, { + data: { text: "plan accepted" }, + model: [{ role: "user", content: "plan accepted", timestamp: 0 }], + }); // final id, inside the closure + tx.task(reminderKind, { background: true, state: { status: "scheduled", about: id } }); + return id; // any value; resolved after commit +}, call); +``` + +```ts +type EntryInput = + Omit & + (E extends ContextHead ? { readonly head: Id | "self" } : { readonly head?: never }); + +interface EntryDraft { + readonly key?: string; + readonly data?: JsonValue; + readonly model?: readonly Message[]; + readonly head?: Id | "self"; + readonly edits?: readonly ContextEdit[]; +} + +// Distribute over the declared variants; orphaned is never a public write target. +type StatesWithRole, Role extends TaskRole> = + S extends TaskStateBase + ? S["status"] extends "orphaned" ? never : R[S["status"]] extends Role ? S : never + : never; + +// Keep status and payload correlated, even when callers pass unions. +type WriteArgs = S extends TaskStateBase + ? [status: S["status"], payload: Omit] + : never; +type ExactWrite> = A & [ + status: A[0], + payload: Record, "status">>, never>, +]; + +interface Tx { + // reads (committed state) + getEntry(id) / getEntry(kind, id) / getEntries(...) / getTask(id) / getTask(kind, id) / getTasks(...) + value(addr: Value): { get(at?): Promise; set(v: T): void; delete(): void }; + list(addr: List): { append(v: T): Id; remove(id: Id): void; clear(): void; read(q): Promise>> }; + // writes; rewindable conversation state before entries (§3.2) + entry(kind: EntryKind, conversationId: Id, input: EntryInput): Id; // §5.8 + write(kind: EntryKind, input: EntryInput): Id; // inputId; appends now or queues (§5.8) + queueInput(input: QueuedInput): Id; // inputId (§8.1) + task>(kind: TaskDefinition, + spec: { state: S; after?: Id[]; background?: true; owns?: Id[] }): Id; + patch, + A extends NoInfer>>>( + task: TypedTask, ...args: A & NoInfer, A>>): void; + settle, + A extends NoInfer>>>( + task: TypedTask, ...args: A & NoInfer, A>>): void; + createConversation(spec): Id; deleteConversation(id: Id): void; +} +``` + +Both operations take an explicit target status and its **complete payload without status**: + +```ts +tx.patch(task, "running", { call, assistant, args, replay }); +tx.settle(task, "done", { call, assistant, output, result }); + +// Same-status update: narrow, remove the tag, then supply the full replacement. +if (task.state.status === "running") { + const { status, ...payload } = task.state; + tx.patch(task, status, { ...payload, jobId }); +} +``` + +There is no status-free or partial-merge overload. Both writes replace the stored state with +`{ ...payload, status }`; fields from the previous variant are not retained implicitly. `patch` +accepts start/inflight targets, `settle` terminal targets and retires scratch. Neither permits +`orphaned`; only open's internal orphan handling writes it. Same-status replacement remains valid +but does not advance `statusEpoch`. Creation still receives the complete initial tagged state. +An invocation's task snapshot does not change after patch; read again when a later replacement +needs state committed since that snapshot, or carry the explicitly constructed next payload. + +`WriteArgs` is a union of status/payload tuples: a union-valued status and an unrelated payload do +not prove a valid pair. `NoInfer` prevents arguments from widening the task's known union/role map. +`ExactWrite` rejects extra top-level keys visible in a variable or spread, not just object literals, +including a duplicated status. These are static checks, not runtime schema validation: casts, erased +extra fields and nested structural typing still follow TypeScript's limits. Wire boundaries validate +untrusted payloads; line validation checks current liveness, invocation ownership and allowed roles. + +The implementation below shows erased runtime signatures; the public signatures are above. It +numbers as it goes, keeps the one ordering rule, and hands storage a batch; storage numbers from its own `lastSeq` and gets the same ids (§7.3). + +```ts +class Tx { + private writes: MainWrite[] = []; + private seq: number; + private sawEntry = false; + constructor(private storage: Storage, private kinds: Kinds) { this.seq = storage.lastSeq; } + + private push(w: MainWrite): number { this.writes.push(w); return ++this.seq; } + private state(w: StateWrite): void { + if (this.sawEntry && "conversation" in w.addr.scope && w.addr.rewind) + throw new Error("rewindable conversation state must precede entries (§3.2)"); + this.push(w); + } + + value(addr: Value) { + return { + get: (at?: number) => this.storage.getValue(addr, at).then(v => v?.value), // committed state + set: (value: T) => this.state({ type: "value.set", addr, value }), + delete: () => this.state({ type: "value.delete", addr }), + }; + } + list(addr: List) { + return { + append: (value: T) => { this.state({ type: "list.append", addr, value }); return this.seq; }, + remove: (element: number) => this.state({ type: "list.remove", addr, element }), + clear: () => this.state({ type: "list.clear", addr }), + read: (q: ListQuery) => this.storage.readList(addr, q), + }; + } + + entry(kind: EntryKind, conversationId: number, draft: EntryDraft): number { + this.sawEntry = true; + const id = this.seq + 1; + const { head, ...fields } = draft; + const entry: Entry = { + id, conversationId, kind: kind.kind, ...fields, + ...(head === "self" ? { head: id } : head === undefined ? {} : { head }), + }; + this.push({ type: "entry", entry }); + return id; + } + task>( + kind: TaskDefinition & { readonly initialStatus: S["status"]; readonly turn?: true }, spec: { + conversationId: Id; state: S; after?: Id[]; background?: true; owns?: Id[]; + }): Id { + const id = this.seq + 1; + if (spec.state.status !== kind.initialStatus) throw new Error(`initial status must be ${kind.initialStatus}`); + const role = getOrThrow(kind.roles[spec.state.status]); + this.push({ type: "task", task: { id, kind: kind.kind, role, turn: kind.turn, ...spec } }); + return id; + } + patch(task: Task, status: string, payload: JsonObject) { + if (Object.hasOwn(payload, "status")) throw new Error("payload must not include status"); + const role = this.roleFor(task.id, status); + if (role === "terminal") throw new Error(`${status} requires settle`); + this.push({ type: "patch", id: task.id, role, state: { ...payload, status } }); + } + settle(task: Task, status: string, payload: JsonObject) { + if (Object.hasOwn(payload, "status")) throw new Error("payload must not include status"); + if (this.roleFor(task.id, status) !== "terminal") throw new Error(`${status} is not terminal`); + this.push({ type: "settle", id: task.id, role: "terminal", state: { ...payload, status } }); + } + private roleFor(id: Id, status: string): TaskRole { + /* kind from transaction view; reject unknown status and reserved orphaned */ + } + + getTask = this.storage.getTask; getTasks = this.storage.getTasks; // reads: committed state + getEntry = this.storage.getEntry; getEntries = this.storage.getEntries; + + batch(): CommitBatch { return { kind: "main", writes: this.writes }; } +} + +// The line owns admission, persistence, indexes and decisions; callbacks run afterward. +async commit(plan: (tx: Tx) => T | Promise, call: Call): Promise { + const completed = await this.line.run(async () => { + const invocation = call.value(invocationKey); + this.checkAdmission(invocation); // phase, exact running identity, method and mark (§6.3) + const tx = new Tx(this.storage, this.kinds); + const result = await plan(tx); + const batch = tx.batch(); + this.validate(batch, invocation); // whole-batch authority and structural invariants + if (batch.writes.length === 0) return { result, actions: [] }; + await this.storage.commit(batch); + const actions = this.driver.applyBatch(batch); // apply ALL writes; then reserve affected work/waiters + this.publishState(batch, actions); // fold views, enqueue delivery; no listener execution + return { result, actions }; + }); + this.dispatch(completed.actions); // start effects, signal controllers, deliver callbacks + return completed.result; +} +``` + +`ConversationTx` is the same with `conversationId` bound. `ScratchTx` is the `value`/`list` half +with the batch tagged `scratch`. Builders may be async to await storage reads on the line; no other +line operation can interleave. Reads (including Tx/ScratchTx value/list/entry/task reads) return +Promises; writes are synchronous, buffered and validated in order. Never await an external effect, +a driver waiter or another line operation inside the builder. Ids are final when returned, and a +throw discards everything. The outer promise resolves +with the closure's value after persistence and publication, which is when ids may escape. +`ConversationHandle.commit` binds the conversation; `TaskRuntime.commit` is the same for task +code. + +### 9.3 Task and tool integration + +**Provisional API:** the sink, tool runtime and output-state shapes below are the current sketch. +Simplify them together with previews before implementing. Preserve streaming text/images, typed +details, bounded capture/spill, diagnostics, usage, control outcomes, jobs and cooperative cleanup; +this section does not settle the final division of responsibilities or arbitrary-work adoption. + +```ts +interface Tool { + readonly name: string; readonly description: string; readonly parameters: JsonSchema; + readonly output?: ShellOutputLimits; // retained window for text output + readonly replay?: "safe" | "never"; + execute(toolCallId: string, params: TParams, + out: ToolOutput, runtime: ToolRuntime, call: Call): Promise; +} + +// the sink; everything a tool produces goes through it (mobile-handoff/01-harness/04-tool-output) +interface ToolOutput { + write(text: string): void; // append to the text block + image(image: ImageContent): void; // images are never windowed + replace(text: string): void; // replace the retained text wholesale + capture(metadata: ShellOutputMetadata): void; // truncation totals / spill path, no text resent + readonly details: TDetails; // the tool's own details object; mutate it + usage(usage: Usage): void; // accumulates + addTools(names: string[]): void; // replaces; post_tools updates selected tools (§5.5) + terminate(value: boolean): void; // replaces; orthogonal to how execution ended + handoff(message: string): void; // replaces; new_context asks for a reset (§8.3) + delegate(job: Id): void; // this call's work continues as that job (§8.6); generic, so UIs can follow it + diag(severity: "info" | "warn" | "error", message: string, code?: string): void; // commentary, see below +} + +interface ToolRuntime { + readonly taskId: Id; // the tool task; identity for origin/for links + readonly budgetMs: number; // blocking budget (§8.3) + readonly env: ExecutionEnv; // FileSystem & Shell + commit(plan: (tx: ConversationTx) => T | Promise, call: Call): Promise; + conversation(id: Id, call: Call): Promise; + abortTask(id: Id, call: Call): Promise; + waitForTask(id: Id, options: { budgetMs?: number } | undefined, call: Call): Promise; + jobOutput(id: Id, call: Call): Promise; +} + +// what the live sink folds to; recorded on the tool task and used to build the result entry +interface ToolOutputState { + content: (TextContent | ImageContent)[]; + details: JsonValue; + usage?: Usage; + addedTools?: string[]; + terminate: boolean; + handoff?: string; + delegated?: Id; + diags: readonly { severity: "info" | "warn" | "error"; message: string; code?: string }[]; + truncation: ShellOutputTruncation; // totals over everything ever written +} + +type ToolResultData = Omit; +``` + +`execute` returns nothing and reports failure by throwing; the harness sets `isError` when it +rejects, and a tool may throw anything. `terminate` is orthogonal to how execution ended, so a +failing tool can still stop the turn (`out.terminate(true); throw error`). Usage and `addTools` go +through the sink rather than a return value so that a replayed tool can seed from durable state. +The sink owns its scratch promises: it handles expected post-mark/close rejections, reports real +persistence errors and drains pending callbacks before the invocation returns. Raw plugin scratch +writes must be awaited/caught. The sink's text ops (`write`, `replace`, and the `slide` the env emits +for a moving tail) are what scratch and the watch stream carry, so a UI shows output live without the tool doing anything. + +**Diagnostics are a channel, not text.** Commentary about a call (it was truncated, the output +spilled to a file, the path was resolved differently, the file changed on disk since it was read, +the search stopped at 500 matches) goes through `out.diag`, never into the text the model reads as +the tool's data. The harness emits the ones it owns: the sink calls `diag` itself when it truncates +or spills, the path resolver when it corrects a path, the blocking budget when a call continues as +a job. A tool adds only what it alone knows. Tool settlement renders the data first and the +diagnostics after it, delimited, and stores that message on the result entry: + +```text +…last line of the file + +[warn] output truncated: 2,000 of 51,204 lines shown; full output at /tmp/pi/out-4421.log + +``` + +so the model can parse tool output as data, a plugin can post-process it without stripping notices +it doesn't know, and a UI renders diagnostics as callouts by severity. `isError` says whether the +call failed; a `warn` diag does not change it. Tool settlement stores the exact rendered result as +entry `model`; the non-message fields of `ToolOutputState` become `ToolResultData` for typed logic +and rendering. The transcript therefore records exactly the commentary the model saw without +copying the message into `data`. + +### 9.4 Watch + +The tool-facing preview API below is provisional (§9.3). Delivery coalescing is deferred, not an +initial implementation requirement; the current per-commit publication rules remain unchanged. + +The transport is the commit stream, gap-free and replayable. The interface is typed events derived +from it by the harness, and a view the harness keeps current, so a client renders and never parses +commits: + +```ts +interface WatchHandle { + readonly view: View; // captured on the line at watch(); folded before each event is delivered + start(listener: (event: Event) => void): void; // delivers everything since the capture, then live + resnapshot(call: Call): Promise; // fresh capture, same subscription (when lagging) + unsubscribe(): void; +} + +interface ConversationView { + readonly conversation: Conversation; + readonly entries: readonly Entry[]; // the last `tail` entries; older ones via entries(id, { before }) + readonly context: readonly Id[]; // the derived context, as ids + readonly tasks: readonly Task[]; // live tasks, typed by kind (retry attempt, deferred handle, ToolOutputState ... in state) + readonly inbox: readonly Element[]; + readonly values: ReadonlyMap; // every value the registered kinds declare in config, plus any asked for + readonly previews: ReadonlyMap; // per live task: the kind's tracked preview (§5.2) + readonly faulted: boolean; + readonly readAt: Id; +} + +type InboxOp = + | { readonly type: "append"; readonly item: Element } + | { readonly type: "remove"; readonly id: Id } + | { readonly type: "clear" }; + +type ConversationEvent = + | { type: "entry"; entry: Entry } + | { type: "task_start"; task: Task } + | { type: "task_update"; task: Task; previous: Task } + | { type: "task_end"; task: Task } + | { type: "task_output"; task: Id; ops: readonly DeltaOp[] } // ops on the task's preview; applied to view.previews + | { type: "value"; addr: Address; value: JsonValue | undefined } + | { type: "inbox"; ops: readonly InboxOp[] } + | { type: "context"; ids: readonly Id[] } // a head or edit changed derived context + | { type: "fault"; error: unknown } | { type: "closed" }; + +const w = await h.watch(c.id, { tail: 100, values: [myPlugin.config.mode] }, call); +render(w.view); +w.start(event => render(w.view, event)); // w.view is already folded when the listener runs +``` + +The view is authoritative and the events are wake-ups: a renderer is `apply(view)`, diffing against +what it last drew (entries keyed by id, tool components keyed by task id), and it may ignore an +event's payload and still be correct. The payload is for skipping work on a hot path (`task_output`: +redraw one component from its preview) and for logs and tests. + +Because every piece of work is a task of a known kind, the four task events carry what the lane +harness needed a name per case for, and cover plugin kinds without anyone adding one: + +| lane event | v3 | +|---|---| +| run_start / run_end | task_start / task_end where `generationKind.is(task)` | +| retry_scheduled, run_suspend | task_update with status `retry_wait` (attempt, notBefore) / `deferred` (handle) in state | +| message_update | task_output on the generation; `view.previews` holds the partial assistant message | +| tool_start / tool_update / tool_end | task_start / task_output / task_end on the tool task; `ToolOutputState` in state | +| compaction_start / compaction_end | task_start / task_end where `collapseKind.is(task)`; reason and summary id in state | +| navigation_start / end | a conversation created (session watch); with `abort`, task_end aborted on the source | +| config_update, value_update | value | +| queue_update | inbox | +| entry_added, message_end | entry | + +Events are delivered per commit, in commit order, all events of one commit before any of the +next, so a terminal task and its successor never show as an idle gap. `watch` captures the view +and registers the subscription in one serialized step on the line, so nothing falls between them. +Task-output watch events travel as Chord delta ops only; durable scratch uses ordinary records (§7.4). +For watch delivery, the folded preview lives in the view, never in the event. `resnapshot` marks a +barrier on the delivery tail while the line holds the new capture, so calling it from inside the +listener neither deadlocks nor refolds stale state. Buffering is bounded; a lagging client gets a +`fault` and calls `resnapshot`. A listener never awaits a commit on the same line. + +Events are proportional to their change, and the view fold is a generic reducer, +`applyEvent(view, event)`, exported by the harness package and needing no kinds: append the entry, +upsert the task, apply task-output ops to `previews[task]`, apply inbox ops, set the value. Inbox +operations are folded across one commit before delivery, so an idle acceptance's same-commit append +and remove emits no inbox event. The harness maintains `w.view` +with it, and a UI in another process runs the same reducer on the same events; nothing is +re-diffed for the wire: + +```ts +// worker // ui process +const w = await h.watch(c.id, { tail: 100 }, call); on("view", m => { view = m.view; ui.apply(view); }); +send({ type: "view", view: w.view }); +w.start(e => send({ type: "event", event: e })); on("event", m => { applyEvent(view, m.event); ui.apply(view, m.event); }); +``` + +The replication contract, so a reducer in another language can't drift: events of one commit +travel as one message; an `Address` serializes to one canonical string key; and preview ops are +Chord delta ops (`packages/chord/src/delta`, mobile-handoff `01-delta`): `r` replace, `s` set, `d` +delete, `a` append, `t` truncate, `p` splice, with the wire codec's interning optional for thin +clients. That applier is already the mobile port; the reducer adds its ten cases on top and the +harness package's implementation is the reference, tested by porting `delta.test.ts` first. + +`task_output.ops` are ops on the task's *preview*, which is a Chord delta tracker the kind mutates +in place (`runtime.preview.state`): the generation applies each stream event to a tracked partial +message, the tool's preview is the sink's tracked state, a job's is the same. The harness flushes +the tracker after each scratch commit; a streamed token is one `["a", path, delta]`, computed from +dirty paths and a memcmp, never a recomputed object (a reassigned preview would flush as a full +replace). On attach or reopen the harness builds the preview once with `preview.init(scratch)` +and the first flush is the base. Fan-out to several in-process watchers uses `applyImmutable` or +copies the batch, since `apply` adopts `r` payloads. + +The lane-shaped +snapshot (`operation`, `runningTools`, `retry`, `deferred`, `streamingMessage`) is a pure function +of `view.tasks` and `view.previews`; a renderer that wants it computes it. `h.watch(c.id, { raw: +true })` delivers the commits themselves, for a receiver that has a harness and runs the fold. + +There is a session-level watch for what isn't one conversation's: + +```ts +interface SessionView { + readonly conversations: readonly Conversation[]; + readonly values: ReadonlyMap; // session-scoped: pi.usage.totals, pi.session.name, ... + readonly faulted: boolean; + readonly readAt: Id; +} +type SessionEvent = + | { type: "conversation"; conversation: Conversation; change: "created" | "deleted" } + | { type: "value"; addr: Address; value: JsonValue | undefined } + | { type: "usage"; row: UsageRow; totals: Usage } + | { type: "report"; task?: Id; kind?: string; method?: string; error: unknown } // hook/task errors + | { type: "fault"; error: unknown } | { type: "closed" }; +harness.watch(call: Call): Promise>; +``` + +### 9.5 End to end + +```ts +const h = await Harness.open(storage, options, call); +const c = await h.root(call); +const w = await h.watch(c.id, { tail: 100 }, call); render(w.view); w.start(e => render(w.view, e)); + +const answer = await c.prompt({ input: "Inspect the parser" }, call); + +const driving = c.drive(call); await c.abort(call); await driving; // durable intent, then cleanup + +const child = await c.spawn({ prompt: "Inspect only tests", context: "fresh", + values: { inherit: [model] } }, call); +await h.drive(call); // drives the child too +console.log(await h.conversation(child, call)); + +const alt = await c.fork({ at: answer.id }, call); +await alt.prompt({ input: "Try a different implementation" }, call); // source remains untouched + +w.unsubscribe(); await h.close(call); +``` + +```ts +const h = await Harness.open(storage, { + models, + tools: [readTool, writeTool, bashTool, ...pluginTools], // subagent and job tools are built in + kinds: { entry: pluginEntryKinds, task: pluginTaskKinds }, // added to the built-ins + replace: { generation: myGenerationKind }, // swap a built-in by name; same statuses, hook names + rootValues, +}, call); +h.kinds.generation; h.kinds.tool; h.kinds.postTools; h.kinds.collapse; h.kinds.job; // whatever is registered under the name +``` + +The built-in entry kinds (`user`, `assistant`, `tool_result`, `system`, `notice`, `summary`, +`handoff`, `reset`) and task kinds (`generation`, `tool`, `post_tools`, `collapse`, `job`) are +registered by `open` itself, because `accept`, `prompt`, `queueInput` and `collapse` need them to write +and type new records. Reading context does not need entry kinds. A task-kind replacement is +registered under the built-in's name and must understand its persisted statuses and keep its hook +names, so existing handlers keep working; a wrapper that delegates to the original is the usual +shape. `h.kinds.` is how handles and plugins refer to whatever is currently registered, so +`c.settings` is the config of the generation kind in use, not of a specific import. + +`Harness.open` takes storage, models, tools, additional kinds, replacements, section definitions and +initial root values. It creates the root only for empty storage and initializes `rootValues` in that +same commit; on reopen it ignores those initial values and preserves durable configuration. Tool +registration never implicitly selects tools. The host explicitly supplies model/thinking/selectedTools +for a fresh root or configures it before generation; missing required configuration reports an error. +Children inherit only the values selected by their spawn policy; historical forks inherit rewindable +values at their entry cutoff. Reopen never creates work by inference. + +### 9.6 Mutable kind and section registration + +Construction accepts initial `kinds: { entry: [...], task: [...] }` and `sections: [...]` collections. +Built-in section tokens are registered initially; their content is supplied by host hooks. These are +initial values, not immutable registries: + +```ts +await h.entryKinds.register(noteKind, call); +await h.entryKinds.replace(noteKindV2, call); +await h.entryKinds.remove(noteKind.kind, call); + +await h.taskKinds.register(reminderKind, call); +await h.taskKinds.replace(reminderKindV2, call); +await h.taskKinds.remove(reminderKind.kind, call); + +await h.sections.register(policySection, call); +await h.sections.replace(policySectionV2, call); +await h.sections.remove(policySection.key, call); +``` + +The registry methods retain the supplied kind/section generics. `register` rejects an existing name; +`replace` requires an existing compatible definition (otherwise migrate); `remove` removes code, +not stored entries/tasks/section values. All changes serialize on the line. Existing task calls and +section preparations retain their captured implementation; future ones use current registrations. +Explicit section-renderer replacement marks that section for recomputation in a later preparation; +never rewrite old entries or clear a newer replacement's pending work from an older preparation. + +Removing a section definition does not remove its instructions. Historical reads and untouched seeded +sections use stored rendered text without the renderer. Registering it again restores typed editing. +Only `sections.delete(tokenOrKey)` on the preparation draft requests durable section removal. Stable +string keys cross restarts; function objects and token identity do not get serialized. The mutable +draft is a local hook API, not a JSON wire DTO; an eventual remote-plugin adapter must provide an +explicit contribution/edit protocol rather than serializing token renderers or draft callbacks. + +Task-kind removal rejects while live tasks of that kind exist. At open, absent task kinds follow +§6.4: foreground tasks become terminal `orphaned`; background tasks remain parked until registration +restores their recovery implementation. Registration does not resurrect orphaned or other terminal +tasks and does not implicitly attach a drive scope. + +## 10. Validation + +### 10.1 Backend conformance + +One validated mutation stream replayed into memory, JSONL (close, reopen) and SQLite (close, +reopen), compared on: current objects and live-task metadata, transcript pages and inherited +prefixes, derived contexts at several boundaries, value tombstones and list clears, request keys, +scratch lifetimes, next allocated sequence. Randomized small histories plus explicit deep-fork +cases: fork before and after a delete, before and after a clear, before a later summary, nested +source cutoffs, sticky state untouched by historical reads. + +### 10.2 Failure and concurrency + +| Scenario | Required | +|---|---| +| Persistence paused after construction | readers see the old complete state | +| Accept commit durable, reply lost | `acceptance(requestId)` returns the original conversation/input id | +| Main and scratch writes in one batch | reject, nothing written | +| Torn final JSONL line | only that line discarded | +| Malformed complete line | open fails; no silent truncation | +| Retirement committed, unlink fails | old scratch invisible | +| Effect returns as abort starts | one invocation per task; fresh abort only after return | +| Post-mark main/scratch write | reject before builder; sinks handle cancellation and drain callbacks | +| Status cycle returns to initial status | epoch counts intermediate committed changes; no false fault | +| Host patches an owned task | reject lifecycle/state mutation; abort marks remain allowed | +| 100,000 historical children, two live tasks | query only live seed/needed ancestry, no history traversal | +| Repeated drive and cancelled waiter | one attachment; remove waiter/listener exactly once | +| Known dependency self-drive | reject before acceptance or cycle-creating admission | +| Close vs task commit | earlier commit completes; later mutation rejects; completion still admitted | +| Shutdown vs accept / idle nextRun inbox | accept before mark batch or reject; queued items/results preserved | +| Shutdown crash before child cleanup | reopened abort marks tasks only; child queues remain unchanged | +| Main 100, live scratch through 150, crash | reopen recovers lastSeq=150; next write=151 | +| Retired scratch still on disk | ignore it even if malformed; later main settlement covers its sequences | +| Parent cleanup vs already-terminal job | already finished is successful cleanup | +| Broken abort handler | reject drives/shutdown, signal/join and close; replace kind before recovery | +| Parallel tools settle in either order | post_tools starts once | +| Abort vs post_tools settlement | no unmarked successor | +| Child finishes while its owning tool is marked | tool settles aborted, not done | +| Two overlapping drives | one execution per task | +| A drive caller cancels its wait | other callers and tasks unaffected | +| Summary lands after a competing head/reset | rejected as stale; intervening edits do not stale it | +| Withdraw vs land of the same inbox item | one wins on the line; one terminal input result | +| Abort vs generation/post_tools group transfer | every active input resolves once; no unmarked owner escapes | +| Several inputs share one generation | every result points to the same final answer entry | +| Overflow retry while collapse runs | no live task waits on the collapse | +| Watch registration races a commit | base includes it or the stream delivers it | + +Faux providers, fake processes and clocks, storage barriers; both orders of every race; failed +commits consume no ids; old task ids remain queryable after compaction. + +### 10.3 Performance + +Workloads: many short turns with and without tools; frequent compaction plus cold reopen; parallel +and sequential tools; large streaming output; deep forks with repeated historical reads; many +finished child conversations with few live tasks; many live background tasks; queued input and +cancellation. Measure CPU, wall latency, RSS and heap, bytes on disk including scratch and WAL, +query counts and rows decoded. Report backend-owned memory separately from live execution data; +JSONL growth with history is expected, SQLite must not keep a full-history copy. + +The pico2 prototype's 9.7 s against 35.9 s on 2,000 faux turns is a reason to measure this design +the same way, not a result for it. Its lessons are the ones above: cheap live queries, batched +reads, small writes, no history scans on the hot path. diff --git a/packages/agent/docs/pico/pico-work.md b/packages/agent/docs/pico/pico-work.md new file mode 100644 index 000000000..29ddb3446 --- /dev/null +++ b/packages/agent/docs/pico/pico-work.md @@ -0,0 +1,622 @@ +# pico implementation plan + +Clean room. Nothing from pico2 is copied; its tests are read for the bugs they caught (a failed +generation that never ended its run, refs inside task patches, a collapse deadlocking appends, +quiescence firing during a retry wait, watch dropping late deltas) so that packages 6, 9, 12 and +16 each carry a case for them. + +The target is bottom-up modules with their own tests and explicit dependencies, green after every +step. The numbered groups below are a provisional coverage map, not final work-package sizes or a +completed dependency audit. After the remaining conceptual blockers are settled, run the independent +whole-document review and final work-package decomposition described below. `pico-v3.md` is the +reference design; `pico-usage-guide.md` shows the intended surface. + +Next design discussions: evaluate status-indexed task authoring and returned plans separately, then +simplify the tool/sink/preview contract. The steps proposal has not been adopted. Groups 10, 14 and 15 +preserve capability/test coverage, but their current API sketches are not ready-to-implement contracts. +Preview delivery coalescing is deferred. Approval/question workflows and durable answer reuse belong +to workspace/plugins; only their access to existing task identity/scratch is a Pico integration question. + +## 1. Types and ids + +`Id`, `EntryIdentity`, `EntryBase`, the composable `EntryData` / `ModelProjection` / +`ContextHead` / `ContextEdits` facets, `Entry`, `EntryKind`, `EntryInput`, `ContextEdit`, `Task`, +`TaskRole`, status-tagged task-state unions and their derived `Orphaned` variant, compiler-only +`TypedTask`/`TaskDefinition` witnesses preserving the complete union and literal role map, stored task roles +and `turn` flag, `Conversation`, `QueuedInput`, `InputResult`, `InboxOp`, acceptance +receipts, `Call` (an alias of Chord Context), `TaskRuntime`/`ToolRuntime`, private typed invocation +identity, `Address`/`Value`/`List`/`Scope`, `Write`/`CommitBatch`, `Page`/`Cursor` and query shapes. +Every async harness/handle/runtime method takes a required final Call; no duplicate signal options, +task-conversation facades or admission gates. + +Add immutable `SystemSection` definitions created by `defineSystemSection`: a durable string +`key` and synchronous pure `render(value: T): string`, with JSON-representable payloads. Tokens provide +typed access without plugin casts; only their keys and payloads are serialized. Export typed built-ins +`systemSections.identity`, `.environment` and `.skills`; plugins may define/register their own. +There is no discovery/read callback on a token and no renderer callback on an entry kind. Payload +shapes are trusted after wire validation; shape changes need migration or compatible replacement. + +`SystemSectionDraft` exposes typed `get(token)`, `set(token, value)`, `delete(token)` and +`wrap(token, rendered => string)`. `get` returns an owned copy; array changes use get+set, not an append +operator. Persist ordered `SectionChange` records: + +```ts +type SectionChange = + | { key: string; action: "set"; value: JsonValue; rendered: string } + | { key: string; action: "remove" }; +interface SystemData { + baseline?: true; + sections?: readonly SectionChange[]; +} +type SystemEntry = EntryBase & EntryData & ModelProjection & Partial; +``` + +Baseline sections are a complete ordered array of set records. Null and empty string are valid +payloads; deletion is an explicit action. Definitions and wrapper closures are never stored. Tool +definitions live only in model fields, never duplicated in SystemData. + +Tests: typed token/draft inference without plugin casts; owned-copy reads; JSON null versus deletion; +every entry facet and built-in combination; persisted payload+rendered text and model-only tool definitions. +Task type tests: `defineTaskKind()({ ... })` infers literal roles; exact role-map keys, start-only +initial status, reserved orphaned, consistent common-field types/optionality; orphaned exposes only +common fields via `Pick>`, preserving common optional fields. Typed reads +include orphaned; kind methods exclude it. No compiler witness is stored or needed by untyped readers. + +## 2. Memory storage + +`Storage` and `MemoryStorage`: `commit` numbering from `lastSeq`, point reads, fork-aware +`scanEntries` / `scanTasks` / `scanConversations` with cursors, target-capped `newestHead` over stored +numeric boundaries, stored entry data/model/edits, value versions and list elements read at a +position, `ownedFrom`, the set of entry kind strings written and session-wide live-task scans. +Use existing batched `getTasks` for named owners; no header/projection APIs or inbox-specific queries. + +Tests: every row of the §7.2 query table against hand-built batches; ids are `lastSeq + 1 + i`; a +head is found without a kind; entry reads/scans/head lookup return full entries after indexed +selection; live-task scans skip terminal rows (named owner reads may fetch them); `remove` and +`clear` hide by position. + +## 3. The line and `Tx` + +`commit(plan, call)` on a serialized line: buffered writes, ids final at call time, rewindable +conversation value/list writes after an entry throw, a throwing plan discards everything, publish +after persist; apply the entire batch to live indexes before scheduling or testing idle. Driver +callbacks, signals and task methods dispatch outside the line. Session and sticky conversation state, task +writes and conversation writes may appear anywhere. `task` / `patch` / `settle` materialize the role +from the kind's status map; status lives only in `state.status`. `patch(task, status, payload)` and +`settle(task, status, payload)` always require an explicit status and its complete payload without +status. Both replace state with `{ ...payload, status }`; no partial merge or status-free overload. +Patch targets start/inflight roles, settle targets terminal roles and retires scratch; neither exposes +orphaned. A bare id must first be read with a kind to obtain the typed witness. Materialize `turn` from the kind and +maintain the indexed `inTurn` predicate. `entry` rejects outside-turn model-visible writes while turn +tasks are live; `write` places immediately or queues at a safe boundary, returning an inputId. +`value` / `list` / `entry` / `task` / `patch` / `settle` build the batch of §7.3. + +Tests: concurrent commits serialize; a rejected commit consumes no ids; each builder verb produces +the expected write; reads inside a plan see committed state only; session and sticky conversation +writes may follow and reference a new entry; rewindable value set/delete and list +append/remove/clear after an entry each reject; exact invocation-token checks run on the line; +post-mark main/scratch mutation rejects before builder; only an owned task's current invocation may +patch its state/status or settle it (host abort marks remain allowed); caller cancellation never +abandons admitted persistence; Tx/ScratchTx reads are asynchronous and builders may await them +without releasing the line; writes remain synchronous. No external effects or nested line entry in builders. +Compile-time cases: required target fields and types, no extra top-level keys on literals/variables/spreads, +no duplicated status, status/payload correlation under union arguments, no role/status widening from +inference, narrowed task retains all transition targets, typed reads cannot write orphaned, no bare-id +escape. Cover valid same-status full replacement and optional fields; document structural typing/cast +limits rather than adding deep exact-type machinery. Runtime cases: replacement drops prior-variant +fields, rejects wrong roles/reserved statuses/duplicated payload status, current terminal tasks cannot +change, and same-status replacement does not advance the epoch. Payload validation stays at wire boundaries. + +## 4. Entry kinds and context + +`EntryKind` as `kind` plus `is`, the registry and typed append helpers for the built-in kinds +(`user`, `assistant`, `tool_result`, `system`, `notice`, `summary`, `handoff`, `reset`). Writers +materialize optional model messages and stored controls; context = newest stored head prepended to +the fork-aware range from its numeric boundary, older heads excluded, stored edits folded in +transcript order, stored model arrays concatenated, then pi-ai tool results ordered by call index. +This package has no managed-system state fold, baseline hoisting or system-kind projection filter. +Generation preparation writes any required system changes and omission edits (group 9); the generic +projector applies stored facets only. Arbitrary plugin head writers need no system-specific behavior. + +Tests: data-only and model-only entries; summary keeps the tail; handoff/reset normalize `"self"`; +repeated compaction subsumes; a stored head below the previous visible boundary is rejected; edits +omit/replace targets and persist across turns; arbitrary managed-system targets reject, while atomic +fresh-baseline supersession is permitted; context is identical with its plugin kind unregistered; +retained system deltas remain visible until a later stored omission applies; tool-result order. + +## 5. Forks and historical reads + +`createConversation` with `parent`, the shared prefix in fork-aware `scanEntries`, capped-source +lookup for values and lists, arbitrary transcript-entry fork points. + +Tests: a fork sees the head, edits and values in force at its entry; heads/results the source adds +later are invisible; deep fork chains; successful incomplete tool exchanges project with missing +results but inherit no tasks; state committed after the entry (a model change) is not in the fork. + +## 6. Task kinds and the driver + +`TaskKind` with `(task, runtime, call)` methods, registry and `TaskRuntime`; one Call-final convention +for operations, existing Context-aware env/provider/hook boundaries. Driver decisions, attachments, +waiters, invocation completion and lifecycle run on the commit line. Only effects/callbacks dispatch +outside it. Seed live tasks once; committed batches update live, reverse-dependency and conversation +indexes. Resolve ownership upward only for live/owned work; evict unused ancestry. Stable attachments +are distinct from temporary waiters. No Wake, polling loop, worker pool, poison graph, gate or parked role. + +One invocation per task ID, concurrent across IDs; reserve before dispatch and release only after +actual return, even if already terminal. Durable abort revokes execute/recover main and scratch writes, +then signals outside line; fresh abort invocation alone writes cancellation outcomes. Status epochs +count actual committed transitions, including same-start/end cycles. Task-contract faults fail-stop: +reject waiters, stop admission, signal/join and close; preserve durable recovery state. + +Tests: one open scan, no full scan per completion; 100,000 historical children with two live tasks; +whole-batch settle/successor publication; exact epoch rules; owned-task exclusive mutation; one +invocation per ID; blocked calls do not block others; abort bypasses dependencies and never overlaps +execute; repeated mark spares running abort; abort-handler failure; cooperative signal window; +known direct/dependency self-waits, including background caller and new cycle-creating admission; +already-aborted waiter registration and every cancellation/idle race; repeated drive uses one +attachment; foreground idle while attached background work continues; reopen recovers marked work. + +## 7. Scratch + +Scratch batches, one task per batch, `ScratchTx`, retire on settle, sidecar-independent (memory). + +Tests: a crash before settle keeps scratch; settle deletes it; writes after mark/settle/retired token +reject; new attempts clear scratch; persisted assistant frames use the pi-ai compact frame encoder, +not raw cumulative provider events; a rejected stream-frame write cancels and joins the producer +before invocation completion (iterator exit is insufficient); harness progress bridges own every promise, suppress expected +late cancellation only, report persistence errors, and drain before invocation completion. Raw task +scratch calls must be awaited/caught; no successful silent no-op. + +## 8. Harness shell and handles + +`Harness.open` (built-in registries, `kinds` and `replace` options, kinds-set check, `inspect`, +`drive`, `close`, `shutdown`), `ConversationHandle` (`commit`, `value` / `list`, `config` / +`settings`, `fork` with `abort`, `abort`, `hooks.on` scoped with `subtree`), `acceptance(requestId)`, +`result(inputId, call)`, `abortTask`, `conversations` with `parent` / independent filtering. Call is +required on every async public/runtime operation, including reads and lifecycle; no raw host-lifecycle +Harness exposed to tasks. No section-order option or separate ordering configuration. No agent +behaviour yet. Root creation applies explicit `rootValues` atomically and only once; reopen preserves +durable model/thinking/selectedTools. Registry contents do not imply tool selection. Children use +explicit value-inheritance policy; missing required generation configuration fails clearly. + +Initial `Harness.open` accepts `sections: [...]` definitions in addition to the built-in tokens. +Mutable registry operations take a required final Call and serialize on the line: +- `h.sections.register(token, call)`, `.replace(token, call)`, `.remove(tokenOrKey, call)`. +- Parallel `h.entryKinds` and `h.taskKinds` register/replace/remove APIs use kind definitions or names. + +Register rejects duplicates; replacement is explicit and must understand the stored shape. Removing +code never deletes durable entries or section state. In-flight preparation retains its captured section +registry snapshot; later preparations see newer definitions, including explicit renderer replacements. +No rendering runs during registry mutation or replay. Task-kind removal rejects while live instances +exist. Open settles missing-kind foreground tasks as `orphaned`, parks missing-kind background tasks, +and leaves terminal history untouched. Registration restores parked recovery in already attached +scopes; it neither implicitly drives other scopes nor resurrects terminal tasks. + +Tests: rootValues applies atomically only to a fresh root, ignored on reopen; no automatic tool +selection from registry changes; required-config errors; open on empty vs existing storage; +initial/custom/built-in section definitions; duplicate +registration rejects; compatible explicit replacement; removal preserves durable state and stored +rendered fallback; registry changes during preparation do not alter its captured definitions, while +later preparation sees the replacement. Unregistered entry kinds are reported without history scans +and stored facets still derive context; missing foreground kinds become orphaned, missing background +kinds remain parked and reported, registration restores recovery, terminal history stays untouched; +removal with live instances rejects. Replace by name keeps `h.kinds.` consistent; `settings` round-trips; scoped hooks run after +harness-wide ones, innermost last; derived Call preserves typed private identity and telemetry; +stale/foreign task token rejects; close stops admission in one nonpersistent line job, joins outside; +shutdown atomically marks live tasks only, then permits abort cleanup; queued items/results remain +unchanged, including idle inbox-only conversations; crash/reopen child cleanup marks tasks only and +preserves queues; repeated close/shutdown share completion; close interrupt rejects shutdown; +lifecycle calls with task identity reject; delete rejects outstanding terminal invocations. + +## 9. Generation kind + +One stable generation task carries `inputs: Id[]` and cycles pending → streaming → retry_wait / +deferred → streaming until done / failed / aborted on a faux provider; explicit terminal results for +its whole input group; captured config (model, thinking, selected tools, profile, budget); +`system_instructions`, `before_request`, `after_response`, `on_yield`; retry sleeps in execute; +recover from frames; usage recorded per attempt. + +**Durable configuration and typed preparation.** Config remains ordinary scoped state, persisted +when changed with its declared rewind/sticky policy. Host files and catalogue contents are not +magically stored as config. The Call-final `system_instructions` hook receives captured config and +one shared mutable `SystemSectionDraft` as `sections`. Sections are mutated, not returned; a handler +may return `{ tools?: readonly Tool[] }` as a full desired loadout. Handlers run sequentially in +registration order, with outer scopes before inner ones. + +Seed each new draft from canonical durable section payloads and rendered text. Typed `get(token)` +returns a copy; changes require `set(token, value)`. Setting an existing key retains its position; +a new key appends; `delete(token)` explicitly removes it. Array payloads use typed get+set, without +an append operator. Missing contributions or registered definitions are not deletions. An untouched +seeded section retains its stored rendered text, including after restart; an unavailable renderer +uses that fallback. Initial sections need a provided base payload or stored fallback. + +Explicit set/wrap or renderer replacement recomputes through the captured registered renderer. +`wrap(token, rendered => string)` requires a registered section; wrappers are synchronous, pure and +applied in registration order after rendering its payload, not on top of already wrapped stored text. +Wrappers belong only to the current draft and reset each preparation. Host handlers refresh full +current base payloads before plugin transformations; this intentionally rebuilds wrappers. Preservation +of a missing wrapper across a refreshed base is not promised. The composed transformation CHAIN must +reach a fixed point under repeated preparation, or use an earlier authoritative base reset. Individually +idempotent handlers are insufficient; the harness does not add a generic convergence loop. + +Disk discovery and caches remain private to host/hook closures or services, refreshed by their own +watcher/TTL policy. Tokens have no source-read callback and entries retain no callback state. Refresh +failure is not deletion. Under the existing skip-failed-handler policy, discard that handler's draft +mutations, including wrappers, while retaining earlier handlers' changes; never publish a partial delete. +After hooks, capture payloads and pure rendered results outside the line. No plugin renderer runs on +replay or on the commit line. + +**Canonical section state, separate from model projection.** Fold fork-visible managed system data +through the target, starting at the most recent baseline, independently of model heads and baseline +supersession omissions. Those facets control model projection, not canonical section payloads. Use existing indexed +kind scans, not unrelated transcript scans or a new storage API. An optional per-handle cache retains +current section state and its prepared cursor, not full history; each fresh baseline checkpoints all +canonical sections. There is no additional sticky full-state write. Missing contributors therefore +survive restart and compaction even when their original baseline is outside the retained model range. + +Compare both payload and final rendered text by key, never by parsing prose. Baseline data contains +complete ordered set records; delta data contains only set/remove changes. JSON null and empty string +are values, not removal. A payload-only change persists a metadata-only managed delta with `model: []` +when there is no independent tool change. A renderer/wrapper change with unchanged payload emits a +model update when rendered text changes. Pure reorder emits no entry or order-change delta. Render +baselines and simultaneous changes in draft order with generic initial/change/remove labels by stable +key; historical messages are never reordered. A system entry records canonical instructions PREPARED +for a request, not proof the provider received them; appending it invokes no provider. + +Tool differences remain independent of section differences. Complete definitions occur only in +SystemMessage `toolsAdded`/`toolsRemoved`, never SystemData. Fold removals before additions; additions +upsert by name, including same-name definition/schema replacement without a removal for that name. +A deletion carries the previous stored full definition, not a lookup in today's catalogue. + +**Epoch preparation, not head-time behavior.** Every generation/provider request preparation checks +the newest visible head and effective managed system entries. A missing current-epoch baseline +requires a fresh baseline. Arbitrary managed-system edits reject; the current baseline's supersession +omissions do not invalidate it repeatedly. Rebuilding uses canonical payloads/rendered fallback from +the independent data fold, never just the model tail. +Append stored omission edits on that same fresh baseline entry for superseded retained managed +baselines AND deltas, never arbitrary system notices. Before preparation, generic context may expose +dangling old deltas; no request bypasses preparation. Head writers stay generic. The fresh baseline +remains at its appended position after the retained tail, not in a hidden prepended slot. + +Apply planned omissions before folding tool declarations from ALL remaining effective SystemMessages, +including non-managed messages. A fresh baseline adds the complete desired tool set and explicitly +removes unwanted remaining declarations; `baseline: true` is not a pi-ai tool-map reset. Never emit +removals derived solely from messages that the same preparation is about to omit. + +Capture canonical section state/cursor and the section registry snapshot before hooks. After hook and +render completion outside the line, verify on the line that no concurrent managed section-state change +occurred, including metadata-only changes. If it changed, restart preparation outside the line from +fresh canonical state. A compaction-only head change does not stale the payload draft: recompute only +the model baseline/delta choice, planned omissions and effective tool differences on the line. + +Atomically persist any baseline/delta (including metadata-only), inflight status and +`state.requestThrough` on the prepared generation variant. Capture the cutoff even when no system +change is needed; unprepared variants have none and deferred variants retain their request's cutoff. After +persistence, the same line operation catches the current model cache up to the cutoff and captures an +immutable array of effective entry references, including immutable replacement projections, before +releasing the line. Current caches advance independently; the invocation keeps its snapshot, not mutable +cache containers. Clone messages only for request-local mutating normalization/hooks. Cold/recovery +reads at an older cutoff derive from storage without rewinding current caches. Release snapshot and +captured registry references after invocation; no version registry. + +**Request-local overrides.** Preserve arbitrary `before_request` message transformations on a +request-local copy; they cannot mutate stored entries or switch Pico out of messages-only mode. +Canonical prepared state remains the basis for later diffs. Exact transformed requests are not +reconstructible from that state unless explicitly captured; no mandatory second request ledger. +Validate returned tool calls against the actual offered definitions after transformation, while +retaining implementation availability and permission checks (group 10). + +Tests: +- Typed built-in/custom tokens; owned-copy get and explicit set; shared-draft registration/inner order; + existing-key replacement retains position, new key appends, explicit delete differs from null/empty + payload, array updates use get+set, and wrappers require registered sections. +- Untouched seeds retain rendered text; explicit set/wrap and renderer replacement recompute; wrappers + compose in registration order and reset per draft. Authoritative base refresh rebuilds wrappers; + missing wrappers need not survive that refresh. Check stability of the complete transformation chain, + not merely its individual handlers, and repeated preparation with earlier authoritative base reset. +- Missing contributor/definition and failed refresh preserve stored payload/text; a skipped handler's + partial mutations cannot leak. Registry remove never deletes section state; re-registration and + compatible replacement work. No renderer runs on replay/line; no closures enter persisted records. +- Persist payload AND rendered text; changed payload/unchanged text gives metadata-only `model: []`; + unchanged payload/changed rendering gives a model update; explicit removals and reorder-only no-op. + Baseline/change rendering follows draft order. Tool changes remain independent, including tool-only + empty content, additions/removals, same-name schema changes and `addTools`; no duplication in data. +- Config change commits before the next system entry; crash after config change preserves it; crash + after system append before invocation does not duplicate unchanged canonical preparation. Canonical + recovery folds metadata-only records and never requires the original contributor/renderer. +- Independent canonical fold across fork cutoffs, model heads and supersession omissions; missing-plugin + sections survive an original baseline outside the model range and appear in the next full baseline. + Repeated compaction uses the latest canonical checkpoint without an unbounded handle history cache. +- Compaction/reset/handoff and arbitrary plugin heads; a retained old model baseline still requires a + new epoch baseline; retained tail order, baseline-time omissions and preserved notices; fork before + and after preparation uses its own canonical state/inherited config without changing the source. +- Arbitrary managed-system omit/replace edits reject; fresh-baseline supersession commits controls and + instructions together without changing the seed's canonical data. Superseded omissions do not trigger + repeated baselines. Compute tool removals after planned omissions; include remaining non-managed tool + declarations. Generic projection runs no system-kind code. +- Concurrent canonical writes during hooks/render, including metadata-only deltas, force preparation + retry outside the line; registry changes preserve the captured snapshot and affect later prepares. + Head-only changes recompute epoch/omissions without discarding the payload draft. A later head during + projection/streaming cannot change `requestThrough`, including when no system entry was appended. +- Preparation snapshots include baseline omissions; later heads, edits and cache updates cannot change + captured entries or replacement projections. Normalization/hooks mutate message clones only. + Cold/recovery derivation at an older cutoff leaves current caches unchanged; invocation completion + releases snapshot references without a version registry. +- Request-local overrides leave stored state unchanged, remain messages-only and determine actual + offered-tool validation; adapter fixtures are gated on the pi-ai prerequisite in group 20. +- In-band provider abort without a durable mark has a kind-level outcome; post-mark execute settlement + rejects and fresh abort writes optional partial, cancelled input results and known usage atomically. + No mark branch in normal settlement; missing post-cutoff usage is unknown; crash while streaming + publishes the partial; retry budget exhausted → failed with no successor. + +## 10. Tools, post_tools, exchanges + +**API redesign pending:** retain the capabilities and behavioral tests below; simplify the tool, +sink and preview authoring interfaces together before implementing them. + +The tool kind with the sink (`ToolOutput`, `ToolOutputState`, limits enforced by the sink, `diag`, +`delegate`, `handoff`, `addTools`, `terminate`), tool-result entries with structured data plus their +materialized model message, `before_tool` (fail-closed) and `after_tool`, +replay policy on recover; post_tools with `after`, carried input groups, terminate / handoff / steer / +next generation; `accept` idle vs busy; `prompt`, `result` and request acceptance lookup. Validate +calls against the actual definitions offered by the prepared request after request-local overrides, +not merely captured selected-tool names or today's catalogue. Preserve registry and permission checks. + +Tests: transformed offered-tool definitions, removed tools and same-name schema changes; parallel tools +completing in either order; sequential via `after`; an aborted generation +creates no tool tasks/results while an aborted existing tool writes its own error result; +`new_context` resets after the exchange, never inside it; `addTools` writes the rewindable loadout +before any handoff/user entry in the settlement commit and appears in the next turn's `toolsAdded`; +a throwing tool → error result, `terminate` still honoured; truncation diag from the sink; a lost +accept response is recovered through `acceptance(requestId)`; a duplicate create reports the first +receipt without comparing payloads or modes; results remain point-readable after further turns. +A missing tool implementation produces an ordinary error result through the registered tool task kind; +the generation/post_tools input owner remains present. Separately, post_tools handles orphaned tool +tasks by writing unavailable-tool results from their common fields. +Turn-task entry appends remain immediate; outside-turn model-visible writes queue while inTurn is +nonempty and land at post_tools/final boundaries. Data-only entries are never blocked. + +## 11. Inbox + +`pi.inbox` as a conversation sticky list whose element id is `inputId` and whose value holds mode, +user content or a write's entry draft and optional request id; append/remove/clear watch operations; +queued/placed/done/unanswered result variants; the three placement points; carried generation/post_tools +input groups; `queueInput` and `abortInput`; abort draining steer and followUp while preserving write +and nextRun. A write reaches done without an answer in its placement commit. Input-group ownership +and results are built-in harness behavior, implemented by generation/post_tools; compatible replacements +retain that contract. No generic plugin-payload inference or new input-owner protocol is needed. + +Tests: idle append/remove is one commit and emits no inbox event; busy image payload is one append +operation; the modes table; steer joins at post_tools but starts a group after a final answer; +followUp starts the next group; nextRun waits for idle accept; writes are placed without joining; +several inputs resolve to one answer; cancel/land and abort/group-transfer in both orders; queued and +placed crash recovery; cancelled unplaced payload is unavailable; input queued during collapse lands. + +## 12. Collapse + +Manual, threshold and overflow; `before_collapse`; publish only if no newer head; the overflow +chain (generation settles → collapse → new generation carrying the attempt). + +Tests: a summary lands under a running generation and later entries stay in context; a competing +head makes a summary stale while intervening edits do not; overflow retries once and no live task +ever waits on the collapse; threshold before a turn; abort of a running collapse leaves appends flowing. +A head adds no implicit system omissions: next request preparation writes them with its fresh baseline. +Test retained deltas before preparation and their stored omission afterward, without invalidating an +already-prepared request at its frozen cutoff. + +## 13. Subagents + +The `subagent` tool (`run`, `spawn`, `send`, `status`, `wait`, `stop`), ownership links, foreground +reach through live owners, `run`'s recover driving the child again, `spawn` initialization of config, +explicit child input results. + +Tests: restart in the middle of `run`; conversation abort reaches a `run` child and spares a `spawn` +child; abortTask marks only owner and fresh cleanup cancels its child; drive waiter observes Call; +cleanup tolerates already-terminal child/job; `stop`; nested children; child's own baseline. + +## 14. Jobs and the budget + +**Authoring API provisional:** job-first execution remains the initial supported approach. How tool +code starts, observes and delegates work is part of the tool/sink/preview simplification; the current +helper/sink signatures are not final. + +`jobKind` on `ExecutionEnv.exec` with output into its scratch, `waitForTask` / `jobOutput`, `bash` +delegating first and waiting with the budget, `notify` and the `notice` entry, the `job` tool, +schedules. + +Tests: job creation atomically stores job id and cancellation policy on the tool; fresh tool abort +uses those durable references, not execute locals or catch writes; budget expiry settles delegated +and job continues; delegation requests notification in separate sticky state, not a patch to an owned +job; both notification/completion orders publish once for exited, lost and killed; schedule cycles one +stable id; foreground drive +ignores background recurrence; recover → lost or safe rerun; abort kills non-detached job. + +Remaining integration design: arbitrary-promise budget adoption must explicitly transfer effect and +sink ownership before the tool invocation releases. A raced, abandoned promise is not permitted. +Preserve the capability (crash outcome lost), but do not turn arbitrary-work adoption into an +implementation package until the simpler tool contract and ownership transfer are settled. + +## 15. Previews + +**API redesign pending:** retain incremental live output and reconstruction from durable scratch. +The tracker/sink-facing API below is a sketch to simplify, not a separate framework to implement first. +Delivery coalescing is deferred; do not add a frame scheduler or delay scratch durability for it. + +`runtime.preview` as a Chord tracker per task; the generation applies stream events to a partial +message, the tool's preview is its sink state, the job's the same; `preview.init` on attach and +reopen; flush after each scratch commit. + +Tests: one token → one `a` op and nothing else; no `r` mid-stream; init after reopen yields a base +equal to the live preview; a sliding tool tail → `t` + `a`. + +## 16. Watch + +`ConversationView`, `ConversationEvent`, the exported kind-free `applyEvent`, `WatchHandle` +(capture on the line, bounded buffering, `resnapshot`, `unsubscribe`), the session watch with +`report` and `usage`, the usage ledger (`pi.usage` + totals). + +Tests: the fold is correct (view after N events equals a fresh capture, randomized); head and edit +entries update derived context; inbox append/remove/clear operations update the view and same-commit +append/remove cancels; all events of one commit delivered together; a thin-client reducer over a +recorded stream with no kinds loaded; +lag → fault → resnapshot; `resnapshot` from inside the listener; usage totals equal the ledger +fold, failed and aborted attempts included. + +## 17. JSONL storage + +Append-only batches, replay on open, torn tail discarded, malformed line fails open, entry +`data`/`model`/`head`/`edits` and the kind-string set from replay. Whole value sets and intrinsic list +operations in main and scratch files; no Chord storage codec. Replay main first, then surviving live +scratch; per-file sequence gaps are valid. Recover lastSeq from maximum complete surviving batch +endpoint, including clear/remove. Retired scratch is ignored; its later settlement covers its ids. +Physically remove torn suffixes before appending; malformed complete replayed batches fail. + +Tests: conformance against memory; full replacement values on disk; compact incremental frames/output +operations avoid repeated growing snapshots; main=100/live scratch=150 reopens at 150; settle=151 +with failed unlink ignores retired scratch, even malformed; clear/remove high-water; torn main and +live scratch tails; nonoverlapping ranges; accepted payload plus placement has two JSONL copies, +including idle acceptance. Backend memory and disk growth claims match whole-value behavior. + +## 18. SQLite storage + +Tables and indexes of §7.5, nullable entry JSON columns for data/model/edits and an indexed integer +head boundary, scratch rows deleted in the settle transaction, storage version and `migrate`. + +Tests: conformance three ways; removed sticky inbox elements may be physically discarded while +input results remain point-readable; a cold reopen decodes live tasks and only named required owner +records, including terminal owners; no retained terminal payload cache; a version mismatch rejects. + +## 19. Race matrix and telemetry + +The §10.2 matrix in both orders with faux clocks, fake processes and storage barriers; spans per +task call and per commit. + +Tests: the matrix; span tree for one turn with a tool and retry; nested Call preserves active telemetry +parent and private invocation identity; drive-caller cancellation does not become the task signal; +providers/env interpret signals, hooks propagate cancellation rather than swallowing it; no callback +runs on the line; stale RPC-bound invocation rejected. Metadata transport never carries task authority. + +## 20. Clients and pi-ai integration prerequisite + +mini (`worker/run.ts`, `worker/lane-service.ts`, TUI `apply(view)`), the experimental agent's four +seam files (`session-worker.ts`, `agent-controller-provider.ts`, `models-provider.ts`, +`transcript-provider.ts`), real providers. The integration milestone remains: system deltas on a live +model, a retry, a spawned subagent surviving a restart, speculative compaction under a running turn. + +**External prerequisite, not landed functionality.** PRs +[#9116](https://github.com/earendil-works/pi/pull/9116) and +[#9117](https://github.com/earendil-works/pi/pull/9117) are open dependencies in this design round. +Their agreed target and fixtures must be delivered and verified before integration; neither an open +PR's types nor this plan establish that the behavior already exists. + +Target: pi-ai enters system-message mode when BOTH `context.systemPrompt` and `context.tools` are +undefined. Pico supplies only `{ messages }`, never parallel top-level instructions/tools. SystemMessage +carries text plus complete JSON `toolsAdded`/`toolsRemoved` definitions. Provider/model translation +belongs to pi-ai: unsupported mid-conversation changes become user messages bracketed with `` +at their historical positions, and adapters derive required bulk wire tool declarations from message +history. Pico does not implement fallback or flatten changes into a rewritten top-level prompt. +Cache preservation is best-effort, not a universal prefix-cache or instruction-priority guarantee. + +Required adapter fixtures: both top-level fields absent versus either supplied; ordered baseline and +section updates; empty-content tool-only changes; removals and same-name upserts; historical calls to +removed/replaced tools; historical-position fallback; compaction with retained tail, stored omissions +and an appended fresh baseline. Include messages-only preparation for summarizer/provider request +paths, not just ordinary generation. Run faux fixtures without provider credentials; live integration +waits for the dependency contract and separately authorized smoke tests. + +The current sketch puts groups 1–16 before this milestone and 17–19 later. That ordering and those +sizes are provisional, to be verified and split at the final planning gate below. + +## Final review and work-package gate + +After the remaining conceptual blockers are settled, perform an independent whole-document audit of +spec, guide and this plan: reconcile APIs, invariants, examples, cross-references, external prerequisites +and coverage while preserving agreed features and explicitly tracking residual integration questions. +That audit is still ahead, not completed by the blocker-7 update. + +Then replace the provisional groups with small, self-contained, testable, human-reviewable work +packages. Each needs explicit prerequisites, bounded scope, interfaces and acceptance tests, with a +green incremental verification step and no unresolved forward dependencies. Do not treat the current +numbering or package sizes as the final implementation decomposition. + +## 21. Runtime schema bundle + +`harness.schema(call)` walks the three registries and emits one document describing everything a +client can see in this session: the core protocol types (`ConversationView`, `ConversationEvent`, +`InputResult`, `QueuedInput`, `DeltaOp`), plus each registered entry kind's `data`, each task +kind's state union and preview, and each tool's parameters and details. Generated at runtime, not +at build time, because the interesting half is per installation: the plugin kinds and tools that +happen to be registered. A client fetches it once per session and can then read `Entry.data`, +`Task.state` and previews from plugins its authors never heard of. + +Schemas are optional per kind. A kind that declares none is still usable; its payload is opaque +JSON to a foreign client, which is the generic-fallback case a renderer already handles. Declaring +is progressive: add a schema to the kinds you want third-party clients to understand. + +Two open options for how a kind declares one, to be picked after trying the first: + +- A small descriptor owned by pico (about twelve cases: scalars, literal, array, object with + optional fields, tagged union, ref, unknown) with `Static` deriving the TypeScript type, plus + adapters `toJsonSchema` / `toTypeBox` / `toZod` and `fromTypeBox` / `fromZod` for authors who + already declare with a validator. Pico then depends on no validation library. Cost: hand-rolled + conditional types with worse errors than TypeBox's, and adapters must reject what the descriptor + cannot express (refinements, formats, dynamic keys) rather than silently dropping it. +- JSON Schema as the descriptor, since tool `parameters` already is one, with adapters only for + authoring convenience. One representation fewer; authors without a validator library write JSON + Schema by hand. + +Schemas describe, they do not validate: stored objects are trusted (§7.3) and validation belongs at +wire boundaries, so a host that wants strict ingest validation opts in. Versioning is per kind, not +global; a kind that changes its shape bumps its own version, which is also the migration signal. + +Needed when a client is not JavaScript. Not part of the gate. + +Not Pico packages: approval/question policy, presentation, durable answers and their replay are +workspace/plugin responsibilities. `before_tool` may block, rewrite args or wait; existing scratch +and scoped values provide persistence. The workspace/plugin layer chooses answer keys and lifetimes; +Pico's remaining integration question is authorized hook access to task identity/scratch, not a new +memo protocol. Session migration remains outside this experimental scope. + +## What comes from the lane harness, and how + +Clean room means no imports from `src/harness/runtime`, `session`, `agent-harness.ts` or the +`dom`/`pico`/`pico2` spikes. Where the old code has something worth keeping, it is **copied** into +`src/harness/pico/` and owned there; pico must build with the rest of `src/harness` deleted. + +Copy (under `packages/agent/src/harness/` unless noted): + +| what | from | into | package | +|---|---|---|---| +| `ExecutionEnv` / `FileSystem` / `Shell` types, the Node env, capture and spill | `types.ts`, `env/`, `tools/tool-context.ts` (03-execenv) | `pico/env/` | 10, 14 | +| shell output limits, `applyShellOutputUpdate`, truncation totals | `utils/` | `pico/env/output.ts` | 10 | +| the built-in tools (`read`, `write`, `edit`, `bash`, `image`) | `tools/*.ts` | `pico/tools/`, adapted after the tool/output interface is simplified | 10, 14 | +| system prompt helpers, skills, context files, templates | `system-prompt.ts`, `skills.ts`, `prompt-templates.ts` | host handlers refresh typed base payloads; pure section renderers format them, without a source-read framework | 9, 20 | +| telemetry span helpers | `telemetry.ts` | `pico/telemetry.ts` | 19 | + +Depend on, as packages (they are not the old harness): + +- `@earendil-works/chord` Context types and `@earendil-works/chord/context` helpers (1, 6, 8, 19) +- `@earendil-works/chord/delta` (15, 16: preview/watch only; not storage) +- `@earendil-works/pi-ai`: `faux` provider for tests, `utils/estimate` for thresholds; SystemMessage and + messages-only adapter behavior depend on the verified PR #9116/#9117 target above (4, 9, 12, 20) + +Read before writing the equivalent, then close the file: + +- `runtime/drive/retry.ts`, `deferred.ts`, `response.ts`: retryable-error classification, overflow detection, deferred polling (9) +- `compaction/`, `runtime/drive/boundary.ts`: the exchange-cut rule and the summarizer prompt (12) +- `runtime/drive/recovery.ts`, `restore.ts`: the per-phase recovery case list (9, 10, 14) +- `runtime/drive/tool-placement.ts`: the result order projection must reproduce (4) +- `hooks.ts`, `docs/harness.md` §before_tool: the exact decision shape (10) +- `session/`, `test/harness/jsonl-*.test.ts`: torn-tail and malformed-line rules (17) +- `session/`, `test/harness/mutation-line.test.ts`: line discipline edge cases (3) +- `agent-harness.ts`: `LaneSnapshot` / `HarnessEventPayload`, for the `toLaneSnapshot(view)` shim only (20) + +Tests to port (`packages/agent/test/harness/`): the three conformance suites as the model for 17–18; +`compaction`, `branch`, `context`, `execution-*`, `values`, `mutation-line` as the parity source for +19; `system-prompt`, `prompt-templates` for the host handler in 9; `tools`, `truncate`, +`output-capture` for 10; pico v1's 27 ported scenarios as the shortest list of behaviours to +re-prove. + +Do not read: `runtime/lane.ts`, `reducer.ts`, `drive/reconcile.ts`, `structural.ts`, +`terminal.ts`, `checkpoint.ts`, and the `dom` spike. They are the operation state machines and the +reconciler this design replaces. diff --git a/packages/agent/docs/pico/v3/hardening-handoff.md b/packages/agent/docs/pico/v3/hardening-handoff.md new file mode 100644 index 000000000..658e75c02 --- /dev/null +++ b/packages/agent/docs/pico/v3/hardening-handoff.md @@ -0,0 +1,473 @@ +# Harness V3 hardening handoff + +## Goal + +Harden the implementation in `harness-v3.zip` without reverting its core design. Return a new standalone zip for review. Do not integrate it into the pi repository yet; the accepted implementation will later be incorporated under `packages/agent/src/harness/pico3/`. + +Baseline archive: + +- `~/Downloads/harness-v3.zip` +- SHA-256: `1c69c5c5d923fe1c80200cc5e49ddb3868158e1b6a844efa3e2f03eb24ff5be7` +- Current runtime suite: 67 passing tests +- Current kernel + kinds: approximately 3,448 lines, excluding storage backends + +The current code is a good executable design spike. Preserve its legible fixed-core, document/view, and phase-map architecture. Correctness takes precedence over preserving the exact line count. + +## Companion documents and supersessions + +This copy in `packages/agent/docs/pico/v3/` is canonical. Two companion +documents in the same folder were written after this handoff and take +precedence where they overlap: + +- `view-and-events.md`: the watch protocol, the single view document, ops by + path, named events, the commit-by-commit mapping, watchers and late joiners. +- `plugins.md`: the harness registration surface, the four durable primitives + (entry, checkpoint, slot, namespace), tool/hook/custom-kind/entry/namespace + extension points with examples, unregister/re-register/reload, and the Chord + facet layer. + +Specifically superseded in this document: + +| here | replaced by | +|---|---| +| Accepted direction #3 (separate `rewindable`/`sticky` view fields with separate op lists) | one flat view document with `config`, `inbox`, `turn`, `compaction`, `tasks`, `plugins` and one op list (`view-and-events.md` §2–§4); the storage split stays | +| Accepted direction #2 "live task slots ... in the view" and §13 last paragraph (`TaskRef` slot typing) | slots remain, but the view publishes kind-described `tasks[id].status` via `describe()`, never raw task records (`view-and-events.md` §2.2, §6.6; `plugins.md` §4.3) | +| §9 Watch protocol: envelope shape, `Seq` ordering and `applyDelta` | `Envelope { revision, ops, events }` with a contiguous per-watch `revision` (storage `Seq` is not exposed), folded with `applyImmutable`; the active-transcript definition and capture rules in §9 here remain normative (`view-and-events.md` §3, §5, §9) | +| §9 bounded capacity / overflow-close | the raw watch invokes its listener synchronously in order; only the pre-`start` buffer is bounded (overflow closes the watch); asynchronous consumers own their queue (`view-and-events.md` §3; `plugins.md` §6.1 for the Chord bridge) | +| §13 plugin state visibility | a namespace is in the view only through its declared `view` projection; memos and slot working state are never in the view (`plugins.md` §1.5, §2, §4.5) | +| §15 `beforeTool` throw = block; hooks by name | hooks are registered with their namespace token (`h.hooks(ns, kind, handlers)`); `BeforeToolApi.waiting(ctx)` is asynchronous; memo keys are `(task, namespace, name)` (`plugins.md` §4.2) | +| §16 / view §6.2 generation failure closures | every generation failure resolves the group `unanswered/failed`, runs the final boundary and starts a successor for queued triggers (`view-and-events.md` §6.2) | +| §4.4 entries / §1 `pi.*` | `pi.*` entry names reject for non-core callers except the allow-listed `pi.notice` (`plugins.md` §4.4) | +| §13 plugin state (`plugins[pluginId]` slices, extra document shard) | `h.namespace(ns, defaults)` tokens, `tx.plugins(ns)`, `tx.emit(ns, …)`, lazy seeding; extra shards stay deferred (`plugins.md` §2, §3, §4.5) | +| §15 tool API (candidate) | `ToolApi` with `memo` (slot memoOnce), `waiting`/`memo` on `BeforeToolApi`, `waitingOn` on the tool slot (`plugins.md` §4.1–§4.2; `view-and-events.md` §2, §6.3) | +| §1 "install fixed built-ins internally" / registration at open | registration capabilities callable while open, before-`resume()` rule, `quiescent()/hold()` for idle reload and `suspend()/reopen` otherwise; no handler-withdrawal machinery in v1 (`plugins.md` §3, §5) | +| §16 `pi.plugin` handlers map | folded into custom task kinds; `pi.plugin` may stay as a convenience kind but is not the plugin mechanism | + +Everything else here (authority explanation, proxy lifetime, prospective busy, +display-only assistants, read semantics, phase-map contract, session line, +waiters, storage atomicity, terminal retention, strict types, context/sections, +tool/process behaviour, known defects, test matrix, deliverable) stands. + +## Accepted direction + +These are decisions, not open questions: + +1. **Fixed core.** `pi.generation`, `pi.tool`, `pi.post_tools`, `pi.collapse`, and the Harness methods built on them are fixed and unreplaceable. `pi.job` and `pi.plugin` are built-ins too, though they use ordinary-task authority. Plugins cannot register any `pi.*` task kind or replace a built-in. +2. **Document state.** Keep Chord-delta-backed session, conversation-rewindable, and conversation-sticky documents. Keep the core rendering fields (`inbox`, `turn`, live task slots, core config) represented in the authoritative conversation view. +3. **Rendering-oriented view.** Keep `rewindable` and `sticky` as separate top-level `ConversationView` fields with separate Chord op lists. Keep commit deltas easy to fold and render. A renderer should not reconstruct turns by correlating unrelated channels. +4. **Phase-map tasks.** Keep `initial` plus an exhaustive phase map. Do not restore a separate handwritten `execute`/`recover` pair. +5. **One implementation, few capability views.** One internal `TxImpl` is desirable. Expose narrow host/ordinary/core transaction types or façades over it rather than a large hierarchy. Runtime checks remain authoritative. +6. **Extensibility.** Preserve custom entries, custom phase tasks, tools, core hooks, system sections, typed config, jobs, owned conversations/subagents, plugin state, and compatibility with separate Chord RPC/presentation services. +7. **Single process.** Do not add cross-process locking. Document that a storage path has one owning process/session at a time. Reject a second Session over the same Storage object in-process; behavior from two processes opening the same JSONL path is unsupported. +8. **Commit-granular document history.** A fork at an entry sees the final rewindable document state of the atomic commit containing that entry. Document this explicitly. Do not reintroduce per-write `Seq` or rewindable-before-entry ordering unless a concrete failure demonstrates it is needed. +9. **Separate stable IDs and commit sequence.** Object IDs remain stable identity. `Seq` orders atomic commits and watch envelopes. +10. **No terminal pruning.** Terminal task records and outcomes remain durably readable. Terminal checkpoints may be dropped once no downstream contract relies on them; `ToolInput.offered` already removes the generation-checkpoint dependency. + +## What `boundary()` and `resolveInputs()` mean + +These are core authority and must not be exposed to ordinary tasks. + +### `boundary(conversationId, at, headBoundary)` + +`boundary()` implements inbox placement at a safe turn boundary. `headBoundary` is the current retained transcript boundary, supplied by core code so `boundary()` performs no storage scan after same-batch appends. It: + +1. reads the ordered sticky inbox; +2. finds a queued self-head write (reset/handoff) and marks older steer/follow-up inputs stale; +3. selects every passive write, steer inputs according to `steeringMode`, and at a final boundary follow-ups according to `followUpMode`; +4. places selected entries strictly in global input-ID order; +5. updates their durable `Input` records; +6. removes selected/stale items from the inbox; +7. returns trigger input IDs and whether a self-head ended the old input group. + +An ordinary task calling it could drain input in the middle of an assistant/tool exchange, stale requests, place a reset at an invalid point, or consume triggers without creating the required successor generation. Only fixed core turn code may call it. + +### `resolveInputs(ids, resolution)` + +`resolveInputs()` controls the durable lifecycle observed by `InputHandle`: done/unanswered, answer ID, reason, and detail. Generation, post-tools, failure, termination, and abort paths call it only when the corresponding transcript/task transition commits. + +An ordinary task calling it could claim another input was answered by a fake entry, abort unrelated requests, or wake waiters before the transcript contains the asserted result. Keep `setInput`, `resolveInputs`, and direct input-table writes core-internal. + +## Proxy lifetime requirement + +`Proxy.revocable(target, handler)` returns `{ proxy, revoke }`. After `revoke()`, every operation through that proxy throws `TypeError`. + +Revoking only the root document proxy is insufficient because a nested proxy can escape first: + +```ts +let escaped: object; +await c.commit((tx) => { + escaped = tx.rewindable(c.id).plugins; +}, ctx); +// Revoking only the root does not necessarily revoke `escaped`. +``` + +Implement a transaction-scoped **revocable membrane** over the cached Chord tracker: + +- every object/array returned from a document proxy is wrapped lazily; +- all wrappers share one active/revoked state; +- preserve object identity inside the transaction with a `WeakMap`; +- mutations forward to the underlying Chord proxy so its tracker still records ops; +- at transaction finish, callback failure, validation failure, or storage failure, revoke every wrapper; +- retained root or nested handles throw after the callback; +- no raw nested Chord proxy may escape through `get`, iteration, descriptors, or array access. + +If a correct membrane is too invasive, add tracker-level revocation to Chord so every nested proxy checks one tracker lifetime flag. Do not rely only on documentation or TypeScript to prevent escape. + +Tests must retain both a root and nested object/array, mutate them after commit/failure, assert a throw, and prove no later unrelated commit persists the attempted mutation. + +## Required hardening + +### 1. Fixed registry and token identity + +- Install fixed built-ins internally. +- Reject duplicate task-kind names. +- Reject plugin task names beginning with `pi.`. +- A user-supplied `{ core: true }` must confer no authority. +- Use current registered token identity for typed task creation and hook registration; a stale/redeclared token rejects. +- Make hook and registry unsubscribe functions idempotent. A second call must not splice/delete an unrelated newer registration. +- Reject or define replacement semantics for duplicate tool, entry, and section names. Unregister must only remove the exact object it registered. + +Tests: attempts to replace `pi.generation`, forge core authority, use a stale token, and unsubscribe twice. + +### 2. Capability surfaces and scope enforcement + +Keep one `TxImpl`, but expose at most these author-facing capability views: + +- **HostTx:** reads; plugin/application document state; passive `write`; create ordinary tasks and conversations. No direct core task creation, entry append, boundary, or input resolution. +- **TaskTx:** reads in the task's conversation/owned subtree; own checkpoint; own typed slot/plugin state; passive write in its own conversation; create ordinary tasks and owned conversations. No admission/boundary/input authority. +- **CoreTx:** internal full authority for fixed turn machinery. +- Abort mode must have its own runtime checks even if its public type is an `Omit`/narrow view. + +Runtime authorization must cover every method, not only `appendEntry` and `createTask`: + +- `send`, `boundary`, `setInput`, `resolveInputs`, raw core document fields, core task creation, and core entry append are internal; +- ordinary tasks cannot mutate `sticky.inbox`, `sticky.turn`, or another task's slot even via a cast; +- a task may access only its conversation and conversations it owns; +- parent selection for an owned conversation must stay in that subtree; +- host APIs invoke fixed core capabilities through internal tokens, not user-controlled strings. + +Add `kind` and an unforgeable invocation token to the internal task invoker. Every runtime commit checks that the token is still the active invocation, the task is live, and an ordinary run invocation is not marked. A captured runtime cannot write after the method returned, after terminalization, or after a durable mark. + +Tests must include malicious casts so runtime checks, not only TypeScript, are exercised. + +### 3. Prospective task/turn state + +Fix `busy()` and boundary decisions to use the prospective state of the current atomic commit. + +Concrete failing trace today: + +```text +generation terminal closure +→ tx.write(notice) +→ busy() sees the same generation as live +→ notice is queued +→ generation terminalizes +→ no boundary remains to place it +``` + +When executing a terminal closure, the current task counts as gone. A same-batch terminal `task.patch` also counts as gone; a same-batch created live turn task counts as present. An idle terminal-closure write appends immediately. + +Regression tests: + +- overflow with nothing collapsible appends its notice and leaves no inbox item; +- an idle job notification appends immediately; +- a same-batch successor generation keeps the conversation busy; +- no input/write becomes stranded when the last turn task terminalizes. + +### 4. Display-only assistant entries + +Provider-error and aborted partial assistant messages are presentation history only: + +- they never enter model context; +- they never create tool work; +- later requests do not include them. + +Prefer omitting `model` and storing the display message in strict-JSON `data`, or add an equally explicit non-context projection facet. Do not make `deriveContext` infer this from an arbitrary error string. + +Tests: abort/error, send another input, inspect the next provider request, and assert no aborted/error assistant message appears. + +### 5. Read semantics + +Tracked document reads have read-your-writes inside one transaction. Keep that advantage. + +Apply one rule to every read: if buffered writes can change its answer, the transaction must either provide a complete explicit overlay or reject it. Do not silently return committed pre-transaction state. + +Maintain complete tx-local overlays for direct object reads: + +- `entry(id)` sees same-batch entry creation; +- `task(id)` sees same-batch task creation and every later patch; +- `conversation(id)` sees same-batch conversation creation; +- `input(id)` and request-key lookup see every same-batch input creation/replacement. + +These direct reads remain valid after writes. This is required because terminal closures append their result entry and then resolve existing inputs. `putInput` must update both the ID and request-key overlays, not only `writes`/`effects`. + +Scan-shaped reads have no general overlay. Reject a scan only after a buffered write that can affect that scan's domain: + +- after an entry append: `newestEntry`, `scanEntries`, and `context` reject; +- after a task create/patch: task scans reject; +- analogous future table scans follow the same rule. + +Use a named `ReadAfterWrite` error and poison the transaction even if the callback catches it. Refactor the known offenders: + +- threshold-collapse selection reads context before appending the assistant and carries the result into the terminal closure; +- `boundary(conversationId, at, headBoundary)` never scans storage: the caller reads the newest head before its first append, or passes the materialized ID of a same-batch self-head entry it just appended; +- while placing queued writes in ID order, `boundary()` updates its local head boundary after each selected head write, so later explicit-head validation sees earlier same-batch heads; +- `send()` performs request-key lookup before writing; +- generation/post-tools compute context-dependent decisions before appending; post-tools passes a same-batch handoff ID as `headBoundary` when it just appended that self-head. + +Document reads remain exempt because the tracked proxy supplies an actual overlay. + +### 6. Phase-map contract + +Keep the phase-map model, but encode phase role rather than relying on comments: + +- a normal phase can be reached by a returned `next` transition; +- an in-flight/recovery phase is written immediately before an external effect and is entered by the scheduler only after reopen; +- the scheduler rejects a normal transition into an in-flight-only phase. + +A transition builder may atomically return: + +- a next checkpoint; +- a typed completion; +- `"retry"` when its snapshot became stale. + +Add an explicit `Carry` facility if it removes repeated unchecked destructuring across phase checkpoints. A kind method or terminal closure that returns an invalid shape is a task-contract fault; do not invent an untyped `{ reason: "threw" }` outside the kind's declared failure type. + +Retain and expand the existing recovery tests for generation, tool, collapse, job, post-tools, marked tasks, and ordinary custom tasks. + +### 7. Session line, failure, and cancellation + +- Every storage interaction, waiter registration, watch capture, registry mutation, reservation, and commit enters one FIFO Session line. +- **No `AsyncLocalStorage` / `node:async_hooks`, anywhere.** Nested-line detection uses Chord contexts only: the Session derives a line context for every callback it runs (`withContextValue(LINE_KEY, marker, ctx)` with a private `createContextKey`), passes it to transaction builders (`commit((tx, ctx) => …)`; task builders receive `(tx, current, ctx)`), and `commit`/`read`/`onLine` reject `NestedLineOperation` when the supplied `ctx` carries `LINE_KEY`. Inside a builder, authors use the builder's `ctx` for everything; calling the Session from inside a builder with an outer ctx is a documented, undetectable self-wait, not something the kernel prevents. +- Check caller cancellation before queuing and again before invoking a transaction callback. +- Once persistence starts, call `Storage.commit` exactly once with `withoutAbortSignal(ctx)`. +- A storage commit rejection faults the Session, closes storage with a non-cancellable context, and makes already-queued/later operations reject `Faulted`. +- Callback or validation failure before `Storage.commit` discards the transaction but does not fault the Session. +- Listener, hook-reporting, watch, and scheduler callback failures never turn a successfully persisted commit into a caller-visible commit failure. +- Do not swallow document/tool-stream persistence failures. The current `flushing.then(...).catch(() => {})` pattern must go. +- Enforce one owning Session per Storage object in-process. Successful close/fault cleanup releases it; failed close keeps ownership. + +### 8. Waiters + +Register waiters atomically on the Session line: + +```text +enter line +read current task/input/idle state +if terminal, return immediately +otherwise install waiter +leave line +``` + +Cancellation only removes/rejects that waiter; it does not cancel durable work. Handle an already-aborted context. Add deterministic tests where completion is forced between the old read and registration points. + +### 9. Watch protocol + +Keep the rendering-friendly `ConversationView` and Chord document ops, but implement the settled watch behavior: + +- capture and subscription are one line operation; +- committed envelopes are delivered in commit order with a contiguous per-watch `revision` (storage `Seq` is not exposed); +- one commit produces one envelope folded into the view before the listener runs; +- listener execution is off-line, synchronous and in order; +- listener throw or decoder failure closes only that watch and reports through `onError`; it never rejects the already-persisted writer; +- envelopes before `start()` are buffered with a bounded capacity (default 256); overflow closes the watch; no other queue exists in the raw watch; +- no cursor, replay, acknowledgement, deduplication, or resnapshot protocol; the client opens a fresh watch; +- `stop`/unsubscribe is idempotent; +- no callbacks after close/stop. + +Define the initial entry capture exactly: + +- Let `H` be the newest fork-visible entry carrying a head. +- If `H` exists, the active transcript is the chronological union of `H` and every fork-visible entry whose ID is at least `H.head`, without duplicating `H`. `H` remains at its actual chronological position. +- If no head exists, the active transcript is the complete fork-visible transcript in chronological order. +- Capture walks `scanEntries` pages newest-first until it passes `H.head`, using the same fork-aware ancestry walk as `deriveContext`, then reverses the collected entries. There is no public capture limit parameter and no internal fixed cap. + +The captured transcript must retain all of the following: + +1. fork-inherited entries when `H.head` points into a parent conversation; +2. chronological order—a summary/head entry stays at its written position and is not moved to the front as context derivation does; +3. inner or older head entries that remain inside the active range; unlike model-context derivation, the view does not drop them; +4. display-only entries, including aborted/error assistants, `pi.usage`, and model-less plugin entries; +5. every entry's edits verbatim; the view is the transcript, not the edited/model-message projection; +6. every later committed entry for the lifetime of the watch—there is no live cutoff. + +When a committed entry carries a new head, `applyEnvelope` truncates the existing active transcript using this same fork-aware boundary definition and then appends the new entry at its chronological position. It must not apply context derivation's special treatment of heads. A fresh capture and a client that folded every committed envelope must produce identical `entries`. + +Add a regression with a child conversation whose newest head points to an entry in its parent and with an older head entry still inside that retained range. Assert that capture includes the inherited target, preserves both heads in chronological order, and equals the result of incremental folding. + +The view keeps fixed core fields for rendering. Plugin RPC/presentation DTOs may filter or project it through normal Chord services. + +### 10. Storage atomicity and JSONL recovery + +No cross-process lock. + +MemoryStorage: + +- validate the complete batch before mutating, or stage into copies and publish only after every write succeeds; +- one failed batch changes no table, document, ID high-water, or commit sequence; +- stable IDs never reuse once committed; callback-minted but uncommitted IDs may be reused only after reopen. + +JSONL multi-file publication: + +1. Assign one commit `Seq`. +2. Append and optionally fsync all sidecar records for that `Seq`. +3. Always append one `main.jsonl` record/commit marker last, even if there are no main-table writes. The marker lists the sidecar refs expected for that commit. +4. The main marker is the publication point. +5. Replay uses stored `Seq` values, validates monotonicity/shape, and applies sidecar records only when the corresponding main marker confirms them. +6. Unconfirmed sidecar tails are ignored or truncated. +7. Physically truncate bytes after the last newline before opening for append; otherwise a later append extends a corrupt torn line. +8. Sticky sidecar rewrite/truncate runs on the Session line and cannot race appends. Use temp file + fsync + rename as appropriate, but no directory/cross-process locking is required. +9. Carry committed ID high-water in every record needed to prevent reuse after retired/unlinked sidecars. +10. A malformed complete record fails open; a stale retired task sidecar cannot resurrect state. + +Add fault-injection tests for a crash after each sidecar append and before/after the main marker. Graceful `Harness.close()` is not a storage crash test. + +Durability modes and failure boundary: + +- `fsync: false` covers callback failures, append failures, and process termination while the OS/filesystem remain alive. Complete published records recover; torn final lines are truncated; unconfirmed sidecar tails roll back. +- `fsync: false` does **not** promise that a returned commit survives power loss, kernel/VM-host failure, storage-cache loss, or a filesystem that loses or reorders completed writes. +- Worst case without fsync: acknowledged tail commits disappear. If sidecars survive without their marker, replay rolls them back. If a marker survives without a required sidecar, open fails rather than exposing a half-commit. If the lost commit was a pre-effect tool/job checkpoint or memo, recovery may attempt the external effect again; external idempotency is still required. +- `fsync: true` flushes each sidecar before flushing the main marker, strengthening file-data durability and ordering. It is not a complete database guarantee until parent-directory metadata for file creation, rename, and unlink is also synchronized, and it cannot strengthen a filesystem or device that provides weaker guarantees. + +### 11. Terminal task retention + +Remove `pruneTerminal` behavior that makes terminal tasks unreadable. `getTask`, `waitForTask`, task scans, dependencies, and typed outcomes must continue to work after arbitrary later activity and reopen. + +Reduce retained size by: + +- clearing terminal checkpoints when safe; +- storing compact complete terminal records or validated patches; +- unlinking live-only task sidecars after the terminal record is durably published. + +Do not infer that every outcome is represented by an entry: job and plugin task results are not. + +### 12. Strict durable types + +Harden durable positions without rebuilding V2's entire address type system: + +- `JsonValue` only; no durable `unknown`; +- `Stored = JsonRepresentation` for imported pi-ai values; +- stored entries/messages/tool calls/usage/deferred handles use `Stored`; +- task input/checkpoint/result/failure/aborted values extend strict JSON; +- tool details, plugin inputs/results, document/plugin state, section payloads, and process state are validated/encoded before persistence; +- readonly durable snapshots at public boundaries; +- no `any`, no inline/dynamic imports, erasable TypeScript; +- use the repository's pinned `typebox` 1.x API after checking its installed types; do not use `@sinclair/typebox` 0.34; +- `ToolDeclaration.execute` receives `Static

    ` rather than `unknown` plus author casts; +- replace `never`-parameter registry erasure and unchecked casts with a small internal erased adapter whose validation boundary is explicit. + +Make the standalone `tsconfig` portable: no `/home/claude` paths, exclude design-history sketches from production type checking, and include a real compile-test command. Remove unused `partial-json` if it remains unused. + +### 13. Config, documents, and plugin state + +Keep fixed core document fields and the flat typed `c.config` facade. + +- Derive/seed defaults from registered kind definitions rather than duplicating them in `defaultRewindable`/`defaultSticky` and kind configs. +- Resetting a key is an explicit operation (`config.reset(keys)`; RPC `configReset`), never assigning `null`: storage deletes the override, the resolved view sets the declared default; only `model`, which has no default, is deleted from the view. +- Do not use `value ?? default`, because durable `null` is a legitimate value. Test property presence. +- Custom kind config defaults must be visible both through `c.config` and to the task itself before any explicit set. +- Reject config-key collisions at compile time where practical and always at installation. + +For plugin state, first implement typed/namespaced slices inside the existing documents instead of recreating arbitrary value/list addresses: + +```text +session.plugins[pluginId] +rewindable.plugins[pluginId] +sticky.plugins[pluginId] +``` + +At minimum, associate an owning namespace/type with a plugin or task-kind token so typos and cross-plugin writes are not silently accepted. If a plugin has large independent state, support a plugin-declared extra document shard using the same Chord fold semantics; do not add this complexity without a test demonstrating the cost/isolation need. + +Associate a task slot type and initializer with its task-kind witness. `TaskRef` should determine the slot type; remove caller-invented `slot()` casts. Core turn/tool slots remain fixed and rendering-oriented. + +### 14. Context, transcript, and sections + +Preserve the current centralized context derivation and full system-section design: + +- head/range/edit chronology; +- newest edit wins; +- tool results reordered to call order; +- missing fork-truncated results synthesized; +- messages-only requests; +- managed `pi.system` baseline/delta chronology; +- preparation snapshot/retry loop; +- historical tool additions/removals; +- section seeds before first preparation. + +Fix and test: + +- transcript/head invariants before persistence; +- display-only assistant exclusion; +- complete active transcript capture; +- private conversation section-seed metadata (do not expose it in public `ConversationView`); +- `SystemEntry` typing permits metadata-only `model: []` deltas; +- ancestry/subtree hook reconstruction after reopen; +- registry changes are line-serialized and preparation revision checks remain correct. + +### 15. Tool and process behavior + +Keep `ToolInput.offered` and the persisted final validated call. Recovery from `started` requires persisted-safe, current-safe, and current-schema-valid evidence; `beforeTool` never reruns after `started`. + +The V3 tool API is a candidate, now no longer an unspecified gap, but harden it: + +- `beforeTool` runs before `started`; a throw blocks; +- kernel owns bounded streaming; +- fix tail retention so a single chunk larger than `maxBytes` is sliced during `push`, counted as dropped, and never retained whole; +- enforce both byte and line bounds for streaming and returned content; +- no swallowed flush errors; +- a tool cannot mutate call identity or protected turn fields through `progress`; +- the final tool-result message is bounded and strict JSON; +- use the stored final call consistently for invocation/recovery evidence; +- preserve fixed five-second job TERM-to-KILL grace, optional `ProcessHost`, and polling capped at one second. + +### 16. Known implementation defects to cover + +Also fix these concrete issues found during review: + +- `Harness.waitForIdle` has the same check-then-register race as input waiters. +- Reopened owned-conversation ancestry is reconstructed incorrectly by looking for a conversation whose ID equals the owner task ID. +- Listener invocation can currently throw after persistence and reject the writer. +- Scheduler drain/abort errors can become unhandled rejections. +- Ordinary kind throws currently manufacture a failure outside the declared failure type. +- `createConversation` and document access omit subtree validation. +- Hook matching by name accepts stale tokens. +- `config.get` treats stored `null` as absent. +- custom kind defaults are returned by `c.config` but are not necessarily present for task document reads. +- `SystemEntry` witness requires one model message although metadata-only deltas use none. +- JSONL replay ignores recorded sequence values. +- sticky truncate currently runs outside the line. +- use `rt.now()` rather than ambient `Date.now()` in kind-controlled durable records where determinism/testing matters. + +## Required test matrix + +Retain all existing behavioral and recovery tests, then add focused regressions for every numbered hardening section. At minimum, the final report must show: + +1. runtime tests for memory and JSONL; +2. strict TypeScript compile tests for public capability surfaces, kind/config/hook/task-ref/slot inference, and strict durable JSON; +3. fault-injected JSONL atomicity at every publication cut; +4. escaped root and nested proxy rejection; +5. stale invocation and post-mark commit rejection; +6. malicious ordinary-task attempts against every core-only operation; +7. waiter and idle-registration race tests; +8. ordered/gap-free watch capture, listener throw, overflow, stop, and reconnect tests; +9. display-only assistant and terminal-closure write regressions; +10. terminal task lookup after many retirements and reopen; +11. real TypeBox validation tests; +12. the existing generation/tool/post-tools/collapse/job/subagent crash matrix. + +Tests must not use real providers, API keys, or paid endpoints. Fake models and process hosts only. + +## Deliverable + +Return a new zip containing: + +- the hardened standalone implementation; +- portable `package.json` and `tsconfig.json` with pinned dependencies; +- all runtime and compile tests; +- updated README and usage guide describing the fixed-core/document/phase design and explicit single-process assumption; +- a concise `CHANGES.md` mapping every item in this handoff to code/tests; +- production LOC count excluding storage backends and tests; +- exact commands and results for install, typecheck, and tests; +- unresolved questions or deliberate deviations, with no silent omissions. + +Do not modify `packages/agent/src/harness/pico` or the approved V2 files. The returned zip will receive another independent review before incorporation as `packages/agent/src/harness/pico3`. diff --git a/packages/agent/docs/pico/v3/harness-v3.zip b/packages/agent/docs/pico/v3/harness-v3.zip new file mode 100644 index 000000000..d938dc224 Binary files /dev/null and b/packages/agent/docs/pico/v3/harness-v3.zip differ diff --git a/packages/agent/docs/pico/v3/plugins.md b/packages/agent/docs/pico/v3/plugins.md new file mode 100644 index 000000000..e8893d372 --- /dev/null +++ b/packages/agent/docs/pico/v3/plugins.md @@ -0,0 +1,603 @@ +# Pico3 plugins + +Status: proposal for how the hardened V3 (`pico3`) harness is extended, and how +those extensions are packaged as Chord facets in the coding agent. Companion of +`view-and-events.md`. Written against `harness-v3.zip`, `pico-handoff-v2.md` +(§8–§12, §16) and `packages/agent/docs/plugins.md` (the coding-agent facet +architecture), read in full. + +Read order: §1–§5 are the harness alone (no Chord); §6 is the Chord layer; §7 tests. + +## 1. Principles + +1. **There is no plugin object.** The harness exposes registration capabilities; + whoever composes the process (a Chord facet, a test, a script) calls them. + The coding agent's "plugin" is a package of host-specific Chord facets that + call these capabilities (§6); the harness never sees the package. +2. **One durability story.** Everything durable is one of four primitives (§2): + entry, task checkpoint, task slot, namespace slice. Tools, hooks and custom + task kinds use the same four; none has a private mechanism. +3. **Nothing durable belongs to a facet.** Chord service instances and + replicated state are live process state; a reload replaces code, never state. +4. **Core authority stays core.** Extensions cannot append entries mid-turn, + touch `turn`/`inbox`, resolve inputs, create core tasks, or write outside their + namespace or their own task's slot. Enforced at runtime on every method, by + type at the surface. +5. **An extension's presentation is only what it projects.** The view shows a + namespace's declared `view` projection (nothing by default), a kind's + `describe()` output, the core-rendered tool slot fields, and entries. Raw + namespace state (caches, keys, secrets) and slot working state (memos, + idempotency evidence) never appear in the view. + +Here, “durable” means recoverable according to the selected storage mode. JSONL +with `fsync: false` covers process termination while the OS/filesystem remain +alive, but acknowledged tail commits may disappear after power, kernel, VM-host, +or storage-cache failure. `fsync: true` strengthens file-data durability; see +`hardening-handoff.md` §10 for the exact boundary and worst cases. Neither mode +makes an external effect exactly once without an external idempotency key. + +## 2. Durable primitives + +| primitive | lives in | lifetime | written by | forks / history | in the view | +|---|---|---|---|---|---| +| **entry** | transcript | forever, immutable | anyone via `c.write` (boundary-safe); core via `appendEntry` | inherited up to the fork point; rewind = `at` | `entries` | +| **task checkpoint** | task record | the task | its kind, whole replacement | no | via `describe()` → `turn.generation`, `compaction`, `tasks[id].status` | +| **task slot** | sticky document, `tasks[id]` (turn tasks: `turn.tools[i]`) | the task; retired atomically with its terminal record | its kind; tools running in it; hooks running in it | no | `tasks[id]`, `turn.tools[i]` | +| **namespace slice** | `plugins[ns]` in the session, rewindable or sticky document | the conversation / session | holders of the namespace token | rewindable slices are inherited and rewindable; sticky/session are present-only | `plugins[ns]` | + +Rules: + +- **Checkpoint** is the kind's recovery program counter, nothing else. It is not + a place for working state (that is the slot) or long-lived state (namespace). +- **Slot** is working state that should die with the task: progress, streamed + output, and **memos**. `memoOnce(slot, key, candidate)` is one operation inside + one commit: if `Object.hasOwn(slot.memos, key)` return the stored value, else + store `candidate` and return it (`??=` is wrong: `null` is a valid value). It + returns the durable winner to every caller. A memo is durable coordination and + evidence; it is **not** exactly-once execution by itself. For an external + effect, persist the idempotency key *before* the effect and replay the effect + with the same key, relying on the external side to dedupe or to answer a + result lookup; persist a result *after* an effect only when a duplicate is + otherwise harmless. Slot contents are never in the view. +- **Namespace slice** defaults are declared at registration, per document, and + seeded lazily (`plugins[ns] ??= defaults` on first access). Choose the + document by semantics: rewindable when a fork should see the value as it was + (plan mode, chosen model overrides), sticky for the present (UI preferences, + caches), session for session-wide. +- **Entries** are the only primitive the model can see (when they carry + `model`); everything else is invisible to the model by construction. +- Protected: `turn`, `inbox`, core config keys and other tasks' slots reject + writes from extensions. + +What is deliberately **not** durable: Chord keyed service instances (an open +approval dialog), hook handler registrations, tool and section registrations. +All are re-established by code at startup or reload. + +## 3. Registration surface + +```ts +const h = await Harness.open(storage, { models, processHost?, onReport? }, ctx); +// nothing runs yet; register, then: +h.resume(); +``` + +All registration methods return an unregister function and may be called while +the harness is open (§5 for what happens then). + +```ts +h.registerTool(declaration: ToolDeclaration): () => void +h.registerSection(section: SystemSection): () => void +h.registerTaskKind(kind: Kind): () => void // ordinary kinds only; config keys checked for collisions +h.namespace(ns: string, defaults: T, opts?: { view?: (slice: T) => JsonValue }): Namespace + // token with idempotent `unregister()`; unique, not `pi.*`; no `view` ⇒ nothing in the view +h.hooks(ns: Namespace, kind, handlers: Partial>, opts?): () => void // harness-wide; bound to the namespace (§4.2) +c.hooks(ns, kind, handlers, { subtree? }): () => void // one conversation (+ owned) +defineEntry(kind: string): EntryKind // stateless; throws on `pi.*` (§4.4) +h.conversation(id, ctx): Promise +h.quiescent(): boolean; h.hold(): () => void; h.suspend(ctx): Promise // §5 +``` + +Token identity: the object passed to `tx.plugins(ns)`, `tx.emit(ns, …)`, +`tx.createTask(kind, …)`, `h.hooks(ns, kind, …)` must be the currently +registered one; a redeclared or unregistered token rejects with +`StaleDefinition`. The namespace *string* is what persists (memo keys, state +slices); the token is only current runtime authority. This is what makes reload +safe: code from a retired generation cannot keep writing, and a replacement +registration of the same string finds the same state and memos. + +Before `resume()`: task kinds whose tasks may exist in storage, and hook +handlers that recovery reruns (`beforeTool`, `systemInstructions`, +`beforeRequest`, `beforeCollapse`). A task whose kind is unregistered at +`resume()` is orphaned (open-time reconciliation, `pico-handoff-v2.md` §13). A +kind registered later only affects tasks created later. Tools, sections, +namespaces and observer hooks may be registered at any time. + +Inside a commit, extensions get: + +```ts +tx.plugins(ns): T // typed slice in the declared document(s), tracked proxy +tx.emit(ns, name: string, data: JsonValue): void // -> { type: `plugin.${ns}.${name}`, data } in this commit's envelope +tx.slot(task): Slot // own task only (kinds); tools/hooks use the api variants +memoOnce(tx.slot(task), key, candidate) // first writer wins; hasOwn semantics (§2) +tx.write(conversationId, entryKind, input): Promise +tx.createTask(kind, input, opts): TaskRef // ordinary kinds +``` + +## 4. Extension points + +Each with what it may do, where its durable state lives, and a worked example +against the raw harness. The three examples are the ones §6 turns into Chord +facets. + +### 4.1 Tools + +`ToolDeclaration` as in the sketch (`name`, `description`, `parameters` TypeBox, +`replay`, `output` bounds, `execute(args: Static

    , api, ctx)`), plus the +tool-facing API: + +```ts +interface ToolApi { + readonly taskId: Id; readonly conversationId: Id; readonly callId: string; + stream(chunk: string | Uint8Array): void; // kernel-bounded; becomes the result content if none returned + progress(update: (slot: ToolSlot) => void, ctx): Promise; // own slot: progress, details, continuedBy + memo(name: string, candidate: T, ctx): Promise; // memoOnce on the own slot + memo(name: string, ctx): Promise; // read + task(kind: K, input: InputOf, opts, ctx): Promise>; + conversation(spec, ctx): Promise; + waitForTask(ref, ctx): Promise>; + slot(ref, ctx): Promise; // another task's slot, read-only snapshot +} +``` + +Durability: the tool's **slot** (`turn.tools[i]`) for streamed output, progress +and memos; **entries** through its result. A tool has no checkpoint; it runs +inside `pi.tool`, whose `started{call}` checkpoint is the effect evidence. + +How memos and `replay` interact: `beforeTool` and the `started` checkpoint +happen before `execute`; a crash after `started` re-invokes only `replay: +"safe"` tools. A safe tool that performs a keyed external effect records the +idempotency key as a memo *before* the effect and reuses it on re-invocation; +the external side dedupes or answers a result lookup for that key. Recording a +result *after* the effect (as the question tool does) is only correct when a +repeated effect is harmless, which asking a human again is. Memos are scoped to +the tool task's slot and retired with it, so they cannot leak into a later call +with the same id. + +Example: a question tool that waits for a human. `ask` is whatever the host +wires in (§6.2 wires a Chord dialog service; a test wires a promise). + +```ts +export function questionTool(ask: (req: QuestionRequest, api: ToolApi, ctx: Context) => Promise): ToolDeclaration { + return { + name: "question", description: "Ask the user a question and wait for the answer.", + parameters: QuestionParamsSchema, replay: "safe", + async execute(params, api, ctx) { + let response = await api.memo("answer", ctx); // answered before a crash? + if (response === undefined) { + const candidate = await ask(params, api, ctx); // may wait for a long time; abort unwinds through ctx + response = await api.memo("answer", candidate, ctx); // first writer wins + } + return response.outcome === "cancelled" + ? { content: [{ type: "text", text: "User cancelled" }], isError: true } + : { content: [{ type: "text", text: `User answered: ${response.answer}` }], details: { ...params, answer: response.answer } }; + }, + }; +} +``` + +Crash before the memo: `pi.tool` recovers `started` (safe), re-invokes, no memo, +asks again. Crash after: re-invokes, memo present, returns without asking. +Two answers racing: both call `api.memo("answer", …)`, both receive the winner. + +### 4.2 Hooks + +Declared per kind, typed (`GenerationHooks`, `ToolHooks`, `PostToolsHooks`, +`CollapseHooks`, plus any custom kind's `H`). Handlers receive +`(…payload, api, ctx)`. `HookApi` is identity only; each point's runner passes +a point-specific info object with exactly the capabilities that point can use: + +```ts +interface HookApi { readonly kind: string; readonly taskId: Id; readonly conversationId: Id } + +interface BeforeToolApi extends HookApi { + readonly callId: string; + waiting(ctx: Context): Promise; // commits turn.tools[i].waitingOn = ; idempotent + memo(name: string, candidate: T, ctx: Context): Promise; // memoOnce on the tool task's slot, keyed (task, namespace, name) + memo(name: string, ctx: Context): Promise; + emit(name: string, data: JsonValue, ctx: Context): Promise; // plugin.. +} +// Hooks are registered with their namespace token (`h.hooks(ns, kind, handlers)`), so `waiting`, `memo` and +// `emit` need no namespace argument and two plugins cannot collide on a memo name. `waitingOn` is cleared by the +// kernel atomically with `started` (handler allowed), with the synthetic result (handler blocked or threw), or in +// its own commit when the handler unwinds on abort or suspend. + +interface ToolHooks { + beforeTool(call: ToolCall, api: BeforeToolApi, ctx: Context): { call?: ToolCall; block?: string } | void | Promise<…>; + afterTool(call: ToolCall, result: ToolResult, api: HookApi & { readonly callId: string }, ctx: Context): ToolResult | void | Promise<…>; +} +// generation points: HookApi & { cutoff } / { attempt } as today; collapse: HookApi. A `memo` on another +// point's hosting-task slot is added to that point's info type when a real case needs it, not by default. +``` + +Durability: a hook has no task of its own. State tied to the invocation it is +running in goes in the **hosting task's slot** via the point's `api.memo`, +keyed by `(hosting task, namespace string, name)` so it survives recovery and +re-registration and cannot collide with another plugin's memo (retired with +that task, no cleanup code); state that outlives tasks goes in a **namespace**. +There is no hook invocation id: nothing durable identifies "the same handler +invocation" across a crash or a reload, and the rerun must find the earlier +decision. Hooks commit through `h.conversation(api.conversationId, ctx)` +handles (never from inside a transaction builder: use the builder's own `ctx` +there, see `hardening-handoff.md` §7) and may wait as long as they like; `beforeTool` handlers rerun after a +crash or a suspend/reopen (they run before `started`), which is why their +decision must be a memo. + +Example: tool approval. + +```ts +export function installApproval(h: Harness, ask: (req: ApprovalRequest, ctx: Context) => Promise): () => void { + const ns = h.namespace<{ autoAllow: string[] }>("approval", { sticky: { autoAllow: [] } }); // long-lived: which tools skip approval; no `view` ⇒ private + + return h.hooks(ns, kinds.tool, { + async beforeTool(call, api, ctx) { + const c = await h.conversation(api.conversationId, ctx); + const auto = await c.commit((tx) => tx.plugins(ns).autoAllow.includes(call.name), ctx); + if (auto) return; + let d = await api.memo("decision", ctx); // decided before a crash / reload? key = (task, "approval", "decision") + if (d === undefined) { + await api.waiting(ctx); // view: turn.tools[i].waitingOn = "approval"; event tool.waiting + const candidate = await ask({ conversationId: c.id, callId: call.id, taskId: api.taskId, name: call.name, args: call.arguments }, ctx); + d = await api.memo("decision", candidate, ctx); // first writer wins + } + return d.decision === "deny" ? { block: `denied${d.by ? ` by ${d.by}` : ""}` } : undefined; + }, + }); +} +``` + +No `afterTool` cleanup: the memo lives in the tool task's slot and is retired +with it. Late joiners see `waitingOn` in the snapshot. Two clients deciding at +once both get the memo winner. Abort or suspend unwinds `ask` through `ctx`; +the kernel clears `waitingOn` when the handler returns for any reason. Asking a +human twice after a crash between `waiting` and the memo is harmless, which is +why a post-effect memo is acceptable here. + +### 4.3 Custom task kinds + +Shape as in the sketch and the hardening handoff: `defineTask({ name, config?, +hooks?, initial, phases, abort, describe? })`, exhaustive phase map, in-flight +phases written before an effect and entered only after reopen, `next` builders +that may checkpoint, terminalize or `"retry"`. + +Durability: + +| need | primitive | +|---|---| +| where to resume after a crash | **checkpoint** (`tx.checkpoint(next)` / returned `next`) | +| progress, partial output, per-run memos | **slot** (`tx.slot(current)`, `memoOnce`) — shown as `tasks[id].status` after `describe()` | +| state that outlives the task, per conversation or session | **namespace** | +| facts for the transcript / model | **entries** via `tx.write` (placed at a boundary when busy) | + +`describe(task): JsonValue` turns checkpoint + slot into the rendering-shaped +status the view publishes; if omitted, the view shows `{ phase }`. + +Example: a background repository indexer, resumable and observable. + +```ts +type IndexInput = { root: string }; +type IndexCheckpoint = { phase: "scanning"; cursor: string | null } | { phase: "writing"; batchKey: string }; +type IndexSlot = { files: number; lastFile?: string; memos?: Record }; + +export const indexer = defineTask({ + name: "repo.indexer", + config: { rewindable: { indexIgnore: [] as string[] } }, + describe: (task) => ({ stage: task.checkpoint?.phase ?? "starting", files: task.slot?.files ?? 0 }), + + async initial(task, rt, ctx) { + await rt.commit((tx) => { tx.slot(task).files = 0; tx.checkpoint({ phase: "scanning", cursor: null }); }, ctx); + return scan(task, null, rt, ctx); + }, + phases: { + scanning: (task, rt, ctx) => scan(task, task.checkpoint.cursor, rt, ctx), + // in-flight: only entered after a crash. The batch key was persisted before the effect; the write is + // repeated with the same key and is safe only because writeBatch(key) is idempotent on the receiving side. + async writing(task, rt, ctx) { + await writeBatch(task.checkpoint.batchKey, ctx); + return { next: { phase: "scanning", cursor: task.checkpoint.batchKey } }; + }, + }, + async abort() { return () => null; }, +}); + +async function scan(task, cursor, rt, ctx) { + const ignore = (await rt.rewindable(task.conversationId, ctx)).indexIgnore; + const batch = await nextBatch(task.input.root, cursor, ignore, ctx); + if (batch === undefined) { + return { done: async (tx, current) => { + await tx.write(current.conversationId, entries.notice, { model: [{ role: "user", content: `Indexed ${tx.slot(current).files} files.`, timestamp: rt.now() }] }); + return { status: "completed", result: { files: tx.slot(current).files } }; + } }; + } + await rt.commit((tx) => tx.checkpoint({ phase: "writing", batchKey: batch.key }), ctx); // idempotency key before the effect + await writeBatch(batch.key, ctx); + return { next: (tx, current) => { + const s = tx.slot(current); s.files += batch.files.length; s.lastFile = batch.files.at(-1); + return { phase: "scanning", cursor: batch.key }; + } }; +} +``` + +Registered with `h.registerTaskKind(indexer)` before `resume()`; started with +`c.commit((tx) => tx.createTask(indexer, { root }, { background: true }), ctx)` +or from a tool via `api.task(indexer, …)`. The view shows +`tasks[id] = { kind: "repo.indexer", background: true, status: { stage, files } }`; +a UI renders it from that alone. + +### 4.4 Custom entries + +```ts +interface CiEntry extends Entry { kind: "ci.result"; data: { run: number; ok: boolean }; model: [UserMessage] } +const ci = defineEntry("ci.result"); + +await c.write(ci, { data: { run: 812, ok: false }, model: [{ role: "user", content: "CI run 812 failed.", timestamp: now }] }, ctx); +const last = await c.commit((tx) => tx.newestEntry(c.id, ci), ctx); // CiEntry | undefined +for (const e of view.entries) if (ci.is(e)) render(e.data.run); +``` + +Omit `model` for bookkeeping the model must not see. `head` and `edits` are +core-only. `defineEntry` throws on a `pi.*` name and `c.write`/`tx.write` reject +`pi.*` at runtime for non-core callers even after a cast, with one allow-listed +exception: `pi.notice` (one user message, optional `data`, no `head`/`edits`), +which hosts, jobs and tools use for notices. No entry registry or schema is +required. Placement is boundary-safe: immediately when idle, otherwise at the +next boundary (`view-and-events.md` §6.1). + +### 4.5 Namespaces + +```ts +const ns = h.namespace<{ planMode: { enabled: boolean }; cache: Record }>( + "plan", + { rewindable: { planMode: { enabled: false } }, sticky: { cache: {} } }, + { view: (s) => ({ planMode: s.planMode }) }, // only this projection reaches the view; `cache` never does +); +await c.commit((tx) => { tx.plugins(ns).planMode.enabled = true; tx.emit(ns, "toggled", { enabled: true }); }, ctx); +// view: plugins.plan.planMode; envelope: ["s", ["plugins","plan","planMode","enabled"], true] + { type: "plugin.plan.toggled", data } +env.own(ns.unregister); // idempotent; state stays, token goes stale +``` + +Defaults are per document; a slice may span documents (`{ rewindable: …, sticky: +… }`) and `tx.plugins(ns)` returns the merged typed object with each key routed +to its document (same mechanism as core `config`). `view` is a pure projection +the kernel re-evaluates when the slice changes; without it the namespace has no +presence in the view. Re-registering the same string with changed defaults adds +new keys lazily and leaves removed keys as inert stored data. + +### 4.6 Owned conversations and jobs + +Subagents (`api.conversation`) and processes (`pi.job` via `api.task(kinds.job, …)`) +are core features with their own durability; extensions use them, they do not +extend them. See `pico-handoff-v2.md` §8, §10.5 and the hardening handoff §15. + +## 5. Unregister, re-register, reload + +Every registration can be undone and redone while the harness is open. + +| contribution | on unregister | on re-register | +|---|---|---| +| namespace (`ns.unregister()`, idempotent) | token stale; `tx.plugins`/`tx.emit`/`h.hooks(ns, …)` with it reject | new token for the same string; stored state and memos untouched; new defaults seeded lazily | +| hooks | removed from future chains; a handler already running continues to completion (it is cancelled only by abort or `suspend`) | joins the chain | +| task kind | pending tasks of the kind are not dispatched (not orphaned; orphaning is open-time only); running invocations keep the kind object they captured | pending tasks dispatch with the new implementation; config keys re-checked | +| tool / section | revision bump; in-flight invocations keep their captured declaration; preparation snapshots retry | same | +| entry kind | nothing | nothing | + +Reload replaces code, never durable state. Two modes, chosen by the host: + +- **Quiescent reload.** `h.quiescent()` is true when the scheduler holds **no + invocation**, running or sleeping: no phase handler, no `retrying`/`deferred` + sleep, no job poll, no waiting `beforeTool`. Live task *records* may exist + (pending, or between invocations). Then: `release = h.hold()` (defers new + dispatch and new hook chains; commits and reads continue) → deactivate old + facets (their disposals unregister) → activate new ones (re-register) → + `release()`. +- **Suspend and reopen** when not quiescent (the first safe version for active + work). `h.suspend(ctx)`: stop dispatch, cancel every in-process invocation + context, await unwind, close storage; task records stay live and nothing is + terminalized. Reopen with the new registrations and let ordinary recovery run: + pre-effect phases rerun, `started` tools replay if safe and still safe, unsafe + ones become interrupted results, other phases recover per their kind. Memos + make reruns cheap (an approval already decided is not asked again). The + injected `ProcessHost` must outlive the harness instance so running jobs + reconcile through `status(key)` instead of becoming `unknown`. + +Cancelling live calls of a tool or kind that the new generation no longer +registers is not a transparent migration: it durably aborts them, cannot undo +external effects, and post-tools observes aborted results. It is an explicit +host policy (`suspend({ abortRemoved: true })`), never the default. Generation +leases (old registrations kept alive until captured invocations drain while new +work uses replacements) are a possible later refinement; they are not required +for v1. + +Registration belongs in `onActivate`, not `setup`: during a shape-preserving +reload the replacement's `setup` runs before the old facet is deactivated, and +two live registrations of one namespace string would collide. + +## 6. The Chord layer + +The coding agent composes the harness and its extensions as Chord facets +(`packages/agent/docs/plugins.md`). Nothing in §1–§5 changes; this section only +says which facet calls what. + +### 6.1 Built-in facets (plumbing) + +Provided once by the coding agent, consumed by every feature: + +```ts +// session worker +export const Harness = defineService("pi.harness", { local: true }); // the in-process handle from §3; never remote +export const ConversationService = defineService<{ // keyed by conversation id; remote + view: ReplicatedState; // view-and-events.md; one publish per commit, on the line + send(input: SendInput, ctx): Promise; write(entry: NewEntry, ctx): Promise; + inputAbort(id: Id, ctx): Promise<"aborted" | "already_placed" | "not_found">; + configSet(patch: Partial, ctx): Promise; + abort(ctx): Promise; reset(handoff: string | undefined, ctx): Promise; collapse(instructions: string | undefined, ctx): Promise; + fork(at: Id | "start", spec: ForkSpec, ctx): Promise; entries(scan: EntryScan, ctx): Promise; +}>("pi.conversation"); + +export const harnessSessionFacet = defineFacet({ + id: "@pi/harness", + setup(env) { + const conversations = env.provideMany(ConversationService); + const handle = createHarnessHandle(); // wraps Harness.open/resume/hold and the registration surface + env.provide(Harness, handle); + env.onActivate(async () => { + await handle.open(storage, { models, processHost }, ctx); + handle.onConversation((c) => { // existing at open, and every fork/subagent created later + const view = env.replicatedState(c.snapshot()); // built on the line + c.attachView(view); // bounded ordered adapter: raw envelopes → view.state + publish(), off the line (below) + env.own(conversations.spawn(String(c.id), { view, send: (i, cx) => c.send(i, cx).then((r) => r.id), /* … */ })); + }); + }); + // resume() is called by the session worker after the facet host reports activation complete (all registrations done) + }, +}); +``` + +`ConversationView` gains `commit: { events }` so events ride inside the +replicated member; a subscriber reads `value.commit.events` after each delivery. +No revision or storage sequence is placed in the view: Chord's +`ReplicatedStateDelivery.sequence` is contiguous per member and plays that role. + +Publication must not happen on the Session line. Chord's `publish()` invokes +source and subscriber listeners synchronously and a subscriber may throw or be +slow, and `subscribe()` itself publishes pending mutations. The bridge is a +bounded ordered adapter: the kernel enqueues raw envelopes from its own view +tracker; the adapter, off the line, applies one envelope's ops to +`view.state`, sets `commit.events`, and calls `publish(ctx)` in the **same +synchronous block** (no await between apply and publish). One raw envelope = +one publish = one Chord sequence. If the queue overflows or `publish` throws, +close that keyed instance and respawn it with a fresh snapshot; a persisted Pico +commit is never turned into a failure. Chord's subscription snapshot then gives +late joiners the member at its current sequence, so §9 of +`view-and-events.md` is implemented by Chord, not by us. + +### 6.2 A feature: contract + session facet + presentation facet + +The rule from `plugins.md`: one shared contract module, one facet per host, +connected by service IDs. The session facet calls §3; the presentation facet +observes services and the view. Facets keep **no durable state**: everything +durable is in the harness through §2, and keyed service instances are channels. + +**Approval** (hook + dialog service): + +```ts +// contract.ts +export interface ApprovalRequest { conversationId: Id; callId: string; taskId: Id; name: string; args: JsonValue } +export type ApprovalDecision = { decision: "allow" | "deny"; by?: string }; +export interface ApprovalDialog { readonly request: ReplicatedState; decide(d: ApprovalDecision, ctx: Context): Promise } +export const ApprovalDialogs = defineService("pi.approval-dialog"); // keyed `${conversationId}:${callId}` + +// session.ts +export const approvalSessionFacet = defineFacet({ + id: "@pi/approval", + setup(env) { + const harness = env.use(Harness); + const dialogs = env.provideMany(ApprovalDialogs); + env.onActivate(() => { + env.own(installApproval(harness, (req, ctx) => { // §4.2, unchanged + const completion = Promise.withResolvers(); + const close = dialogs.spawn(`${req.conversationId}:${req.callId}`, { + request: env.replicatedState(req), + async decide(d, _ctx) { completion.resolve(d); }, + }); + return awaitAbortable(completion.promise, ctx.abortSignal).finally(close); + })); + }); + }, +}); + +// tui.ts +export const approvalTuiFacet = defineFacet({ + id: "@pi/approval", + setup(env) { + const tui = env.use(Tui); + env.observe(ApprovalDialogs, async (dialog, ctx) => { // one task per open instance, incl. hydrated ones + const r = dialog.request.value!; + const modal = await tui.acquireModal(ctx.abortSignal); + try { await dialog.decide({ decision: (await modal.confirm(`Allow ${r.name}?`, pretty(r.args))) ? "allow" : "deny", by: "tui" }, ctx); } + finally { modal.close(); } + }); + }, +}); +``` + +The decision memo (`api.memo("approval", …)`) is written by the hook in the +session worker, so `decide` only resolves the local promise; the first-writer +rule is applied where the durability is. A second client's `decide` resolves an +already-resolved promise: harmless. + +**Question tool** (tool + dialog service): identical shape with `questionTool` +from §4.1 as the session contribution (`harness.registerTool(questionTool(ask))`) +and `ToolApi.memo` instead of `BeforeToolApi.memo`. This is the `packages/agent/docs/plugins.md` +question example with `memoOnce` replaced by the slot memo. + +**Indexer** (custom kind, no service of its own): + +```ts +// session.ts +export const indexerSessionFacet = defineFacet({ + id: "@pi/indexer", + setup(env) { + const harness = env.use(Harness); + env.onActivate(() => { env.own(harness.registerTaskKind(indexer)); }); // before resume(): tasks may exist in storage + }, +}); +// tui.ts: nothing to provide; render view.tasks[id].status for kind "repo.indexer" from ConversationService.view, +// and offer a /index slash command that calls conversation.commit-equivalent RPC (or a tiny IndexerService.start method). +``` + +### 6.3 What the session worker does at reload + +```text +if (harness.quiescent()) { + release = harness.hold() + FacetHost.reload(candidates) // old facets' env.own() disposals unregister; new onActivate re-registers + release() +} else { + await harness.suspend(ctx) // cancel invocations, close storage; task records stay live + FacetHost.reload(candidates) + await harness.reopen(ctx) // recovery runs against the new registrations +} +``` + +Durable state, task records and open conversations are untouched either way +(§5). + +## 7. Tests the hardening must include + +- Memo: two concurrent `api.memo` writers receive the same winner; `null` is a + stored value (hasOwn, not `??=`); a crash after the memo skips `ask`, before it + re-asks; the memo is gone after the tool task terminalizes; two namespaces + using the same memo name on one task do not collide. +- Hooks: `await api.waiting(ctx)` sets `waitingOn` in its own commit and it is + cleared atomically with `started`, with the synthetic result on block/throw, + and in its own commit on abort/suspend; the decision memo survives a crash + between `waiting` and `started`; memos never appear in the view. +- Custom kind: crash in the in-flight phase re-enters it and repeats the keyed + effect (test with a fake idempotent sink and a crash between effect and the + next checkpoint); `describe()` output appears in `tasks[id].status`; + unregistered kind leaves pending tasks pending and orphans only at open. +- Namespaces: writes outside `plugins[ns]` reject; stale token rejects; + `unregister` is idempotent; re-register keeps values and memos and seeds new + defaults; only the `view` projection appears in the view; rewindable slice is + inherited by a fork and readable at an entry. +- Entries: `defineEntry("pi.x")` throws; `write` of any `pi.*` except `pi.notice` + rejects at runtime after a cast; `pi.notice` with `head`/`edits` rejects. +- Reload: `quiescent()` is false while any invocation exists (including a + waiting `beforeTool` and a polling job); `hold()` defers dispatch and hook + chains but not commits; suspend/reopen with the approval facet lets a pending + call be approved through the new generation without asking twice when a + memo exists; `ProcessHost` survives suspend/reopen and a running job + reconciles. +- Chord: loopback facet tests for approval and question (session + presentation + fixtures), late-joiner hydration of an open dialog instance, and + `ConversationService.view` sequence equals commit sequence. diff --git a/packages/agent/docs/pico/v3/view-and-events.md b/packages/agent/docs/pico/v3/view-and-events.md new file mode 100644 index 000000000..983256a88 --- /dev/null +++ b/packages/agent/docs/pico/v3/view-and-events.md @@ -0,0 +1,525 @@ +# Pico3 view and events + +Status: proposal for the hardened V3 (`pico3`) watch protocol. Replaces the +`ConversationView { entries, tasks, rewindable, sticky }` shape and the per-channel +delta of the V3 sketch. Written against `harness-v3.zip` and the Codex app-server +protocol (`codex-rs/app-server-protocol/src/protocol/v2/*`), read in full. +Companion: `plugins.md` (harness extension surface, durability, Chord facets). + +## 1. Principles + +1. **One snapshot, then one envelope per commit.** A watch captures the view and + subscribes in one line operation; every later commit that touches the + conversation produces exactly one envelope, delivered in commit order with a + contiguous per-watch `revision`. No cursor, replay or acknowledgement: a gap + means "open a fresh watch". +2. **Ops are the truth, events are annotations.** The envelope carries Chord ops + that transform the view document, and named events the kernel attaches because + it produced those ops and knows why. Ops fully determine the replica: a client + that folds only ops is never wrong. Events annotate cause and outcome and may + carry transient facts the view deliberately does not retain; a client that + consumes only events never scans the view. +3. **Path = event.** Every field of the view has a stable path, so an op on that + path is a typed notification. Nothing in the view is ever discovered by + diffing. +4. **The view is rendering-shaped, not kernel-shaped.** No task records, no + checkpoints, no documents. In-progress work is described by the kind that owns + it, in the shape a renderer wants. +5. **Atomic by construction.** Everything a commit changes lands in one envelope: + the assistant entry, the cleared streaming slot, the resolved inputs and the + `turn.ended` event are never observable separately. This is the one property + Codex's protocol lacks (`item/completed` and `turn/completed` are separate + messages). + +## 2. The view document + +One JSON document per conversation. The kernel keeps a Chord tracker for it per +watched conversation, mutates it through the proxy after each commit, and +`flush()`es the ops into the envelope. Clients fold with `applyImmutable`. + +```ts +interface ConversationView { + conversation: { id: Id; parent?: { conversationId: Id; at: Id }; owner?: Id }; + + /** Active transcript: newest head H, then every fork-visible entry with id >= H.head, + * chronological. All fork-visible entries when there is no head. See §2.1. */ + entries: Entry[]; + + /** Resolved configuration, flat: every built-in and registered kind's declared keys, + * defaults applied. `model` is the only optional built-in. */ + config: { + model?: ModelRef; thinkingLevel: ThinkingLevel; selectedTools: string[]; profile: string; + threshold: number; keepRecent: number; + retry: RetryPolicy; steeringMode: QueueMode; followUpMode: QueueMode; + [pluginKindKey: string]: JsonValue; + }; + + /** Queued steer / followUp / passive writes, admission order. */ + inbox: QueuedInput[]; + + /** Present while an input group is live (from placement to its end); absent when idle. */ + turn?: TurnView; + + /** Present while a pi.collapse task is live. */ + compaction?: { taskId: Id; reason: "threshold" | "manual" | "overflow"; stage: "summarizing" | "retrying"; attempt: number; retryAt?: number }; + + /** Non-turn live tasks (jobs, plugin kinds), described by their kind. Removed on terminal. */ + tasks: { [taskId: string]: { kind: string; background?: true; marked?: true; status: JsonValue } }; + + /** Plugin presentation slices, by namespace: the namespace's declared `view` projection, never raw state. */ + plugins: { [namespace: string]: JsonValue }; +} + +interface TurnView { + inputs: Id[]; // the input group + generation?: GenerationStatus; // present while a pi.generation is live + message?: AssistantMessage; // streaming assistant message (frames applied in place) + tools: ToolSlot[]; // tool calls of the newest assistant message in this turn, by call index +} + +type GenerationStatus = + | { stage: "waiting"; on: "compaction" } // replacement generation pending behind a collapse + | { stage: "preparing" } // before `prepared` + | { stage: "requesting"; attempt: number } // request sent, no frame yet + | { stage: "streaming"; attempt: number } // first frame applied + | { stage: "retrying"; attempt: number; retryAt: number; lastError: string } + | { stage: "deferred"; attempt: number; pollAt: number }; + +interface ToolSlot { + callId: string; name: string; args: JsonValue; + status: "pending" | "running" | "done" | "error" | "aborted"; + waitingOn?: string; // namespace of the beforeTool handler that awaited BeforeToolApi.waiting(ctx); cleared when it returns + output?: string; // bounded stream so far (api.stream) + progress?: string; details?: JsonValue; + continuedBy?: Id; // background task that took over; see tasks[continuedBy] + entry?: Id; // the pi.tool_result entry once landed +} +// Memos and other slot working state are never in the view (plugins.md §2); only the fields above are rendered. +``` + +### 2.1 Active transcript + +- `H` = newest fork-visible entry with a `head`; `from` = `H.head`. +- `entries` = every fork-visible entry with `id >= from`, chronological. Inherited + parent entries are included when `from` points into a parent. +- Older head entries inside the range stay (a renderer may mark them). Display-only + entries (`pi.assistant` with `stopReason` `error`/`aborted`, `pi.usage`, model-less + plugin entries) stay. `edits` are carried verbatim; the view is the transcript, + not the model projection. +- A new head entry truncates: the kernel emits one splice removing everything + before `head` and one appending the entry (§4, `entries`). +- No `limit` anywhere on the public surface; the capture pages internally. + +### 2.2 What is deliberately not in the view + +- `Task` records, checkpoints, outcomes: available through `getTask`/`scanTasks` + for tooling and tests, never pushed. +- The two storage documents (`rewindable`, `sticky`): their split is a fork/history + property; the view flattens their keys into `config`, `inbox`, `turn`, `tasks`, + `plugins`. Which key lives in which document is declared per kind and matters + only to `stateAt(entry)` / `fork(at)`. +- Section seed metadata on conversations. + +## 3. Envelope + +```ts +interface Envelope { + revision: number; // contiguous per watch: snapshot.revision + 1, +2, …; storage Seq is not exposed + ops: Op[]; // Chord ops over ConversationView (§4) + events: ViewEvent[]; // named annotations for this commit (§5); may be empty +} +``` + +`revision` is per watch, not the storage commit sequence: a conversation watch +sees only commits that touch its conversation, so the global `Seq` would have +legitimate gaps. Storage `Seq` stays internal unless a public commit-correlation +use appears. + +Delivery: folded into the kernel's view tracker on the line; the listener is +invoked **synchronously, in order, off the line**, one envelope per commit that +changed anything visible to this conversation. Envelopes arriving before +`start()` are buffered (bounded; overflow closes the watch). Asynchronous +consumers (the Chord bridge, application queues) own their own queue and +backpressure policy (`plugins.md` §6.1). A listener throw closes that watch and +reports through `onError`. `stop` is idempotent; nothing is delivered after +`stop`. + +Base flush: after the kernel rebases the view tracker (reopen of a cold watch is a +fresh snapshot instead), an envelope may carry `["r", view]` as its only op. +Clients must handle it as "replace everything". Events in that envelope are +still valid. + +## 4. Ops by path + +Chord op vocabulary: `["r", v]` replace root, `["s", path, v]` set, `["d", path]` +delete, `["a", path, str]` string append, `["t", path, n]` string truncate, +`["p", path, index, deleteCount, items]` array splice. + +| path | ops you will see | meaning | +|---|---|---| +| `["entries"]` | `["p", ["entries"], n, 0, [e…]]` | entries appended (always at the end) | +| `["entries"]` | `["p", ["entries"], 0, k, []]` then `["p", …, n, 0, [head]]` | new head: transcript truncated, head entry appended | +| `["config", key]` | `s` / `d` | config changed; `config.reset(keys)` sets the declared default (only `model`, which has none, is deleted) | +| `["inbox"]` | `p` | queued, placed (removed), aborted (removed) | +| `["turn"]` | `["s", ["turn"], {…}]` / `["d", ["turn"]]` | input group started / ended | +| `["turn","generation"]` | `s` / `d` | generation stage changes / generation ended | +| `["turn","message"]` | `s` (first frame), `a`/`s` on content paths, `d` | streaming assistant message | +| `["turn","message","content",i,"text"]` | `a` | text delta | +| `["turn","message","content",i,"arguments"]` | `s` | tool-call arguments materialized at checkpoint/end | +| `["turn","tools"]` | `s` (whole array, with the assistant entry) | tool slots created for a new assistant message | +| `["turn","tools",i,"status"]` | `s` | pending → running → done/error/aborted | +| `["turn","tools",i,"waitingOn"]` | `s` / `d` | a `beforeTool` handler (namespace = value) is waiting on a human/service; cleared when it returns | +| `["turn","tools",i,"output"]` | `s` | bounded stream flush (whole string; retention may drop the head) | +| `["turn","tools",i,"progress"]`, `[…,"details"]`, `[…,"continuedBy"]` | `s` | `api.progress` | +| `["compaction"]` | `s` / `d` | collapse started / stage changed / ended | +| `["tasks", id]` | `s` / `d` | non-turn task appeared / status changed / terminal | +| `["tasks", id, "status", …]` | any | kind-described status detail (jobs: stdout, exitCode, …) | +| `["plugins", ns, …]` | any | the namespace's `view` projection | + +Rule for kernel authors: mutate the tracked view through its proxy so the +tracker records minimal ops; never rebuild subtrees. Streaming applies each +frame to `turn.message` in place (same reducer as pi-ai's frame reducer). + +## 5. Events + +Events are emitted by core code inside the commit that causes them +(`tx.emit(event)`, core authority). Extensions emit `{ type: "plugin..", … }` +through their namespace token from their own commits. Rule: ops fully determine +the replica; events annotate the cause and outcome of the commit and may carry +transient facts the view does not retain (`turn.ended.reason`, +`task.ended.outcome`, `compaction.failed`, `warning`). An event must never be the +only place a late joiner could learn something needed to render steady state. + +```ts +type ViewEvent = + // transcript + | { type: "entry.added"; entry: Entry } // one per appended entry, in order + | { type: "head.moved"; entry: Entry } // summary / reset / handoff landed + // input group + | { type: "turn.started"; inputs: Id[] } + | { type: "turn.ended"; inputs: Id[]; status: "done"; answer: Id } + | { type: "turn.ended"; inputs: Id[]; status: "unanswered"; reason: "terminated" | "aborted" | "failed" | "stale"; detail?: string } + | { type: "input.queued"; input: Id; mode: "steer" | "followUp" | "write" } + | { type: "input.placed"; input: Id; entry: Id } + | { type: "input.aborted"; input: Id } + // generation + | { type: "generation.started"; taskId: Id; attempt: number } // request sent + | { type: "generation.retrying"; taskId: Id; attempt: number; retryAt: number; error: string } + | { type: "generation.deferred"; taskId: Id; pollAt: number } + | { type: "generation.completed"; taskId: Id; entry: Id; toolCalls: number } + | { type: "generation.failed"; taskId: Id; reason: "provider" | "overflow" | "retries_exhausted" | "no_model"; detail: string; entry?: Id } + // tools + | { type: "tool.waiting"; taskId: Id; callId: string; on: string } // a beforeTool handler is waiting; `on` is its namespace + | { type: "tool.started"; taskId: Id; callId: string; name: string } + | { type: "tool.finished"; taskId: Id; callId: string; entry: Id; isError: boolean; control?: ToolControl } + | { type: "tool.aborted"; taskId: Id; callId: string; entry: Id } + // compaction + | { type: "compaction.started"; taskId: Id; reason: "threshold" | "manual" | "overflow"; through: Id } + | { type: "compaction.retrying"; taskId: Id; attempt: number; retryAt: number; error: string } + | { type: "compaction.finished"; taskId: Id; summary: Id } + | { type: "compaction.failed"; taskId: Id; reason: "stale" | "declined" | "provider" | "retries_exhausted" | "no_model"; detail: string } + // other tasks + | { type: "task.started"; taskId: Id; kind: string; background?: true } + | { type: "task.ended"; taskId: Id; kind: string; outcome: "completed" | "failed" | "aborted" | "orphaned" } + // settings and diagnostics + | { type: "config.changed"; keys: string[] } + | { type: "warning"; source: string; message: string } // hook skipped, tool output truncated, selected tool missing, … + | { type: `plugin.${string}`; data: JsonValue }; +``` + +`warning` has no state counterpart at all; it exists for exactly the +occurrences Codex models as `warning` / `error{willRetry}` notifications. + +## 6. Commit-by-commit mapping + +What each kernel moment writes, the ops it produces, and the events attached. +Ops are abbreviated; every row is one atomic envelope. + +### 6.1 Admission + +| moment | ops | events | +|---|---|---| +| `send`, idle | `entries +pi.user`; `turn = { inputs, tools: [] }`; (older queued items placed first: `inbox` splice, `entries +…`) | `input.placed`, `entry.added`, `turn.started` | +| `send`, busy (`steer`/`followUp`) | `inbox +item` | `input.queued` | +| `write`, idle | `entries +entry` (+ head truncation if it carries a head) | `entry.added` (+ `head.moved`) | +| `write`, busy | `inbox +item` | `input.queued{mode:"write"}` | +| `InputHandle.abort` while queued | `inbox -item` | `input.aborted` | +| `config.set` | `config.` set/delete | `config.changed{keys}` | + +### 6.2 Generation + +| moment | ops | events | +|---|---|---| +| task created (successor/replacement) | `turn.generation = { stage: "preparing" }` (turn already present) | `task.started` is **not** emitted for turn kinds | +| `prepared` commit | optional `entries +pi.system`; `turn.tools = []`; `turn.message` deleted; `turn.generation = { stage: "requesting", attempt }` | `entry.added` (system) | +| `requesting` checkpoint | `turn.generation.attempt` | `generation.started` | +| frames (coalesced ≥256 B / 100 ms) | first: `turn.message = partial`, `turn.generation.stage = "streaming"`; then `a`/`s` on content paths | none | +| retry decision | `entries +pi.usage`; `turn.message` deleted; `turn.generation = { stage: "retrying", … }` | `entry.added`, `generation.retrying` | +| deferred | `turn.generation = { stage: "deferred", pollAt }` | `generation.deferred` | +| response with tool calls (closure) | `entries +pi.assistant`; `turn.message` deleted; `turn.generation` deleted; `turn.tools = slots(status:"pending")`; (threshold: `compaction = {…}`) | `entry.added`, `generation.completed{toolCalls:n}`, (`compaction.started`). Slots are `pending`; `tool.started` fires at each tool's `started` checkpoint (§6.3) | +| final answer, no triggers, no continuation (closure) | `entries +pi.assistant`; `turn` deleted; boundary placements (`inbox` splices, `entries +writes`) | `entry.added`, `generation.completed`, `turn.ended{done, answer}`, `input.placed`/`entry.added` for placed writes | +| final answer, `onYield` continuation | `entries +pi.assistant`, `entries +pi.user{continuation}`; `turn.generation = { stage: "preparing" }` | `entry.added`×2, `generation.completed`; **no** `turn.ended` (same group continues) | +| final answer with queued followUp/steer triggers | as "no triggers", then `entries +pi.user`, `turn = { inputs: triggers, tools: [] }`, `turn.generation = { stage:"preparing" }` | `turn.ended{done}`, `input.placed`, `entry.added`, `turn.started` | +| terminal error / aborted-by-provider (closure) | `entries +pi.assistant` (display-only; `model` absent, message in `data`); group resolved `unanswered/failed`; **final boundary runs**: placed writes, selected triggers; `turn` deleted, or replaced by a successor turn for triggers | `entry.added`, `generation.failed{provider|retries_exhausted, entry}`, `turn.ended{unanswered, failed}`, then `input.placed`/`entry.added`/`turn.started` as in "with triggers" | +| `no_model`, overflow with nothing collapsible | as above; overflow additionally `entries +pi.notice` (placed immediately: the closing task counts as gone). `no_model` is not special-cased: a successor for queued triggers fails fast and resolves its own group | `generation.failed`, `turn.ended{unanswered, failed}`, `entry.added`, … | +| overflow with collapse (closure) | `compaction = { reason: "overflow", … }`; `turn.generation = { stage: "waiting", on: "compaction" }` (replacement created `after` collapse) | `generation.failed{overflow}`, `compaction.started`; turn continues | +| abort (fresh abort closure) | optional `entries +pi.assistant` (display-only partial, only if content was streamed); `turn` deleted | (`entry.added`), `turn.ended{unanswered, aborted}` | + +### 6.3 Tool + +| moment | ops | events | +|---|---|---| +| `beforeTool` handler awaits `api.waiting(ctx)` (`BeforeToolApi`, `plugins.md` §4.2) | `turn.tools[i].waitingOn = ns` (its own commit) | `tool.waiting{callId, on: ns}` | +| `started` checkpoint (after `beforeTool` allowed, before invocation) | `turn.tools[i].waitingOn` deleted; `turn.tools[i].status = "running"` (one commit) | `tool.started` | +| handler blocks/throws | `waitingOn` deleted atomically with the synthetic error result below | | +| handler unwinds on abort/suspend | `waitingOn` deleted in its own commit | none | +| stream flush (≤ every 100 ms) | `turn.tools[i].output = text` | none | +| `api.progress` | `turn.tools[i].{progress,details,continuedBy}` | none | +| result (closure) | `entries +pi.tool_result`; `turn.tools[i].status = done|error`; `turn.tools[i].entry` | `entry.added`, `tool.finished{isError, control}`, (`warning` if truncated) | +| synthetic error before invocation (not offered / missing / invalid / blocked) | same as result with `status: "error"` | `entry.added`, `tool.finished{isError:true}` | +| abort closure | `entries +pi.tool_result(aborted)`; `turn.tools[i].status = "aborted"` | `entry.added`, `tool.aborted` | +| recovery: `started` + unsafe/current-unsafe/schema-invalid | as synthetic error (`interrupted`) | `tool.finished{isError:true}` | + +### 6.4 Post-tools + +| moment | ops | events | +|---|---|---| +| continuation (default) | boundary placements (`inbox` splices, `entries +pi.user` for steer, `+writes`); `turn.inputs += triggers`; `turn.generation = { stage: "preparing" }` | `input.placed`…, `entry.added`…; no turn event | +| synthesized missing results (orphaned/aborted tool) | `entries +pi.tool_result` per missing call | `entry.added`, `warning{source:"post_tools"}` | +| `addTools` | `config.selectedTools` | `config.changed{["selectedTools"]}` | +| `terminate` | `turn` deleted; boundary(final) placements; successor turn if triggers | `turn.ended{done, answer: assistant}`, then as §6.2 "with triggers" | +| `handoff` | `entries` truncated + `+pi.handoff` (head); `turn` deleted; placements; successor turn | `head.moved`, `entry.added`, `turn.ended{done}`, … | +| queued self-head write ended the group | `entries` truncated + `+reset/handoff`; `turn` deleted; earlier queued steer/followUp removed from `inbox` | `head.moved`, `turn.ended{unanswered, terminated}`, `input.aborted`… (stale) | +| abort closure | `turn` deleted | `turn.ended{unanswered, aborted}` | + +### 6.5 Collapse + +| moment | ops | events | +|---|---|---| +| task created | `compaction = { taskId, reason, stage: "summarizing", attempt: 1 }` | `compaction.started{through}` | +| retry | `compaction.stage = "retrying"`, `retryAt` | `compaction.retrying` | +| summary lands (closure) | `entries` truncated to `head` + `+pi.summary`; `compaction` deleted | `head.moved`, `entry.added`, `compaction.finished{summary}` | +| stale / declined / failed | `compaction` deleted | `compaction.failed{reason}` | +| hook-supplied summary | same as "summary lands" without a provider stage | same | + +Manual collapse (`c.collapse`) is `background`; it appears in `compaction` +exactly the same way. At most one live collapse and exactly one live generation +per conversation are core invariants (rejected before persistence, never +generalized to maps in the view): a manual collapse while one is live rejects +`CollapseInProgress`; a threshold trigger while one is live is skipped; +`NothingToCollapse` rejects the call and produces no envelope. An overflow +replacement generation waits behind its collapse as `turn.generation = { stage: +"waiting", on: "compaction" }`. + +### 6.6 Jobs and plugin kinds + +| moment | ops | events | +|---|---|---| +| task created | `tasks[id] = { kind, background, status: describe(task) }` | `task.started` | +| checkpoint / slot update | `tasks[id].status…` (job: `stage`, `stdout`, `stderr`, `exitCode`, `occurrence`) | none | +| job notice (`notify: true`) | `entries +pi.notice` (immediately if idle, else via `inbox`) | `entry.added` or `input.queued{write}` | +| terminal | `tasks[id]` deleted | `task.ended{outcome}` | +| marked | `tasks[id].marked = true` | none | + +`describe(task)` is declared on the kind next to its checkpoint union; the +kernel calls it whenever the task record changes and writes the result to +`tasks[id].status`. Turn kinds do not use it: their status lives in `turn`. + +### 6.7 Conversation-level + +| moment | ops | events | +|---|---|---| +| `abort(conversation)` | `inbox` steer/followUp removed; then per-task envelopes as above | `input.aborted`…, then `turn.ended{aborted}` etc. | +| fork created | (new conversation; own watch) | none on the parent | +| owned conversation created by a tool | `turn.tools[i].details` if the tool records it; nothing else on the parent | none | + +## 7. Codex mapping + +For anyone coming from `codex app-server`. + +| Codex | pico3 | +|---|---| +| `Thread` | conversation; `thread/status/changed` ≈ `turn` present/absent (+ `tasks` non-empty for background activity) | +| `Turn { id, items, status, error }` | `turn` (live) + `turn.ended{status}`; items are the entries appended during the group | +| `turn/started`, `turn/completed` | `turn.started`, `turn.ended` (in the same envelope as the last entry, never separate) | +| `ThreadItem::UserMessage` | `pi.user` entry | +| `ThreadItem::AgentMessage` + `item/agentMessage/delta` | `turn.message` (streaming) → `pi.assistant` entry; ops on `["turn","message",…]` | +| `ThreadItem::Reasoning` + deltas | thinking blocks inside `turn.message.content` / the assistant entry | +| `ThreadItem::CommandExecution { status, aggregatedOutput, exitCode }` + `item/commandExecution/outputDelta` | `turn.tools[i]` (`status`, `output`) → `pi.tool_result` entry | +| `ThreadItem::McpToolCall`, `DynamicToolCall`, `FileChange` | same: a tool slot; the tool declaration decides `details` | +| `ThreadItem::SubAgentActivity`, `CollabAgentToolCall` | tool slot with `details`/`continuedBy`; the child is its own conversation/watch | +| `ThreadItem::ContextCompaction` | `compaction` (live) + `pi.summary` entry + `compaction.finished` | +| `error { willRetry: true }` | `generation.retrying` **and** `turn.generation.stage = "retrying"` (visible to late joiners) | +| `error { willRetry: false }` | `generation.failed`, `turn.ended{failed}` | +| `thread/tokenUsage/updated` | `usage` on the assistant entry; `pi.usage` entries for failed attempts | +| `warning`, `deprecationNotice` | `warning` event | +| `thread/queue/changed` | ops on `["inbox"]`, `input.*` events | +| `thread/settings/updated` | ops on `["config",…]`, `config.changed` | +| `thread/resume` (history + `active_turn_snapshot`) | `watch` snapshot: `entries` + `turn` captured atomically with the subscription | +| `thread/items/list`, `thread/turns/list` pagination | `h.entries(scan)` for history beyond the active transcript | +| `process/outputDelta`, `process/exited` | `tasks[id].status` for `pi.job` | +| `item/started` → `item/completed` (same id) | tool: `callId` is stable across the slot and the result entry; assistant: atomic swap, no id needed | + +What pico3 has that Codex does not: atomic multi-field envelopes; retry and +compaction state visible in a snapshot; one replica protocol for state-driven and +event-driven clients. + +What Codex has that pico3 does not: turn-level unified diff (`turn/diff/updated`), +plans, approvals (`waitingOnApproval`). Approvals are a `beforeTool` hook waiting +on user input; the kernel shows that as `turn.tools[i].waitingOn` and the +`tool.waiting` event (see `plugins.md` §4.2 for the approval plugin). + +## 8. Consumption patterns + +Event-driven (imperative UI, third-party client): + +```ts +w.start(({ ops, events }) => { + view = applyImmutable(view, ops); // keep the replica current for lookups + for (const e of events) switch (e.type) { + case "entry.added": transcript.append(e.entry); break; + case "turn.started": status.set("working"); break; + case "turn.ended": status.set(e.status === "done" ? "idle" : `stopped: ${e.reason}`); break; + case "generation.retrying": status.set(`retrying at ${e.retryAt}`); break; + case "compaction.started": banner.show("compacting…"); break; + case "compaction.finished": banner.hide(); break; + case "tool.started": transcript.showRunning(e.callId); break; + case "tool.finished": transcript.settle(e.callId, e.entry); break; + } +}); +``` + +State-driven (React/Solid): `useSelector(v => v.turn?.message)`, +`useSelector(v => v.compaction)`. `applyImmutable` preserves identity of untouched +branches, so selectors are O(1) and no traversal happens. + +Path router (when neither fits): dispatch each op on its first one or two path +segments; the table in §4 is the complete list of paths. + +Presentation services (Chord RPC): keep a derived tracked document in the shape +the client wants (e.g. Codex-style `turns[]`), update it from envelopes, ship its +own ops. Project image blobs out of `entries[].model` there, not in the kernel. + +## 9. Watchers, late joiners, multiple clients + +### 9.1 One view per conversation, many watchers + +- The kernel keeps **one** tracked view document per conversation that has at + least one watcher (not one per watch). It is built on the line the first time + a watch is opened and dropped when the last watch stops. +- After every commit that touches the conversation, the kernel mutates that one + document through its proxy, `flush()`es once, attaches the commit's events, and + hands the same `Envelope` to every watcher. Fan-out cost is per conversation; + the kernel never knows how many clients exist or what they render. +- `watch(ctx)` returns `{ view, revision, start, stop }` where `view` is a + `structuredClone` of the tracker target taken **on the line**, `revision` is + this watch's starting revision, and the subscription is installed in the same + line operation. The first envelope a watcher receives is therefore exactly + `revision + 1`; there is no window in which a commit can land between snapshot + and subscription. + +### 9.2 Late joiners + +A client that attaches mid-flight needs no history and no events: everything +in progress is state and is in the snapshot. + +| in flight at attach time | where the late joiner finds it | +|---|---| +| assistant message streaming | `turn.message` (partial), `turn.generation.stage === "streaming"` | +| provider backoff | `turn.generation = { stage: "retrying", retryAt, lastError }` | +| tool running | `turn.tools[i].status === "running"`, `output` so far | +| tool waiting for approval / a question | `turn.tools[i].waitingOn === ns` (+ the plugin's own Chord service instance, `plugins.md` §6) | +| compaction | `compaction` | +| background job | `tasks[id].status` | +| queued follow-ups | `inbox` | + +This is the concrete difference from an event protocol: Codex must answer the +same question with `thread/resume` + `active_turn_snapshot()` + `activeFlags`, +and still cannot show "retrying". + +### 9.3 Reconnect + +A client keeps `lastRevision`. If an envelope arrives with +`revision !== lastRevision + 1` (only possible on a transport drop between a +Chord RPC service and its client; in that transport Chord's own +`ReplicatedStateDelivery.sequence` plays this role), the client discards its +replica and opens a fresh watch. Nothing is client-specific, so there is nothing +to resume, acknowledge or replay. + +### 9.4 Concurrent decisions + +Any action that answers something the view shows (approve a tool, abort an +input, change config) is a commit on the line. Two clients acting on the same +item are serialized: the first commit records the outcome, the second sees the +recorded outcome inside its own transaction and rejects with a typed error +("already decided", `already_placed`, …). Both then receive the same envelope +and converge. Plugin APIs must therefore do their check-and-record inside one +commit (`plugins.md` §2, `memoOnce`), never check-then-commit. + +### 9.5 Owned conversations + +A subagent is its own conversation with its own watch. The parent view shows +only what the parent's tool put in its slot (`details`, `continuedBy`); a UI that +wants to render the child watches the child. Presentation services may merge the +two into one derived document if a client wants a tree. + +## 10. Plugins + +Everything an extension can contribute (tools, hooks, task kinds, entry kinds, +namespaces), where its durable state lives, how it appears in this view, and how +it is packaged as Chord facets is specified in `plugins.md`. The view-facing +rules are: + +- an extension's presentation is only what it projects: the namespace's + declared `view` projection under `plugins[ns]` (nothing by default), the kind's + `describe()` output under `tasks[id].status`, the core-rendered tool slot fields + (`status`, `waitingOn`, `output`, `progress`, `details`, `continuedBy`, `entry`), + and entries. Raw namespace state, memos and other slot working state never + appear; +- extension events are `plugin..*`, emitted with the namespace token from the + extension's own commits; core events remain core-only; +- nothing about Chord services is visible in the view. A keyed service instance + (an open approval dialog) is a channel, not state; the corresponding state is + `waitingOn` on the slot. + +## 11. Kernel implementation notes + +- One tracked view document per watched conversation (§9.1); the kernel applies + each commit's effects to it (entries, config keys, inbox, turn, compaction, + tasks, plugins) through the proxy and flushes. +- Core closures record events with `tx.emit(event)`; the kernel attaches the + list to the envelope. Events are core authority; extensions emit only through + a namespace token (`plugins.md` §3). +- `turn.generation`, `compaction` and `tasks[id].status` are derived from task + records by kind-owned `describe(task)`; the raw records never leave the kernel. +- Display-only assistant entries carry the message in `data.display` and no + `model`, so context derivation needs no special case. +- Envelope assembly happens on the line; listener dispatch off the line, + synchronous and in order; a listener throw closes that watch only. +- No `AsyncLocalStorage`: line re-entry is detected through the Chord context + the Session hands to its callbacks (`hardening-handoff.md` §7). +- Chord bridge (`plugins.md` §6.1): a bounded ordered adapter applies one raw + envelope's ops to the Chord `MutableReplicatedState` and calls `publish()` in + the same synchronous block (Chord's `subscribe()` publishes pending mutations + itself, so apply and publish must not be separated by an await); overflow or + publish failure closes and respawns that instance; a persisted commit is never + failed by a listener. +- Tests: for every row in §6, assert the exact ops and events of that commit + (memory backend, fake provider), plus snapshot-equals-fold after every + envelope and a head-in-parent capture case. + +## 12. Open items + +- `warning` payload shape (source enum vs free string). +- Application-level backpressure for slow asynchronous consumers is TBD; the + raw watch itself is not (synchronous listener, bounded pre-start buffer). +- Whether `turn.tools` should keep finished slots of the previous assistant + message when the continuation generation prepares (current: cleared at + `prepared`; Codex keeps all items of the turn). Recommendation: keep them, keyed + by call index within `turn.exchanges[]` only if a UI needs it; otherwise + entries already hold the results. +- Whether plugin events need a schema registry for RPC clients (the plugin + token could carry TypeBox schemas per event name). +- Whether plugin-declared **extra documents** are needed for large plugin state; + start with slices in the core documents and add shards only with a + demonstrated cost. diff --git a/packages/agent/docs/pico2.md b/packages/agent/docs/pico2.md new file mode 100644 index 000000000..1e1188d5a --- /dev/null +++ b/packages/agent/docs/pico2.md @@ -0,0 +1,2492 @@ +# pico v2 + +A harness for running agents. This document is the design under review, not an implementation +claim. Decisions still needed before implementation are listed in §25. + +Assumed knowledge: pi-ai (`Models`, `streamSimple`, tool definitions, thinking levels, deferred +handles, `AssistantMessage`/`UserMessage`/`ToolResultMessage`). Everything else is defined here. +Interfaces and examples are proposed TypeScript shapes, not existing package exports or a claim of +implementation completeness. JsonValue/JsonObject mean strict JSON. AssistantMessage, UserMessage, +AssistantMessageFrame, DeferredHandle, Usage and Models reuse pi-ai; AgentMessage/ThinkingLevel and +Context reuse the existing agent harness. Host capabilities and explicitly open contracts are not +silently implemented by these sketches. Normative words: MUST, MUST NOT, SHOULD, MAY. + +History: `pico.md` (v1) modelled a session as one resident tree of nodes; `pico-v2-tree-draft.md` +tried to page that tree; `pico-v2-folds-draft.md` replaced the tree with a transcript and used +"folds" to bound the model's context. This version keeps v1's driver, kinds, scratch, line, hooks, +typed values and watch, and replaces both the tree and the folds with three separately-lived things and an +explicit context list. + +--- + +## 0. In one page + +A **session** stores conversations, entries, tasks, values and lists. Its committed writes share +one ordered sequence; the backend decides physical representation. JSONL keeps main history and +separate named working scopes (§4). A conversation is three things: + +1. a **transcript**: an append-only list of immutable **entries** (a user message, an assistant + message, a tool result, a summary): what happened; +2. **tasks**: small mutable records with a status and code (a generation in progress, a tool + executing, a background job, a subagent, an approval waiting): what the driver runs; +3. **typed values and lists**: settings and plugin state with immutable versions at journal + positions: what forks and rewinds read as of a committed boundary. + +Session-scoped values hold metadata such as entry labels and request identities. They do not rewind. +Conversation values and lists follow history, independently of transcript selection (§5). + +The model does not read the transcript. It reads the conversation's **context**: a short list of +entry ids, edited by three ops (`append`, `replace`, `reset`), that always has the shape +"one head entry (a summary or a handoff), then selected entries in transcript order". + +A **kind** defines what an entry or task is and, for tasks, the code that runs, +recovers and aborts it. Values and lists use typed bound addresses, not kind registration. A **read model** (tables for SQLite; rebuilt in memory for JSONL) answers +"what is live", "what is this task now", "the newest value of key K before position P" and pages +the transcript, without replaying the journal. Memory holds each live conversation's context +entries and its live tasks; everything else is read on demand. A **subagent** is a task that +owns a conversation; the only nesting is `conversation ⊃ task ⊃ conversation`. Clients **watch** a +conversation and receive a transcript page, the context list, live tasks, and then journal records +as they commit. + +--- + +## 1. Definitions + +- **seq**: the position of a record in the journal; a session-global positive safe integer minted + while building a command on the line (§7). Every entry, task and conversation is identified by + the seq of the journal record that created it. Storage preserves these seqs; it does not assign + or remap them. Internal identities need no UUIDs or separately allocated ids. An optional external + request identity maps to the accepted input's numeric id through a session value (§18.1). +- **conversation**: an identity (its creation seq) with a mutable **conversation state** (§4.1), a + transcript, work, typed values/lists, and a context list. The **root** is the conversation with no parent. +- **entry**: an immutable transcript element (§2). Written once; never patched; never reordered. +- **task**: a mutable element with a lifecycle (§3): a generation, a tool execution, a job, a subagent, a collapse, an approval. Patched by `set`. +- **value**: a scalar at a typed address; session-scoped (latest only) or conversation-scoped + (versioned, with historical reads). A conversation **list** stores immutable sequenced elements (§5). +- **history boundary**: the last seq of a complete commit, never a position inside one. +- **transcript order**: the order of a conversation's entries by id (seq). There is no other order. +- **context**: the ordered list of entry ids the model reads (§6). Lives in the conversation state. +- **live**: a task whose status role is not `terminal` (§3.2). A conversation is live if it has live + tasks, a live owner task, or an attached watcher/handle. +- **the line**: the session's single command queue (§7). +- **scratch**: ordinary values/lists in a named working scope, durable until explicit retirement (§3.5). +- **projection**: the messages the model receives, built from the context list (§6.4). +- **canEditContext** (§8.1): no generation request in flight and no unresolved foreground calls. + A safe context edit is not a request to generate. +- **generation task**: the durable obligation to generate; never inferred from the context tail. + +--- + +## 2. Transcript + +### 2.1 Entry + +```ts +interface Entry { + id: number; // seq of its journal record; also its transcript order + conv: number; // owning conversation + kind: string; // "user" | "assistant" | "tool_result" | "summary" | plugin content kinds + by?: number; // the task that wrote it, if any + callIndex?: number; // tool results: index of the call within the assistant message they answer + through?: number; // summaries: the last entry the summary replaced (§6.2) + key?: string; // optional indexed content key; not a typed value address + meta: JsonObject; // small resident fields: timestamp, preview, sizes + payload?: JsonValue; + commitEnd: number; // containing MAIN commit; supports entry-based historical selection +} +``` + +Entries are immutable and the transcript is append-only: no journal record targets an existing +entry and nothing is inserted before an existing one. An entry is reconstructed by one addressed +read of its journal record. + +### 2.2 Content kinds + +```ts +defineContent({ kind, meta: M, payload: P, keyed?: true, context: "select" | "none" }, + { project(entry, ctx): Promise }) +``` +- `context`: whether inserting an entry of this kind also appends its id to the conversation's + context list (§6.1). Core: `user`, `assistant`, `tool_result` → `select`; `summary`, + `custom` → `none` (a summary enters the context only through the `ctx_replace` issued by the + command that creates it; a reset's bootstrap enters through `ctx_reset`). There is no per-entry override. +- `project`: what a selected entry contributes to the model's messages. May read the payload + (`ctx.payload(id)`) and nothing else. A selected entry MAY project nothing. + +--- + +## 3. Tasks, scheduling and cancellation + +### 3.1 Task record + +```ts +type TaskRole = "start" | "inflight" | "waiting" | "terminal"; +interface Task { + id: number; // creation-record seq + conv: number; + kind: string; + at?: number; // originating entry, including an inherited entry + for?: number; // spawning task; provenance, not an execution dependency + after: number[]; // all must be terminal before normal effect starts + status: S; + state: D; // schema-validated JSON; no promises, closures or mutable aliases + owns?: number; // child conversation, reciprocal with conversation.owner + abort?: true; // internal durable cancellation target; not a public task-cancel API +} +``` + +`after` expresses a durable wait. It means **settled**, not succeeded. There is no automatic +failure propagation along edges, no per-edge cancellation policy, and no kind-level concurrency +flag. Ordinary errors are outcomes. Concrete coordinators interpret the outcomes they need. + +```text +parallel tools: T1, T2, T3; P.after = [T1,T2,T3] +sequential tools: T1; T2.after = [T1]; T3.after = [T2]; P.after = [T1,T2,T3] +collapse: G.after = [C] +foreground child: tool.finishing.after = [subagent] +``` + +Dependencies must exist, remain in the same ownership tree, and form an acyclic graph. Check +creation and dependency edits against earlier records in the command. `for` is not a dependency +edge: G may spawn C and then wait for C without creating a cycle. Independent forks never become +implicitly runnable because another scope refers to them. + +### 3.2 Roles and lifecycle + +| Role | Meaning | Scheduler action when unowned | +|---|---|---| +| `start` | Work or a local continuation is ready once dependencies/time permit | `effect` | +| `inflight` | External work was begun; an unowned task may have been interrupted | `recover` | +| `waiting` | An external observer or owned child will supply the next transition | Re-arm once after open | +| `terminal` | Immutable outcome | None | + +A kind declares its statuses and allowed transitions. Status roles are interpreted by the +harness, not by storage. Every nonterminal status must have a recovery/cancellation story. + +```text +external work: commit inflight intent → perform effect → commit outcome +local work: read/prepare → commit outcome +park: commit new status/dependencies → RETURN +settle: output + usage + terminal status + required successors, ONE COMMAND +``` + +A task MUST NOT await another task's lifetime inside `effect`, `recover` or `abort`. It either +settles and leaves successors, or records a dependency/continuation and returns. A control task +such as `post_tools` needs no fake inflight write when its entire action is a command. + +### 3.3 Kind and execution interfaces + +These are proposed interface shapes, not existing exported APIs. Schemas validate strict JSON; +concrete status/state fields are specified in §3.9–§3.15. + +```ts +interface JsonSchema { + parse(value: unknown): T; +} +interface TaskDefinition { + kind: string; + state: JsonSchema; + initial: S; + roles: Record; + transitions: Record; + effect(task: Readonly>, ctx: TaskContext): Promise; + recover(task: Readonly>, ctx: TaskContext): Promise; + abort(task: Readonly>, ctx: TaskContext): Promise; +} +interface TaskPatch { + status?: S; + state?: Partial; + after?: number[]; +} +interface TaskContext { + readonly signal: AbortSignal; + readonly conv: ConversationView; + readonly services: TaskServices; + readonly working: WorkingScope; // ordinary values/lists, §5.6 + set(patch: TaskPatch): Promise; // this task only + settle(status: S, state?: Partial): Promise; + commit(build: (tx: Command) => void | Promise): Promise; + commit(scope: WorkingScope, build: (tx: WorkingCommand) => void | Promise): Promise; + getValue(address: Value): Promise; + readList(address: ValueList, options: ListReadOptions): Promise[]>; + payload(entryId: number): Promise; + emit(kind: string, payload: JsonValue): Promise; + sleep(ms: number): Promise; // this effect's timer, interrupted by signal +} +``` + +`TaskServices` is the typed host service bundle (§12), not another durable object. Provider/tool +exceptions are classified by their kinds. Storage and invariant failures escape and fault the +session; a broad catch around effect plus settlement must not convert them into tool errors. +An unexpected kind exception is reported and faults the session rather than endlessly relaunching +unchanged start-role work. Faulted handles cannot continue cancellation writes. + +### 3.4 Foreground, ownership and busy state + +```text +local foreground: + generation, or task whose for names a generation + // tools, post_tools, and automatic collapse are foreground + +required foreground work: + local foreground plus its unfinished after dependencies + plus foreground work in conversations owned by those required tasks + +background: + other work; e.g. a detached job/subagent, standalone approval, manual collapse + +busy(conv): + local foreground remains OR cancellation targets remain in its required ownership subtree +``` + +`post_tools` is foreground even after every tool has settled. This prevents acceptance between +tool completion and continuation from accidentally creating a second generation. Background +work does not request generation. Automatic collapse uses `for: G`; a standalone manual collapse +does not. A waiting foreground tool keeps its required subagent/approval inside cancellation. + +```text +conversation.parent = provenance and historical inheritance +conversation.owner = subagent task that owns this conversation, if any +scope(conv) = conv + descendants reached through task.owns, not parent links +``` + +Root and every ownerless fork are independent drive scopes. A foreground wait on a subagent is +represented by the tool's dependency, not inferred from its parent's transcript or tool name. + +### 3.5 Scratch and publication + +Scratch is a use of ordinary values/lists in a named working scope (§5.6), not a storage subsystem. +Kinds reduce bounded list pages into partial output. Task outcomes reference immutable entries; +control tasks may have no output entry. A task uses `workingScope(String(task.id))` by convention. Keys distinguish attempt/poll identities; +recovery reads only the current request's keys. Superseded attempt data may remain unreachable +until retirement, but cannot be mistaken for the active attempt's frames/candidate. + +```text +progress: working-scope-only commit → publish preview +settlement: stop producer/sink admission → drain admitted progress writes + main commit: output + terminal task + retireScope(working) +cleanup: backend may remove physical working files AFTER that commit +``` + +Retirement is logically atomic with settlement. Failed unlink is harmless: committed retirement +makes the old contents invisible. Storage never infers retirement from owner absence/status. Close +and fault retire nothing. Invocation fencing rejects late writes; generic scopes allow reuse. + +### 3.6 Scheduler and process-local ownership + +One scheduler per session services the union of active drive scopes. Multiple callers may overlap; +they never install competing executors. Removing a caller does not cancel durable work. + +```ts +interface OwnedRun { + controller: AbortController; + promise: Promise; // current effect/recovery and any joined cancellation cleanup +} +const running = new Map(); +const recovered = new Set(); // inherited waiting tasks already re-armed in this process +const openedAt = committedHead; +``` + +These structures are process-local. The map is the owned set; no duplicate owned bitmap, durable +lease, or promise registry is needed. Each retry/phase gets a fresh controller. The scheduler owns +any promise replacement/cleanup; effect completion cannot delete a newer ownership claim. + +```text +pass: + scope = union of active drive scopes + for marked tasks in scope: + signal EVERY owned affected execution before waiting on any one + arrange exclusive cancellation reconciliation (§3.7) + for unmarked, unowned live tasks in scope: + start: + if all after tasks terminal, notBefore elapsed, required poll permit present: + claim synchronously; launch effect; do not await its whole lifetime in this pass + inflight: + claim synchronously; launch recover + waiting: + if task.id <= openedAt and not recovered: + mark recovered before launch; claim; launch recover to re-arm and return + resolve drive observations from committed state + wait for commit | effect completion | timer | external observer notification +``` + +External observers for jobs/approvals are disposable subscriptions, not promises held by a task +waiting for another task. Each observer writes through the line and checks the current task's +status/cancellation before publishing. The host retires observers when their task ends/closes. + +The scheduler reads task metadata, not transcript payloads or sibling tool outcomes. Terminal +dependency states use indexed point reads and may be cached while referenced by live tasks; +never repeatedly replay history. No global reverse-dependency index is required initially. +No conversation `next()`, tail inference, or open-time successor repair exists. + +### 3.7 Conversation cancellation + +Public cancellation is `conv.abort()`. **There is no public `abortTask(id)`.** A task's internal +`abort` bit records membership in a conversation cancellation; it is not a second user-facing +cancellation mechanism. `cancelQueued(id)` only withdraws unconsumed input. + +```text +abort(conv), ONE COMMAND: + if this conversation already has outstanding cancellation targets: join; do not drain again + select current foreground + required dependency/owned-child closure + mark every selected nonterminal task abort = true + drain currently queued steer/followUp in affected foreground conversations + preserve write/nextRun; return drained input identities/content + +normal admission/settlement: + marked work cannot start an external action or create normal successors + input arriving during cancellation queues; do not consume it during cleanup + new foreground tasks cannot bypass outstanding cancellation in that conversation +``` + +Selection is durable in the task marks. Do not recompute a smaller target set when an earlier +parent becomes terminal. Detached background work is not selected by ordinary foreground abort. +Deletion/explicit whole-conversation shutdown selects all live tasks in the ownership subtree; +this is still a conversation operation, not arbitrary graph-node cancellation. + +```text +reconcile a marked task: + retain ONE ownership claim through the entire sequence + signal its controller if an effect/recovery is owned here + await that task's own execution returning, whether fulfilled or rejected + report rejection; storage/invariant faults still prevent further writes + re-read task; if already terminal, do nothing + otherwise run kind.abort under the SAME exclusive claim + release claim only when that invocation finishes +``` + +That await is allowed: it joins this process's current execution of the SAME task. It prevents +its effect and abort handler from writing concurrently. It is not a wait for dependency tasks. +An uncooperative external effect can delay cancellation; no timeout may silently permit a second +writer. Driver passes can still signal/cancel other independent tasks. + +Never-started sequential tools can abort immediately, regardless of their `after` dependencies. +Each writes its required error result. `post_tools.abort` creates no generation and performs no +unsafe context edit; queued writes may remain queued. For a marked conversation owner, defer abort invocation until marked owned descendants are terminal +(§3.13); no effect promise waits for that condition. Normal child completion checks the owner's +marker before settling it, so cancellation cannot become success. + +```text +abort before P settlement: P is marked → no G2 +P settlement before abort: P + G2 commit together → abort selects G2 +crash after marking: open reports marked tasks → next covered drive reconciles them +crash after effect returned: marked inflight task remains → abort reconciliation resumes +``` + +The host is the sole live writer. Another process may reopen after handoff/crash; concurrent +independent writers are not supported. A remote client sends cancellation through the owning host. + +### 3.8 Open, drive, suspension and close + +```text +open: + hydrate storage/residency; validate kinds/statuses/dependencies + start nothing; write no repair generation; expose inspect() +inspect(): session-wide { start, inflight, waiting }, including cancellation marks +conv.drive(): run only its ownership tree +harness.drive(): run every independent tree in the session +``` + +A task in `start` can still be blocked by a dependency, retry time or deferred-poll permission. +`inspect()` reports durable roles, not a promise that every start-role task is runnable now. + +```text +conv.drive → idle: no foreground or outstanding cancellation in its ownership scope +harness.drive → idle: no live tasks anywhere +suspended: work remains, but progress requires an outside event/permission +closed: harness closed before requested condition +``` + +While a conversation drive is active it also services background work in that scope, but it does +not keep waiting solely for detached background tasks after foreground quiescence. A host wanting +to supervise all background work keeps a session drive active. Inherited waiting-task recovery is +once per session process, not once per caller or scope. + +`close()` seals commands and scratch admission, signals owned executions, detaches observers, +and drains admitted persistence. It writes no cancellation marks and no task settlements. Effects +must release resources/cooperate with close; the current cooperative-wait policy and a possible +bounded-close alternative remain explicit in §25. A caller cancelling its wait is not `abort()`. + +### 3.9 Generation + +**Responsibility:** one provider generation, including its retries/deferred polls. It publishes a +response and either tool work plus a join, or the normal final-answer boundary. It never awaits tools. + +```ts +type GenerationStatus = "pending" | "streaming" | "retry_wait" | "deferred" | "polling" + | "done" | "failed" | "aborted"; +interface GenerationState { + settings: GenerationSettings; // captured when this generation is created + attempt: number; // starts at 1 + notBefore?: number; + requestAsOf?: number; // committed context boundary used by current request + collapse?: { id: number; reason: "threshold" | "overflow" | "manual" }; + overflowRetried?: true; + deferred?: DeferredHandle; + poll?: number; + produced?: number; // immutable assistant entry + calls?: number[]; + postTools?: number; + error?: string; +} +``` + +Roles: pending/retry_wait/deferred = start; streaming/polling = inflight; others terminal. +Transitions: pending/retry_wait → streaming or failed/aborted; streaming → retry_wait/deferred or +terminal; deferred → polling or aborted; polling → deferred/retry_wait or terminal. Adding a +collapse dependency can preserve the current start status. Terminal states accept no patches. + +```text +creation: idle acceptance, a final-answer continuation, or post_tools settlement + capture model/thinking/tools/tool order/options/retry policy from current settings + task pending, attempt = 1 + +pending/retry_wait effect: + if captured collapse dependency exists: + it is terminal (scheduler readiness) + overflow collapse unsuccessful → fail this generation; no replacement + threshold/manual failure → proceed using unchanged context + else if automatic compaction is indicated: + decide before_collapse outside line + revalidate context and cancellation inside line + if approved: + C = create collapse { for: G, captured prefix/model } + set G { after: [C], collapse: { id:C, reason:threshold } } + RETURN; no promise waits for C + capture conversation's MAIN commitEnd as context boundary; commit streaming intent + build projection + request; run before_request outside line + re-check cancellation/close before provider call + stream provider; scratch receives frames + classify outcome below +``` + +A live manual collapse is attached as a dependency before admitting a new request, rather than +creating a competing automatic collapse. A generation always exists while automatic compaction +runs. There is no state where consumed input is represented only by a background collapse. + +```text +provider outcome, in precedence order: + close → leave task recoverable; no terminal write + marked cancellation → aborted partial + matching error results; no successors + deferred → reported usage + set deferred(handle, poll, notBefore) + overflow, not already retried: + decide collapse; commit usage + existing G.retry_wait + C + G.after=[C] + declined/impossible → usage + failed G + other error → reported usage + bounded retry_wait, else failed G + successful response with calls → settleWithCalls + successful response without calls → settleFinal +``` + +```text +settleWithCalls, ONE COMMAND: + A = assistant entry with call list + for each call in source order: + Ti = tool { for:G, at:A, callIndex, captured active-tool authorization } + parallel: Ti.after = [] + sequential: Ti.after = [previous tool] except first + P = post_tools { for:G, at:A, after: all Ti } + usage(G, reportedUsage) + settle G { done, produced:A, calls:all Ti, postTools:P } +``` + +The response, all tools, and exactly one P commit together. Stop reasons other than toolUse with +calls create truncated tool tasks: they produce error results but do not execute external tools. +Projection later sorts results by callIndex, regardless of their transcript completion order. + +```text +settleFinal: + prepare on_yield outside line only if eligible queued input is absent + ONE COMMAND, checking latest cancellation and queue state: + A = assistant entry; usage; settle G + select safe writes and eligible steering/followUp (§18) + if continuing: + place selected items in admission order; dequeue + place hook continuation if applicable + create next generation with freshly captured settings + else: + place safe writes only + finishOwnedConversation(tx, conv, { outcome:done, result:A }) +``` + +Stale hook decisions are discarded/recomputed before committing. A new queued trigger takes +precedence over an old yield decision. `finishOwnedConversation` is ordinary settlement code +(§3.13), not a later scheduler pass. Terminal failure uses the same owner notification with a +failed outcome and no successor. Input-result attribution remains separately open (§18.2). + +**Deferred:** admitted only with `pollDeferred`. Commit polling intent and increment poll before +fetching; still-deferred/fetch-error outcomes return to deferred with backoff on the same task. +Abort attempts provider cancellation best-effort, then completes locally. No new generation is +created for a deferred poll or retry. Retry delays use the captured bounded exponential policy. + +**Recovery:** polling becomes deferred; streaming with committed frames publishes an interrupted +aborted partial and matching error results; streaming without frames retries within budget or +fails. This preserves the current draft's policy; old-lane interrupted-stream parity is pending +(§25). Recovery must never resurrect a marked generation or create a fresh retry budget. + +**Abort:** stop/drain frames, cancel deferred handle if present, write required partial/error +entries and known usage, settle aborted. A never-started generation has no fabricated assistant +response. No tools, join, yield continuation or provider retry follows cancellation. + +### 3.10 Tool execution + +**Responsibility:** one call. No sibling reads, last-sibling checks, queue draining, context reset, +or next-generation creation. The same handler serves parallel and sequential execution. + +```ts +type ToolStatus = "planned" | "truncated" | "running" | "waiting" | "finishing" | "done" | "aborted"; +interface ToolState { + callIndex: number; + callId: string; // provider call identity; task.id is harness invocation identity + name: string; + args: JsonObject; + allowed: boolean; // captured from generation's active-tool names + replay: "safe" | "never"; + approved?: true; + approval?: number; + delegated?: number; // a subagent/job whose outcome supplies this tool result + produced?: number; + terminate?: boolean; + handoff?: string; // requested reset; interpreted by post_tools only + error?: string; +} +``` + +Roles: planned/truncated/finishing = start; running = inflight; waiting = waiting; done/aborted = +terminal. Planned can become waiting/running/done/aborted; running can become finishing/done/aborted; +waiting can become planned/done/aborted; finishing becomes done/aborted. Safe recovery can return +running to planned while preserving effective arguments/memos. Tool and sink interfaces: §16. + +```text +planned effect: + unavailable/inactive tool → settle own error result + validate arguments against tool schema + before_tool outside line, unless already approved: + block → settle own error result + hold → create approval + set this tool waiting, ONE COMMAND; RETURN + commit running intent with effective args and replay declaration + execute with own signal, invocation identity, scratch/sink and application context + drain output; run after_tool outside line + ONE COMMAND: + re-check cancellation + append own result; reported usage; active-tools additions + settle this tool, retaining terminate/handoff control metadata + +truncated effect: + settle own "not executed" result; never invoke tool +``` + +A definition merely present in the registry is insufficient authorization: the generation must +have offered it. Arguments and replay declarations are captured before the external invocation. +An ordinary tool throw becomes `isError`; storage errors do not. Unknown addTools names remain +dropped under the existing draft policy. Final text is capped; truncation is explicit (§16). + +**Delegation without awaiting another task:** a harness-native tool can return a durable wait action. + +```text +ONE COMMAND: + S = create subagent/job with for = this tool + set tool { finishing, delegated:S, after:[S] } +RETURN from current execution + +S eventually terminal → scheduler can run this SAME tool's finishing phase +finishing effect: + load S's committed result + apply after_tool outside line + commit this tool's own result + terminal outcome +``` + +This reuses the tool task rather than inventing an operation wrapper. The extra phase is needed +when async after_tool processing must occur outside the child settlement command. No in-process +waitFor(S) exists, either initially or on recovery. Background delegation instead settles the tool +with the task id immediately; it has no dependency on the detached work. + +**Recovery:** replay only when both captured and current declarations permit safe replay and no +cancellation won. Preserve task id, effective arguments and memos. Otherwise settle an interrupted +error result from the last checkpoint, without applying uncertain addTools/terminate/handoff effects. +Waiting holds are restored as holds; finishing dependencies survive without re-arming a promise. + +**Abort:** after joining its own execution, drain scratch and settle an error tool result with +aborted metadata. Unstarted calls also get results, completing the assistant exchange. No dependency +must finish first. This handler does not inspect siblings or invoke post_tools itself. + +### 3.11 Post-tools + +**Responsibility:** join one generation's tool results and decide what follows. This is a real +exchange coordinator, not a conversation-level progression loop. + +```ts +type PostToolsStatus = "pending" | "done" | "aborted"; +interface PostToolsState { + generation: number; + produced?: number; // optional terminal result reference, never payload +} +``` + +Created by generation settlement, `for:G`, `after:all calls`. Pending = start; done/aborted = +terminal. It has no external effect and no inflight status. Its effect is one serialized command: + +```text +P.effect, ONE COMMAND: + require all after tasks terminal and P still pending + read completed call outcomes // ONLY this coordinator reads siblings + if marked cancellation: settle P aborted; RETURN + select safe writes, mode-selected steering, and any valid handoff + place selected content in admission order; dequeue consumed items + apply handoff reset after full exchange completion, if requested + if exchange was aborted or termination policy says stop: + do not consume generation-triggering input + settle P done + finishOwnedConversation(tx, conv, terminal outcome) + else: + G2 = create generation with current model/thinking/tools/options + settle P done +``` + +Selection precedes mutation: on a stopping path only safe writes are consumed. Follow-ups wait +for a normal final answer, not this tool boundary. A handoff is an explicit trigger, not tail +inference. Compaction belongs to the already-created G2; it can attach a collapse dependency. + +Current draft termination policy is any completed call requesting terminate, with all sibling +results retained. Its difference from the old lane's all-results policy remains listed in §25. +An ordinary error result is not cancellation. Conflicting handoffs require a defined selection +policy before implementation; do not silently pick completion order (§25). + +**Recovery:** there is no unknown external action. If the commit did not land, P remains pending; +if it landed, P is terminal and G2 already exists. **Abort:** settle P aborted, create nothing, +perform no context edit. Remaining writes stay queued; other marked tasks finish independently. + +```text +T2 settles → P blocked +T3 settles → P blocked +T1 settles → P eligible, already durable +P settles + G2 created → one commit +``` + +No tool knew which one finished last. One additional task/terminal record per exchange buys a +single owner for the exchange policy and removes orchestration from every tool implementation. + +### 3.12 Collapse + +**Responsibility:** summarize a captured prefix and publish a validated replacement. It never +creates a generation. A pending generation already represents automatic continuation. + +```ts +type CollapseStatus = "pending" | "summarizing" | "retry_wait" | "publishing" | "done" | "failed" | "aborted"; +interface CollapseState { + through: number; + prefixVersion: number; + contextAsOf: number; + settings: GenerationSettings; + instructions?: string; + attempt: number; + notBefore?: number; + produced?: number; + error?: string; +} +``` + +Pending/retry_wait/publishing = start; summarizing = inflight; others terminal. Pending/retry_wait +→ summarizing; summarizing → publishing/retry_wait/done/failed/aborted; publishing → done/failed/ +aborted. A publishing task can replace its after list to wait for a current exchange boundary. + +```text +creation: + capture complete-exchange cut, prefixVersion, context boundary and summary settings + one live collapse per conversation; reuse existing task rather than race duplicate insertion + +effect: + commit summarizing intent + project captured prefix, including prior summary; call summarizer + complete candidate → persist in task scratch + ONE COMMAND: + usage + outcome + cancelled/stale prefix → terminal outcome, no context publication + context editable → summary entry + ctx_replace + settle done + otherwise → set publishing with after = current request/exchange coordinator + RETURN + +publishing effect, ONE COMMAND: + re-check cancellation, prefixVersion and canEditContext + if another exchange is now open: update after to its generation/post_tools; RETURN + otherwise publish candidate + ctx_replace + settle done +``` + +The durable publishing phase is needed only when a speculative summary finishes during a busy +exchange. It waits through dependencies, never an effect-held promise. Reported provider usage +commits once with the attempt outcome, not again when publishing. Scratch remains until terminal +settlement. Prefix changes reject publication; appends beyond through survive unchanged. + +**Recovery:** reuse a complete persisted candidate when available; otherwise retry interrupted +summarization within budget or fail. Publishing reads its candidate and revalidates. **Abort:** +stop summarizer, drain scratch, record known uncommitted usage and settle aborted, without summary +publication. Automatic G waits on this same task and interprets its terminal outcome (§3.9). +Token-budget/split-turn parity remains an explicit compaction-policy decision (§25). + +### 3.13 Subagent + +**Responsibility:** own one child conversation and expose its terminal foreground result. + +```ts +type SubagentStatus = "planned" | "running" | "done" | "failed" | "aborted"; +interface SubagentState { + prompt: AgentInput; + context: "fresh" | "inherit"; + values: { + inherit: { namespace: string; key: string }[]; + set: { namespace: string; key: string; value: JsonValue }[]; + }; // serialize selections; resolve them in child creation command + produced?: number; // may reference result in owned conversation + error?: string; +} +``` + +Planned = start; running = waiting; others terminal. Planned → running/failed/aborted; running → +terminal. `owns` identifies the child. Scalar initialization is precisely defined in §8.3. + +```text +planned effect, ONE COMMAND: + create child { parent history, owner:S, inheritValues:false } + write selected initial values + overrides + append prompt + create child's initial generation + set S running { owns:child } +RETURN +``` + +No external action requires a fake inflight status here. The child's final-generation or stopping +post_tools settlement performs `finishOwnedConversation` in the SAME command: + +```text +finishOwnedConversation(tx, child, outcome): + if child.owner is absent: return + S = read owner; require S owns child and is still running + settle S with outcome/result entry reference + // A foreground launching tool already exists in finishing, after:[S]. + // It is now eligible; no lost event or new task is required. +``` + +**Recovery:** a planned owner creates child once; a running owner starts nothing and returns. +Its child tasks carry all durable continuation. No waitFor or child-completion subscription needs +rebuilding. Completed result payload is not copied merely to settle the owner. + +**Abort:** conversation cancellation already marked required child work in the same marking +command. Scheduler defers owner abort invocation until its marked owned descendants are terminal; +it does not hold an effect promise while they finish. Then settle S aborted. If a child is aborted +directly, its live owner is included for cancellation completion; a parent-side finishing tool +can subsequently report the child's aborted outcome when its drive scope runs. Background child +work is untouched by unrelated parent foreground cancellation. + +### 3.14 Job + +**Responsibility:** run/adopt a process with durable output, without keeping a foreground tool +alive unless that tool explicitly depends on it. + +```ts +type JobStatus = "planned" | "spawning" | "running" | "exited" | "killed" | "lost"; +interface JobState { + command: string; + logPath: string; + pid?: number; + processIdentity?: string; // host-specific adoption identity, not pid alone + notBefore?: number; + produced?: number; + exitCode?: number; +} +``` + +Planned = start; spawning = inflight; running = waiting; others terminal. Planned → spawning/killed; +spawning → running/lost/killed; running → exited/lost/killed. + +```text +effect: + commit spawning with durable log/adoption location + spawn/adopt through host process service + commit running with process identity + install exit/output observers; RETURN + +exit observer: + drain bounded output/checkpoints + commit immutable job-result entry + usage if reported + settle exited + a dependent tool.finishing becomes eligible through after +``` + +**Recovery:** spawning reconciles the uncertain launch through the host adoption identity; never +blindly spawn a second process. Running recovery, once after open, reattaches output/exit observers +if the same process is alive, otherwise records lost. A pid by itself is not proof of identity. +Exact durable adoption/exit-status capability is a host contract, pending finalization (§25). + +**Abort:** when included by required foreground cancellation or whole-conversation shutdown, +stop the process through its service, drain output, write result and settle killed. A detached job +survives ordinary parent foreground abort and harness close. Explicit background shutdown/deletion +is conversation-scoped. Domain-specific process control is not a generic task-cancel endpoint. + +A schedule uses planned/notBefore and, for repeated execution, a successor job created atomically +with the preceding outcome. The schedule definition must supply a finite/bounded recurrence policy; +there is no hidden recurring-work scheduler or task held across another job's lifetime. + +### 3.15 Approval + +**Responsibility:** hold a durable human decision. The waiting tool, not a promise, blocks its +exchange and sequential followers. + +```ts +type ApprovalStatus = "pending" | "waiting" | "granted" | "denied" | "cancelled"; +interface ApprovalState { + tool?: number; + prompt: string; + produced?: number; + decision?: JsonValue; +} +``` + +Pending = start; waiting = waiting; others terminal. Pending → waiting/cancelled; waiting → +granted/denied/cancelled. + +```text +before_tool hold, ONE COMMAND: + A = approval { for:tool, tool:tool.id, prompt } + set tool waiting { approval:A, after:[A] } + +approval.effect: + commit waiting; publish human prompt through observation; RETURN + +approve(A), ONE COMMAND: + verify waiting and not marked + record decision + settle A granted + set matching waiting tool planned { approved:true } + +deny(A), ONE COMMAND: + record decision + settle A denied + write matching tool's own blocked result + settle tool done +``` + +Decision commands identify the existing approval but are not cancellation APIs. If asynchronous +result hooks are required on denial, ready the existing tool's finishing phase with the decision +instead of invoking hooks on the line; that phase settles only that tool. This is the same durable +continuation pattern as subagent results. + +**Recovery:** re-announce/reattach the human decision surface once; do not grant or rerun the tool. +**Abort:** record cancelled approval; the independently marked foreground tool writes its aborted +result. Denial/grant versus cancellation is decided on the line. Late/repeated decisions return the +existing outcome or a rejection; they never restart terminal work. + +--- + +## 4. Journal, conversation state, read model + +### 4.1 Journal records + +``` +ins_conv { parent?: {conv, at?, asOf, inheritValues}, owner?, context: number[] } +set_conv { id, patch } // inbox, label, queue modes +del_conv { id } +ins_entry { conv, kind, by?, callIndex?, through?, key?, meta, payload? } +ins_task { conv, kind, at?, for?, after, state } +set { id, patch } // tasks only +settle { id, patch (status terminal) } // tasks only; a task's result is an entry +ctx_append { conv, ids } +ctx_replace { conv, through, with, expectPrefixVersion } +ctx_reset { conv, ids } +value_set { conv?, workingScope?, namespace, key, value } +value_delete { conv?, workingScope?, namespace, key } // conversation deletion = tombstone +list_append { conv?, workingScope?, namespace, key, value, tag? } +list_delete { workingScope?, namespace, key } // session/working lists only +scope_retire { workingScope } // MAIN-scope write +// usage(task, conv, usage) appends to the main session list pi.usage; no special storage verb +``` +`parent.asOf` is a committed parent-history boundary; optional `at` records an entry-based selection. +Historical forks set `inheritValues: true`. Subagents set it to false and store selected initial +values as local writes (§8.3). Parent lineage and context inheritance do not imply value inheritance. +Value/list records are journal records, not transcript entries, and never implicitly enter context. + +The command supplies final numeric seqs (§7); storage never remaps them. All scopes share one +session-global sequence space. Each admitted batch is contiguous; retained history and individual +physical files may have gaps after working writes/retirement. Backend encoding is private. JSONL +uses one complete line per commit in exactly one file; SQLite uses one transaction. Main-scope +history is retained for historical reads; obsolete working data need not be retained. + +**Conversation state** (derived; in memory; in a table for SQLite): +```ts +interface ConversationState { + id: number; + commitEnd: number; // last MAIN write affecting this conversation + parent?: { conv: number; at?: number; asOf: number; inheritValues: boolean }; + // model, thinkingLevel and activeTools are conversation values (§5), not fields here + inbox: InboxItem[]; // write | steer | followUp | nextRun; each names stored content + owner?: number; // owning subagent task, absent on independent forks + label?: string; + context: number[]; // §6 + prefixVersion: number; // bumped by ctx_replace / ctx_reset only + steeringMode: "all" | "one-at-a-time"; + followUpMode: "all" | "one-at-a-time"; +} +``` + +### 4.2 Stored data and indexes + +Storage exposes conversations, entries (including conv), tasks, values and lists. Scratch and usage +are uses of values/lists. It does not expose a parallel ReadIndex or a raw journal/payload API. +`Entry` includes its payload and the end seq of its creation commit, so entry-based forks need no +separate commit-boundary lookup. Conversation historical reads include their context list. + +```text +current: conversations, tasks, session values/lists, working values/lists +history: entries, conversation versions/context edits, conversation value/list versions +indexes: by conversation/id; live tasks; owner/parent; value/list address + seq +``` + +Historical scalar/list lookup follows §5.3–§5.4. Index and payload layouts are backend choices; +these queries must use bounded indexed reads, not full-history scans. Main-history indexes may +reference retained records; never retain deleted working payloads just to preserve a dense array. +Shared SQLite containers prefix every key with session id. No closure table or journal prev pointers. + +### 4.3 Storage interface and backends + +Follow the existing lane Storage: one atomic commit, entity reads/scans, values/lists. No separate +scratch interface, raw-record reader, or public index layer. `Context` is the existing harness +invocation/cancellation context. Types below are proposed interfaces, not current package exports. + +```ts +interface CommitResult { firstSeq: number; lastSeq: number } +interface HistoricalRead { asOf: number } // complete MAIN commit boundary; 0 = empty +interface Scan { + after?: number; before?: number; // exclusive id/seq bounds + order?: "asc" | "desc"; limit: number; // default asc; 1..10_000 +} +interface ConversationScan extends Scan { parent?: number; owner?: number } +interface EntryScan extends Scan { conv?: number; kind?: string; key?: string; asOf?: number } +interface TaskScan extends Scan { conv?: number; kind?: string; live?: boolean } +interface ListReadOptions extends Scan { asOf?: number; stopAtTag?: string } +interface ListElement { seq: number; value: T; tag?: string } +interface StoredValue { value: T; seq: number } + +interface Storage { + readonly head: number; // highest committed GLOBAL seq, including working writes + commit(writes: Write[], context: Context): Promise; + getConversations(ids: number[], context: Context, at?: HistoricalRead): Promise>; + scanConversations(query: ConversationScan, context: Context): Promise; + getEntries(ids: number[], context: Context): Promise>; + scanEntries(query: EntryScan, context: Context): Promise; + getTasks(ids: number[], context: Context): Promise>; + scanTasks(query: TaskScan, context: Context): Promise; + getValue(address: Value, context: Context, at?: HistoricalRead): Promise | undefined>; + scanValues(prefix: Value, context: Context): Promise[]>; + readList(address: ValueList, options: ListReadOptions, context: Context): Promise[]>; + close(context: Context): Promise; +} +``` + +Storage addresses include their bound conversation or working-scope identity (§5.6). Historical +reads apply to main conversation data only; session/working values expose current state. Current +multi-query snapshots use the session line. Every single logical query, including ancestor walks, +observes one committed state. Reads return immutable snapshots; callers cannot mutate owned data. +Filters precede limits. `stopAtTag` includes its matching element and never searches beyond limit. + +```text +commit: + validate one write scope: main OR one exact working id + require final supplied seqs == head+1, head+2, ...; reject unsafe integers + validate/prepare touched rows; persist one atomic batch + publish rows/indexes/head together; only then resolve + no callbacks, task handlers or plugin reducers inside storage + +scope retirement in a MAIN commit: + invalidate working data through the retirement seq atomically with sibling main writes + physical cleanup may follow; cleanup failure does not undo the committed outcome + +failure: + pre-admission rejection → no changes or consumed seqs + admitted persistence failure → fault; outcome may already be durable; reopen, never blind retry +close: + seal admission; drain admitted persistence; release resources; retire no scopes +``` + +**Memory:** maps for current entities/scoped values/lists, ordered references for historical data. +Prepare only touched rows; publish synchronously without awaits/callbacks. Retirement drops current +working maps and releases obsolete payloads. Main history remains available. Binary-search ordered +value/list references; ancestor traversal costs fork depth, not unrelated session history. + +**SQLite:** one transaction updates entities, historical versions, global head and any retirement. +Working value/list rows are ordinary scoped rows, not a journal of every old checkpoint. + +```text +BEGIN IMMEDIATE + validate supplied seqs against stored head + apply entity/value/list writes and required history/index changes + retirement: DELETE working scalar/list rows WHERE session_id=? AND scope_id=? + advance global head +COMMIT +``` + +Use `(session, conv, address, seq)` historical indexes, `(session, workingScope, address, seq)` +working-list indexes, entity primary keys, live-task and conversation-owner indexes. Read compound +queries in one synchronous SQL read transaction; no awaits while holding it. WAL with +`synchronous=NORMAL` is the baseline. Rebuildable indexes are private, not another API contract. + +**JSONL:** adopt the lane scoped-storage protocol described in +[`implementation-handoff.md`](mobile-handoff/01-harness/02-scopes/implementation-handoff.md). +That document is design evidence, not a claim that scoped storage is implemented in this tree. + +```text +mainPath main entity/history commits + scope_retire records +mainPath.scopes/scope-.scope one named working scope, value/list commits only + +commit: + one session queue and global counter for BOTH kinds of file + encode complete batch; writeAll to exactly ONE file, handling short writes + publish in-memory state only after complete newline reached OS + main retirement committed → serialize attempted unlink before later scope reuse + failed unlink → old data still logically retired; retry cleanup on reopen + +open: + validate main/sidecar identity and complete transactions + compute high-water mark from header + ALL complete main/sidecar records BEFORE cleanup + find latest main retirement seq per working id + retain only scoped writes newer than that retirement + replay retained transactions in global order; preserve boundaries; reject duplicate/overlapping seqs + discard only unterminated final transactions; malformed complete data fails open + remove files containing no writes newer than their retirement +``` + +One batch has consecutive seqs, but files/history have legal gaps. Never use array position as a +seq. No per-scope counter, range reservation, cross-file acknowledgement, owner-derived sweeping, +or scope-creation transaction. First scoped write creates the sidecar atomically with a validated +header; names are encoded, fixed-prefix components, never raw paths. Follow the linked handoff's +Unicode/length validation and `.scope` layout so repository listing cannot mistake them for sessions. + +Scopes are reusable. Retirement ends writes through its global seq, not every future use of the +name. A delayed unlink must never delete a later lifetime. Main compaction must retain retirement +boundaries while old physical files could otherwise become live. Repository deletion removes the +whole scope directory. Close/fault never retire. Main and sidecar complete writes provide process- +crash durability; power-loss durability/fsync is a separate explicit policy, not a cross-file protocol. + +### 4.4 Memory + +```ts +conversations: Map // live conversations only +tasks: Map // live tasks, plus each conversation's newest generation (§8.1 reads it) +entries: Map // every id on a live conversation's context list +history: LRU // pages a UI asked for; bounded +``` +This is **residency** and it is the same on every backend; it is not the read model (§4.2) and +does not depend on how the backend indexes. After each commit the harness keeps exactly the entries +some context list names and the tasks that are live or a newest generation, and drops the rest; a +commit that puts an old entry on a list (a fork) loads it first. +Objects are values (a write yields a new object). All reads are async; a hit is a microtask, a +miss uses the ordinary Storage entity/value/list reads. Execution residency is proportional to (live conversations × +context size) + live tasks, plus the currently needed settings. Active plugin replicas additionally +hold their current state and tracking data; they do not retain every historical state revision. +The process's total memory also includes retained main history/indexes on memory/JSONL, live working +scope contents, bounded query/cache state on SQLite, and active dependency metadata. A bounded execution working set is not a claim of bounded +total process memory on memory/JSONL. + +--- + +## 5. Typed values, lists and plugin state + +### 5.1 Bound addresses + +Keep the old harness's typed bound-address approach: namespace and optional key are bound once; +subsequent operations take that address, not another string key. Scope and kind are part of its +TypeScript type. No registration, declaration merging, runtime token catalog, or per-write rewind +flag is required. `declareVar` is replaced by these constructors: + +```ts +const model = conversationValue("pi.model"); +const thinkingLevel = conversationValue("pi.thinking"); +const activeTools = conversationValue("pi.active-tools"); +const planMode = conversationValue("my-plugin.plan-mode"); +const moves = conversationList("my-game.moves"); + +const sessionName = sessionValue("pi.session.name"); +const entryLabel = (id: number) => sessionValue("pi.entry.label", String(id)); +const requestInput = (requestId: string) => sessionValue("pi.request-input", requestId); + +await conv.setValue(planMode, true); +const enabled = await conv.getValue(planMode) ?? false; +await conv.deleteValue(planMode); +await conv.appendList(moves, move); +await harness.setValue(entryLabel(42), "before migration"); +``` + +Namespace must be non-empty; namespace/key cannot contain NUL; the empty key is legal. Equal +(scope, kind, namespace, key) tuples name the same address within the receiver's scope. `pi` and +`pi.*` are reserved for core. Constructing the same address with incompatible value types is a +programming defect, not a reason for a runtime registry. Value types are determined by the address +(`NoInfer` on writes); wrong scope, wrong kind and wrong value type are compile-time errors. +Defaults belong in caller code, not token definitions. Stored payloads are JSON values. + +### 5.2 Scope and atomic writes + +- **Session values:** latest value only through the public API; setting replaces and deletion + removes. Session name, entry labels and request identities do not change on rewind or fork. + Their journal records remain durable; the current-value index needs only the latest set's seq. +- **Session lists:** current append/page/delete semantics; no rewind or fork inheritance. Usage + is a main session list. Working lists use the same operations with explicit lifetime (§5.6). +- **Conversation values:** each set appends an immutable version at its journal seq. Deletion + appends a tombstone that masks both earlier local and inherited versions. JSON `null` is a value, + not deletion. Historical reads select a complete commit boundary. +- **Conversation lists:** each append writes one immutable sequenced element. No per-element + update, insertion or deletion. Forks share prefixes; they do not truncate parent lists (§5.4). + +Model identity, thinking level and active tool names are ordinary rewindable conversation values, +not sticky configuration fields. Root creation writes their initial values in its creation command; +subagent initialization supplies its selected settings (§8.3). Generations capture the effective +settings they use; later setting changes do not rewrite an already-created generation. + +Convenience setters issue one command each. Read-modify-write uses the serialized command line: + +```ts +await conv.command(async (tx) => { + const todos = await tx.getValue(todoBoard) ?? []; + tx.setValue(todoBoard, [...todos, newTodo]); +}); +``` + +Reads use the pre-command committed view (§7); buffered writes apply in order at commit. A bound +conversation command supplies its conversation id for conversation-address operations. The +session-level command uses an explicit conversation id, allowing input creation, request mapping, +child creation and initial values to commit atomically. Value/list writes never select transcript +content or trigger generation by themselves. + +### 5.3 Historical scalar lookup + +Only local versions are stored. To read address K in conversation C through boundary U: + +1. Seek the newest local version of K with `seq <= U`. +2. A set returns its value; a tombstone returns absent and MUST NOT fall back to the parent. +3. If no local version exists, stop if C has no parent or `parent.inheritValues` is false. +4. Otherwise continue in the parent with `U = min(U, parent.asOf)`. + +For example, A writes false at 40 and true at 80; B forks A through commit 60. B reads false. +Parent writes after 60 never leak into B, including through further nested forks. A local deletion +in B masks false; a later local set recreates the value only in B's history. + +**Memory/JSONL:** map conversation and address to an ordered array of version references; binary +search the cutoff. References point to journal records rather than copying payloads. Parent lookup +is a map lookup. Worst case is one binary search per ancestor, not a scan of journal history. + +**SQLite:** use the `value_versions` primary key (§4.2), one indexed seek per ancestor: + +```sql +SELECT seq, deleted +FROM value_versions +WHERE conv = ? AND namespace = ? AND key = ? AND seq <= ? +ORDER BY seq DESC +LIMIT 1; +``` + +Fetch the winning version's payload by its indexed reference (or directly from that version row). Parent metadata is a point read and may be +cached for active conversations. No copied inventory, ancestor-closure table or recursive journal +scan is required. The baseline cost depends on fork depth; it is not constant-time. Query-plan +tests must show indexed searches without a temporary ordering b-tree. + +### 5.4 Historical lists and paging + +```ts +const page = await conv.readList(moves, { after: cursor, asOf, limit: 100 }); +// [{ seq, value }, ...], oldest first; continue with the last returned seq +``` + +`after` is exclusive; `asOf` is an inclusive committed boundary. Omit `asOf` for the latest boundary +at that read; pass a fixed boundary across pages for a stable historical read. Limits are positive, +bounded, and required in this API sketch. A cursor is a sequence filter for the same conversation +and address, not a replication revision or a transferable replica binding. + +Traverse the same parent chain, carrying the minimum cutoff at every boundary, and read ranges +oldest ancestor first. A historical fork sees its ancestor's visible prefix followed by its own +appends. Subagents stop value/list traversal at their initialization boundary (§8.3). Descendants' +local writes follow their inherited prefixes in journal order, so concatenating these ranges needs +no global sort or copying. Skip ranges outside the requested cursor/cutoff and stop when the page +limit is filled. + +Memory/JSONL binary-search each local element array. SQLite uses `list_elements` with equality on +conversation/address, `seq > after AND seq <= cutoff`, `ORDER BY seq ASC LIMIT remaining`, then +fetches only those element payloads. An ordered-index outer loop plus addressed payload lookups +must not become a journal scan or temporary sort. Baseline traversal has fork-depth overhead plus +indexed reads of the requested elements, not work proportional to unrelated history. + +### 5.5 Checkpoints and replicated state + +Plugins own their delta shapes, reducers, validation and checkpoint cadence. A checkpoint is an +ordinary rewindable value; it contains the state and the last included list-element seq: + +```ts +const checkpoint = conversationValue<{ through: number; state: CanvasState }>("my-canvas.checkpoint"); +const deltas = conversationList("my-canvas.deltas"); +``` + +Hydration reads the checkpoint visible at the target, then pages visible deltas after `through`. +Checkpoint and delta writes that belong together share a command. An active wrapper retains the +current state so ordinary updates do not replay history on every write. Reopen and rewind reconstruct +from the selected checkpoint and suffix, without inverse operations. Checkpoint cadence bounds the +replay suffix only when maintained across the inherited history, not merely counted anew per child. +Checkpoints do not authorize deleting older deltas while arbitrary rewind is promised. + +Replication is a layer over these values/lists, not another durable store (§9.1). A task is needed +only for an actual lifecycle; plugin state alone does not create one. + +### 5.6 Named working scopes + +History scope (session/conversation) and working lifetime are distinct concepts. Working addresses +are session-owned, durable until explicitly retired, and never inherit/rewind. Use a named group, +not a boolean retention hint, so related checkpoints/memos can commit and retire together. + +```ts +interface MainScope { readonly kind: "main" } +interface WorkingScope { readonly kind: "working"; readonly id: string } +type WriteScope = MainScope | WorkingScope; +declare function workingScope(id: string): WorkingScope; + +declare const valueType: unique symbol; +declare const addressScope: unique symbol; +declare const writeScope: unique symbol; +interface Value { + readonly kind: "value"; + readonly scope: "session" | "bound-conversation" | "working"; + readonly namespace: string; readonly key: string; + readonly conv?: number; readonly workingScope?: string; + readonly [valueType]?: (value: T) => T; // invariant value type + readonly [addressScope]?: () => Sc; // covariant scope for reads +} +interface ValueList extends Omit, "kind"> { + readonly kind: "list"; +} +// Write is the §4.1 tagged union, with final seq and this invariant phantom: +interface ScopedWrite { + readonly seq: number; + readonly [writeScope]?: (scope: Sc) => Sc; +} +type SessionValue = Value & { readonly scope: "session" }; +type SessionList = ValueList & { readonly scope: "session" }; +interface ScalarSelection { + readonly scope: "conversation"; readonly kind: "value"; + readonly namespace: string; readonly key: string; +} +interface ConversationValue extends ScalarSelection { + readonly [valueType]?: (value: T) => T; +} +interface ConversationList extends Omit, "kind"> { readonly kind: "list" } +interface InitialValueWrite { address: ScalarSelection; value: JsonValue } +declare function conversationValue(namespace: string, key?: string): ConversationValue; +declare function conversationList(namespace: string, key?: string): ConversationList; +declare function sessionValue(namespace: string, key?: string): SessionValue; +declare function sessionList(namespace: string, key?: string): SessionList; +declare function setValue(address: ConversationValue, value: NoInfer): InitialValueWrite; +declare function value(namespace: string, key?: string): SessionValue; +declare function value(namespace: string, key: string, scope: WorkingScope): Value; +declare function list(namespace: string, key?: string): SessionList; +declare function list(namespace: string, key: string, scope: WorkingScope): ValueList; +``` + +`sessionValue`/`sessionList` are explicitly named main-session constructors. Conversation tokens +bind a conversation at the harness boundary; the backend receives a numeric conv on the address. +Only main addresses may carry conv. Equal scope id + namespace/key + kind means equal address; +object identity does not matter. Retention information is stored, not reconstructed from tokens. + +```ts +const work = workingScope(String(taskId)); +const frames = list("pi.pending.frames", "", work); +const memo = value("pi.pending.memo", "step", work); + +await harness.command(work, tx => { + tx.appendList(frames, frame); + tx.setValue(memo, checkpoint); +}); + +// After sealing/draining that invocation's progress: +await harness.command(tx => { + const resultId = tx.entry(convId, "tool_result", meta, result); + tx.settle(taskId, { status: "done", state: { produced: resultId } }); + tx.retireScope(work); // MAIN write, not a direct working delete +}); +``` + +All writes in a command are main OR one working id. Type invariance rejects main+working mixes; +a runtime check rejects two different working ids (their static tag is identical). Main commands +may retire multiple working scopes. Reads may use either lifetime; only writes are restricted. +Never mix a direct working append/set/delete with an entity settlement, even on SQLite. + +Retirement is the sole storage authority: no task-name parsing, missing-owner heuristic, TTL or +implicit cleanup at close. Reusing a scope after retirement is legal. Harness invocation fencing +must prevent a late progress job from accidentally becoming that new lifetime. State coordinating +a task transition remains main-scoped; only independently committed working data goes in the scope. +A typed address alone does not make a cross-scope atomic update possible. + +--- + +## 6. Context + +### 6.1 The list and its three ops + +`context: number[]` on the conversation state. Edited only by: +- `ctx_append { ids }`: emitted by the insert of a `select` kind, in the same command, in + transcript order. Never called directly. +- `ctx_replace { through, with, expectPrefixVersion }`: remove the prefix up to and including + entry `through`, put entry `with` (a summary) at the head. Rejected if `prefixVersion` moved. +- `ctx_reset { ids }`: replace the whole list with a bootstrap: `[]` (a user's `/clear`: waits for + input), a `user` handoff requested by `new_context` (post_tools resets and creates its generation), + or a `summary`-kind note. Reset alone never requests generation. + +Both `replace` and `reset` bump `prefixVersion`; `append` does not. Consequently the list always has +the shape **[head?, then selected entries in transcript order]**, and a summarization that started +earlier survives appends (they land after `through`) but is rejected by a competing replace or +reset. Validation (§7.2) checks, on every `ctx_*` op, that every id exists in this conversation or +in the inherited prefix, that the resulting list has that shape, and that `through` does not split +an exchange: no `tool_result` after `through` may answer an assistant entry at or before it. +The context as of an earlier commit is reconstructed by replaying `ctx_*` records (§6.6). + +### 6.2 Summaries and handoffs + +A `summary` entry is appended by the `collapse` task and carries `through`. A handoff is a `user` +entry appended by post_tools for the model's `new_context` call and selected by `ctx_reset`. Both +are ordinary transcript entries at their append position; their place *in the context* is the +head. UIs draw a summary immediately after its `through` (so retained turns follow it) and a +handoff at its own position (§9.2). + +### 6.3 Boundaries + +```text +provider request: generation task exists; no unresolved foreground calls +replace/reset: canEditContext (§8.1), checked at committing boundary +historical fork: safe source boundary; subagent capture is separate (§8.3) +queued write: safe message boundary; never splice into an open tool exchange +``` + +Pending/retry-waiting generation may permit edits, never duplicate generation. Settlements complete +exchanges, including required error results. `through` must not split an exchange; reset drops +whole exchanges. Empty/note-only resets create no work. + +### 6.4 Projection + +``` +messages := for each id in conv.context: entries[id].kind.project(entry) (tool results ordered by callIndex within their message) + then before_request hooks edit the request +``` +Nothing else. Projections never write. + +### 6.5 Collapse policy + +```text +generation.pending, before provider intent: + if captured collapseId exists: observe that task; do not recreate it + else if usage > threshold × contextWindow and more than keepRecent turns remain: + decide before_collapse outside line; revalidate context inside line + if approved: commit collapse task + generation.after=[collapse] and captured reason + RETURN; generation stays pending and scheduler waits for its dependency + build request from resulting context + commit streaming intent → call provider + +provider overflow: + decide collapse outside line; revalidate before outcome commit + commit attempt usage + generation.retry_wait + collapse task/id together + existing generation waits for that collapse, then retries once + declined/impossible/failed overflow collapse → fail existing generation; no replacement + +manual collapse: create collapse task only; does not request generation +collapse success: commit summary entry + ctx_replace + usage + terminal collapse +``` + +The id is a specific task dependency, not a progression flag. Driver can run collapse while its +generation waits; reopen observes the same task. Retry timers never bypass unfinished dependencies. +Threshold decline proceeds with existing generation, not another conversation check. Threshold +collapse failure proceeds with unchanged context; overflow collapse failure ends the generation. +Summary captures `through` and `prefixVersion`, includes the previous summary, and subsumes it. + +### 6.6 Context as of a commit + +```text +contextAsOf(conv, asOf): + require complete commit boundary, with conv already created + base = newest ctx_reset <= asOf, otherwise conv's ins_conv + context = copy(base.ids or base.context) // inherited initial context included + page context_ops in (base.seq, asOf], oldest first: + ctx_append: append ids + ctx_replace: replace prefix through named entry; retain EXISTING suffix + ctx_reset: replace entire list + return context +``` + +Replacement is not a full checkpoint: `[u1,a1,u2,a2] → [summary,u2,a2]` retains u2/a2 although their +appends preceded replacement. Indexed baseline replay excludes unrelated history but may span the +conversation since creation/reset. Compaction-bounded reconstruction remains an optimization to +review. Current context is a materialized point read; plugin hydration uses value/list indexes. + +--- + +## 7. The line + +One command queue per session. `session.command(build)` (exposed to task handlers as +`ctx.commit(build)`) runs the callback with a command-local buffer and the previous command's +committed view. Reads may be async; no other command runs until this one finishes. + +The buffer starts its sequence counter at the last committed seq + 1. Adding a record mints its +seq immediately and advances that counter. Creation methods return the seq as the new object's id; +later records use it directly. These methods only build the command: they perform no storage write +and publish no state. + +```ts +interface ValueReader { + getValue(address: Value): Promise; + readList(address: ValueList, options: ListReadOptions): Promise[]>; +} +interface ValueCommand extends ValueReader { + setValue(address: Value, value: NoInfer): number; + deleteValue(address: Value): void; + appendList(address: ValueList, value: NoInfer, tag?: string): number; + deleteList(address: ValueList): void; // reject historical conversation-list deletion +} +interface WorkingCommand extends ValueCommand {} +interface EntryOptions { by?: number; callIndex?: number; through?: number; key?: string } +interface TaskOptions { for?: number; at?: number; after?: number[] } +interface Command extends ValueCommand { + readonly view: ConversationView; // committed pre-command reader + conversation(options: Pick): number; + setConversation(id: number, patch: Partial): void; + deleteConversation(id: number): void; + entry(conv: number, kind: string, meta: JsonObject, payload?: JsonValue, opts?: EntryOptions): number; + task(conv: number, kind: string, state: JsonObject, opts?: TaskOptions): number; + set(id: number, patch: TaskPatch): void; + settle(id: number, patch: { status: string; state?: JsonObject }): void; + ctxReplace(conv: number, through: number, withId: number, expectPrefixVersion: number): void; + ctxReset(conv: number, ids: number[]): void; + retireScope(scope: WorkingScope): number; // always a MAIN write + usage(taskId: number, conv: number, usage: Usage): number; // session-list append convenience + bindValue(conv: number, address: ConversationValue): Value; + bindList(conv: number, address: ConversationList): ValueList; +} +``` + +Bound `conv.command` supplies conversation-token reads/writes without repeating conv. Main command +reads can inspect working values; its writes cannot directly change them. WorkingCommand exposes +only value/list writes, and runtime validation pins all writes to the supplied working id. + +```ts +await ctx.commit((tx) => { + const assistantId = tx.entry(convId, "assistant", meta, { message }, { by: generationId }); + const calls = message.content.filter(block => block.type === "toolCall"); + const toolIds: number[] = []; + for (const [callIndex, call] of calls.entries()) { + toolIds.push(tx.task(convId, "tool", { + status: "planned", callIndex, callId: call.id, name: call.name, + args: call.arguments, allowed: activeToolNames.includes(call.name), replay: "never", + }, { + for: generationId, at: assistantId, + after: sequential && toolIds.length ? [toolIds[toolIds.length - 1]!] : [], + })); + } + const postTools = tx.task(convId, "post_tools", { status: "pending", generation: generationId }, { + for: generationId, at: assistantId, after: toolIds, + }); + tx.usage(generationId, convId, message.usage); + tx.settle(generationId, { status: "done", state: { produced: assistantId, calls: toolIds, postTools } }); + tx.retireScope(workingScope(String(generationId))); +}); +``` + +`entry` for a selected kind also buffers its `ctx_append` immediately, with its own seq (§6.1). +Callers neither append it themselves nor calculate offsets around it. All methods use the same +counter, including patches and context operations; ids require no `{ref}` placeholders, reserved +ranges, deep traversal, or fix-up pass. + +Normal callback return commits the buffer; returning without adding records is a no-op. A throw +or validation rejection discards the whole buffer without advancing the committed sequence. Minted +ids MUST NOT be published or used for external effects before the command succeeds. The command +object is valid only during its callback; it cannot be retained to add records later. + +After the callback returns, the line loads the rows the records target, validates them in order +against committed state plus earlier records in the buffer (§7.2), and appends the whole buffer as +one atomic commit. Storage verifies the supplied seqs are contiguous and start immediately after +its global committed head; it never renumbers them. Storage publishes its read model as part of commit +(§4.3), not via a second harness write. The line then updates execution residency (§4.4), delivers +to listeners, and resolves `true` (`false` for a no-op). A storage failure faults +the session (§7.3); reopening derives the next seq from the last complete durable commit. Sequence +exhaustion rejects rather than allocating an unsafe integer. Nothing else writes the journal. + +### 7.1 Guarantees +1. No two commands interleave; a callback sees the previous command's committed state. +2. A caller resolves after delivery; a listener sees the state its records describe; a listener + MUST NOT await a command on the same session inside its callback; a throwing listener is + reported as `handler_error` and does not affect the commit. +3. Terminal settlement seals/drains working writes, then commits outcome + scope retirement together. + Physical unlink follows; no task-status-based orphan inference exists (§3.5). +4. A completed task and its successors commit together; no later driver/event pass must create them. + +### 7.2 Validation (rejects the whole commit) +- `ins_entry`: the conversation exists; `through` (if any) is an entry of it. +- `ctx_replace` / `ctx_reset`: the resulting list (§6.1) has the head+ordered-suffix shape; `through` + does not split an exchange; `expectPrefixVersion` matches. +- `ins_task`: conversation exists; optional at belongs to it or its inherited prefix; for is valid; + initial status is allowed; after targets exist in the same ownership tree and are acyclic. +- Every command writes main OR one working id. Main retirement is the sole cross-lifetime action; + it retires logical scope contents without requiring a second-file transaction. +- `set`/`settle`: target exists; keys are the kind's; a status change is an allowed transition; + `settle` reaches a terminal status; a terminal task accepts no patch. +- `ctx_*`: every id exists in this conversation or its inherited prefix. +- Conversation value/list writes: the target conversation exists, including when created earlier + in the same command. Parent history boundaries are complete commits; a historical fork's + inheritance link and a subagent's independent initialization survive reopen unchanged. +- `del_conv`: ownership subtree has no live tasks; cancellation was driven before deletion. + Retain immutable history required by independent forks; reject new work in deleted conversations. + +### 7.3 Faults +A failure after a plan's decision faults the session: later commands reject; the process should +exit and reopen. Rejections (§19) are not faults. + +--- + +## 8. Conversations and admission + +### 8.1 Context editing is not scheduling + +Scheduler, drive scopes and cancellation are defined only in §3.6–§3.8. + +```text +canAcceptNow = no foreground work and no outstanding cancellation (§3.4) +canEditContext = no streaming/polling request and no unresolved call in the current exchange +``` + +Pending post_tools blocks acceptance, even after all calls settled. Pending/retrying/deferred +generation can permit safe edits but never another generation. A held call blocks replacement/reset. +Background jobs do not block edits. Context-only writes never become generation requests on open. + +### 8.2 Acceptance and the chain + +```text +idle accept, ONE MAIN COMMAND: + place eligible queued content + input in selected admission order; dequeue + capture settings; create pending generation + write optional external request mapping + +busy/cancelling accept, ONE MAIN COMMAND: + store unselected input + queue followUp + write optional external request mapping +``` + +Acceptance starts no provider effect. The chain is generation → tools + post_tools → next generation +(§3.9–§3.11). Each tool settles only itself. The join exists before any call starts; the last tool +merely makes it eligible. Post_tools and final-answer generation own exchange/turn-end decisions. + +### 8.3 Subagents and jobs + +A subagent is a `subagent` task with `owns`. Its creation is **initialization**, not historical +value inheritance: the creating tool chooses scalar addresses to copy and typed overrides. Context +inheritance is independent. Public API sketch: + +```ts +await conv.spawn({ + prompt: "inspect the parser", + context: "inherit", + values: { + inherit: [model, thinkingLevel], + set: [setValue(activeTools, ["read", "grep"])], + }, +}); +``` + +`setValue(address, value)` here constructs a typed initial write; it does not commit independently. +There is no universal `inherit` flag on a token: one tool may need a value another must exclude. +Common selections are ordinary arrays of typed addresses, not a registry. Selection is scalar-only; +canvas/game lists are not implicitly copied into subagents. + +Inside one serialized creation command, capture the parent boundary, read selected effective +values, create the child with `owner` and `parent: {conv, at?, asOf, inheritValues: false}`, write the selected +present values and overrides, insert the prompt AND its initial generation task, and mark the +owner task `running` with `owns`. +An absent selected value stays absent unless overridden. Copy values, not mutable object aliases. +Only the resulting local values are stored, not token objects or a durable selection policy. +The child cannot drive before this command commits. It has no parent value/list fallback: plan +mode and plugin state not selected are absent even when its context was inherited. Model, thinking +level and tool names become ordinary rewindable local values. A later historical fork of the child +inherits these values normally and stops at the child's independent initialization boundary. + +The child gets an empty or inherited context per `context: "fresh" | "inherit"`. Initialization may +happen while the parent runs; an inherited context must end at a complete exchange and does not edit +the parent's context. The exact context-capture policy for an executing parent remains under review +(§25). The child finishing command settles its owner (§3.13). A foreground tool is already parked +in finishing with after:[owner]; it becomes eligible without a waitFor promise. Background launch +settles the tool with the id immediately. Job behavior is defined in §3.14. + +### 8.4 Forks and rewind + +A historical fork records `parent: {conv, at?, asOf, inheritValues: true}`. It inherits **all** +conversation values and lists visible at that boundary; no per-token filter applies. Fork creation +copies neither value inventories nor list prefixes. Local writes override inherited scalars, and +local list appends follow inherited prefixes (§5.3–§5.4). Session values remain session-wide. + +A target may be `{ at: entryId }` or `{ asOf: commitEndSeq }`. Entry selection resolves to the commit +boundary that **contains** the entry, so a tool result and active-tools update in the same command +are inherited together. An explicit commit target supports moves or canvas changes that contain +no message entry. A seq inside a commit is rejected, not rounded to a partially visible state. + +With inherited context, copy the parent's context as of the selected boundary (§6.6); for an +entry selection, also restrict to the selected entry position as in the original fork contract. +Trim to the last complete exchange. The fork evolves independently of later parent edits. +Historical forks inherit context/values, not live tasks. Continuing requires explicit generation-task +creation or acceptance; driving copied user-ended context must not infer work. + +Unless keepRunning, `rewind` aborts/drains current conversation foreground work before creating +the fork. It does not selectively cancel graph nodes by at. keepRunning leaves the source chain +untouched. Entry/commit targets select inherited history, not cancellation membership. `/tree` +lists by parent and distinguishes independent forks by absent owner. + +### 8.5 Open, resume, close, delete + +- `open`: conversation states, live tasks, the entries on live conversations' context lists, scratch + of live tasks. Start nothing; report live tasks by role. +- `drive(opts)`: schedule the selected ownership scope; session drive covers all trees (§3.8). +- `abort(conv)`: mark foreground targets and drain eligible queues once (§3.7). +- `close()`: seal admission, signal executions, drain persistence; retire/settle nothing. +- `shutdown(conv)`: conversation cancellation including background work. +- `delete(conv)`: report work; on confirm, shutdown/drain ownership subtree, then del_conv. + Retain history needed by independent forks; physical repository deletion is separate. + +--- + +## 9. Observation + +### 9.1 Watch + +`watch(conv, { limit })` runs on the line: it registers the listener and reads the base in one +step, so there is no gap. Base: +``` +{ asOf: seq, + conversation: ConversationState, // includes context and prefixVersion + entries: [ last `limit` transcript entries in transcript order, with payloads ], + tasks: [ live tasks ], + scratch: { [taskId]: reduced } } +``` +Then deliver relevant main writes in commit batches, retaining each write's seq (`ins_entry`, +`ins_task`, `set`, `settle`, `set_conv`, context/value/list edits and scope retirement; owned +conversations if requested). Usage is a main-list append. Working progress is delivered as a preview, +and ephemeral deltas `{ frame | progress | kind-emitted, taskId }`, plus `quiescent` and `fault`. +Older pages: `entries(conv, { before, limit })`, each with its own `asOf`; the client ignores stream +records with `seq ≤ asOf` for entries it paged in. Remote clients RPC into the host, so the same +one-step base applies to them. + +**Replicated plugin state** is a wrapper over conversation checkpoints and delta lists (§5.5), +exposing Chord's `ReplicatedState` read surface. Persistence and conversation identity remain +Pi/application concerns, not Chord concepts. + +- **No durable replication revisions.** Storage records only journal seqs. At fresh hydration, + the adapter establishes stream revision 0, then numbers emitted update batches 1, 2, 3, …. + Journal updates at 104 and 117 can therefore be stream revisions 1 and 2. Chord's consecutive + revision check remains unchanged; no stored `stateSequence` or second allocation index exists. + The adapter retains the journal cutoff for catch-up independently of the live stream counter. +- **Commit before publication.** Read the current committed state and derive/validate changes on + the line; buffer delta/checkpoint writes; commit; then publish the corresponding state before + the next command runs. Concurrent updates cannot compute from the same stale base. A failed + command publishes nothing and its private draft must not contaminate a subsequent update. + Retain the current committed state while active; hydration is not repeated per write. +- **Hydration and binding.** Read the visible checkpoint and delta suffix, capture the journal + boundary and register update capture in one line step. On rewind, reconstruct the selected + history, invalidate the old binding, hydrate the new one and restart its live counter. Reject + deliveries from retired bindings. A periodic checkpoint does not by itself restart an existing + replication stream or require a client-visible update when the state is unchanged. +- **Actual Chord boundary.** `replicatedState(initial)` exposes tracked `.state`, published `.value` + and synchronous `.publish(context)`; its public subscription delivers complete values. Its + decoded-operation subscription and producer registration are private. Moreover, subscribing to + a producer or its service can flush pending mutations. Therefore mutating an exposed producer + and then awaiting storage is NOT a safe persistence adapter. Draft tracking must remain private + until commit; the exact supported Chord adapter seam is still to be designed (§25). +- **Delta ownership and encoding.** Chord Delta publicly supplies `track`, `apply`, `applyImmutable`, + `encoder` and `decoder`. Decoded `Op[]` carry complete paths; compressed `WireOp[]` may depend on + prior path definitions. A complete replacement resets those dictionaries. Arbitrary wire batches + cannot be independently replayed or spliced across histories without their matching codec state. + Durable encoding and codec handling at checkpoints/forks remain an explicit design choice (§25), + not something list storage infers. Mutable replay must own its payloads; it must not mutate the + authoritative in-memory journal. Immutable application shares unchanged subtrees but copies + changed-path containers, whose cost can depend on container size. + +The wrapper owns checkpoint cadence and state/batch bounds. Rewind reconstructs; it does not send +inverse operations. Historical data cannot be pruned merely because a newer checkpoint exists. + +### 9.2 Rendering + +Transcript order, with one rule: a `summary` is drawn immediately after its `through` (at the top +of the loaded page if `through` isn't loaded); everything else where it is. The context list marks +what the model sees (dim the rest). +A client-side `toLaneSnapshot(base)` (transcript, streaming message from the generation's reduced +scratch, running tools from live tool tasks, queues from the inbox) serves renderers that want the +old shape. + +### 9.3 Events + +Subscriptions over journal records by kind and transition; no separate event vocabulary. + +--- + +## 10. Hooks + +Points and decisions: `before_request` (edit), `after_response` (observe), `before_tool` +(`allow | block(text) | hold`), `after_tool`, `on_yield` (`pass | continue(message)`), +`before_collapse` (`decline | {meta}`). `hooks.on(point, handler, { priority })`, ascending; first +`block`/`hold`/`decline` wins; first `continue` wins. Hooks own no state. + +--- + +## 11. Concrete task index + +§3 is the authoritative task contract; this index introduces no second lifecycle definition. + +| Task | Responsibility | Durable continuation | +|---|---|---| +| generation (§3.9) | provider response/retry/poll | tools + post_tools, or final boundary | +| tool (§3.10) | one call and own result | no sibling/turn orchestration | +| post_tools (§3.11) | join calls, apply exchange policy | next generation or finish | +| collapse (§3.12) | summarize and safely replace prefix | existing generation becomes eligible | +| subagent (§3.13) | own child conversation | child terminal command settles owner | +| job (§3.14) | spawn/adopt process, observe exit | dependent tool or scheduled job | +| approval (§3.15) | human decision | resume/finish existing tool | + +`new_context` is a tool outcome, not another task kind: the tool stores handoff intent; post_tools +appends the handoff, resets context and creates the next generation atomically. + +--- + +## 12. Public API and usage + +### 12.1 Shared types and host services + +```ts +type AgentInput = string | UserMessage | readonly AgentMessage[]; +interface ModelIdentity { provider: string; modelId: string } +interface RetryPolicy { maxAttempts: number; baseDelayMs: number } +interface CollapsePolicy { threshold: number; keepRecent: number } +interface GenerationSettings { + model: ModelIdentity; + thinkingLevel: ThinkingLevel; + activeToolNames: string[]; + toolOrder: "parallel" | "sequential"; // creates after edges; not a scheduler lock + streamOptions: JsonObject; + retry: RetryPolicy; +} +interface DriveOptions { pollDeferred?: boolean } +type DriveOutcome = "idle" | "suspended" | "closed"; +interface Inspection { start: Task[]; inflight: Task[]; waiting: Task[] } +interface Accepted { entryId: number } +interface AbortReport { drained: InboxItem[] } +type HistoryTarget = { at: number } | { asOf: number }; +interface TaskServices { + models: Models; + tools: ReadonlyMap; + hooks: Hooks; // typed points in §20 + resources: ReadonlyMap; // host capabilities; checked by consuming kind + retry: RetryPolicy; + collapse: CollapsePolicy; +} +interface SpawnOptions { + prompt: AgentInput; + context: "fresh" | "inherit"; + values: { inherit: readonly ScalarSelection[]; set: readonly InitialValueWrite[] }; +} +interface RegisteredKind { readonly kind: string } // opaque result of validated registration +declare function defineTask(definition: TaskDefinition): RegisteredKind; +interface HarnessOptions extends TaskServices { + initial: GenerationSettings; + kinds: readonly RegisteredKind[]; + systemPrompt: string | (() => string | Promise); +} +``` + +`RegisteredKind` is the validated/erased registration returned by defineTask/defineContent; each +constructor preserves its schema's state/status typing. This is not an arbitrary object registry +for values. Provider option patching and process-adoption capabilities remain host seams (§25). + +### 12.2 Harness and conversation + +```ts +interface ConversationView { + state(): Promise; + entry(id: number): Promise; + task(id: number): Promise; + entries(query: Omit): Promise; + tasks(query: Omit): Promise; + getValue(address: ConversationValue, at?: HistoricalRead): Promise; + readList(address: ConversationList, options: ListReadOptions): Promise[]>; +} +interface ConversationCommand { + readonly view: ConversationView; + getValue(address: ConversationValue): Promise; + setValue(address: ConversationValue, value: NoInfer): number; + deleteValue(address: ConversationValue): void; + appendList(address: ConversationList, value: NoInfer, tag?: string): number; + readList(address: ConversationList, options: ListReadOptions): Promise[]>; + entry(kind: string, meta: JsonObject, payload?: JsonValue, options?: EntryOptions): number; + task(kind: string, state: JsonObject, options?: TaskOptions): number; + set(id: number, patch: TaskPatch): void; +} +interface Conversation extends ConversationView { + readonly id: number; + accept(input: AgentInput, options?: { requestId?: string }): Promise; + drive(options?: DriveOptions): Promise; + prompt(input: AgentInput, options?: { requestId?: string }): Promise; + result(inputId: number): Promise; // contract still open in §18.2 + appendMessage(message: AgentMessage): Promise; // context only, deferred if unsafe + steer(input: AgentInput): Promise; + followUp(input: AgentInput): Promise; + nextRun(input: AgentInput): Promise; + cancelQueued(id: number): Promise<"cancelled" | "already_consumed" | "not_found">; + abort(): Promise; // commit intent; drive performs cancellation + shutdown(): Promise; // mark + drive whole ownership subtree, including background + waitForIdle(): Promise; // observe only; starts no work + runWhenIdle(callback: () => Promise): Promise; // host-side admission; semantics to finalize (§25) + collapse(options?: { instructions?: string }): Promise; // create task, not provider effect + resetContext(ids: number[]): Promise; + fork(options: HistoryTarget & { context: "fresh" | "inherit"; label?: string; summary?: string }): Promise; + rewind(target: HistoryTarget, options?: { keepRunning?: boolean }): Promise; + spawn(options: SpawnOptions): Promise; // create owner task; return immediately + setValue(address: ConversationValue, value: NoInfer): Promise; + deleteValue(address: ConversationValue): Promise; + appendList(address: ConversationList, value: NoInfer, tag?: string): Promise; + command(build: (tx: ConversationCommand) => void | Promise): Promise; +} +interface Harness extends ValueReader { + root(): Promise; + conversation(id: number): Promise; + conversations(query: ConversationScan): Promise; + inspect(): Inspection; + drive(options?: DriveOptions): Promise; + command(build: (tx: Command) => void | Promise): Promise; + command(scope: WorkingScope, build: (tx: WorkingCommand) => void | Promise): Promise; + setValue(address: Value, value: NoInfer): Promise; + deleteValue(address: Value): Promise; + watch(conv: number, options: { limit: number }, listener: (event: WatchEvent) => void): Promise<() => void>; + decideApproval(id: number, decision: "grant" | "deny", details?: JsonValue): Promise; + delete(conv: number, options: { confirm: boolean }): Promise; + close(): Promise; +} +declare const Harness: { + open(storage: Storage, options: HarnessOptions): Promise; +}; +``` + +Invocation Context is supplied by the host binding; Storage receives it explicitly. Public async +wait cancellation must remain observer-only; it never silently calls abort. Expected-input fencing +and exact observer Context overloads remain open (§25). `InputResult` needs the explicit attribution +contract in §18.2, not chronological answer guessing. WatchEvent is the wire union in §21. + +Conversation tokens carry scope/kind/value type but no conversation id until bound. SpawnOptions +contains prompt, fresh/inherit context, scalar address selections and typed initial writes (§8.3). +Heterogeneous selections erase only after each write has passed its address's NoInfer check. + +### 12.3 Accept, inspect, drive and cancel + +```ts +const h = await Harness.open(storage, options); // no effects +const root = await h.root(); +const accepted = await root.accept("Inspect the parser", { requestId: "web-request-42" }); +console.log(h.inspect()); // includes pending generation +const outcome = await root.drive(); +if (outcome === "idle") console.log(await root.result(accepted.entryId)); + +// Another caller can cancel the conversation while drive is active: +const driving = root.drive(); +const cancelled = await root.abort(); // durable marks; no per-task cancel +await driving; // provided this scope remains driven +console.log(cancelled.drained); + +// After a restart, inspect for display, then drive; do not iterate tasks to resume each one. +console.log(h.inspect()); +await h.drive({ pollDeferred: true }); // all independent trees +``` + +### 12.4 Queueing and independent scopes + +```ts +const first = await root.accept("Analyze A"); +await root.accept("Then analyze B"); // first generation exists: queues followUp +await root.steer("Use read-only tools"); +await root.nextRun("Apply this on the next explicit idle acceptance"); +await root.drive(); + +await root.appendMessage({ role: "user", content: "Context only", timestamp: Date.now() }); +await root.drive(); // creates nothing merely from this message + +const fork = await root.fork({ at: first.entryId, context: "inherit" }); +await fork.accept("Try a different approach"); +await fork.drive(); // parent and sibling forks remain parked +``` + +### 12.5 Values, child initialization and working state + +```ts +const model = conversationValue("pi.model"); +const thinking = conversationValue("pi.thinking"); +const tools = conversationValue("pi.active-tools"); +const plan = conversationValue("my-plugin.plan"); +await root.setValue(plan, true); +await root.spawn({ + prompt: "Inspect only the parser", + context: "inherit", + values: { inherit: [model, thinking], set: [setValue(tools, ["read", "grep"])] }, +}); +await root.drive(); + +// Working value/list writes use the SAME command path, pinned to one named lifetime. +const work = workingScope("upload-preview"); +const chunks = list("my-plugin.chunks", "", work); +await h.command(work, tx => { tx.appendList(chunks, "first chunk"); }); +await h.command(tx => { tx.retireScope(work); }); +await h.close(); +``` + +The examples are API illustrations, not benchmark results or executable integration tests. + +--- + +## 13. Invariants and tests + +1. The read model equals a replay at every commit, on every backend. +2. Every external effect is preceded by committed inflight intent; settlement and successors are one command. +3. Every non-terminal status has a driver action; an unknown status fails at open. +4. After abort and a completed cancellation drive, all selected foreground/required tasks are terminal + and their working scopes retired. Detached background tasks survive; shutdown includes them. +5. Delivery order and the listener rule (§7.1). +6. Projections never write; the driver never reads a payload. +7. Entries are never patched or reordered; a terminal task accepts no patch. +8. The context list is always `[head?, then entries in transcript order]`; every id on it exists; `through` never splits an exchange; `ctx_replace` with a stale `prefixVersion` rejects; a summarization survives appends and is rejected by a competing replace/reset; the context as of any commit boundary is reconstructible by replaying `ctx_*` records from the newest reset. +9. **Residency**: execution holds live conversation states, live tasks, their scratch, context entries and needed settings. Backend history and active plugin replicas are separate, explicitly counted costs (§4.4). +10. **Bound**: resident entries = union of context-list ids; resident tasks = |live| plus newest generations not already live; independent of journal length on every backend. Verify at ≥100k turns with cold reopens; this is a required measurement, not a validated result. Count full journal/index memory on memory/JSONL and active plugin tracking state separately. +11. A fork's context never changes when its parent summarizes or resets later. +12. A task older than any summary survives a cold open, is recovered, settles, and is readable by id. +13. Watch has no gap: base and subscription are one line step; a paged-in entry never regresses a streamed update (`asOf`). +14. Session metadata survives navigation unchanged; historical scalar reads respect every ancestor cutoff and deletion tombstone. +15. Historical forks copy no value/list prefixes; subagents store only selected initial scalar values and overrides, with no parent fallback. Both behaviors survive reopen. +16. List pagination and checkpoint replay produce the same visible history across nested forks as a reference traversal; no page leaks sibling or later-parent writes. +17. Input acceptance and external identity mapping are one atomic commit; a lost response cannot leave just one of them durable. +18. Replication publishes only committed state. Journal gaps require no durable replication counter; fresh bindings hydrate before consecutive emitted updates. +19. Acceptance/settlement/explicit caller commands create generation tasks, never driver tail inference. +20. Generation creates its tools and exactly one post_tools atomically. Tools settle only themselves; + post_tools remains foreground and creates exactly one successor with its own settlement. +21. Context-only writes request no generation, before or after reopen; pending/retrying/deferred work prevents duplicate foreground admission. +22. Reported usage commits with its attempt outcome, including retries and stale collapse publication. +23. Main/working mixed writes reject before mutation. Retirement commits with task outcome; physical + unlink failure cannot resurrect retired data. Late progress cannot reopen a finished task's scope. +24. Overlapping drive scopes share ownership; effect/recovery and abort never write concurrently. +25. Open starts nothing, invents no successors, and restores only explicitly live working lifetimes. + +Scenarios: the v1 suite re-expressed; the walkthroughs in §15; the adversarial list: rewind after +replacing a conversation value; a plugin writing thousands of versions (execution residency flat, +backend history counted separately); nested forks before/after deletes and checkpoints; selective +subagent settings without inherited plan mode; compact +repeatedly (one head, never a chain); an ancient job waiting across many summaries; fork before a +later parent summary; a reset while a summarization runs (summary rejected); page while updates +arrive (`asOf`); render a summary with a retained tail at `limit: 3` (§15.6). + +--- + +## 14. Contrast + +| | runtime/ (lanes) | dom/ and pico v1 | pico v2 | +|---|---|---|---| +| unit | lane over an entry branch | node tree | conversation = transcript + tasks + values, with a context list | +| in-flight state | one 13-leaf op record per lane | status on nodes | status on tasks; one live set | +| model context | scan back to the compaction entry | collapse node / folds | explicit list: head + suffix; three ops | +| memory | window per lane | whole tree (v1) | context entries + live tasks | +| history | branch index, `getEntry` | resident tree | transcript pages, typed historical value/list reads | +| subagents | child lanes | nodes | tasks owning conversations | +| plugin state | session values/lists + custom entries | nodes | typed session values and rewindable conversation values/lists | +| rewind | move a cursor | fork node | historical fork; optionally cancel source conversation foreground | +| storage | entries + branch index | ops + checkpoints | journal + read model; JSONL rebuilds by scan, SQLite keeps tables | + +--- + +## 15. Walkthroughs + +Task names below stand for numeric creation seqs. Braces group ONE atomic main commit. Progress +commits go only to the named working scope; terminal commits include its retirement. + +### 15.1 One tool + +```text +accept: { user U; create G1 } +request: { G1 streaming } → provider +response: { assistant A; create T; create P after:[T]; settle G1; retire G1 scope } +execution: { T running } → tool +result: { tool result R; settle T; retire T scope } // P remains live, no idle gap +coordination: { settle P; create G2 } +answer: { assistant B; settle G2; retire G2 scope } // no continuation: idle +``` + +T knows nothing about P or other tools. P exists before T starts, so a crash after T's result +requires no repair: P is already ready. Crash during T executes its normal recovery policy. + +### 15.2 Parallel and sequential calls + +```text +parallel response: + { create T1,T2,T3; create P after:[T1,T2,T3]; settle G1 } +completion order: + { R2; settle T2 } → P blocked + { R3; settle T3 } → P blocked + { R1; settle T1 } → P ready + { settle P; create G2 } + +sequential response: + { T1; T2 after:[T1]; T3 after:[T2]; P after:[T1,T2,T3]; settle G1 } +execution: + T1 terminal → T2 ready → T2 terminal → T3 ready → T3 terminal → P ready +``` + +Both use the same tool and join handlers. The transcript preserves completion order; model +projection emits results in call order. A blocked tool produces an ordinary error result and +settles. Only P interprets completed exchange outcomes and decides whether generation continues. + +### 15.3 Hold, restart and conversation cancellation + +```text +T1 held: { create approval A; T1 waiting after:[A] } +T2 after:[T1] and P remain blocked +open: inspect reports all tasks; starts nothing +approve: { A granted; T1 planned approved:true } → covered drive runs T1 + +instead, conv.abort(): + { mark T1,T2,T3,P and required approval A; drain steer/followUp once } + signal owned effects; join each task's own execution before abort handler + each tool writes its error result; unstarted tools need not await predecessors + P aborts without generation; every terminal task retires its working scope +``` + +Only conversation cancellation is public. Deleting a queue item is separate from cancelling live +work. Detached background tasks survive foreground cancellation; shutdown/delete includes them. + +### 15.4 Foreground/background subagent + +```text +foreground tool T delegates: + { create owner S; T finishing after:[S] } // effect returns +S starts: + { create child C owner:S; initial values; prompt; Gchild; S running owns:C } +child finishes: + { final assistant B; settle Gchild; settle S result:B } // existing T is now eligible +T finishes: + { parent tool result from B; settle T } // parent P waits for all calls + +background tool T delegates: + { create S; tool result containing S.id; settle T } // no dependency on S +``` + +No effect holds a promise across child lifetime. Recovery drives existing child tasks and stored +dependencies. Child completion does not execute parent work outside the caller's drive scope; +it only settles its owner as part of that completion. Parent tool finishing runs when covered. +Forks have no owner and are never pulled into a parent's scope merely through provenance. + +### 15.5 Background job across restart + +```text +{ create job J; tool result J.id; settle launching tool } +{ J spawning, durable adoption/log location } → spawn/adopt +{ J running, verified process identity } → install output/exit observers, return +parent P creates next generation without waiting for J + +open → inspect J running; covered drive re-arms observer once +exit → { immutable result E; settle J exited produced:E; retire J scope } +later tool → indexed read J and E, even if J.at is long outside current context +``` + +The external process log and bounded working checkpoints have explicit host lifetime rules. +Neither context compaction nor ordinary foreground cancellation makes a detached job disappear. + +### 15.6 Collapse, speculative, subsuming, and the rendering at `limit: 3` + +Transcript `u1 a1 u2 a2` (ids 1..4), context `[1,2,3,4]`, budget exceeded, `keepRecent` keeps `u2 a2`: +``` +20 T collapse C {pending, through:2, prefixVersion:0} 21 set 20 {summarizing} + — meanwhile generation 22 runs and appends 23 (u3), 25 (a3): ctx [1,2,3,4,23,25] — +26 E summary C "…" through:2 by:20 (context: "none": not auto-appended) ┐ one commit +27 ctx_replace {through:2, with:26, expectPrefixVersion:0} → ctx [26,3,4,23,25] │ prefixVersion 1 +28 settle 20 {done, produced:26} ┘ +``` +A reset that had landed between 21 and 26 would have bumped `prefixVersion` and 27 would reject; +20 would settle `failed`. A second collapse later captures `through: 23` and summarizes the prefix +`[26, 3, 4, 23]`, which includes summary 26: the new summary subsumes it; the list stays one head +plus a suffix. + +Rendering: transcript order is `u1 a1 u2 a2 u3 a3 S` (S = 26 was appended last). A watch with +`limit: 3` receives `[u3, a3, S]`; S is drawn after its `through` (a1), which isn't loaded, so at the +top: `S u3 a3`. Paging `before: u3` returns `[u2, a2]`, drawn below S because their positions are +after a1: `S u2 a2 u3 a3`. Paging again returns `[u1, a1]`: `u1 a1 S u2 a2 u3 a3`. Pagination uses +transcript order as stored; only the summary's display position is relocated. + +### 15.7 Reset (Codex-style) + +The new_context tool settles its own result with handoff intent. It never checks siblings or edits +context. Once all calls settle, post_tools applies the handoff: +```text +{ append handoff H; ctx_reset [H]; create G2; settle post_tools } +``` +This is explicit continuation. `/clear` alone creates no task; changing context while a generation +is already pending does not implicitly cancel that task. +Transcript unchanged; jobs unchanged; conversation values/lists unchanged. The next generation projects `[40]` +plus subsequent appends. A watcher draws the handoff at its position and dims everything above it. + +### 15.8 Fork and rewind + +Fork at `a1` (entry 2) after §15.6 published: the fork copies C's context restricted to ids `≤ 2`: +`[26]`? No: 26 has `through: 2`, its head stands in for `1..2`; restricted to `≤ 2` that head is not +applicable (it was published after the fork point and the fork's `asOf` is earlier), so the fork +reads C's context **as of `asOf`** = `[1, 2]`, trims to the last complete exchange, and starts from +`[1, 2]`. Later parent summaries never reach it (invariant 11). `rewind(C, at: 2)` while tool 7 runs +under generation 2: cancel/drain the source conversation's foreground chain, then fork. +Values/lists in the fork follow the parent link through the commit containing entry 2; later parent +versions are invisible. A separate `{ asOf }` target can select a value-only or delta-only commit. + +### 15.9 Plugin state and replicated state + +Plan mode enabled at commit 80: `value_set C my-plugin.plan-mode=true`. A historical fork through +commit 60 sees the preceding value (or the caller's default if absent); parent changes after 60 +are invisible. Deleting plan mode in the fork masks the inherited value without altering C. +A subagent can copy C's model/thinking level but omit plan mode and override active tools; all +initial settings commit with child creation (§8.3). + +A canvas stores a checkpoint value at creation and periodically, plus one list append per delta +batch. Hydration finds the checkpoint visible through the selected commit and replays only its +visible suffix, including inherited ranges. A fork appends locally without copying parent strokes. +An entry label or external request mapping remains a session value and does not rewind. + +If committed deltas have journal seqs 104 and 117, a freshly hydrated replication binding emits +revisions 1 and 2. Neither revision is stored. Rewind replaces the binding and hydrates the selected +state afresh. Un-journaled preview (cursors) is scratch on the canvas's live task, if it has one. + +### 15.10 Open, in general + +Read live tasks (`terminal = false`) and their conversation states; for each live conversation read +the entries on its context list (one batched read); read named working values/lists. Then drive the desired scope. +Execution-state reads use indexes rather than replaying the journal. JSONL first imports the +complete journal; active plugin state hydrates separately from checkpoints and bounded delta pages. + +--- + +# Part II — interface details and validation + +The task definitions are in §3. The following sections supplement them without defining another +lifecycle. Open design questions are explicit in §25. Templates and skills remain a layer above. + +## 16. Tools + +```ts +interface Tool { + name: string; description: string; + parameters: JsonObject; // provider-compatible JSON Schema + replay?: "safe" | "never"; // default never + output?: { retain?: "head" | "tail"; maxBytes?: number }; // default tail, 64 KiB + execute(args: JsonObject, ctx: ToolExecutionContext): Promise; +} +interface ToolExecutionContext { + signal: AbortSignal; + invocationId: number; callId: string; + conv: ConversationView; + out: ToolSink; + resources: ReadonlyMap; +} +type ToolDelegation = + | { kind: "subagent"; options: SpawnOptions; background?: boolean } + | { kind: "job"; options: { command: string; notBefore?: number }; background?: boolean }; +interface ToolSink { + write(text: string): void; // appended to scratch; capped per `output`; truncation is marked in the result + details(json: JsonValue): void; // result details, replaced on repeat + image(image: ImageContent): void; // image block in final tool result + handoff(text: string): void; // intent for post_tools, never edits context here + usage(u: Usage): void; // journaled atomically with attempt outcome + addTools(names: string[]): void; // updates the activeTools conversation value at settlement; unknown names are dropped + terminate(v: boolean): void; // the run stops after this exchange; the assistant's other calls still settle + progress(partial: JsonObject): Promise; // durable checkpoint in scratch; a `progress` delta to watchers; reported by recovery + memo: { get(key): Promise; set(key, v): Promise }; // durable per-execution memos, retired at settlement +} +``` +Rules. A tool's throw becomes an error result (`isError: true`, the message text); a throw after +`signal.aborted` becomes an aborted result. Only `execute` is inside the harness's try; a storage +failure while settling propagates (§7.3). Results are `tool_result` entries `{ callIndex, isError, +bytes }` with the capped text (`[truncated: N bytes total]` appended when capped), details, and +images. A call for a tool not in `tools` settles with `not found`; a `truncated` call (stop reason +other than toolUse) settles with `not executed` and never runs. `addTools` names not present in +`tools` are ignored. `terminate` ends the run after the current exchange: no new generation, the +turn is quiescent, `drive` returns `idle`. Calls are parallel unless +the captured `toolOrder` is sequential: generation creates after edges in callIndex order, and a +held predecessor remains nonterminal. The scheduler has no tool-specific concurrency lock. + +## 17. Provider request boundary + +Generation lifecycle, classifications, retry, polling, recovery and cancellation are defined once, +in §3.9. Request construction projects the captured context (§6.4), resolves captured model/tools, +resolves the system prompt and applies before_request. Tools must still enforce captured active-tool +authorization, not merely registry presence. Capture the MAIN context boundary before request intent; +working progress seqs are not navigation targets. Provider payload/cache/option parity remains in §25. + +## 18. Queues and context-only writes + +```ts +interface InboxItem { + id: number; + kind: "write" | "steer" | "followUp" | "nextRun"; + entry: number; // durable unselected content, not yet placed in context +} +``` + +```text +API idle busy +accept(input) place + create generation enqueue followUp +appendMessage / selected place safely; no generation enqueue write + context-only insertion +steer / followUp / nextRun enqueue its tag enqueue its tag + +boundary eligible items +idle acceptance writes + nextRun + mode-selected steer/followUp, then accepted input +post_tools writes + mode-selected steer; successor unless stopped +normal final answer writes; mode-selected steer/followUp if continuing +idle context-only append earlier writes, then new write; no generation +abort drain steer/followUp; retain write/nextRun +``` + +Preserve admission order within selected items. `steeringMode`/`followUpMode` default to `all`; +`one-at-a-time` selects the oldest item of that tag, leaving others for later boundaries. `write` +placement never requests generation, even when it leaves a user-shaped context tail. `nextRun` +is consumed only by explicit idle acceptance. On failure/termination, place safe writes but do +not consume input requiring a successor that will not be created. + +```text +enqueue: unselected content + inbox item in one command +consume: selected placement + dequeue + required generation in one command +cancelQueued: queued → cancelled; already placed → already_consumed; absent → not_found +``` + +Representation linking unselected content to placement/cancellation history remains open with +result attribution (§18.2). Admission position is not necessarily model-context position; do not +silently duplicate payloads or assume old entry ids can be appended out of transcript order. +The four queue behaviors and atomic consumption do not depend on that representation choice. + +### 18.1 Caller-known acceptance identity + +Ordinary `accept(input)` returns its numeric input id. An optional caller-supplied `requestId` +exists for recovery when acceptance commits but its response is lost. It is an opaque external key, +not an internal entity id, and need not be a UUIDv7. Only keyed acceptances pay its storage cost. + +Inside the serialized acceptance command, look up `requestInput(requestId)` (§5.1). If absent, +create the accepted input and set that session value to its id in the same commit. If present, +resolve the existing acceptance rather than inserting another input. There is no crash window +between two records in one atomic command and no separate request-specific storage abstraction. + +Memory/JSONL maintain the current keyed map from journal replay; SQLite uses the session-values +primary key to resolve the request key, then fetches its numeric input id from the journal payload. +No UUID field or secondary UUID index is needed on every entry, task or conversation. The mapping +is session-wide, survives rewind and is not copied as child conversation state. Core owns these +mappings; accepting a request must not overwrite an existing identity association. + +Exact behavior for reusing a key with different input or a different conversation must be settled +before implementation; it must not silently accept a new request or return another conversation's +answer. Lookup itself is simply `harness.getValue(requestInput(requestId))` followed by input/result +lookup. A convenience result overload is not required for this mechanism. + +### 18.2 Pending result attribution contract + +The original rule, "first final assistant after the input", is invalid. If B is queued while A +runs, A's final answer can appear after B's acceptance entry without answering B. An explicit durable +input-to-completion association is needed, including queued consumption, cancellation, failure, +and multiple inputs consumed together. Its representation and exact result outcomes remain open; +request identity lookup does not solve this second association. Do not implement chronological +result inference as a substitute. + +## 19. Outcomes, reports, errors + +- Drive outcomes and ownership scope are defined in §3.8. Conversation idle concerns foreground; + session idle means no live tasks. Suspended includes dependency-blocked work whose prerequisite + awaits an outside event. Observe committed predicates after commits/passes; events only wake. + Callers share execution, not authority to cancel it. +- `inspect()` → `{ start: Task[], inflight: Task[], waiting: Task[] }` from the live set; `open()` + reports the same and starts nothing. +- Errors: `Closed` (command after `close`), `Faulted` (command after a fault; `.reason`), + `Rejected` with a code (`invariant`, `busy`, `unknown_target`, `version_mismatch`, `not_found`); + none of these is a fault. A listener that throws is reported through the `handler_error` signal. +- `close()` waits for in-process effects to observe cancellation (each `abort` signal fires; effects + return), does not settle anything, does not wait for jobs. + +## 20. Hooks, precisely + +```ts +interface HookContract { input: I; decision: D } +interface ProviderRequest { + systemPrompt: string; messages: AgentMessage[]; tools: Tool[]; streamOptions: JsonObject; +} +interface HookContracts { + before_request: HookContract<{ conv: ConversationView; generation: Task; request: ProviderRequest }, ProviderRequest>; + after_response: HookContract<{ generation: Task; message: AssistantMessage }, void>; + before_tool: HookContract<{ call: Task; args: JsonObject }, { kind: "allow" } | { kind: "block"; text: string } | { kind: "hold" }>; + after_tool: HookContract<{ call: Task; result: JsonValue }, void>; + on_yield: HookContract<{ conv: ConversationView; generation: Task; message: AssistantMessage }, { kind: "pass" } | { kind: "continue"; message: AgentMessage }>; + before_collapse: HookContract<{ conv: ConversationView; through: number; prefixVersion: number }, { kind: "decline" } | { kind: "allow"; meta: JsonObject }>; +} +interface Hooks { + on

    ( + point: P, + handler: (input: HookContracts[P]["input"]) => HookContracts[P]["decision"] | Promise, + options?: { priority?: number }, + ): () => void; +} +``` + +| point | input | decision | combination | +|---|---|---|---| +| `before_request` | `{ conv, generation, request: { systemPrompt, messages, tools, streamOptions } }` | `request` | applied in order | +| `after_response` | `{ generation, message }` | — | all run | +| `before_tool` | `{ call: Task, args }` | `allow` \| `block(text)` \| `hold` | first block/hold wins; later hooks do not run | +| `after_tool` | `{ call, result }` | — | all run | +| `on_yield` | `{ conv, generation, message }` (completed final response candidate) | `pass` \| `continue(message)` | first continue wins | +| `before_collapse` | `{ conv, through, prefixVersion }` | `decline` \| `{ meta }` | first decline wins; metas merged | +A hook that throws: `before_request` fails the generation; `before_tool` blocks with the error text; +`before_collapse` declines; `on_yield` passes; observers are reported as `handler_error` and ignored. Hooks are called by effects, outside the line, so a hook +MAY `await ctx.commit(...)` (append entries, write typed values/lists, insert work); those commands interleave +with other actors under §24. A `before_request` hook affects *this* request only through the +request it returns; an entry it appends is projected from the next generation on. Only listeners +(watch/`on`) are forbidden from issuing commands inside their callback. + +## 21. Watch wire shapes + +``` +base { type: "base", asOf, conversation: ConversationState, entries: Entry&{payload}[], tasks: Task[], scratch: { [taskId]: JsonValue } } +commit { type: "commit", firstSeq, lastSeq, writes: [ { seq, change } ] } // relevant MAIN writes, §4.1 +delta { type: "delta", taskId, kind: "frame" | "progress" | string, payload } +signal { type: "quiescent", conv } | { type: "fault", message } | { type: "handler_error", taskId?, message } +page entries(conv, { before | after, limit }) → { asOf, entries: Entry&{payload}[] } +``` +Base and subscription are one line step. Commit batches retain boundaries even when filtering +out unrelated writes; clients reduce a whole batch before publishing. Thus tool/result retirement +or post_tools/successor creation cannot become a partial visible transition. Preview/signal fields +are not durable work requests. Pages carry their own asOf for deduplication. del_conv ends the watch; +context replacement/reset cues model-context rendering. WatchEvent is this base/commit/delta/signal union. + +## 22. Usage ledger and telemetry + +**Usage** is journaled with the corresponding attempt outcome, not independently written to a ledger. + +```text +provider completes: commit reported usage + outcome/retry/deferred transition together +collapse loses prefix race: commit reported usage + failed collapse, without summary/context edit +tool completes: commit usage + own result + own settlement + working-scope retirement +post_tools completes: commit exchange policy + successor generation, if continuing +identity: usage record seq, NOT taskId; a task can have several attempts +query: read the main session usage list; any aggregate/filter indexes are backend implementation details +fork: costs remain attributed to original conversations; no copied billing rows +cleanup: task completion/scratch retirement never removes usage +``` + +An interrupted request with no provider report has unknown cost; no exactly-once billing claim. +Reject stale/repeated outcomes instead of appending duplicate usage. Usage is an ordinary main +session list; no separate ledger authority/file or redundant full usage payload in task patches. + +**Telemetry** uses pi's callback `TelemetryContext` (typed schemas, no second contract). Spans and +their parents follow the procedure nesting: +``` +harness.open · harness.drive (per pass) + acceptance / task settlement (explicit successor creation) + task.effect / task.recover / task.abort (attributes: kind, taskId, status from→to, attempt) + ai.request (model, usage, stop reason, durations; never prompts or completions) + tool.execute (name, callIndex, bytes, isError; never args or results) + hook (point, name, decision) + line.command (op count, seqs, duration) ← parent: the command's caller + watch.deliver (listener count, duration) +``` +A pre-aborted signal starts no span. Attributes are ids, names, counts, durations, statuses and +usage; never message text, arguments, results, file contents, provider payloads, headers or handles. +Each command and each effect carries its own telemetry parent and abort signal; cancellation ends +only that caller's observation. + +## 23. Storage conformance + +Each backend passes: +1. commit preserves command-minted global seqs and atomic batches. Each admitted batch starts at + global head+1 and is contiguous; main files, working files and retained history have legal gaps. +2. Read model equals a replay of the journal after every commit (entries, tasks, conversations, + context lists, `prefixVersion`). +3. Transcript pages and content queries equal a scan; typed value reads and list pages equal + reference history traversal, including nested fork cutoffs and deletion tombstones. +4. Working values/lists use ordinary reads/writes, survive restart, and disappear only through + explicit retirement. Result + task settlement + retirement survive or disappear together. + Crash before physical unlink leaves logically retired files, not recoverable task state. +5. JSONL: one line per commit; a torn final line is truncated on open; rebuild-by-scan yields the + same read model as SQLite's tables for the same journal. +6. Usage survives atomically with its outcome, including multiple attempts under one task id. +7. Storage open creates no conversation/tasks. Harness open on empty storage creates root + initial + values in one command; nonempty open starts no work and writes no journal records. +8. Command construction: creation methods return their final numeric ids before append; subsequent + records reference them directly, including automatic context appends. No-op and rejected commands + consume no seqs; concurrent commands allocate disjoint consecutive ranges; reopen continues after + the last complete commit. Payloads such as `{ ref: 0 }` round-trip unchanged. +9. SQLite value/list queries use ordered indexes without journal scans or temporary sorts; + sparse keys and deep fork chains do not require reading unrelated history. Fork creation adds + no inherited value/list rows. Subagent creation stores only selected present values and overrides. +10. Reopen preserves metadata, historical versions, list cursors and subagent initialization. + Checkpoint payloads in the memory journal remain unchanged after mutable replay by a wrapper. +11. Acceptance and request mapping survive or disappear together; a retry after a lost response + does not create a second input. Conflicting reuse tests follow the policy still pending in §18.1. +12. Paused persistence exposes old complete state, never proposed indexes. Short/failed JSONL writes + and crash after newline/SQL commit but before publication reopen to one complete prefix. + Malformed complete JSONL lines fail open; only unterminated final bytes are discarded. +13. Close drains admitted main/working writes and retires nothing. Reused scope ids retain only + records newer than latest retirement. Old cleanup cannot delete a reused lifetime. +14. Crash around acceptance, tool and post_tools/final settlement: join always exists; no consumed + trigger without generation, no duplicate successor on reopen; usage agrees with outcomes. +15. Mixed main/working and two-working-id commits reject. Memory, JSONL and SQLite agree on logical + retirement. High-water computation includes retired physical records before cleanup. +16. Late progress is rejected by invocation fencing, not by a global ban on scope-id reuse. + +## 24. Race catalog (each has exactly two durable orders; tests assert both) + +| race | orders | +|---|---| +| `accept` vs `accept` on an idle conversation | first is the prompt; second queues as followUp | +| keyed `accept` vs process loss before response/`drive` | input and mapping both absent → retry may accept; both present → resolve existing input, never duplicate it | +| `drive` vs `drive` | both join one pass | +| `abort` vs generation settlement | marker first → aborted, no successor; settlement first → output and successors commit, then abort reaches any still-live successor | +| `abort` vs tool outcome | marker first → the effect's signal fires and it settles `aborted`; outcome first → `done`, result preserved | +| `abort` vs `retry_wait` | settles `aborted` without waiting the timer | +| `abort` vs collapse settlement | marker first → collapse aborted, list unchanged; settlement first → list replaced | +| `abort` vs `on_yield` continuation | marker first → the continuation entry is not inserted; entry first → the run continues and abort settles it | +| `cancelQueued` vs boundary consumption | cancellation first → cancelled; consumption first → already_consumed | +| conversation setting write vs generation creation | old value or new; the generation captures what it uses | +| `nextRun` vs `accept` | captured by this prompt or stays for the next | +| collapse settlement vs `ctx_reset` | reset first → `version_mismatch`, collapse `failed`; settlement first → the reset applies to the new list | +| collapse settlement vs appends | appends survive either order | +| frame/progress write vs settlement | seal/drain first; outcome + retirement commit together; failed unlink leaves logically dead data | +| watcher registration vs commit | one line step: old base + all later records, or new base | +| `close` vs settlement | settlement committed before close, or the task stays inflight and recovers next open | +| fork vs parent summary | the fork copied its list as of `asOf`; the summary is not applied to it | +| subagent initialization vs parent setting write | selected old values or selected new values at one captured boundary; no mixed initialization | +| plugin update vs plugin update | second command derives its update from the first committed state | +| replica binding vs committed delta | hydration includes it, or the new stream emits it after hydration; never a gap | +| parallel tool outcomes | either order settles only those tools; existing post_tools becomes ready when all dependencies are terminal | +| steering vs generation creation | included with creation, or queued for next boundary | +| context-only write vs drive/reopen | context may change; neither order creates generation | +| settlement vs uncertain persistence outcome | output/usage/successors/retirement all absent or all durable on reopen; old handle faults | +| post_tools settlement vs conversation abort | marked join creates nothing, or successor commits first and becomes a cancellation target | +| scope retirement vs reuse | later writes survive; older unlink is serialized before new lifetime | +| working progress vs retirement | admitted old writes drain before retirement; stale producer cannot recreate task data | +| overlapping drive callers vs cancellation | one shared ownership claim covers effect join and abort handler | + +## 25. Remaining design decisions + +These are not settled by the value/list and identity decisions above: + +- **Result attribution:** durable input-to-completion association and conflicting request-key reuse + (§18). Caller-known identity must not be confused with result ownership. +- **Historical context optimization:** §6.6 defines correct creation/reset-based replay; faster + compaction-bounded reconstruction still needs review. Replacement alone is not a checkpoint. +- **Subagent context:** safe inherited-context capture while parent tool exchange is open; caller + selections are copied at child creation, not implicitly at address construction. +- **Host and background lifecycle:** durable process adoption/exit status, bounded schedules, and + whether close retains cooperative waiting or gains a bounded host policy. Finalize runWhenIdle + admission/observer cancellation without introducing a second execution owner. +- **Queue/response policy:** exact input normalization, steering precedence/mode capture, conflicting + handoffs, retained failed/deferred attempt messages and projection. Keep representation and policy + separate from the settled chain mechanism. +- **Old-harness parity:** intentionally decide differences in tool termination aggregation, hook + transformations/error handling, interrupted-stream recovery, retryable/overflow classification, + deferred poll permits/external retry scheduling, token-budget/split-turn compaction, summarized + navigation/import, usage adjustments/totals and complete watch hydration. These are not silently + removed by rewriting tasks. Expected-input cancellation/result fencing remains tied to attribution. +- **Chord adapter:** choose a supported producer/operation seam and durable delta encoding, including + checkpoint/fork codec state. Do not mutate an exposed producer while a commit is pending. Stream + revisions remain process-local emission bookkeeping, regardless of this choice. + +Implementation, stronger regression coverage, and equivalent faux-provider benchmarks follow design +review. CPU time, total process memory, and actual disk bytes (including scratch and usage) must be +measured separately; the spike's existing figures are not validated comparisons for this design. diff --git a/packages/agent/docs/plugins.md b/packages/agent/docs/plugins.md new file mode 100644 index 000000000..cc1ab59f3 --- /dev/null +++ b/packages/agent/docs/plugins.md @@ -0,0 +1,1158 @@ +# Coding-Agent Application Hosts and Facets + +The application-neutral facet, service, and replicated-state runtime is provided by +`@earendil-works/chord`. This document specifies how Pi composes that runtime with Pi-owned +service contracts, process roles, routing, and lifecycle policy. + +> **Status:** Design specification for the experimental facet and service architecture. + +This document assumes you already understand `AgentHarness`, `AgentLane`, `Session`, `Branch`, `SessionRepo`, and invocation `Context`. Read `rpc.md` for service transport semantics and `telemetry.md` for the telemetry model. + +## Bird's-eye view + +The coding agent is assembled independently in several processes. Three layers: + +1. The **facet kernel** owns service-aware lifecycle mechanics: synchronous setup, dependency assembly, local and connected service binding, activation, scoped resource ownership, setup-failure cleanup, reload, and reverse-order disposal. It knows about services and remote service sources, but not about Harness, tools, TUI components, or coding-agent policy. +2. An **application host** owns one concrete runtime and contributes runtime facets that provide its concrete services. The **session host** normally runs in a dedicated session worker and owns session authority — the real Harness. A **presentation host** (TUI today, web later) owns a user interface. A **server host** owns server-wide authority: session records (`SessionRepo`), session-worker management, authentication, attachment, and routing between presentations and session workers. +3. An **extension** may distribute independent host-specific bundles containing **facets**. No aggregate extension object is loaded into all processes. Each host loads only facets built for that process and those facets can use only services available in that host graph. + +The initial topology has one server and no server-to-server links: + +```text +server +├─ TUI A +├─ web B +├─ session worker S0 +└─ session worker S1 +``` + +A presentation and a session worker each connect to the server. There is no direct presentation→session-worker connection; the server routes service calls to the selected worker. The server lists and manages only its own sessions. Multi-server routing and server hierarchies are out of scope. + +A session worker normally owns one session, and each session facet is instantiated for that session. A server facet is instantiated once per server process and is shared across every session and presentation connected to it. Server facets should therefore be rare and limited to inherently server-wide concerns. Per-session feature state belongs in session facets; dedicated workers provide the preferred lifecycle, crash, and state isolation. Future co-location may preserve the same logical service graph without changing what objects a facet can access. + +**Host** and **client** are roles per connection, not fixed process kinds. The server hosts presentation and session-worker connections. A session worker serves its provided session services and may consume server-provided services over the same RPC binding mechanism. "Client" below always names the connection role, never a kind of extension. + +## Why this shape + +- **Authority stays where it belongs.** Provider credentials, tool execution, and per-session extension data exist only in the session worker; session records and worker control exist only in the server. Nothing reaches a presentation except through a deliberate contract. +- **One feature stays coherent.** The question extension's tool, dialog, and renderer ship in one package around one JSON contract, yet each facet is host-native code. +- **A new surface is presentation-only work.** A web facet for the question dialog or the session picker registers against existing tokens; session and server code do not change. +- **Server state stays server-wide.** A server facet is shared by all sessions and clients, so features use one only when their authority is inherently global to that server. +- **One facet mechanism.** Built-ins, runtime capabilities, and extensions use the same facet environment in every host. +- **Testable in pieces.** A facet tests against service-providing fixtures, a contract against loopback, and a routed TUI → server → session-worker path against a real transport — independently. + +## One feature, several independently loaded facets + +There is deliberately no `CodingAgentPlugin` runtime interface. A server, Session worker, TUI, and future web host execute different bundles in different processes, so they cannot share one loaded object containing all host facets. + +The in-process unit is one facet: + +```ts +interface Facet { + readonly id: string; + setup(env: FacetEnvironment): void; +} +``` + +Each process loads an ordered `Facet[]` appropriate to that process. Setup is synchronous declaration; asynchronous initialization belongs in `onActivate()`. A feature may consist of a shared contract bundle plus zero or more separately resolved server, Session, TUI, or web bundles. Their shared service IDs and wire contracts connect them, not an aggregate JavaScript object or a `definePlugin()` wrapper. + +A package keeps shared wire contracts separate from host dependencies: + +```text +question-extension/ + contract.ts JSON DTOs and service tokens + session.ts dialog-service authority and tool contribution; imports agent/session code + tui.ts terminal dialog and renderer; imports TUI code + web.ts optional browser dialog and renderer + package exports unresolved mapping from host kind to independently loadable bundles +``` + +The browser build never imports `session.ts`; the session process never imports TUI or DOM code. + +The question extension is this document's end-to-end example: + +```text +model calls the question tool (session facet) +→ session facet adds one invocation-keyed dialog service (session authority) +→ every connected TUI/web facet observes the service instance (keyed service) +→ the first accepted answer settles it for everyone +→ session facet returns the durable tool result +→ closing the instance closes every presentation's dialog +``` + +With no presentation connected, the question remains pending. A TUI or web facet that connects later obtains the same pending question. + +The models service in the next sections illustrates services and replicated state; the [server section](#the-server-directory-management-and-routing) covers server-wide services and session routing; the [question section](#session-owned-deferred-interactions-the-question-extension) makes the full round trip concrete. + +## Loading and connecting hosts + +The loader abstraction is intentionally smaller than an extension manifest: + +```ts +interface LoadedFacets { + readonly facets: readonly Facet[]; + dispose(): Promise; +} + +interface FacetLoader { + load(): Promise; +} +``` + +Each host receives one or more static, combined, or extension-backed loaders. A loader owns the resources for one loaded module generation; the facet host owns the active facet environments. Initial startup loads facets, assembles the service graph, activates it, and disposes the loaded generation only after the host retires. + +An extension resolver may add identity, ordering, version selection, package isolation, and process-specific source resolution. Its output remains independent `FacetLoader` inputs for each process rather than one cross-process extension object. + +After transport setup, a host gives the kernel its loaded facets, runtime facets that provide concrete local services, and any host-selected remote service sources. The kernel runs every `setup()` in loader order, validates the complete service graph, binds dependencies, and then activates providers before consumers. Setup failure and normal shutdown dispose resources in reverse dependency order. + +Exactly one process owns a Session's authority at a time. Worker replacement must close the old owner before a new process opens the same durable Session. Each presentation and Session worker uses one multiplexed connection to its server; facets never open private sockets or handle request IDs, cancellation frames, routing namespaces, or reconnect buffering. + +Out of scope: arbitrary undeclared object remoting, serialized functions/classes/`Map`/`Set`, remote hook or tool execution, offline presentation writes or automatic mutation replay, a universal remote `AgentHarness`, and a serialized UI tree. + +## Services connect host facets + +Facets communicate across processes through **services**. One token type gives a service contract its identity: + +```ts +function defineService(id: string, options?: { local?: boolean }): Service; +``` + +The declaration lives in the shared contract module and creates nothing. Services are remotely publishable by default; a process-local token declares `{ local: true }`. `provide(service, implementation)` adds one singleton to the host service graph. `provideMany(service)` registers ownership of a multi-instance service during facet setup and returns a `ServiceSpawner` whose later `spawn(key, implementation)` calls publish instances. The host publishes every non-local provision across its process boundary. Consumers select the same modes with `use(service)` or `observe(service, handler)`. Within one facet generation, a token must stay in one mode: mixing `provide`/`use` with `provideMany`/`observe` is an assembly or protocol error. + +```ts +interface ServiceSpawner { + spawn(key: string, implementation: T): () => void; +} +``` + +TypeScript types cannot produce runtime member metadata. Facet authors nevertheless declare no parallel member descriptor. When an exposed `provide()` implementation or `ServiceSpawner.spawn()` instance reaches the remote-service boundary, the runtime classifies functions as remote methods and recognizes Chord-created `ReplicatedState` values. It rejects unsupported members and announces the resulting member table over the transport. Process-local services may use arbitrary object contracts. + +`use()` on a singleton returns a stable lazy proxy synchronously, even before a remote provider is attached. Member access creates local method or state slots as they are used; attachment validates those slots against the provider-announced kinds. A mismatch is an assembly or protocol error. This runtime mechanism is implemented once by the host rather than repeated in every service declaration. + +### Dependency declaration and assembly + +Service API calls made during facet setup are the dependency declarations. The kernel does not reflect on erased TypeScript interfaces, and facet authors do not maintain parallel `requires` and `provides` lists. A `Service` retains its stable ID at runtime, and the API call supplies the mode. `use()` and `observe()` initially return source-independent disconnected handles. After all setup completes, the host matches unresolved requirements against provider-generated connection catalogues and binds each token to its local provision or exactly one connection. + +The host records a private generation-scoped ledger: + +```text +env.provide(Models, implementation) +→ @pi/providers-builtin:session provides pi.models/singleton + +env.use(Models) +→ @pi/model-selection:tui requires pi.models/singleton + +env.provideMany(QuestionDialogs) +→ @pi/question:session provides pi.question-dialog/keyed + +env.observe(QuestionDialogs, handler) +→ @pi/question:tui requires pi.question-dialog/keyed +``` + +The first `provide()`, `provideMany()`, `use()`, or `observe()` for a token must occur during facet setup. Commands, hooks, event handlers, and activation callbacks use handles acquired during setup; they cannot introduce an undeclared service dependency later. Dynamic instances use the setup-owned `ServiceSpawner`, so spawning and closing instances do not change the graph. + +After every facet has registered, the host generates its outgoing catalogue from non-local provisions, obtains catalogues from its remote service sources, resolves requirements to local or connected provisions, rejects missing providers, duplicate offers or singleton owners, singleton/keyed mismatches, invalid dependency cycles, and invalid remote service implementations, then records consumer-to-provider edges for lifecycle ordering. `use()` and `observe()` declare hard requirements; optional dependencies require a future distinct acquisition API rather than inference from call failure. The ledger and resulting graph are private kernel machinery, not a facet-facing plan or second declaration format. + +Only dependencies acquired through `env.use()` or `env.observe()` belong to this lifecycle graph. Importing another extension's live implementation bypasses ownership and is unsupported. The module loader separately owns the ordinary source import graph. Reload therefore needs both loaded-source ownership and the generated service graph; see [Reloading facets](#reloading-facets). + +The models service — the authority behind the model picker and thinking-level control — exercises methods, replicated state, and multiple consumers. + +### Shared contract + +```ts +export interface ModelRef { + provider: string; + modelId: string; +} + +export interface ModelsState { + catalog: { revision: number; availableModels: Array }; + configuration: { model: ModelRef | null; thinkingLevel: "off" | "low" | "high" }; + refresh: + | { status: "idle" | "refreshing" | "done" } + | { status: "warning"; errors: Record }; +} + +export interface Models { + readonly state: ReplicatedState; + cycleThinking(context: Context): Promise; + refresh(context: Context): Promise; + select(model: ModelRef, context: Context): Promise; +} + +export const Models = defineService("pi.models"); +``` + +Everything transported in a remote contract is strict JSON: arguments, results, and replicated state. Business-level absence uses JSON `null`, never `undefined`. An unhydrated `ReplicatedState.value === undefined` is local control-plane readiness, not a transported state value. `Context` is control-plane data in a declared position; the proxy strips it and it is never serialized. + +### Session facet + +The snippets below use the facet shape but compress application details. + +```ts +export const providersBuiltinSessionFacet = defineFacet({ + id: "@pi/providers-builtin", + + setup(env) { + const providers = new ProviderRegistry(); // process-local, non-JSON + const state = env.replicatedState(initialModelsState()); + + env.provide(Models, { + state, + + async cycleThinking(context) { + const { catalog, configuration } = state.value; + if (configuration.model === null) return; + const spec = findSpec(catalog, configuration.model); + if (spec === undefined || !spec.reasoning) return; + state.set( + { + ...state.value, + configuration: { + ...configuration, + thinkingLevel: nextThinkingLevel(configuration.thinkingLevel), + }, + }, + context, + ); + }, + + async select(model, context) { + const spec = findSpec(state.value.catalog, model); + if (spec === undefined) throw new Error(`Unknown model: ${model.provider}/${model.modelId}`); + const thinkingLevel = spec.reasoning ? state.value.configuration.thinkingLevel : "off"; + state.set({ ...state.value, configuration: { model, thinkingLevel } }, context); + }, + + async refresh(context) { + state.set({ ...state.value, refresh: { status: "refreshing" } }, context); + const errors = await providers.refresh(context.abortSignal); + state.set({ ...state.value, catalog: providers.snapshot(), refresh: toRefreshStatus(errors) }, context); + }, + }); + + env.onActivate(() => providers.rebuild()); + }, +}); +``` + +### TUI facet + +This shows the generic command-service pattern. + +```ts +export const modelSelectionTuiFacet = defineFacet({ + id: "@pi/model-selection", + + setup(env) { + const models = env.use(Models); + const tui = env.use(Tui); + + tui.commands.register("models.select", async (context) => { + const current = models.state.value; + if (current === undefined) return; + const selected = await tui.select( + "Models", + current.catalog.availableModels.map((model) => ({ + label: model.name, + value: { provider: model.provider, modelId: model.modelId }, + })), + { signal: context.abortSignal }, + ); + if (selected !== undefined) await models.select(selected, context); + }); + tui.commands.register("models.cycle-thinking", (context) => models.cycleThinking(context)); + env.own(models.state.subscribe((next) => renderModelSelector(next))); + }, +}); +``` + +The TUI facet has no credentials, registry, or refresh logic: it calls a typed lazy proxy with the contract's method signatures and renders replicated state after hydration. A web facet would do the same through its web facet environment. + +### Service semantics + +A service has **one owner and many consumers**. In singleton mode, `providersBuiltinSessionFacet` provides `Models` and both model-selection commands consume it. In multi-instance mode, one owner may spawn instances `A` and `B`, and every observer sees the same two instances. + +`use()` behaves differently by locality: + +- **Local:** `use()` returns a stable lazy proxy backed by a direct process-local implementation slot. During synchronous setup it is disconnected; after assembly it binds to the local implementation without requiring provider-before-consumer setup order. Reload unbinds and rebinds that same slot. +- **Remote:** across a connection, `use()` returns the same kind of stable lazy proxy. Calls made while disconnected fail when invoked; state has no value until hydrated. Concurrent consumers of one token in one process share one proxy, one state replica, and one remote subscription. + +Multi-instance services use `provideMany()` and `observe()`. The service is empty until its setup-owned `ServiceSpawner` calls `spawn()`; observing it never creates an instance. `spawner.spawn(key, implementation)` returns an idempotent close function, and the key must be unique among that service's live instances. Local observers use a direct process-local instance registry; non-local provisions additionally publish the same instance through RPC. `observe(service, handler)` reconciles current instances and then ordered additions, replacements, and removals. The handler receives the same `T` proxy shape as `use()`; instance keys remain provider-side addressing details. After an instance's initial state members hydrate, the host starts one handler task with a fresh `Context`. The facet lifecycle owns that observation. Closing the instance aborts its task context, rejects new calls, and lets already-admitted calls return. Cancellation from the instance context is normal task cleanup; other handler failures follow host failure policy. Reusing a closed key creates a new host-owned generation, so stale proxies cannot address the replacement. + +An added instance member has structural identity `(service, key, generation, member)`. Its `ReplicatedState` members therefore need no independent IDs. The instance directory is control-plane metadata, not a facet-visible `ReplicatedState` containing proxies. Switching sessions aborts all observed instance tasks before hydrating the selected session's current instances. + +Every facet uses the same unqualified `env.use()` and `env.observe()` operations. A presentation host combines its local services with connected server and selected-Session services, then routes each token internally. Provider facets resolve services from the same host graph. Transport binding and routing remain host infrastructure rather than facet API. + +## What each facet kind grants + +This is the most important boundary in the design. + +**Session facets run beside the real thing.** They execute in the process that owns the concrete `AgentHarness`, `AgentLane`, `Session`, and Branches, and receive direct, process-local, scoped capabilities backed by those instances — not RPC proxies. Calls preserve real method signatures, `Context` propagation, `Result` types, and object identity. A session facet never RPCs back into its own process. + +```ts +interface ScopedSessionData { + readonly metadata: SessionMetadata; + getValue(address: Value, context: Context): Promise | undefined>; + setValue(address: Value, value: T, context: Context): Promise; +} + +interface AgentFacetScope { + readonly identity: SessionIdentity; + readonly session: ScopedSessionData; + readonly hooks: ScopedHooks; + lane(name: string, context: Context): Promise; +} + +const Agent = defineService("pi.local.agent", { local: true }); +const Providers = defineService("pi.local.providers", { local: true }); +const Tools = defineService("pi.local.tools", { local: true }); +``` + +"Local" and "unrestricted" are separate decisions. The scope narrows authority for lifecycle and composition — hooks and event subscriptions registered through it are automatically owned by the facet and disposed with it. `AgentLaneFacetView` exposes Branch methods directly alongside agent operations. `ScopedSessionData` exposes purpose-bounded durable operations. The host keeps the unrestricted concrete instances and reserves: `AgentHarness.close()` and `Session.close()`; raw `Session.mutate()`, `beginMutation()`, and `SessionMutator` (unless a narrowly trusted durability extension explicitly owns them); `idGenerator` and backend/storage objects; Branch creation; whole-registry setters such as `setTools()`; unscoped hook/event registration; transport exposure and remote-reference registration. This is a composition and lifecycle boundary, not a security sandbox: session facets are trusted code in the authoritative process. A future extension policy may explicitly grant broader local capability, but built-ins should receive no implicit bypass. + +**Presentation facets hold none of this.** A TUI or web facet never receives the raw Harness, Session, tree, tool registry, hooks, or credentials. It uses host-local presentation services plus the semantic services and replicated state deliberately exposed by a Session or server facet. + +```ts +interface FacetEnvironment extends FacetLifecycle { + use(service: Service): T; + observe( + service: Service, + handler: (service: T, context: Context) => void | Promise, + ): void; + provide(service: Service, implementation: T): void; + provideMany(service: Service): ServiceSpawner; + replicatedState(initial: T): MutableReplicatedState; +} + +type AttachmentState = { status: "detached" } | { status: "attaching" | "attached" | "degraded"; sessionId: string }; + +interface SelectItem { + label: string; + description?: string; + value: T; +} + +interface TuiModal { + select(title: string, items: SelectItem[]): Promise; + input(title: string): Promise; + close(): void; +} + +interface TuiHost { + readonly attachment: ReplicatedState; + readonly commands: CommandContributions; + readonly toolRenderers: ToolRendererContributions; + acquireModal(signal: AbortSignal): Promise; + select(title: string, items: SelectItem[], options: { signal: AbortSignal }): Promise; +} + +const Tui = defineService("pi.local.tui", { local: true }); +``` + +The first implemented presentation hookpoint is narrower than this eventual `TuiHost`: a process-local `SlashCommands` registry. Built-in presentation facets and plugin presentation facets acquire the same registry and add command metadata plus callbacks during activation. The returned cleanup removes the contribution, so facet reload and unload update autocomplete and dispatch without rebuilding the TUI. Command callbacks receive narrow selection, status, and prompt-submission operations rather than the raw renderer or editor. + +Each plugin host facet is an independent loader entry. The example `/hello` presentation facet has one default facet export; a future package build emits that facet as one pre-bundled file. Session, server, web, and other presentation facets from the same plugin are separate bundle entries connected by shared service IDs, not one aggregate runtime plugin object. + +`acquireModal()` waits in one presentation-owned queue and holds the modal slot across a multi-step interaction. Its signal removes a queued request or dismisses an active one, and `close()` is idempotent. `select()` is the one-step acquire/select/close convenience. Both return selected values directly, so feature code never recovers identity from a display label. + +The TUI loads all of its facets into one generation. Its host routes `env.use(SessionDirectory)` to the connected server and `env.use(Models)` to the selected Session. While detached, Session calls fail with `session_not_attached` and replicated state has no value. Connection and attachment health are host-local services because they describe presentation control state. A future web host similarly binds local services for routes, views, and DOM dialogs. Its server and Session facets still use unqualified service operations. + +`AgentController` is the presentation-safe command facade over the worker-owned main `AgentLane`. It exposes prompt, queue, abort, resume, compaction, and navigation operations as JSON-safe results. The Session runtime constructs it directly from the lane; it does not publish the raw Harness or lane as local facet services. + +The runtime form is: + +```ts +export function createAgentControllerRuntimeFacet(lane: AgentLane) { + return defineFacet({ + id: "@pi/agent-controller-runtime", + setup(env) { + env.provide(AgentController, createAgentController(lane)); + }, + }); +} +``` + +Its TUI facet consumes `AgentController` through `env.use()` exactly as the model picker consumes `Models`. It does not reveal the Harness object behind the controller; there is no universal remote Harness for arbitrary plugins. `rpc.md` may still define generic Harness proxies for other trusted integrations (an IDE bridge, an orchestrator) — deliberate, separate exposures, not the plugin boundary. + +## Local services and narrow remote facades + +Not every dependency should be remotely reachable. A **local service** is a token declared with `{ local: true }` and confined to its providing process. It may use synchronous methods and hold functions, classes, native objects, credentials, filesystem handles, or other non-JSON values. Remote `use()` cannot resolve it, and local services are never discoverable remotely. Local and non-local provisions share dependency ordering, stable handles, keyed generations, activation, disposal, and provider-facet reload; non-local services only add validation, replication, and RPC publication. The pattern for sensitive state is a local full service plus a narrow remote facade: + +```ts +const Credentials = defineService("credentials", { local: true }); // get/set provider secrets + +interface Accounts { + readonly state: ReplicatedState<{ providers: Array<{ provider: string; configured: boolean }> }>; + remove(provider: string, context: Context): Promise; +} +const Accounts = defineService("pi.accounts"); +``` + +The auth extension's Session facet uses `Credentials` directly; presentations see provider IDs and `configured` booleans — never secrets. If some settings must not be remotely writable, split them the same way; do not rely on presentation-side convention. + +## Replicated state: `ReplicatedState` + +`Models.state` is a `ReplicatedState`: **authoritative latest-value replication** — not event history, durable storage, a CRDT, or a multi-writer mechanism. + +```ts +interface ReplicatedState { + /** Borrowed immutable value, or `undefined` until hydration. Do not mutate or retain it. */ + readonly value: T | undefined; + /** Listener values are borrowed and must not be mutated or retained. */ + subscribe(listener: (value: T, context: Context) => void): () => void; +} + +interface MutableReplicatedState extends ReplicatedState { + /** A providing state is always initialized. */ + readonly value: T; + /** Transfers the JSON value to the state; the caller must not subsequently mutate it. */ + set(value: T, context: Context): void; +} +``` + +Required behavior: + +1. The providing host owns one initialized authoritative value; remote consumers call methods rather than writing the replica. +2. A cold remote replica has no value. Its `.value` is `undefined`, and `subscribe()` registers the listener without invoking it. This `undefined` is local readiness state and never crosses the wire. +3. **Hydration** installs a complete snapshot atomically before updates flow. Subscribing before hydration is valid, and updates emitted concurrently with the snapshot are buffered, so the listener observes snapshot then updates with no gap. +4. Once hydrated, `.value` is synchronously readable and `subscribe()` immediately reports the current value, then future updates. Snapshot hydration uses a fresh delivery context parented to the subscription; later updates reconstruct fresh delivery contexts from source trace metadata. +5. State values are borrowed immutable JSON. The state runtime does not defensively clone reads, writes, snapshots, or listener deliveries. Callers transfer ownership to `set()` and must not mutate or retain values returned by `.value` or passed to listeners; copy explicitly when ownership is required. Process and transport serialization may naturally produce a detached value, but callers must not depend on object identity or detachment. +6. Disconnect, provider withdrawal, and route switching clear readiness, so `.value` becomes `undefined`. Reconnect or singleton replacement installs a complete fresh snapshot in the existing member facade before later updates flow. A presentation that wants stale display data must retain it separately alongside connection or attachment health. +7. `set(value, context)` passes its context to local source listeners and publishes source trace metadata. Remote delivery reconstructs a fresh local `Context`; it never retains the source context object. + +Anything a consumer must recover after reconnect is exposed as replicated state or pulled through a remote method. Replicated state is latest-value replication, not by itself durable session storage; the providing facet must reconstruct its authoritative value after a worker restart. + +Prefer several coarse independent cells over one giant value or a universal patch language, so a catalogue refresh does not retransmit unrelated configuration. High-frequency data such as transcript streaming needs a future snapshot-and-delta design rather than overloading `ReplicatedState`. Revision metadata, gap recovery, unchanged-value suppression, and demand-driven subscription belong to that future protocol, not individual facet authors. + +## Contribution registries: many contributors, one result + +Services fit one owner, many consumers. Providers and tools invert that: **many extensions contribute to one host-owned result**. A mutable global registry would make composition order-dependent and removal impossible. A contribution registry instead replays ordered contributions over a fresh draft: + +```text +fresh ProviderDraft +→ built-in provider contribution (@pi/providers-builtin) +→ remote catalogue contribution (@pi/providers-catalog) +→ models.json transformation (@pi/providers-models-json) +→ authentication/availability marking (@pi/auth) +→ validated ProviderState +``` + +Removing an extension removes its contribution and rebuilds; nothing runs an inverse mutation. Tools follow the same model, including wrapping: + +```ts +sessionContext.tools.add((draft) => { + draft.set("review_add", reviewAddTool); + draft.wrap("bash", (next) => async (invocation) => { + await authorize(invocation); + return next(invocation); + }); +}); +``` + +Ordered wrappers compose deterministically — `telemetry(permission(sandbox(coreBash)))` — and if the permission extension disappears, rebuilding yields `telemetry(sandbox(coreBash))`. Only the host finalizes the draft and applies the complete registry to the Harness; facets never call `setTools()`. Contributions configure rebuilt behavior; hooks intercept live operations — separate mechanisms. + +## Context, cancellation, and telemetry for facet authors + +Every remote method receives a fresh local `Context` in its declared position. The proxy strips the caller's context from JSON arguments and maps `context.abortSignal` to cancellation of that one request. The receiving endpoint constructs a request-local abort signal; it never deserializes the sender's `Context` or arbitrary typed values. Shared service objects must not retain a caller's context. + +The model refresh shows the whole author-visible surface: + +```ts +const controller = new AbortController(); +await uiTelemetry.startSpan({ name: "ui.models.refresh" }, async (span) => { + const context = withAbortSignal(controller.signal, withTelemetryContext(span, BACKGROUND_CONTEXT)); + await models.refresh(context); +}); +``` + +RPC telemetry composes with application spans as: + +```text +ui.models.refresh +└─ rpc.client models.refresh + └─ rpc.server models.refresh + └─ plugin.models.refresh +``` + +`controller.abort()` cancels only that one request: the server's reconstructed `context.abortSignal` aborts, and no other caller is affected. + +Three cancellation domains must never blur: + +1. **Invocation cancellation** aborts one remote call or wait — the `controller.abort()` above. +2. **Service-owned cancellation** is an explicit method such as `job.cancel()` that stops a service-owned task. +3. **Durable Harness cancellation** — `requestAbort()`/`abort()` — writes durable `cancel_requested` and drives durable settlement. + +A transport disconnect performs only the first for active requests and closes that client's subscriptions. It must not silently cancel service-owned work or write durable cancellation. Work intended to outlive its initiating request must deliberately detach into a service-owned task with its own controller and telemetry root. + +## Service-owned jobs + +Private returned references are outside the initial service contract. Prefer `provideMany()` for discoverable live instances. Add caller-private references only after a concrete feature establishes their ownership and collection requirements. + +A possible long-running job contract is: + +```ts +interface IndexJob { + readonly progress: ReplicatedState; + wait(context: Context): Promise; // aborting this context cancels only this wait + cancel(context: Context): Promise; // cancels the job itself, for everyone +} +``` + +An `IndexService.start(root, context)` returning an `IndexJob` validates the root, creates its own `AbortController` and a detached telemetry root, and returns the job. The job crosses the wire as a private **remote object reference** (`rpc.md`) known only to that caller. If every attached presentation must discover a job, register a multi-instance service with `provideMany()` during setup and spawn an instance instead. Discovery is the distinction: returned references are passed explicitly; spawned instances appear in `observe()` hydration. Both make the cancellation domains concrete, and both need explicit lifetime cleanup. + +## The server: directory, management, and routing + +The server host does two jobs. It **owns server-wide services**—listing, creating, deleting, and attaching to sessions—and it **routes session traffic** between attached presentations and the session workers it manages. Routing is host infrastructure that facet code does not implement. + +A server facet is shared by every session and presentation connected to the server. It should be used only for inherently server-wide features. Per-session feature data belongs in session facets. + +### Server host services + +```ts +interface FleetFacetScope { + readonly managed: ManagedSessionsView; // sessions managed by this server + readonly attachments: AttachmentsView; // bind/unbind a client's selected session +} + +const Fleet = defineService("pi.local.fleet", { local: true }); +``` + +The raw `SessionRepo`, storage handles, unrestricted process-kill authority, routing map, and routing machinery stay with the server application: + +```ts +interface ManagedSessionRecord { + sessionId: string; + title: string; + workspaceId: string; + ownerId: string; + cwd: string; // ownerId and cwd never leave the server +} + +type ManagedSessionChange = { type: "created" | "changed" | "deleted"; record: ManagedSessionRecord }; + +interface ManagedSessionsView { + snapshot(): ManagedSessionRecord[]; + onChanged(listener: (change: ManagedSessionChange, context: Context) => void): () => void; + create(options: { title: string; workspaceId: string }, context: Context): Promise; + remove(sessionId: string, context: Context): Promise; +} +``` + +### Shared contract + +The directory is read; management mutates and selects. Both are presentation-safe: `ownerId` and `cwd` are stripped from summaries. + +```ts +export interface SessionRecordSummary { + sessionId: string; + title: string; +} + +export interface SessionDirectory { + readonly state: ReplicatedState<{ revision: number; sessions: SessionRecordSummary[] }>; +} + +export const SessionDirectory = defineService("pi.session-directory"); + +export interface SessionManagement { + create(options: { title: string }, context: Context): Promise; + remove(sessionId: string, context: Context): Promise; + attach(sessionId: string, context: Context): Promise; + detach(context: Context): Promise; +} + +export const SessionManagement = defineService("pi.session-management"); +``` + +### Server facet + +```ts +// server.ts +export const sessionDirectoryServerFacet = defineFacet({ + id: "@pi/session-directory", + setup(env) { + const { managed, attachments } = env.use(Fleet); + const state = env.replicatedState({ revision: 0, sessions: [] as SessionRecordSummary[] }); + + function publish(_change: ManagedSessionChange, context: Context) { + state.set({ revision: state.value.revision + 1, sessions: managed.snapshot().map(toSummary) }, context); + } + + env.own(managed.onChanged(publish)); + env.onActivate(() => + state.set({ revision: 1, sessions: managed.snapshot().map(toSummary) }, BACKGROUND_CONTEXT), + ); + + env.provide(SessionDirectory, { state }); + env.provide(SessionManagement, { + async create(options, context) { + const client = requireClientIdentity(context); + return toSummary( + await managed.create({ title: options.title, workspaceId: client.workspaceId }, context), + ); + }, + async remove(sessionId, context) { + authorizeTarget(requireClientIdentity(context), managed.snapshot(), sessionId); + await managed.remove(sessionId, context); + }, + async attach(sessionId, context) { + const client = requireClientIdentity(context); + authorizeTarget(client, managed.snapshot(), sessionId); + await attachments.bind(client.clientId, sessionId, context); + }, + async detach(context) { + await attachments.unbind(requireClientIdentity(context).clientId, context); + }, + }); + }, +}); + +function authorizeTarget(client: ClientIdentity, records: ManagedSessionRecord[], sessionId: string) { + const record = records.find((candidate) => candidate.sessionId === sessionId); + if (record === undefined || record.workspaceId !== client.workspaceId) { + throw new RemoteServiceError("not_authorized", `Not accessible: ${sessionId}`); + } +} + +function toSummary({ sessionId, title }: ManagedSessionRecord): SessionRecordSummary { + return { sessionId, title }; +} +``` + +Every call is authorized against the client identity that transport policy installed server-locally, never against identity supplied in ordinary arguments. + +### TUI facet: the picker + +```ts +// tui.ts +export const sessionPickerTuiFacet = defineFacet({ + id: "@pi/session-picker", + setup(env) { + const directory = env.use(SessionDirectory); + const management = env.use(SessionManagement); + const tui = env.use(Tui); + + tui.commands.register("sessions.switch", async (context) => { + const current = directory.state.value; + const attachment = tui.attachment.value; + if (current === undefined || attachment === undefined) return; + const selected = await tui.select( + "Sessions", + current.sessions.map((session) => ({ + label: pickerLabel(session, attachment), + value: session.sessionId, + })), + { signal: context.abortSignal }, + ); + if (selected !== undefined) await management.attach(selected, context); + }); + + env.own(directory.state.subscribe((next) => renderSessionList(next))); + }, +}); +``` + +The TUI facet consumes a service provided by the one connected server. There is no session facet in this plugin because sessions do not own discovery or attachment. + +### Attaching and switching + +`attach(sessionId)` selects the session for this presentation connection: + +1. the server authorizes the client for one of its managed sessions; +2. it closes the client's previous session-scoped requests, subscriptions, and observed instance tasks; +3. it binds the presentation host's Session services to the selected Session worker; +4. the Session worker hydrates singleton state and current keyed instances from complete fresh snapshots; attachment state becomes `attached`. + +Session service handles are stable across switches: a proxy returned once by a Session facet's `env.use(Models)` keeps working against the new Session, and `env.observe(QuestionDialogs, ...)` reconciles the new Session's instances. Frames belonging to closed subscriptions or requests are dropped. + +### Routed session call + +```text +TUI A (selected session S1): rpc.client agent-controller.prompt +server: authorize client for S1; route to session worker S1 with authenticated client identity +S1: rpc.server agent-controller.prompt — fresh local Context, validated JSON args → lane.prompt(...) +response returns S1 → server → TUI A +``` + +Aborting the TUI request sends cancellation through the server to S1, aborting the session-side request controller. `Context` and trace metadata are reconstructed at the service endpoint. + +### Routing is host infrastructure + +The server routes session traffic contract-agnostically. It parses protocol envelopes—frame kind, request ID, service ID, optional instance key/generation, and selected session—but not service business payloads. Validation happens at service endpoints, so the server can route a Session service without loading that session facet. + +The server stamps routed calls with its authenticated client identity. The Session worker keys connection-owned requests per presentation route, preventing request-ID collisions and cross-client cancellation. No server facet participates in routing or re-provides session services. + +## Session-owned deferred interactions: the question extension + +Some session-side work must ask users for a decision. The existing `examples/extensions/question.ts` shows the experience: the model calls a `question` tool, a user selects an option or types an answer, and the tool returns that answer with a compact rendering. + +A question is not a reverse RPC routed to one eligible presentation. The Session adds one temporary dialog service keyed by the invocation ID. Every connected TUI or web presentation observes that instance, a presentation connecting later discovers it through instance hydration, and the instance remains open while no users are connected. + +### Shared contracts + +```ts +const QuestionParamsSchema = Type.Object({ + question: Type.String(), + options: Type.Array( + Type.Object({ + label: Type.String(), + description: Type.Union([Type.String(), Type.Null()]), + }), + ), +}); +type QuestionRequest = Static; + +type QuestionResponse = + | { outcome: "selected"; index: number } + | { outcome: "custom"; answer: string } + | { outcome: "cancelled" }; + +interface QuestionDetails { + question: string; + options: string[]; + answer: string | null; + wasCustom: boolean; +} + +interface QuestionDialogs { + readonly request: ReplicatedState; + submitAnswer(response: QuestionResponse, context: Context): Promise; +} + +const QuestionDialogs = defineService("pi.question-dialog"); +``` + +`QuestionDialogs` declares only the contract. Each invocation explicitly adds one keyed instance. Its `request` state is addressed by the service, invocation key, hidden generation, and member name. + +The tool-result helper remains session-local: + +```ts +function questionResult(request: QuestionRequest, answer: string | null, wasCustom: boolean, text: string) { + return { + content: [{ type: "text", text }], + details: { question: request.question, options: request.options.map((o) => o.label), answer, wasCustom }, + } satisfies AgentToolResult; +} +``` + +### Session facet: add one dialog service + +`memoOnce(name, candidate)` is an atomic invocation-memo operation. It keeps the first value, returns that durable winner to every caller, and reports all failures by rejecting its promise rather than throwing synchronously. `awaitAbortable()` is an ordinary shared cancellation utility. + +```ts +// session.ts +export const questionSessionFacet = defineFacet({ + id: "@pi/question", + setup(env) { + const dialogs = env.provideMany(QuestionDialogs); + const tools = env.use(Tools); + + tools.add((draft) => { + draft.set("question", { + label: "Question", + description: "Ask users a question and wait for an answer.", + executionMode: "sequential", + replay: "safe", + parameters: QuestionParamsSchema, + + async execute(_toolCallId, params, _onUpdate, _toolContext, invocation, context) { + if (params.options.length === 0) { + return questionResult(params, null, false, "No options provided"); + } + + const memoName = "pi.question.answer"; + let response = (await invocation.getMemo(memoName)) as QuestionResponse | undefined; + + if (response === undefined) { + const completion = Promise.withResolvers(); + const request = env.replicatedState(params); + const close = dialogs.spawn(invocation.invocationId, { + request, + async submitAnswer(candidate, _answerContext) { + if (candidate.outcome === "selected" && params.options[candidate.index] === undefined) { + throw new Error("Question response selected an invalid option"); + } + const committed = invocation.memoOnce(memoName, candidate); + completion.resolve(committed); + await committed; + }, + }); + + try { + response = await awaitAbortable(completion.promise, context.abortSignal); + } finally { + close(); + } + } + + if (response.outcome === "cancelled") { + return questionResult(params, null, false, "User cancelled the question"); + } + if (response.outcome === "custom") { + return questionResult(params, response.answer, true, `User wrote: ${response.answer}`); + } + const selected = params.options[response.index]; + if (selected === undefined) throw new Error("Question response selected an invalid option"); + return questionResult(params, selected.label, false, `User selected: ${response.index + 1}. ${selected.label}`); + }, + }); + }); + }, +}); +``` + +`dialogs.spawn()` installs the instance before `execute()` waits. The returned close function is the single normal, cancellation, and error cleanup path. Concurrent submissions call `memoOnce()`, whose atomic first-writer rule prevents overwrite and returns the same durable winner. `completion.resolve(committed)` makes the local wait follow that durable operation: success resumes the tool, while failure rejects it and runs the same cleanup instead of leaving it suspended. Each service call also awaits its own `committed` promise, so it cannot report success before durability or leave an ignored rejection. Calls through a closed instance or an old generation fail as stale service calls. + +### TUI and web facets: observe every dialog instance + +```ts +// tui.ts +type QuestionChoice = + | { outcome: "selected"; index: number } + | { outcome: "custom" }; + +export const questionTuiFacet = defineFacet({ + id: "@pi/question", + setup(env) { + const tui = env.use(Tui); + env.observe(QuestionDialogs, async (dialog, context) => { + const request = dialog.request.value; + if (request === undefined) throw new Error("Question dialog was observed before hydration"); + + const modal = await tui.acquireModal(context.abortSignal); + try { + const choice = await modal.select( + request.question, + [ + ...request.options.map((option, index) => ({ + label: option.label, + ...(option.description === null ? {} : { description: option.description }), + value: { outcome: "selected" as const, index }, + })), + { label: "Write a custom answer", value: { outcome: "custom" as const } }, + ], + ); + + let response: QuestionResponse; + if (choice === undefined) { + response = { outcome: "cancelled" }; + } else if (choice.outcome === "selected") { + response = choice; + } else { + const answer = await modal.input(request.question); + response = answer === undefined ? { outcome: "cancelled" } : { outcome: "custom", answer }; + } + + await dialog.submitAnswer(response, context); + } finally { + modal.close(); + } + }); + + tui.toolRenderers.add("question", questionRenderer); + }, +}); +``` + +`observe()` runs one abortable task per open instance, including instances present in the hydration snapshot. Three concurrent tool invocations therefore produce three tasks keyed by their invocation IDs. The TUI modal queue displays them one at a time; a web host may render all three. Closing one instance aborts only its task in every presentation. + +With no connected presentation, the added instance and unresolved tool remain Session-owned. A web facet observes the same service; a headless client may ignore it. Similar features—permissions, OAuth, or editor requests—may add their own service instances when all presentations need to discover temporary instances. Secrets still require narrow methods and presentation-safe state. + +### Durability and worker replacement + +The service instance is live process state; the invocation memo is the replay receipt. The Harness already persists a safe tool's effective arguments, stable invocation ID, `effect_pending` state, and memos. `memoOnce()` synchronously enters one atomic read-or-write on the invocation's Session mutation line and verifies that the same operation, turn, source position, and invocation still own the effect. It returns the existing value or commits and returns the candidate. + +If the worker dies before the answer commit, the old instance and promise disappear. Safe replay reads no answer and adds the same logical key with a new generation. If it dies after the commit, replay reads the answer and returns without adding an instance. A client cannot answer while the worker is absent; calls through the old generation fail instead of locating an invocation by bare ID. + +The memo has the existing invocation lifetime. Staging the tool result as `outcome_ready` atomically deletes it; cancellation and external finalization use the same cleanup. The question request is not copied into another memo because the Harness already persisted the effective tool arguments. Source reload uses this same durable worker-reconstruction path. + +## Lifecycle and disposal + +A facet environment owns service provisions, `provideMany()` instances, observations, and resources explicitly registered through `own()`. Facets must register state subscriptions, watchers, timers, subprocesses, overlays, and other external resources themselves. The host deactivates consumers before providers and runs each facet's owned cleanups in reverse registration order. + +Already-admitted inbound RPC calls may continue while their providing facet deactivates. Withdrawing a provider rejects new calls. Code that requires stronger fencing needs an explicit lifecycle-owned controller. + +## Reloading facets + +Reload means replacing loaded facet source. It is not a transaction over durable Session state or external effects. + +### Shape-preserving provider replacement + +`FacetHost.reload()` replaces facets by `Facet.id` only when each replacement declares exactly the same service requirements, provisions, and singleton/keyed modes as the active facet. The tested sequence is: + +```text +load replacement facets +→ run setup and validate the unchanged service shape +→ withdraw replaced singleton provisions +→ deactivate old facets in reverse dependency order +→ activate replacements in dependency order +→ rebind local implementation slots +→ publish complete RPC singleton replacement snapshots +→ dispose the retired LoadedFacets generation +``` + +The loader disposal is deliberately outside `FacetHost.reload()`: the coordinator that loaded a module owns that module. It must dispose the old `LoadedFacets` only after the old active facets retire, and dispose a failed candidate generation if setup or validation fails. + +A singleton facade belongs to its consumer rather than a provider generation. Existing local proxies and captured local methods dispatch to the replacement implementation. Existing RPC proxies, captured methods, and replicated-state facades retain identity. During replacement they are unavailable: calls fail instead of queueing, and state becomes unhydrated until the complete replacement snapshot arrives. + +Reload has no rollback guarantee after old-facet deactivation begins. A failed replacement leaves affected services unavailable or degraded. Already-admitted calls are not automatically replayed or cancelled; callers must reconcile uncertain durable outcomes through authoritative state or stable operation IDs. + +### Shape changes and process replacement + +Changing requirements, provisions, modes, facet membership, or process authority is structural. It requires a newly assembled graph or an ordinary process restart; `FacetHost.reload()` intentionally rejects it. + +A reload coordinator lives outside the facet graph it replaces and follows these rules: + +1. Reload control stays outside the facet graph being replaced. +2. It chooses a desired source generation, then independently loads the affected server, Session, and presentation bundles. There is no aggregate cross-process extension object. +3. Shape-preserving facets use the existing host reload primitive. Structural changes build and validate a candidate graph before switching. Session-authority changes stop the old worker and release Session ownership before opening the replacement. +4. Hosts may converge at different times, so shared service contracts and durable records must tolerate temporary source-generation skew. +5. Failure is reported without pretending to roll back committed Session records, filesystem writes, subprocess effects, or hosts that already switched. + +`ReplicatedState` is a projection rather than storage. A replacement provider reconstructs authoritative state from durable Session records, configuration, or another owned source and publishes a complete snapshot. Keyed instances that must survive a worker restart likewise need durable application records and return as fresh live generations. Provider-local transport sequences and keyed generations may restart after rebinding and are never globally monotonic. + +Exactly one worker may own an open Session. Worker replacement therefore has a route gap: + +```text +old worker stops and releases Session ownership +→ selected Session remains logically selected but unavailable +→ replacement worker opens the Session and reconstructs services +→ server creates a fresh attachment binding +→ presentations hydrate fresh singleton and keyed snapshots +``` + +Reload never blindly retries an interrupted mutation. A request may have committed before its response was lost. Durable resumability belongs to Harness and application contracts, not facet cleanup. + +## Connection loss, errors, and security + +Disconnect behavior, from the facet author's perspective: + +- **A presentation disconnects.** Its server aborts the client's active requests and closes its observed instance tasks and other session-routed resources. Session-owned work continues per application policy. An added question dialog remains Session-owned; the disconnected presentation loses its proxy and any in-flight `submitAnswer()` call fails. +- **A Session worker disconnects or crashes.** Its server fails routed in-flight calls and closes that worker's observed instance tasks. Attached presentations see `attachment.status === "degraded"` while the server connection stays healthy, so the directory still works and the user can attach elsewhere. +- **A process loses its server connection.** Its connected server and Session services become unavailable. A Session worker loses server services and all attached presentations at once; unattended-Session policy decides whether it exits. +- Reconnect and reattach always hydrate from a fresh authoritative snapshot; prior keyed proxies and attachment-bound frames are invalid. **Never blindly replay a mutation after an uncertain disconnect** — a replayed `select()` is harmless, a replayed `prompt()` is not. Reconnect, hydrate, and reconcile, or design the operation around a stable operation ID with explicit lookup semantics. + +Errors cross the wire as a JSON envelope `{ code, message }` with stable protocol and service codes. Unexpected exceptions become `internal_error` without exposing stacks. Authentication and application errors use registered stable codes. + +Boundary rules: + +- remotely publishable service IDs come from trusted loaded service tokens; the remote boundary accepts only implementation functions and branded replicated-state members, instance generations are host-owned, and `{ local: true }` services are never discoverable remotely; +- business arguments, results, and state are validated as JSON; protocol envelopes cannot be forged as ordinary values; +- clients cannot choose context position, instance generations, selected-Session routing fields, or cancellation targets other than their own requests; and +- credentials, prompts, completions, tool arguments/results, and filesystem contents are not exposed unless an explicit contract permits them. + +## Host composition + +The facet kernel is service-aware but application-agnostic. A complete product should supply independently loaded facet sets for server authority, each Session worker, and each presentation. Shared contracts contain service tokens and JSON-safe DTOs; they do not imply that the providing and consuming facets share a bundle. + +## Open decisions + +Before the extension layer becomes normative: + +- what an `Extension` identifies and how it maps a version to independently bundled host facets; +- the manifest/source-selection format, ordering rules, package export convention, trust policy, and cross-process version skew; +- how structural graph replacement preserves host control and reports partial convergence; +- the concrete scoped server, Session, TUI, and future web capabilities; +- whether directory state is projected per authenticated client or globally presentation-safe; +- authentication, authorization, protocol version negotiation, and expected application error registration; +- optional service dependencies, multi-Session presentations, replicated-state flow control, and gap recovery; +- whether private returned references are needed after keyed services cover concrete features; and +- package boundaries between the facet kernel, service RPC, coding-agent host integration, and extension contracts. + +## Required tests + +The test matrix covers: + +- setup-derived provisions and requirements, late-access guards, missing and duplicate providers, mode validation, cycles, activation order, and reverse disposal; +- local and connected singleton calls, token-driven publication, strict JSON values, keyed instance hydration, generation fencing, cancellation, and selected-Session routing; +- cold state, snapshot/update races, buffering, update order, disconnect cleanup, and complete replacement snapshots; and +- static/combined loaders plus stable local and RPC singleton handles across shape-preserving provider reload. + +It also covers extension discovery and isolated process-specific bundle loading; structural graph reassembly and reload coordination; worker handoff with preserved logical selection and fresh attachment fencing; scoped host capabilities and contribution-registry rebuilds; authenticated routing and telemetry propagation; keyed-provider replacement and activation failure; and the question and collaborative-review examples below. + +## Collaborative diff review: a durable shared sidebar + +A diff review starts in a presentation rather than in a tool invocation. A user asks to review the current working-tree diff; the session snapshots it and opens one shared review. Every attached TUI and web presentation renders the same patch and comments, and any authorized user may add a comment or submit the whole review as one prompt. + +This uses two service modes: + +```text +DiffReviewManager singleton service + createReview() + → persist immutable patch + → add DiffReviews[reviewId] + +DiffReviews[reviewId] keyed service + document immutable patch state + activity durable comments + status state + addComment() commit, then publish + submit() freeze, enqueue one prompt, close +``` + +The keyed instance is the live, reactive projection. An extension-owned record is the durable authority. Pending comments are not weakly persisted: each acknowledged comment survives a worker restart, but the record is deleted after its prompt is durably accepted. + +### Shared remote contract + +```ts +interface DiffCommentInput { + commentId: string; // stable across an uncertain retry + path: string; + side: "old" | "new"; + line: number; + body: string; +} + +interface DiffComment extends DiffCommentInput { + author: { userId: string; displayName: string }; + createdAt: string; +} + +interface DiffReviewDocument { + reviewId: string; + patch: string; +} + +interface DiffReviewActivity { + revision: number; + comments: DiffComment[]; + status: "open" | "submitting"; +} + +interface DiffReviewManager { + createReview(context: Context): Promise; +} + +interface DiffReviews { + readonly document: ReplicatedState; + readonly activity: ReplicatedState; + addComment(input: DiffCommentInput, context: Context): Promise; + submit(context: Context): Promise; +} + +const DiffReviewManager = defineService("pi.diff-review-manager"); +const DiffReviews = defineService("pi.diff-review"); +``` + +The client never supplies a patch, author, or review ID. The session computes a bounded immutable patch, creates the ID, and derives each author from the authenticated identity in `Context`. `commentId` is only an idempotency key; it grants no authority. + +### Narrow local durability capabilities + +Unlike a question, this interaction has no invocation memo. The Session facet uses three process-local capabilities: a diff source that snapshots the working tree, a review store that serializes record mutations, and a prompt queue with idempotent `enqueueOnce()`. One durable review record contains the immutable patch, revisioned comments, status, and an optional frozen `{ submissionId, prompt }`. These local capabilities are ordinary `{ local: true }` services; their repository APIs are not part of the extension's shared contract. + +`DiffReviewRecords` serializes mutations per review. `addComment()` validates the anchor against the stored patch, stamps the authenticated author, deduplicates `commentId`, commits, and then returns the new revision. `freezeForSubmission()` atomically excludes later comments and stores a stable submission ID plus a prompt containing the immutable patch and that exact comment snapshot. If submission was already frozen, it returns the same record. `PromptQueue.enqueueOnce()` returns only after that logical prompt is durably accepted; retrying its submission ID cannot enqueue a second prompt. + +### Why record mutations need a critical region + +`DiffReviewRecords` builds on the facet's scoped Session data (`values.md`): typed durable values in an extension-owned namespace. Each storage call is atomic, but an application read-modify-write cycle spans multiple calls and therefore multiple awaits. Concurrent service calls can interleave between them. + +Concrete failure without serialization — two users press submit at the same time: + +```text +submit A: getValue(record) → status "open" +submit B: getValue(record) → status "open" +submit A: setValue(frozen, subm-A) +submit B: setValue(frozen, subm-B) → overwrites A's freeze +→ enqueueOnce(subm-A) and enqueueOnce(subm-B) both run: two prompts for one review +``` + +Each `setValue()` was atomic; the *cycle* was not. The same window exists in `addComment()` between checking the status and replacing the record. + +In the one-authoritative-worker model, the simplest fix is a per-review **critical region**: a FIFO, non-reentrant async mutex whose `run(signal, fn)` admits one pending function at a time. Every operation that reads and mutates an existing review — including `addComment()`, `freezeForSubmission()`, and `complete()` — uses the same region for that review ID: + +```ts +async freezeForSubmission(reviewId, context) { + return regionFor(reviewId).run(context.abortSignal, async () => { + const stored = await session.getValue(reviewRecord(reviewId), context); + if (stored === undefined) throw new RemoteServiceError("review_not_found", `Unknown review: ${reviewId}`); + + const current = stored.value; + if (current.status === "submission_pending") return current; // idempotent retry + + const submissionId = newSubmissionId(); + const frozen = { + ...current, + revision: current.revision + 1, + status: "submission_pending", + submission: { + submissionId, + prompt: renderReviewPrompt(current.patch, current.comments), + }, + }; + await session.setValue(reviewRecord(reviewId), frozen, context); + return frozen; + }); +} +``` + +`revision` is application-owned and monotonic per review. The region makes `current.revision + 1` unambiguous; the session-global storage `seq` remains storage ordering metadata and is not projected into the record. `publish()` may therefore compare returned record revisions directly. + +A caller aborted while queued is removed from the FIFO and rejects without invoking `fn`. Once admitted, the region releases in `finally`; cancellation and storage failure may reject the operation, while each individual storage transition remains atomic. Stateful validation stays inside the region, but user interaction and unrelated I/O stay outside it. A repository method must not call another method that acquires the same non-reentrant region, and region entries may be discarded after a completed review has no owner or waiters. + +The requirement is one linearizable read-modify-write path per review, not specifically a mutex. A storage compare-and-swap operation or a repository capability that serializes mutations could replace the process-local region. Durable settlement idempotency (`memoOnce()`, `enqueueOnce()`) solves crash and retry behavior after the record transition; it does not replace serialization of the transition itself. + +### Session facet + +The Session facet follows a short reconstruction algorithm: + +```text +activation +→ list pending review records +→ add one DiffReviews instance per record +→ publish document and activity state +→ resume any frozen submission through enqueueOnce() + +createReview() +→ snapshot the diff +→ create the durable record +→ add its DiffReviews instance + +submit() +→ atomically freeze comments and submission ID +→ publish "submitting" +→ enqueueOnce(submission ID, prompt) +→ delete the completed record and close the instance +``` + +Every mutation commits before `publish()`. Concurrent comment and submit calls are ordered by the record repository: a comment committed first is in the frozen prompt; a comment arriving after the freeze receives `review_closed`. `complete()` deletes only the matching frozen record, and `close()` is idempotent. + +The startup scan reconstructs every open keyed instance from durable records. A `submission_pending` record resumes delivery through `enqueueOnce()` and then closes. Thus a crash before prompt acceptance retries the prompt, while a crash after acceptance but before cleanup observes the same submission ID and only completes cleanup. + +### TUI and web facets + +The extension owns its TUI and browser widgets. TUI and web facets both `observe(DiffReviews, ...)`. Each observer opens a native panel from the hydrated document, subscribes to activity, forwards comment and submit actions to the service, and closes the panel when the instance context aborts. The TUI also registers a command that calls `DiffReviewManager.createReview()`; a web surface may expose the same operation as a button. + +`observe()` begins only after both state members hydrate. The panel receives the immutable document once, and subscribing to `activity` immediately renders current comments without retransmitting the patch on every edit. A late client sees the same pending review. Activity updates continue while `nextAction()` waits. Each sidebar action carries a fresh presentation-created `Context`; the longer-lived observation context controls only panel lifetime. When submission closes the keyed instance, every panel's observation context aborts and its `finally` block disposes the subscription and widget. + +The submitted prompt contains the immutable patch and all frozen comments in one request. Abbreviated: + +```text +Review this patch and address all comments: + + + +- src/parser.ts, new line 42 — Armin: Preserve the original error cause. +- src/ui.ts, new line 18 — Jane: Keep this state visible after reconnect. +``` + +Comment authors give the sidebar its basic multiplayer presence. A current-viewer roster or cursors would be separate live state and would not be written to the review record. + +This is a shared review, not a generic room primitive. Keyed services provide discovery and reactive lifetime; the record repository provides temporary durability; the prompt queue provides idempotent handoff into the session. + +## Deferred: delta-based replicated state + +> **Deferred:** `DeltaState` is not part of the initial facet-service or RPC contract. Add it only after a concrete feature demonstrates that full-value `ReplicatedState` updates are too expensive and the same pattern appears in more than one feature. + +A real replication gap remains. Some authoritative values are large, change frequently, and must support late joiners. `ReplicatedState` hydrates and reconnects correctly but sends a complete value on every update. + +A canvas is one possible example: joining requires the complete document, while dragging a shape should ideally send only that operation. With today's primitives, the facet must accept complete `ReplicatedState` updates or keep the high-frequency projection process-local. A concrete feature should establish the snapshot, delta, gap-recovery, and flow-control requirements before another remote primitive is added. + +### Possible future primitive + +If repeated implementations justify extraction, a future `DeltaState` could retain `ReplicatedState`'s synchronous value and snapshot hydration while delivering typed deltas after hydration. The provider would expose only `apply(delta, context)` and `replace(value, context)`; a shared pure reducer would update consumer replicas. + +The shared reducer is pure and deterministic. `apply()` synchronously reduces the provider's value and publishes one delta; `replace()` publishes a new authoritative snapshot. Business snapshots and deltas contain no transport revision. The host stamps revisions within the provider binding, buffers updates racing hydration, applies only consecutive frames, and requests a fresh snapshot after a gap or reconnect. + +Supporting this requires an explicit RPC member kind. The providing object carries the `DeltaState` definition ID; the provider announces that ID in member metadata; and the consuming host resolves the same imported definition before applying deltas locally. The contract is adopted only together with that registration and hydration protocol. + +`DeltaState` would solve only live replication. It would not provide durable storage, mutation serialization, multi-writer merging, offline editing, or automatic mutation replay. A durable canvas would still serialize its own mutations, persist an admitted delta before publishing it, and coordinate log compaction with appends. Durable log cursors remain application/storage metadata and are independent of the host's transport revision. diff --git a/packages/agent/docs/post-wp05-roadmap.md b/packages/agent/docs/post-wp05-roadmap.md new file mode 100644 index 000000000..1c8f6fc9d --- /dev/null +++ b/packages/agent/docs/post-wp05-roadmap.md @@ -0,0 +1,254 @@ +# Post-WP05 roadmap audit + +**Audit baseline:** `5507d76ee` (`dev`, 2026-08-27). + +**Status:** Planning inventory, not a behavior contract. [`harness.md`](harness.md) remains normative where it agrees with the current product boundary. Contradictions listed below must be resolved explicitly; this document does not silently choose one side. + +## Scope and method + +This audit covers the durable AgentHarness and its directly coupled Session backends and presentation path: + +- `packages/agent/src/harness`, `packages/agent/src/search`, and their tests/docs; +- `packages/session-backends/sqlite-node`; +- the current `packages/protocol`, `packages/client`, and `packages/server` presentation path; +- `packages/coding-agent/src/experimental` and its focused tests where they host that path; +- telemetry plumbing in `packages/telemetry`, `packages/ai`, and `packages/agent`. + +The inventory was checked against current source, tests, package READMEs, the complete `harness.md`, completed WP00–WP07 status, the actionable WP08 handoff, explicit stubs/TODOs/skips, and the shipped package boundaries. Historical handoffs are not treated as backlog when current source and the completed WP05 contract supersede them. + +## Executive result + +WP05 is complete through M10. Its remaining assistant-output work is owned by the [mobile assistant-output handoff](mobile-handoff/01-harness/05-assistant-output/message-update.md) and its numbered prerequisites. The current Harness execution graph has no unfinished runtime path: `watchSession()` is the sole `SliceNotImplemented` Harness method. + +That does **not** mean the surrounding durable system is complete. The remaining audit findings are: + +1. normative JSONL snapshot-compaction behavior with no implementation; +2. one required Harness method stub (`watchSession`); +3. a deliberate removal of raw `RemoteSession` that conflicts with later normative WP06/`harness.md` text; +4. a public search type skeleton that conflicts with the newer search design and has no implementation; +5. a full telemetry vocabulary whose only production span is the tool-hook span; +6. smaller repository, client-watch, query-bound, documentation, and end-to-end-test gaps. + +WP07 completed SQLite host-ownership alignment and live-source fork support after the audit baseline; its historical handoff is [`work-packages/07-sqlite-host-ownership-live-forks.md`](work-packages/07-sqlite-host-ownership-live-forks.md). WP08 now owns the separate named-branch, tree-state, and bounded-memory fork redesign; its actionable handoff is [`work-packages/08-named-branch-streaming-forks.md`](work-packages/08-named-branch-streaming-forks.md). + +## Required missing functionality and contract contradictions + +### R12 — Session-wide Harness watch + +**Evidence** + +- `AgentHarness.watchSession(context)` is public in `src/harness/agent-harness.ts`. +- `Harness.watchSession()` throws `SliceNotImplemented("watchSession")` in `src/harness/runtime/harness.ts`. +- `SessionSnapshot` currently contains only `{ lanes: LaneInfo[]; faulted: boolean }`. +- Lane watch, event buffering, delivery-tail barriers, and `resnapshot()` already exist in `src/harness/events.ts` and `src/harness/runtime/lane.ts`. + +**Remaining boundary** + +Define one coherent capture and fold for dynamic lane inventory and fault state. Decide whether the intentionally small `SessionSnapshot` stays small or gains session metadata/stats/global configuration. Then implement snapshot-before-events, lane creation, resnapshot, listener reentrancy, close/fault behavior, and a session reducer if event-only replication is promised. + +**Dependency** + +Independent of the mobile assistant-output handoff and SQLite internals. It should precede any revisioned Transcript service or remote session-wide observation built on it. + +### JSONL snapshot compaction + +**Evidence** + +`harness.md` §1.7 normatively specifies temp-file-and-rename snapshot compaction, preserved sequence high-water marks/list cursors, threshold checks on open, and reclamation after terminal/outcome deletions. `JsonlStorage` implements atomic creation, torn-tail repair, legacy-v3 rewrite, append, and fork snapshots, but no current-state snapshot rewrite or dead-byte accounting exists. + +**Consequence** + +Superseded `pi.op.state`, deleted pending payloads, deleted tool checkpoints, and deleted assistant-frame lists remain physical bytes indefinitely. Generic compaction and the [mobile assistant-output handoff](mobile-handoff/01-harness/05-assistant-output/message-update.md) are complementary. Compaction reclaims dead session-scoped write history after the fact; the handoff moves pending assistant/tool output into ephemeral scopes so it never becomes main-log history, and replaces full/per-frame replication with Chord op batches. + +**Dependency** + +The assistant-output handoff does not depend on J1: scoped storage is the intended lifetime mechanism for pending output, while J1 remains the reclamation mechanism for superseded session-scoped state. Measure both independently so their effects are not conflated. + +### Remote Session contract contradiction — decision required + +**Current product boundary** + +Commit `f8a6e670d` deliberately deleted `RemoteSession`, its raw Session RPC protocol, and the server mutation-scope manager, replacing them with attachment-fenced routed semantic services. Current protocol/server READMEs explicitly state that real `Session` and `AgentHarness` objects remain process-local. The shipped path supports Session discovery/creation/attachment, main-lane prompt/watch, and allowlisted plugin-service calls; it does not expose `Session`, `SessionMutation`, values/lists, branches, or storage over RPC. + +**Conflicting contract** + +WP06, written after that deletion, requires the “current keyless RemoteSession mutation transport,” includes remote begin/read/commit/publication/end in required tests and stop conditions, and forbids its removal. `harness.md` §§2.8 and 9.1 likewise state that local and remote implementations preserve that lifecycle. No such implementation, protocol schema, client facade, server-held scope, worker adapter, or conformance test exists at the audit baseline. + +**Required decision** + +Choose one before scheduling implementation: + +1. **Process-local Session remains intentional:** remove the false RemoteSession requirements from normative current-state docs while preserving semantic service RPC; or +2. **RemoteSession is required:** commission a dedicated package for mutation begin/read/commit/end, disconnect/timeout cleanup, publication-before-end, values/lists/branches/entries/stats, protocol validation, client/server/worker adapters, and remote conformance. + +Do not count the current lane-watch compatibility RPC or plugin-service RPC as RemoteSession. Do not restore the deleted 507-line facade unchanged: it predates the keyless Session/Branch contract and relied heavily on untyped decoding. + +### Telemetry contract exceeds implementation + +**Evidence** + +`src/harness/telemetry.ts` and generated `docs/telemetry-schema.md` declare `pi.ai.request`, operation, checkpoint, turn, step, tool, hook, sleep, event-handler, and session-write spans. Production source starts only `pi.harness.hook`, and only for registered `before_tool`/`after_tool` handlers. AI options propagate `telemetryContext`, but no provider path starts `pi.ai.request`. Server request ingress has cancellation but no trace carrier or client/server RPC spans. `TODO_CONTEXT` remains at transport/worker lifecycle and event-delivery boundaries. + +**Remaining boundary** + +Treat this as separate packages: + +1. local Harness/Session/AI instrumentation and runtime tests; +2. RPC trace-carrier/client/server propagation; +3. an optional application-selected exporter/adapter. + +First reconcile whether every declared span is still wanted. If retained, implement it; if not, remove the unsupported public schema surface and correct `harness.md`. Do not mix telemetry with Context/RPC cancellation, which already has independent request-ID signaling. + +### S3 — Search + +**Evidence** + +`src/search/index.ts` exports a public `SessionSearchService` skeleton with `sync()`, `notify()`, and array-returning `searchEntries()`. `harness.md` §2.8 instead specifies a standalone service, separate catch-up/notify utilities, generation-aware cursors, and optional `AsyncIterable` entry search. There is no factory, sync utility, cursor store, projection, or source SQLite FTS implementation. At the audit baseline the SQLite README advertised nonexistent `createSqliteSessionSearch()` behavior; this audit corrected that README rather than treating the absent API as implemented. + +**Remaining boundary** + +Before implementation, replace or reconcile the draft public interface and decide metadata filtering (`cwd`), candidate restriction, or indexed metadata. Post-filtering after ranked `limit` is unsound. Then implement standalone catch-up and a separate SQLite FTS5 projection; do not add repository search methods. + +### R11 — Schema migrations, activation-gated + +No current format-4 migration is required. Memory is current-only; JSONL and SQLite reject unsupported storage versions; SQLite runs only idempotent `001_initial.sql`. R11 becomes required immediately before the first incompatible durable storage version/address/state change after format 4 is stabilized. It is not prerequisite work for the mobile assistant-output handoff or current WIP format replacement. + +When activated, it must provide ordered transactional migrate-on-open under exclusive ownership, version-specific JSONL decoding plus post-migration compaction, and total mappings for every reachable open operation leaf and surviving value/list. + +## Correctness and data-safety debt + +### SQLite host ownership and live-source forks — completed by WP07 + +The authoritative product rule is in `plugins.md`: exactly one host-assigned process owns writable Session authority; normally it is the Session worker, while the server may temporarily own a newly created or forked destination before closing it and handing it off. Storage backends do not implement writer ownership. The server closes a worker before destructive repository administration. + +SQLite now follows that rule: the writer-lease schema/module, claims, renewal timer, lease-loss path, and pre-commit callback are gone, with no replacement lock or ownership primitive. Create/open/fork/delete retain repository-local ID reservation. Metadata open and deletion use a true no-create read-write mode; listing and external fork sources use no-create read-only connections. + +Same-repository forks retain the source `commitQueue` ordering seam. A source owned elsewhere, including a live worker, is read from its exact canonical container through one independent read-only deferred WAL transaction. Focused per-file and shared-container tests commit a complete later source transaction after the reader establishes its snapshot and before it closes: the first fork excludes the transaction wholly and a later fork includes it wholly. + +WP07 also completed canonical `(containerPath, sessionId)` active identity, safe explicit-ID filenames, custom `databasePath` parent creation, Session-scoped shared deletion, WAL/SHM cleanup, and all-settled SQLite repository close. Writable open/delete reject foreign metadata; foreign fork sources are read only from their exact path. Equal-`createdAt` list ordering still has no deterministic tie-break and remains a later behavior-preserving cleanup. + +### Repository close ownership is unspecified + +`JsonlSessionRepo.close()` contains the only active Agent source TODO and closes no open Session handles. Memory still uses fail-fast `Promise.all`; SQLite now performs backend-local all-settled cleanup of currently open handles, but an already-admitted create/open/fork can still register a handle after repository close captured that set. `SessionRepo` itself declares no `close()` method and shared conformance does not define repository-to-handle ownership or draining of admitted repository operations. Resolve ownership and common cleanup in one repository-lifecycle package; do not patch one backend further without deciding the common contract. + +### Client watch staleness after disconnect + +`Client` clears its active watch-listener map on disconnect, but existing `LaneWatch` objects keep local `ready`/`started` state and old watch IDs. After reconnect/reattach they can call `start()` or `resnapshot()` and fail remotely instead of deterministically rejecting as stale, contrary to `packages/client/README.md`. Service-subscription objects have the sibling problem: their listeners are cleared and surviving objects become silently dead. Fix both with connection/attachment incarnation fencing and focused reconnect tests. This is independent of R12: the current client method is a compatibility main-lane watch. + +### Query bounds and SQLite bind limits + +- SQLite `getEntries(ids)` emits one placeholder per requested ID and can exceed the engine’s variable limit. +- Entry, usage, and branch limits use ad hoc `Math.max(0, limit)` behavior. Memory and SQLite diverge for `NaN`, infinities, fractions, and extreme values; unlike list reads, there is no shared normalization contract. + +Define cross-backend query-limit semantics in agent conformance, then chunk SQLite ID lookups. This is a storage-contract hardening package, not part of WP07. + +### Harness contract and conformance closure + +- The public `OperationStatus` includes `"running"`, but lane inspection, snapshots, and `reduceLaneSnapshot` currently produce only `"open"` or `"aborting"`. Define and implement its producer or remove the dead variant. +- The pre-rewrite abort contract bound/published `operation_abort` before resolving the cancellation promise and signalling the live gate. Current `Lane.command()` materializes the result — resolving/signalling — before constructing and binding the event batch, although it still binds recipients before releasing the Session mutation line. Decide whether to change the implementation or retain/document the current no-interleaving order; add an explicit ordering test. +- The production gate-close contract permits only `HarnessClosed | HarnessFault`, but the private source primitive accepts any `Error` and isolated tests use that wider type. Narrow the source declaration and fixtures or explicitly retain the private widening. +- Part 9 of `harness.md` is the required conformance matrix. Existing focused tests cover the graph extensively, including cancellation reconciliation over all 13 leaves, but there is no audited one-to-one proof that every close/reopen leaf case and every race row has both deterministic orders. Audit the matrix and add only the missing cases rather than claiming blanket completion. + +Keep this package separate from telemetry, RemoteSession, and the mobile assistant-output handoff; it is local contract/test closure. + +### Disabled real worker persistence regression + +`packages/coding-agent/test/experimental-remote-runtime.test.ts` still skips “completes and persists a prompt through the worker-owned Harness” with the obsolete note “Re-enable with runtime no-tool execution.” No-tool execution now exists. Re-enable or replace it with a deterministic faux-provider real-worker persistence test; do not use a real paid provider. + +## Performance debt + +### Mobile assistant-output handoff — durable and replication amplification + +The user-supplied motivating mini Session outside the repository is 303,920 bytes across 569 physical lines. These external measurements are evidence, not a reproducible checked-in fixture: + +- 477 assistant-frame appends totaling about 118,418 serialized write bytes; +- 12 frame-list deletes; physical lines mentioning the frame namespace total 148,214 bytes; +- about 51,568 bytes of superseded `pi.op.state` writes and 26,192 bytes of one structural preparation, showing why generic JSONL compaction and frame-specific bounding are distinct. + +The authoritative design is the [mobile assistant-output handoff](mobile-handoff/01-harness/05-assistant-output/message-update.md), following the numbered `01-harness` prerequisites in [`mobile-handoff/README.md`](mobile-handoff/README.md). Chord delta tracking has landed; scoped storage, tool-output integration, and assistant-output integration have not. + +Implementation must add a deterministic repository fixture and measurement script that reproduces or replaces these figures, then measure Memory logical elements, SQLite rows/pages/WAL, JSONL peak sidecar/main-log bytes, and reopen/replay time. The handoff must preserve unknown-outcome recovery, invocation fencing, non-blocking provider streaming, and settlement retirement while eliminating per-frame durable writes and quadratic `message_update` replication. Numerical budgets belong in its implementation handoff/tests, not in a competing standalone design. + +### SQLite branch divergence + +`createDivergentBranchForEntry()` copies every row after the newest compaction; with no compaction it copies root-through-parent. A first divergence from a long uncompacted transcript is therefore O(history) writes. This exposes an internal contradiction in `harness.md` §2.6: its opening bounded-prefix promise conflicts with its own compaction-based copy algorithm, which the implementation follows. Change both the specification and segment representation so a divergence can reference a covering segment at the parent boundary. Preserve shared-container support and add large uncompacted divergence plus chain-soundness tests/benchmarks. + +### Fork contract and materialization — WP08 actionable + +All current backends materialize source-sized fork snapshots. Branch scope defaults to `main` without named-Branch ancestry validation; tree scope omits application values/lists; JSONL closed-source fork replay may repair its source. WP08 replaces that contract with required named-branch or tree scope, one closed built-in-state policy, complete current application state for tree forks, and backend-specific bounded-memory copy procedures. It preserves WP07's host ownership, physical identity, same-repository ordering, and independent live-source WAL boundary. + +### SQLite catalog, statements, stats, and reclamation + +- Default per-session `list()` synchronously opens/configures every SQLite file serially and silently skips failures. A shared container or external catalog is the scalable deployment choice; bounded async scheduling does not make `DatabaseSync` nonblocking. +- Most hot queries prepare a fresh statement each call. Cache narrowly owned statements after measuring. +- Each usage row parses and rewrites the full aggregate JSON usage payload. +- Shared-container row deletion does not reclaim pages. Define maintenance/VACUUM policy separately; never add `VACUUM` to ordinary deletion casually. + +These are measurable optimization/operations packages, not correctness fixes. + +### Pending-payload amplification and mutation-line parallelism + +Queued payloads deliberately write `pendingEntry → immutable entry`; measure pathological payloads before changing it. Keyed Session mutation lines remain optional and require profiling plus a fresh mutable-ownership audit. Neither is current correctness work. + +## Behavior-preserving cleanup and documentation repair + +Completed by this audit: + +- rewrote the SQLite README’s nonexistent `SqliteSessionRepository`/search APIs, wrong `await using` and `appendMessage` example, FTS trigger/rebuild claims, and “one shared connection” claim; +- reconciled `harness.md`’s default one-file SQLite wording with the supported optional shared container; +- reconciled `harness.md` Part 8 with this audit and corrected stale drive-ownership/RPC-cancellation status in `telemetry.md`; +- corrected coding-agent settings docs: install telemetry configuration also controls selected provider attribution headers. + +Remaining cleanup: + +- **Harness-owned DTO boundary:** the durable Harness still imports legacy agent-loop DTOs from `src/types.ts`: `AgentMessage`, `AgentToolResult`, `AgentToolCall`, `AgentTool`, `QueueMode`, and `ThinkingLevel`. Make a later one-shot cut to independent `HarnessMessage`, `CustomHarnessMessages`, `HarnessToolResult`, `HarnessToolCall`, `HarnessQueueMode`, and `HarnessThinkingLevel` definitions; keep `AgentHarnessTool` as the executable Harness configuration type. Update Harness internals, root exports, declaration-merging tests, documentation, and experimental coding-agent consumers together. Do not alias the new message/tool DTOs back to legacy types, because that preserves the coupling this work is intended to remove. +- Preserve shared-container support; do not remove it incidentally. +- Remove or use unused `insertEntryRow()` and `insertUsageLedgerRow()`. +- Consolidate duplicated SQLite branch payload/structure scan plumbing only after correctness tests pin both paths. +- Decide whether unused `sessions.metadata` and unmeasured indexes have a future owner before schema removal; do not change schema casually. +- Consolidate `startAiSpan()`/`startHarnessSpan()` implementation only if the telemetry surface is retained. +- Mark historical durability documents as delivered where they still read as implementation queues. WP00–WP07, `runtime-simplification.md`, `values.md`’s old consumer deferrals, and external-finalization designs are not active runtime backlog. + +## Optional or deferred product capabilities + +These are not blockers for the durable Harness: + +- standalone S3 search after its API decisions; +- Accounts removal and revisioned Transcript production; +- authenticated workspace/client authorization for the experimental local server; +- private returned references, service flow control, multi-pane presentation, plugin kernel/reload completion, and version-skew negotiation; +- administrative precise rewrite tooling; +- a partitioned Postgres backend/retention policy; +- generic remote Harness/object capabilities; +- DeltaState or delta replication before concrete snapshot pressure justifies it; +- a production telemetry exporter; +- schema migrations before an incompatible stabilized-format change activates R11. + +## Recommended dependency order + +The order is by data safety first, then dependencies. Independent tracks may proceed in parallel only when they do not edit the same contracts. + +1. **Harness contract/conformance closure.** Resolve `OperationStatus.running`, abort signal/event binding order, gate-close typing, and the Part 9 coverage matrix. +2. **Remote Session decision (decision only).** Resolve the false normative boundary early. If process-local wins, repair the docs. If raw RemoteSession wins, later create a dedicated protocol/client/server/worker package; do not fold it into telemetry or R12. +3. **Client watch/subscription staleness** and **repository lifecycle contract.** Small independent correctness packages; complete them before expanding server/worker lifecycle semantics. The lifecycle package must also address Memory's fail-fast repository close. +4. **[Mobile Harness handoff](mobile-handoff/README.md).** Follow its numbered prerequisites through scoped storage, tool output, and assistant output; preserve all recovery boundaries and land deterministic amplification measurements. +5. **JSONL snapshot compaction.** Implement the already-normative physical reclamation path and metrics for remaining session-scoped history. +6. **R12 session-wide watch.** Complete the only Harness method stub before building revisioned Transcript/session-wide remote observation. +7. **Telemetry, if retained:** reconcile schemas, then local instrumentation, then RPC propagation, then an optional exporter. RPC propagation follows the Remote Session/product-boundary decision. +8. **WP08 — named-branch and streaming forks.** Implement the actionable handoff without reopening WP07 ownership or lifecycle decisions. +9. **SQLite branch/query performance hardening.** Keep separate from completed WP07 ownership alignment and WP08 fork semantics; require benchmarks. +10. **S3 search.** Resolve its API/filter/cursor decisions, then implement catch-up and the standalone FTS projection. +11. **R11 migrations.** Activate immediately before the first incompatible stabilized durable schema change, not earlier. + +## Stop conditions for roadmap accuracy + +This inventory must be updated when any of these facts changes: + +- `watchSession` stops being the sole Harness `SliceNotImplemented` method; +- raw RemoteSession is either recommissioned or removed from the normative contract; +- JSONL snapshot compaction lands; +- telemetry schemas are implemented or removed; +- S3’s public API is reconciled; +- a durable format change activates R11; +- WP08 lands or its fork contract changes; +- the host-authority contract changes. diff --git a/packages/agent/docs/rpc.md b/packages/agent/docs/rpc.md new file mode 100644 index 000000000..56802dd9e --- /dev/null +++ b/packages/agent/docs/rpc.md @@ -0,0 +1,215 @@ +# Facet Service RPC + +Chord owns the application-neutral service semantics and pluggable strict-JSON connection +boundary. Pi owns the concrete wire envelope, routing, attachment state, and error adapters +described here. The current implementation treats `JsonValue` as a static contract and defers +runtime rejection of unsupported values to the concrete serializer. + +> **Status:** Design specification for experimental facet-service RPC semantics. + +## Role + +`provide()`/`use()` and `provideMany().spawn()`/`observe()` are the facet system's hidden RPC. Facets share TypeScript service contracts, while the transport carries service/member identifiers, strict JSON values, request/subscription correlation, keyed generations, and binding control messages. Hosts construct the typed local implementation or facade; TypeScript types and arbitrary objects never cross the wire. Independently loaded processes may temporarily run different source generations, so their service contracts must remain forward compatible across the supported skew window; version negotiation remains deferred. + +```text +session/server facet: provide() / provideMany().spawn() + ↕ hidden service RPC +presentation/session facet: use() / observe() +``` + +The service system is the extension boundary. Presentation facets receive semantic services and replicated state; they never receive a raw Harness, Session, tool registry, hook registry, credential store, or storage handle. + +## Non-goals + +Do not serialize `Context`, `AbortSignal`, telemetry objects, callbacks, tools, hooks, functions, or arbitrary object graphs. Do not make core Harness or Session implementations aware of transport mechanics. Do not make disconnect perform durable or service-owned cancellation. Do not build per-method codecs for values already constrained to JSON. + +## Service contracts and typed facades + +A service token is a shared TypeScript contract and stable service ID. It is not a generated descriptor and creates no provider. Tokens are remotely publishable by default; process-local tokens declare `{ local: true }`. `provide()` adds one singleton implementation to the host graph. `provideMany()` registers one multi-instance service owner during facet setup and returns a `ServiceSpawner` whose later `spawn()` calls publish instances. The host automatically publishes every non-local provision. A token has one mode in one host service graph: mixing singleton and keyed use is an error. + +The provider classifies every exposed implementation member as a method or Chord-created `ReplicatedState` and publishes that member table in subscription snapshots. The consumer obtains member names from ordinary property access—for example, a JavaScript `Proxy` receives `"state"` for `models.state` and `"refresh"` for `models.refresh(context)`. Accessed slots are validated against the provider-announced kind when the facade binds. + +Local and remote `use()` both return a stable, lazy typed facade shared by consumers of that token. During synchronous facet setup the facade is disconnected, so setup can capture it but cannot invoke methods, read state, or register member subscriptions. After assembly, a local facade resolves through a direct process-local implementation slot and a remote facade binds through the host's connected services. Reloading the providing facet temporarily marks that same facade unavailable, then swaps its target; RPC singletons clear readiness and install a complete replacement snapshot on their existing subscription so captured methods and member facades address the replacement. While no provider is bound, invoking a method fails and state remains unhydrated; no call is queued merely because it was made through a proxy. + +Remote methods return promises and accept and return strict JSON apart from their declared `Context`; `void` is a successful response without a result field. Private returned references are not supported. The client removes the context before transport and the receiving host constructs a fresh local context. The contract position is host-controlled and must be consistent; the examples use one required trailing `Context`. Business absence is JSON `null` or an options object, never transported `undefined`. + +Use static assertions and runtime validation. Static checks constrain remote methods and replicated-state members; runtime boundaries reject unsupported members and non-JSON arguments, results, and state values. TypeScript supplies typed facades but does not authenticate a peer or create runtime metadata. + +`{ local: true }` removes only remote publication and its wire-contract restrictions. Local and non-local provisions otherwise use the same dependency ledger, activation order, stable singleton slots, keyed-instance generations, observer cancellation, disposal, and provider-facet reload. Local singleton slots and the local keyed registry hold arbitrary object contracts directly; non-local provisions additionally install their implementations in the remote provider. + +## Dependency ledger + +Type erasure does not hide service identity: every service token retains its stable ID at runtime. Facet environments are created by the host with a non-forgeable owner identity, and their setup-time service methods append to a generation-scoped ledger: + +- `provide()` records a singleton provision; +- `provideMany()` records a keyed provision; +- `use()` records a singleton requirement; and +- `observe()` records a keyed requirement. + +Facets always call unqualified `env.use()` or `env.observe()`; routing is not encoded in the call. These operations return source-independent disconnected handles during setup. After setup, each connection returns its provider-generated service catalogue, and the host binds every requirement to its local provision or exactly one connected provider. The method supplies the mode and the token supplies the ID. The host therefore needs no reflection over the erased `T` and no handwritten parallel dependency list. + +First acquisition or provision is permitted only during facet setup. Later commands, hooks, and activation callbacks use setup-acquired singleton facades, observer registrations, or `ServiceSpawner` capabilities. In particular, dynamic instances are spawned through the capability returned by `provideMany()`; late spawning cannot introduce a previously undeclared provision. + +After setup, each host privately resolves the recorded requirements against local provisions and connection catalogues to reject missing providers, duplicate remote offers, and mode mismatches and to derive lifecycle edges. A selected-Session connection with no live attachment may provisionally accept unresolved requirements as unavailable; attachment validates them against the worker's generated catalogue and caches that catalogue for later detached generations. Connection bindings are generation-owned and include only selected requirements, so failed or retired generations release their subscriptions without disposing the underlying transport connection. This internal service graph is distinct from the module loader's source import graph; it is not a facet-authored or facet-visible plan. + +## Bindings and identity + +A presentation host combines services from its connected server and selected Session in one graph. All of its facets use the same unqualified environment API. A Session-service call never accepts a client-selected durable `sessionId`; the server authorizes and routes the presentation's selected Session binding to its worker. + +### Server control plane + +Session listing and management are ordinary server singleton services, not generic remote `Session` methods. `SessionDirectory` exposes presentation-safe session summaries as replicated state. `SessionManagement` exposes `create`, `remove`, `attach`, and `detach` methods. A TUI or web facet consumes both through its environment: + +```ts +const directory = env.use(SessionDirectory); +const management = env.use(SessionManagement); +``` + +The server binds one service provider to each presentation connection. It derives workspace and client authority from locally authenticated connection identity, not from summary fields or method arguments. It may project directory state per client; either way, summaries never expose server-private fields such as owner IDs or working directories. + +`management.attach(sessionId, context)` changes the selected-Session services in the presentation host. The server closes the presentation's previous Session-scoped requests, subscriptions, and observer tasks, binds the presentation's Session services to the worker, then hydrates their singleton state and keyed-instance directory. The server authorizes the selected Session against the connection identity. Attachment state is host control state reporting this selection and its health; it is not a directory service. `detach()` performs the same cleanup without a replacement. + +The host needs a private, host-owned binding incarnation for that route. It changes when the presentation attaches, detaches, switches session, or replaces a failed worker. Its representation is deliberately unspecified. The binding prevents a delayed frame for the old selected session from being applied to the new one; it is not a facet-visible service value or a substitute for authorization. + +A replicated-state source has structural identity: + +```text +(provider binding, service ID, optional instance key + generation, member name) +``` + +There is no separately discoverable state ID. An added instance key is an application-level logical key. Its host-owned generation changes when a closed key is reused, so a stale proxy cannot call the replacement. `requestId` identifies one transport invocation for response and cancellation. A Harness/tool `invocationId` may be a useful instance key—as in the question example—but it does not replace the service, binding, or generation parts of the live address. + +## Calls, context, and routing + +A call carries enough control-plane information to select a provider binding, service, optional keyed instance, and member, plus a request ID, JSON arguments, and trace carrier. The server may parse those control-plane fields to route a Session call, but it does not parse facet business payloads or load facet contracts. The service endpoint validates the member and values, creates a request-local abort controller and `Context`, installs authenticated identity, and invokes the local implementation. + +The client maps `context.abortSignal` to cancellation of that one request. Disconnect cancels that connection's active calls and closes its subscriptions. Neither action cancels service-owned work or writes durable Harness cancellation. Per-client request correlation reaches the worker so request IDs from different presentations cannot collide. + +## Replicated state and keyed instances + +`ReplicatedState` is authoritative latest-value replication, not event history, durable storage, a CRDT, or multi-writer state. A cold replica has `value === undefined`; subscribing before hydration records a listener without invoking it. Hydration installs a complete snapshot atomically before later updates are delivered, so there is no snapshot/update gap. Once hydrated, `value` is synchronous and subscribing reports the current value followed by later updates. The first snapshot callback uses a fresh hydration context; an already hydrated replica uses a fresh local delivery context rather than retaining the original write context. State values are borrowed immutable JSON and are not defensively cloned; callers must not mutate or retain them. + +Remote hydration uses a fresh delivery context parented to the subscription, while updates reconstruct fresh delivery contexts from source trace metadata. Disconnect, provider withdrawal, replacement, and route switching clear readiness. Reconnect or replacement installs a complete snapshot before later updates. The transport buffers updates that race hydration and checks their sequence. Acknowledgements, flow control, and gap recovery remain separate protocol mechanics. + +`observe()` is keyed-instance discovery, not a `ReplicatedState` containing proxies. It reconciles a complete initial directory with ordered additions, replacements, and removals. Each instance's initial state members hydrate before its observer task starts. Closing an instance rejects new calls, aborts only that instance's observer task, and allows admitted calls to settle. A Session facet's `env.observe()` registration aborts old tasks when that facet generation closes; the replacement generation reconciles the fresh directory. + +## Private returned references + +Private returned references are outside the initial service contract. Prefer keyed services for discoverable live instances. If a concrete feature requires caller-private remote identity, its reference must be passed explicitly rather than discovered by `observe()` and scoped to the recipient and provider binding. + +No generic Harness projection is part of this design. Raw Harness, Session, lane, tool, hook, and storage objects remain local authority. If a future integration needs a remote callback or a general object capability, it needs a separate explicit protocol and policy; it is not an extension of service RPC. + +## Context, cancellation, and telemetry + +Every remote method receives a fresh local `Context`; the sender's object, signal, telemetry implementation, and arbitrary typed values never cross the wire. The client maps the call's abort signal to that request and injects a trace carrier. The endpoint constructs a request-local abort signal and telemetry parent. Cancellation is forwarded through the server to Session workers and remains isolated by client plus request ID. + +The span relationship is: + +```text +caller +└─ rpc.client + └─ rpc.server + └─ service implementation +``` + +Three cancellation domains remain separate: aborting one RPC invocation; explicitly cancelling service-owned work such as `job.cancel()`; and durable Harness cancellation such as `requestAbort()`. Transport cancellation and disconnect perform only the first. Work that outlives a call must detach into a service-owned task with its own controller and telemetry root. + +## Security and lifecycle + +Only loaded service tokens not marked local may be registered at the remote boundary. Only the owning `ServiceSpawner` may spawn instances. Services marked `{ local: true }` are never discoverable remotely. Local services may use unrestricted object contracts. Remote providers validate member kinds, while concrete serializers enforce JSON business values. Clients cannot forge control envelopes as ordinary values, choose instance generations, select a different Session route in a service call, or cancel another client's request. + +The server authenticates connections, authorizes attachment, and reconstructs client identity in the service `Context`. Ordinary business arguments never carry authority. Credentials, prompts, completions, tool data, filesystem contents, and other sensitive values require an explicit presentation-safe contract. + +Facet environments own registrations, added service instances, observations, and resources explicitly registered through `own()`. Connection bindings own their transport subscriptions and active request controllers. Already-admitted inbound calls are not attached to the providing facet lifecycle: provider withdrawal rejects new calls, but an admitted method may continue while the old facet deactivates. The provider's own Session work remains alive unless its lifecycle policy stops it. + +## Tests + +The facet-facing semantics are tested over loopback and framed transports: + +- setup-time dependency-ledger ownership, rejection of late acquisition, local and remote `use()`, keyed-provider ownership, singleton/keyed mode validation, token-driven RPC publication, lazy member access, stable local and RPC singleton facades across provider-facet replacement, and `{ local: true }` services remaining unreachable remotely; +- strict JSON boundaries, method context reconstruction, and request cancellation isolation without serializing context values; +- server/Session facet isolation, selected-Session switching, stale-frame rejection, and worker-side per-client request correlation; +- cold and hydrated `ReplicatedState`, snapshot/update race freedom, fresh delivery contexts, and clearing/re-hydration on disconnect, reconnect, and provider replacement; +- instance directory hydration, ordered reconciliation, state hydration before observer tasks, generation-based stale rejection, and task cleanup on close or switch. + +Additional tests cover authenticated attachment and identity, telemetry propagation, flow control and gap recovery, plus the question and shared-review application patterns. If private references are added, they require separate lifetime and isolation coverage. + +## Open protocol mechanics + +Exact service call, cancellation, subscription, snapshot/update, keyed-instance, unavailable, and replacement frames are defined in `packages/protocol/src/protocol.ts`. Provider and namespace layers own member classification, lazy facades, buffering, and sequencing. + +Still open are acknowledgements, flow control, sequence-gap recovery, reference collection if references are added, protocol-version negotiation, and how a future multi-pane presentation represents more than one selected Session. Singleton provider replacement must continue to install a complete replacement snapshot on the existing subscription so method and state member slots retain identity. + +## Example: directory and selected session + +The directory and management services are normal server services. Their contracts carry only presentation-safe values: + +```ts +interface SessionSummary { + serverId: string; + sessionId: string; + createdAt: string; +} + +interface SessionDirectory { + readonly state: ReplicatedState<{ revision: number; sessions: SessionSummary[] }>; +} + +interface SessionManagement { + create(options: { id?: string }, context: Context): Promise; + remove(sessionId: string, context: Context): Promise; + attach(sessionId: string, context: Context): Promise; + detach(context: Context): Promise; +} + +const SessionDirectory = defineService("pi.session-directory"); +const SessionManagement = defineService("pi.session-management"); +``` + +A server facet derives the client from an authenticated `Context`, authorizes the requested Session, and performs the binding transition: + +```ts +serverContext.provide(SessionDirectory, { state: directoryState }); +serverContext.provide(SessionManagement, { + async attach(sessionId, context) { + const client = requireClientIdentity(context); + authorizeSession(client, sessionId); + await attachments.bind(client.clientId, sessionId, context); + }, + async detach(context) { + await attachments.unbind(requireClientIdentity(context).clientId, context); + }, +}); +``` + +A presentation facet renders and selects Sessions: + +```ts +setup(env) { + const directory = env.use(SessionDirectory); + const management = env.use(SessionManagement); + const tui = env.use(Tui); + + tui.commands.register("sessions.switch", async (operation) => { + const snapshot = directory.state.value; + if (snapshot === undefined) return; + const sessionId = await tui.select( + "Sessions", + snapshot.sessions.map((session) => ({ label: session.sessionId, value: session.sessionId })), + { signal: operation.abortSignal }, + ); + if (sessionId !== undefined) await management.attach(sessionId, operation); + }); +} +``` + +Another facet in the same presentation acquires Session services through the same API: + +```ts +setup(env) { + const models = env.use(Models); + // After attach() settles, `models` addresses the selected worker. +} +``` + +The presentation never routes `models` with a selected `sessionId`; its host routes each service token and transport retains the selected-Session binding. The server closes the prior Session binding's resources before hydrating the new one. diff --git a/packages/agent/docs/runtime-simplification.md b/packages/agent/docs/runtime-simplification.md new file mode 100644 index 000000000..b043d1a73 --- /dev/null +++ b/packages/agent/docs/runtime-simplification.md @@ -0,0 +1,391 @@ +# Existing AgentHarness runtime simplification + +## Scope + +Rework the real implementation under `packages/agent/src/harness/runtime/` and the canonical durable types in `packages/agent/src/harness/session/types.ts`. This is not the isolated scratch spike. + +The execution graph is total and public drive is enabled. Format 4 is still work in progress, so the durable type replacement required no migration or compatibility representation. + +## Implementation status + +Pre-M6 completion measurements: + +- pre-simplification runtime at `eb1185d93`: 5,358 TypeScript lines; +- runtime after the first simplification at `417905647`: 4,667 lines; +- simplified substrate at `0e77e57d9`: 4,654 lines; +- total pre-M6 reduction: 704 lines (13.1%). + +Implemented by the simplification pass: + +- explicit `ContinueOperationResult` with `cancel_requested` instead of implicit `undefined`; +- concrete generation, deferred-poll, and tool-call phases: prepare immutable input, publish durable intent, perform the effect, publish the durable outcome; +- shared assistant stream protocol consumption and Drive response lifecycle; +- `publishResponse` for common assistant/deferred durable publication and `publishConfigurationFailure` for their identical pre-intent failure transition; +- source-order tool publication, one-read immutable source validation, and shared tool-batch reconstruction isolated in `runtime/drive/tool-placement.ts`; +- same-call tool status defenses, unchecked `toolOperation()` casts, and their result unions removed; genuine sibling/progress/memo checks retained; +- centralized lane-storage classification without duplicate projection reads; +- checkpoint dead-check removal and exact checkpoint preservation; +- shared transcript mechanics in `runtime/transcript.ts`; +- narrower `LanePatch`, no redundant Drive promise-settlement flag, and no unused operation reject arm. + +At simplification completion, the largest files were `runtime/lane.ts` (~996), `runtime/drive/tools.ts` (~663), `runtime/drive/checkpoint.ts` (~452), and `runtime/drive/response.ts` (~417). + +### WP05 structural status + +M6 established structural execution on the simplified substrate. Its first coherent slice added: + +- shared compaction-bounded transcript/context reads and committed-entry event decoration; +- caller-owned one-provider-request seams for compaction and branch summaries while preserving retrying behavior for existing non-harness callers; +- direct procedures for every structural `at` leaf in `runtime/drive/structural.ts`; +- once-per-trigger threshold routing and atomic overflow preparation publication; +- per-request structural intent and usage settlement, attempt retry/recovery, hook decisions, compaction/navigation publication, and terminal cleanup; +- focused coverage for threshold/overflow entry, split-turn request accounting, generated and hook results, retry/cap/recovery, model absence, mid-request durable cancellation, navigation, and preparation corruption. + +R1a, R2, and R3 then moved queued input to the lane, replaced hydrated family outcomes with immutable operation records, and collapsed the family cross-product to 13 neutral leaves. R1b replaced transitional checkpoint drains and marker fields with one shared atomic boundary planner, a transcript-derived threshold guard, and off-line finish-hook replanning. M7 removed drained control, added atomic drain-and-return cancellation, reconciled all 13 leaves, and installed the total direct dispatcher. + +M8 added primitive/convenience lane surfaces, one ordered tagged inbox, idle ownership, snapshot/event replication, `reduceLaneSnapshot`, and remote resnapshot. M9 reconciled the normative specification. M10 forwards exactly one stable ordinary-provider identity derived from Session metadata id plus lane name; structural summaries keep fresh request identities. At M10 completion the runtime is 7,410 lines, `runtime/drive` is 4,147, `lane.ts` is 1,992, `structural.ts` is 1,221, and the new normative reducer is 193 lines. Relative to the reviewed M7 checkpoint, the public/replication surface added 1,130 runtime lines; the durable 13-leaf procedure model stayed unchanged. + +Keep the visible durable procedure order: +`prepare → publish intent → perform effect → publish outcome`. +Do not introduce a generic Procedure interface, runner, scheduler, graph, callback plan, or dependency facade. Historical work-package documents remain unchanged. + +## Core model + +A Lane is one process-local actor over one durable lane projection. + +- `Lane.state` is authoritative for tip, configuration, current operation, control, inbox IDs, and the latest operation ID. +- Every supported mutation commits through the Session mutation line and publishes the matching `Lane.state` before releasing it. +- One lane-owned Drive is the sole writer that advances an operation state. +- Same-operation callers observe the same Drive. No caller owns it. +- Invocation cancellation stops only that caller's observation after Drive installation. +- `requestAbort` is the only durable operation cancellation. +- Close seals mutation admission. It neither mutates operation state nor replaces a Drive. +- Process loss destroys all live continuations; recovery starts from the durable state after attachment. +- Session reads during execution dereference content named by `Lane.state`; they never rediscover control state. + +## Durable state + +`OperationState` has one discriminator, `at`, with 13 direct leaves: + +- `starting` +- `checkpoint` +- `assistant.ready` +- `assistant.effect_pending` +- `assistant.retry_wait` +- `tools` +- `deferred.suspended` +- `deferred.effect_pending` +- `summary.deciding` +- `summary.ready` +- `summary.effect_pending` +- `summary.retry_wait` +- `navigation.ready_to_commit` + +Every leaf carries one uniform `OperationScope`. The summary quadruple carries one `SummaryTask`; its closed `ResultBoundary` selects in-run checkpoint resumption, standalone finish, or navigation commit. The summary algorithm is derived from that boundary rather than duplicated in state. `Control` remains orthogonal. `ToolBatch` remains a child state machine because parallel tool children genuinely mutate sibling call statuses concurrently. + +## State and content boundary + +Procedures never read these addresses to decide execution: + +- `laneState` +- `operationMeta` +- `operationState` +- `branchTip` +- `laneConfig` +- `operationResult` + +They use the current `Lane.state` supplied by the mutation line. + +Storage reads remain only for content and cleanup: + +- prompt, assistant, deferred-source, final-assistant, and completed-tool entries; +- compaction-bounded branch context; +- `pendingEntry` payloads; +- assistant-frame lists; +- tool arguments, memos, and checkpoints; +- structural preparations; +- staged tool outcomes; +- operation-owned prefix scans required for terminal cleanup. + +## Concurrency model + +### Operation advancement + +While a Drive procedure is alive, no supported concurrent actor can change or remove its operation state: + +- inbox methods change only inbox fields; +- `requestAbort` changes only `control`; +- close prevents later mutation admission; +- another same-operation caller joins the existing Drive; +- another operation cannot be accepted while the lane is busy; +- process crash removes the continuation itself. + +Consequently, ordinary procedure transitions do not recheck operation existence, operation ID, operation kind, `at`, attempt identity, or nested status. Those are procedure preconditions established by dispatch, not supported races. + +The remaining real races are: + +1. cancellation arriving before effect admission or before settlement; +2. inbox arrival before checkpoint routing or terminal finish; +3. parallel tool children staging and materializing sibling outcomes; +4. queued frame/checkpoint/memo writes racing effect settlement; +5. retry timers racing cancellation or close; +6. deferred permit consumption; +7. accept/claim serialization on the Session line. + +### Cancellation boundaries + +Cancellation is checked in three places only: + +1. the drive loop before ordinary dispatch; +2. the gate immediately before an external effect starts; +3. effect settlement, which sees current `control` and commits the appropriate cancelled result. + +An ordinary transition helper may centrally decline progress when current control is `cancel_requested`. Procedures do not repeat that branch manually. + +### Close + +Closing the harness: + +1. marks harness/lane admission closed; +2. seals and drains the Session mutation line; +3. rejects client observations through a harness-close observation promise; +4. keeps every detached pass promise observed so a later rejection is never unhandled; +5. closes the Session after admitted mutations drain. + +Close performs no durable write, installs no replacement Drive, and creates no ownership-loss state. A late effect may return, but its mutation is rejected with `HarnessClosed`. + +Whether close also signals process-local provider/tool work is a resource-cleanup policy, not durable state-machine behavior. Do not couple it to Drive replacement or recovery. + +## Small concrete mutation API + +Replace repeated `LaneCommand` ceremony in procedures with two concrete Lane operations. They are not a scheduler, graph, or action interpreter. + +### `continueOperation` + +Used for ordinary non-terminal progress. + +- Enters the Session mutation line. +- Receives the current authoritative Lane projection and operation state. +- If control is cancelled, returns explicit `cancel_requested` without invoking the semantic planner; callers cannot confuse cancellation with a planner value. +- The planner supplies procedure-specific writes, the next complete `OperationState`, materialization, and events. +- The helper appends the `operationState` write and publishes the matching process-local operation projection. +- It does not verify an expected state or `at` value. + +### `settleOperation` + +Used after an admitted provider/tool/structural effect and for genuine parallel child transitions. + +- Enters the Session mutation line even when control is cancelled. +- Supplies current control/inbox fields and the process-local effect result to the semantic settlement planner. +- Atomically commits payload, usage, tip movement, cleanup, and the classified next state. +- Appends the canonical operation-state write and publishes the matching projection. +- When the planner returns a terminal decision, appends the immutable `operationResult`, idle `laneState` with `lastOperationId`, and the idle process-local projection. + +The caller supplies the typed outcome, cleanup/publication writes, last result, and event. Terminal business decisions remain visible in the owning procedure. + +## Durable procedure shape + +Effectful procedures expose four concrete phases: + +```text +prepare immutable inputs +→ publish durable effect intent +→ perform the external effect +→ publish one durable outcome +``` + +The intent phase must remain visible and precede the external effect. Otherwise a crash after the effect but before its intent would leave no recoverable unknown-outcome marker. Each procedure uses concrete functions such as `prepareGeneration`, `publishGenerationIntent`, `performGeneration`, and `publishResponse`; there is no generic Procedure abstraction. + +## Drive lifecycle + +Delete the installer-owned model. + +- Remove `installerSignal`. +- Remove `DriveAbandoned`. +- Remove `LostOwnership` and `lost_ownership` from procedure results. +- Remove exact-object ABA fencing and `commandDriveOwned`. +- Remove `finalizedOutcome` and planned external-finalization owner retention unless the Flue investigation establishes a concrete requirement. +- Keep `activeDrive` only as the lane's install/join slot. +- Installation transfers work to the lane. Every caller, including the installer, observes completion with its own invocation Context. +- A caller signal abort before installation installs nothing. After installation it only rejects that caller's observation. +- The Drive is removed after its pass settles or reaches a durable wait. No live pass is replaced in-process. + +`requestAbort` retains the two-part gate ordering: + +```text +beginAbort before cancellation mutation +commit cancel_requested +signalAbort after commit +``` + +This prevents a new effect from entering while the durable marker is being committed. + +## Checks that remain + +Do not remove validation of external or referenced content: + +- required entry existence and role; +- pending payload kind; +- deferred handle identity; +- provider stream protocol ordering; +- response stop-reason invariants; +- UUIDv7 follower timestamp parsing; +- configured model/tool availability; +- tool-call source index and staged result identity; +- parallel tool call status and ready-prefix placement; +- progress/memo invocation identity; +- retry timestamp and deferred permit arithmetic. + +These validate data or genuine child concurrency. They are not defensive revalidation of the operation state machine. + +## Completed staged implementation plan + +### Stage 1 — Canonical flat durable types + +Files: + +- `src/harness/session/types.ts` +- compile-only consumers in `src/harness/runtime/`, restore, conformance helpers, and focused tests +- `docs/harness.md` +- `docs/work-packages/05-direct-durable-drive.md` + +Actions: + +- Replace nested operation state declarations with the flat `at` union. +- Preserve every durable datum without compatibility aliases. +- Update pattern matching mechanically without changing behavior. +- Keep ToolBatch/ToolCall nested. +- Update normative documentation in the same change. + +Exit: `npm run check`; existing focused runtime tests pass; no `phase.kind`, generation `status`, deferred `status`, or structural decision `status` remains in canonical operation state. + +### Stage 2 — Add canonical transition operations + +Files: + +- `src/harness/runtime/lane.ts` +- `src/harness/runtime/types.ts` +- focused Lane tests + +Actions: + +- Add `continueOperation` and `settleOperation`, including the terminal-decision suffix. +- Pair every durable operation-state write with process-local projection publication in one implementation. +- Centralize ordinary cancellation diversion. +- Trust the dispatcher-established current leaf; perform no expected-state check. +- Keep procedure-specific writes and event builders visible at call sites. + +Exit: focused tests prove durable state and `Lane.state` remain identical after every helper commit. + +### Stage 3 — Convert starting, checkpoint, and assistant + +Files: + +- `runtime/drive/checkpoint.ts` +- `runtime/drive/generation.ts` +- `runtime/drive/recovery.ts` +- `runtime/progress.ts` + +Actions: + +- Remove repeated operation/null/kind/state checks and `same*` predicates. +- Convert ordinary progress to `continueOperation`. +- Convert assistant settlement to `settleOperation`. +- Keep only checkpoint inbox/finish races and progress-channel ownership checks. +- Fold assistant recovery into the effect-pending handler if that is smaller. + +Exit: no control-state storage reads; no repeated assistant state verification; focused generation tests pass. + +### Stage 4 — Convert deferred and tools + +Files: + +- `runtime/drive/deferred.ts` +- `runtime/drive/tools.ts` +- `runtime/progress.ts` + +Actions: + +- Share assistant/deferred response-entry, usage, and tool-plan construction where their semantics match. +- Preserve deferred permit and handle checks. +- Remove top-level operation-state revalidation. +- Preserve per-call tool status merging, completion-order staging, source-order placement, memo fencing, and progress fencing. +- Use one tool-batch procedure for live, recovery, and cancellation modes instead of separate ownership-result paths. + +Exit: tool status checks exist only for genuine sibling concurrency; focused deferred/tool/progress tests pass. + +### Stage 5 — Remove ownership-loss machinery + +Files: + +- `runtime/types.ts` +- `runtime/lane.ts` +- `execution/effect-gate.ts` +- all existing `runtime/drive/*.ts` +- associated focused tests + +Actions: + +- Remove `LostOwnership`, `commandDriveOwned`, exact Drive checks, `installerSignal`, `DriveAbandoned`, and `finalizedOutcome`. +- Make all drive callers observation peers. +- Add observation-only Context cancellation to Drive completion waiting. +- Keep `activeDrive` only for install/join arbitration. +- Remove ABA/replacement tests and replace them with install/join/observation-cancellation tests. + +Exit: grep finds no ownership-loss or installer-abandonment vocabulary in production runtime. + +### Stage 6 — Close and fault + +Files: + +- `runtime/harness.ts` +- `runtime/lane.ts` +- Drive observation helper +- lifecycle tests + +Actions: + +- Seal mutation admission and drain admitted mutations. +- Reject client observations on close/fault without replacing a Drive or changing durable operation state. +- Observe detached pass failures. +- Verify a late effect cannot commit after close. +- Decide separately whether to signal local effects for resource cleanup. + +Exit: close and process loss leave the same durable restart point; no close path writes cancellation or synthetic settlement. + +### Stage 7 — Finish WP05 on the simpler substrate + +Files: + +- `runtime/drive/structural.ts` +- `runtime/drive/reconcile.ts` +- `runtime/drive.ts` +- `runtime/lane.ts` public surfaces + +Actions: + +- Implement structural generation directly over flat leaves and the transition operations. +- Implement cancellation reconciliation as one flat-state switch. +- Implement the total drive switch as one `state.at` switch. +- Wire public claim/join/observation and convenience methods only after every leaf is total. + +External finalization is excluded unless the Flue investigation identifies a concrete, current caller that cannot be expressed through close, explicit abort, recovery, or offline administration. + +## Validation + +After every code stage: + +```bash +npm run check +``` + +Run each modified focused test file from the package root. Do not run the full Vitest suite directly. Public drive was enabled only after the final stage made every leaf total. + +Final audit: + +```bash +rg 'lost_ownership|LostOwnership|DriveAbandoned|commandDriveOwned|installerSignal|finalizedOutcome' packages/agent/src/harness +rg 'operationState\(|laneState\(|branchTip\(|laneConfig\(' packages/agent/src/harness/runtime/drive +``` + +The second audit may match write constructors, but no reader call may use those control addresses. diff --git a/packages/agent/telemetry-schema.md b/packages/agent/docs/telemetry-schema.md similarity index 100% rename from packages/agent/telemetry-schema.md rename to packages/agent/docs/telemetry-schema.md diff --git a/packages/agent/docs/telemetry.md b/packages/agent/docs/telemetry.md new file mode 100644 index 000000000..e820685dc --- /dev/null +++ b/packages/agent/docs/telemetry.md @@ -0,0 +1,251 @@ +# Invocation Context and Telemetry Design Notes + +> **Status:** Design input, not a normative contract. The context primitives and required trailing `Context` parameters have landed across the harness, sessions, execution capabilities, and hosted-harness adapters. Local propagation is scaffolding rather than proof of complete telemetry semantics: most runtime spans and cross-process trace propagation remain design or implementation work. Drive ownership is now harness-owned and request-ID RPC cancellation is implemented; neither supplies distributed trace parentage. Fold accepted final behavior into `harness.md`. `telemetry-schema.md` remains the generated reference for span names and attributes. + +## Goal + +`Session`, `Branch`, `AgentLane`, and `AgentHarness` receive invocation-scoped control data explicitly through a required trailing `Context` parameter. The same receiver may serve concurrent local callers or RPC clients, so it cannot retain a mutable or default caller context. + +The invocation context must solve two related problems without `AsyncLocalStorage`: + +1. preserve correct telemetry parentage through concurrent asynchronous work; +2. carry an `AbortSignal`, when one exists, that an RPC adapter can map to request cancellation. + +This work must reuse `@earendil-works/pi-telemetry`. It must not introduce another span abstraction. + +## Context model + +The implemented public types are: + +```ts +interface ContextKey { + readonly token: symbol; + readonly valueType?: (value: T) => T; +} + +interface Context { + readonly abortSignal: AbortSignal | undefined; + readonly telemetryContext: TelemetryContext; + value(key: ContextKey): T | undefined; + toString(): string; +} +``` + +`valueType` is a type-only marker. Runtime lookup uses the key's symbol token. `createContextKey(description)` creates and freezes a key with a unique token. + +A context is immutable. Derivation creates a parent-linked, copy-on-write layer. Helper arguments put the value first and parent context last: + +```ts +const requestContext = withAbortSignal(requestSignal, parentContext); +const spanContext = withTelemetryContext(span, requestContext); +const tenantContext = withContextValue(tenantKey, tenantId, spanContext); +``` + +The implemented behavior is: + +- `BACKGROUND_CONTEXT` and `TODO_CONTEXT` are distinct empty roots whose `abortSignal` is `undefined`; +- `telemetryContext` is always available and falls back to `NOOP_TELEMETRY_CONTEXT` when no telemetry value has been installed; +- `withAbortSignal(signal, context)` preserves the supplied signal when the parent has none and otherwise combines it with the parent signal using `AbortSignal.any()`; +- `withCancel(context)` returns an independently cancellable child context and a `cancel(reason?)` function; parent cancellation still reaches the child; +- typed values use symbol identity and immutable copy-on-write layers, and a newer value for the same key shadows its parent value; +- the built-in abort-signal and telemetry keys are private; callers use the named properties instead of retrieving those values by key; +- `toString()` is diagnostic and records the root plus each layered key description. + +Context values are cross-cutting request metadata, not business dependencies. Suitable typed values include request IDs, authenticated principals, tenant IDs, and diagnostic metadata. Storage, models, tools, durable state, and business payloads do not belong in the context. Contexts, signals, telemetry objects, and backend-native span objects are never durable data. + +## Receiver ownership + +Shared receivers retain identity and durable/process state, not invocation context: + +```text +AgentHarness receiver ── no caller context +AgentLane receiver ── no caller context +Session receiver ── no caller context +Branch receiver ── no caller context +``` + +Every invocation supplies its own context. This prevents concurrent callers from overwriting each other's telemetry parent or cancellation signal. + +A process-local object representing one ongoing invocation may retain its derived context. Examples are an active drive task or an event subscription. This is different from storing a default context on the shared harness or session receiver. + +`AgentHarnessOptions.telemetryContext` has been removed. A harness-level default cannot represent two concurrent callers with different parents. + +## Existing typed telemetry remains authoritative + +The design retains: + +- `TelemetryContext` and `TelemetrySpan`; +- callback-owned span lifetime; +- `AI_TELEMETRY_SCHEMA` and `HARNESS_TELEMETRY_SCHEMA`; +- typed span names, start attributes, completion attributes, and events; +- `startAiSpan()`, `startHarnessSpan()`, and `createTypedSpanStarter()`; +- adapter conformance behavior. + +`startAiSpan()` and `startHarnessSpan()` package span derivation by giving their callbacks both the typed span and a derived invocation context: + +```ts +return startHarnessSpan( + "pi.harness.run", + attributes, + async (span, runContext) => { + return runDrive(runContext); + }, + context, +); +``` + +The helpers delegate to `context.telemetryContext.startSpan()` and install the callback-owned span in a child context with `withTelemetryContext(span, context)`. Lower work must receive that child context rather than the parent invocation context. + +Do not mutate a context to install an active span. Do not use a process-global or receiver-global current span. + +## Concurrent parentage + +Explicit propagation supports concurrent sibling calls: + +```ts +await parent.telemetryContext.startSpan({ name: "caller" }, async (callerSpan) => { + const callerContext = withTelemetryContext(callerSpan, parent); + await Promise.all([ + laneA.drive(optionsA, callerContext), + laneB.drive(optionsB, callerContext), + ]); +}); +``` + +Each invocation derives its own child context. Nested work receives the child belonging to that invocation. Correct parentage does not depend on promise scheduling or ambient state. + +Tests must cross concurrent branches deliberately so accidental receiver-level context is visible. A sequential parent/child test is insufficient. + +## Callbacks, hooks, and events + +Host-local callbacks invoked as part of an operation receive the current invocation context in their declared trailing position: + +```ts +handler(event, context); +tool.execute(toolCallId, params, onUpdate, toolContext, invocation, context); +mutation(mutator, context); +``` + +Current propagation preserves context through callbacks and gives `before_tool` and `after_tool` handlers a child context derived from `pi.harness.hook`. Extending that span behavior to every hook type remains work; handlers without an installed hook span currently receive the operation context directly. + +Within the harness process, events preserve the context that caused each event, and buffered event watchers store `{ event, context }` rather than only `event`. Starting `pi.harness.event_handler` from that event context and passing its child context to each listener remains work. Event registration itself is host-local configuration and has no operation parent. + +Session mutation callbacks and commits receive the same explicit invocation context. Starting `pi.session.write` from the committing invocation and passing its child context through the storage commit remains work. + +## Drive execution and joiners + +Several callers may call `drive()` for the same durable operation. Arbitration decides which call installs process-local execution and which calls join it. This is a core runtime concern, not an RPC concern; concurrent local callers have the same issue. + +One active execution has one telemetry parent. It cannot be reparented when another caller joins. + +```text +installer caller +└─ drive.execute + └─ provider/tool work + +joiner caller +└─ drive.join +``` + +A joiner span describes that caller's wait. It carries at least lane name, durable operation ID, and a process-local execution ID. It ends with an outcome such as `settled`, `caller_cancelled`, `execution_stopped`, or `harness_closed`. + +The joiner must not overwrite the active execution context. Correlate the two spans using operation/execution attributes. Telemetry span links would model this relationship better, but the current telemetry contract has no links. Adding links is an optional telemetry-package design question, not a reason to invent multiple parents. + +Distributed traces permit an execution span to outlive the installer RPC span. Parent and child spans may overlap or settle in either order once the child has started. + +## Invocation cancellation versus durable cancellation + +An aborted invocation signal is process-local control. It does not mean that durable cancellation was requested. + +```text +context.abortSignal is present and aborts +→ stop only that caller's observation; an installed lane-owned Drive continues +→ do not write cancel_requested +→ preserve the same durable operation state +``` + +An undefined `abortSignal`, as exposed by both empty roots, means that the invocation has no cancellation signal. + +Only `requestAbort()`/`abort()` writes durable `cancel_requested` and permits durable aborted settlement. + +The runtime must track the stop cause instead of interpreting every aborted provider response as durable cancellation: + +```ts +type ExecutionStopCause = + | "no_drive_waiters" + | "invocation_cancelled" + | "harness_closed" + | "durable_cancel_requested"; +``` + +Only `durable_cancel_requested` may normalize and commit a durable aborted outcome. An invocation/disconnect abort must not produce an assistant `stopReason: "aborted"` settlement while durable control remains `running`; that path would violate the durable state machine. + +Drive ownership is resolved as **harness-owned**: once installed, execution survives caller cancellation/disconnect until durable settlement or wait, explicit durable cancellation, close, fault, or process loss. Joiner signals control only their own observations. Signals from unrelated joiners must never be combined with `AbortSignal.any()` and attached directly to shared execution. One canceled joiner cannot cancel every other caller. + +## RPC trace propagation + +Client and server spans can belong to one distributed trace: + +```text +caller +└─ rpc.client + └─ rpc.server + └─ harness/session operation +``` + +The client does not serialize `TelemetryContext`. It injects a backend-neutral trace carrier from the `rpc.client` span. The server extracts that carrier into a fresh local `TelemetryContext` and starts `rpc.server` from it. + +A transport-facing adapter boundary is required: + +```ts +interface TelemetryPropagation { + inject(context: TelemetryContext): JsonValue | undefined; + extract(carrier: JsonValue | undefined): TelemetryContext; +} +``` + +A production implementation may use W3C `traceparent`/`tracestate`. The current telemetry package has no carrier injection/extraction API, so the accepted design must decide whether this adapter belongs in the telemetry package, RPC infrastructure, or a backend integration package. It must still reuse the existing `TelemetryContext` span contract. + +RPC cancellation and telemetry propagation are independent control-plane channels: + +- trace metadata reconstructs telemetry parentage; +- request ID plus cancel/disconnect messages controls the server request signal; +- neither channel appears in serialized method arguments. + +## Interface migration scaffolding + +Receiver methods now use a required trailing `Context`. Concrete implementations, calls, callback adapters, and object-literal façades have been migrated rather than relying only on interface assignability. + +`TODO_CONTEXT` remains a temporary migration marker, not a semantic root. Current uses cluster at unresolved transport and worker boundaries that cannot yet reconstruct a caller context, notably Pi protocol request ingress and worker RPC ingress. `BACKGROUND_CONTEXT` means intentionally start without a caller. + +Continue to inventory `TODO_CONTEXT` separately. Replace each transport-boundary use only when the boundary can construct a request-local cancellation context and telemetry parent; substituting `BACKGROUND_CONTEXT` would hide unfinished propagation. Compilation still does not prove telemetry or cancellation correctness. + +## Required tests for the later handoff + +Current tests cover immutable typed-value layering and shadowing, distinct empty roots, parent/child abort composition, sibling cancellation isolation, and tool-hook child parentage. Remaining handoff coverage includes: + +- crossed concurrent telemetry branches on one shared receiver; +- every hook type, tools, event handlers, and session writes receive the intended child context; +- buffered events retain their emitting context under delayed delivery; +- no receiver-level telemetry default; +- pre-aborted invocation starts no external effect; +- installer and joiner cancellation isolation under lane-owned execution; +- invocation abort leaves the installed Drive and durable state unchanged; +- durable abort commits the durable aborted outcome; +- close and disconnect do not masquerade as durable cancellation; +- client → server trace reconstruction; +- event delivery reconstructs source trace metadata; +- missing/malformed trace carriers degrade to no-op/root telemetry without affecting business behavior. + +## Resolved migration decisions + +- Receiver methods use one required trailing `Context`. +- Shared Harness, AgentLane, Session, and Branch receivers retain no default invocation context. +- `Context`, `AbortSignal`, and `TelemetryContext` objects are never serialized across RPC boundaries. + +## Open decisions before the telemetry handoff + +- whether telemetry links are required for joiners; +- trace-carrier adapter ownership and shape; +- which context values, if any, may cross an RPC boundary; +- exact span names/outcome attributes for RPC calls and drive join waits. diff --git a/packages/agent/docs/tool-durability.md b/packages/agent/docs/tool-durability.md new file mode 100644 index 000000000..332df21b7 --- /dev/null +++ b/packages/agent/docs/tool-durability.md @@ -0,0 +1,704 @@ +# Tool durability — implementation handoff + +This document specifies the durable tool-call lifecycle and the minimal harness-specific progress-checkpoint extension to the current tool API. It does not otherwise finalize the harness-native public tool interface; that interface must provide the capabilities required here without exposing raw session storage. + +The design has two independent additions: + +1. a durable `outcome_ready` state between external-effect settlement and source-ordered conversation placement; +2. opt-in durable replacement checkpoints for complete bounded `onUpdate` snapshots. + +## Problem + +Parallel tool effects finish in completion order, while tool-result entries must enter the conversation in assistant source order. + +Without an intermediate durable state: + +```text +calls: A, B, C +B finishes +C finishes +A is still running +process crashes +``` + +B and C exist only in process memory because A prevented source-ordered placement. Recovery treats them as unresolved and may replay or interrupt effects that already completed. + +The solution separates two orders: + +1. **outcome durability:** actual completion order; +2. **entry materialization:** assistant source order. + +A complete finalized result becomes durable immediately in `pi.pending.entry`; the call becomes `outcome_ready`; placement happens later when every earlier source position is complete or ready. + +## Goals + +1. Never rerun a tool after its complete finalized outcome is durable. +2. Durably retain out-of-order parallel outcomes without violating transcript order. +3. Preserve the existing whole-tool `replay: "safe" | "never"` contract. +4. Support invocation-scoped durable memoization for Flue-style `step.do`. +5. Preserve tool-selected bounded progress checkpoints for reconnect and unsafe interruption recovery. +6. Fence late tool writes after outcome settlement, cancellation, or external finalization. +7. Keep final tree entries canonical, complete, bounded independently of progress snapshots, and immutable. + +## Non-goals + +- Exactly-once arbitrary external effects. +- A nested durable state machine for each `step.do` call. +- Inferring tool completion from progress output. +- Treating partial output as the canonical final result. +- Giving tools raw `Session`, `SessionMutator`, or bound storage-address access. +- Finalizing the public harness-native tool type in this document. + +## Durable identities + +Each call already reserves its result entry ID before execution. Use that as the stable public invocation identity: + +```text +invocationId = resultEntryId +``` + +It is session-unique, survives safe replay, and is distinct from the provider's batch-local `toolCallId`, which a later assistant message may reuse. + +The surrounding operation state continues to provide: + +- `operationId`; +- `turnId`/generation step ID; +- `sourceIndex`; +- assistant entry ID; +- captured configuration and execution mode. + +## Storage + +### Existing bound values + +```ts +operationToolArgs(operationId, turnId, sourceIndex) +// Effective validated arguments, persisted before effect admission. + +pendingEntry(resultEntryId) +// Complete finalized ToolResultMessage while outcome_ready awaits placement. +``` + +### Invocation memos + +Define the operation-owned address constructor in `session/values.ts`: + +```ts +export const operationToolMemo = ( + operationId: string, + invocationId: string, + memoName: string, +) => value( + "pi.op.tool_memo", + `${operationId}:${invocationId}:${memoName}`, +); +``` + +`memoName` must be non-empty and contain no `:`. Names may use dots or slashes for application-local grouping. `setMemo(name, undefined)` deletes the exact bound value. + +`scanValues(operationToolMemoPrefix(operationId))` permits defensive operation cleanup. `scanValues(operationToolMemoPrefix(operationId, invocationId))` permits atomic invocation cleanup when one outcome becomes ready. Core cleanup uses these owner-defined prefix constructors rather than repeating raw reserved namespace/key grammar. + +### Partial tool output + +The durable recovery value is the latest complete bounded progress snapshot selected by the tool. That is total current state, so use one bound value address: + +```ts +export const pendingToolOutput = ( + operationId: string, + invocationId: string, +) => value>( + "pi.pending.tool_output", + `${operationId}:${invocationId}`, +); +``` + +This is auxiliary observation data. It never proves the effect succeeded or completed. The stored value has exactly the same content/details/usage shape as the live `partialResult`; recovery does not need a tool-specific progress codec. + +The tool owns snapshot bounding, checkpoint cadence, and duplicate suppression. The harness owns synchronous enqueue, promise tracking, invocation fencing, and cleanup. The first API has no generic byte cap and does not truncate or reinterpret typed tool data; in-process tools are trusted to honor the bounded-snapshot contract. There is no tool-visible `flush()` method and no durable list of progress updates. + +A crash may lose live updates newer than the latest committed checkpoint. JSONL physical growth is proportional to the size and frequency of distinct requested checkpoints until compaction; Memory and SQLite retain one current value. At 50 KiB every two seconds, JSONL's uncompacted worst case is approximately 15 MiB per ten minutes of continuously changing output. + +## Tool update API + +Keep the existing full-snapshot update callback and add harness-specific options: + +```ts +export interface AgentHarnessToolUpdateOptions { + /** Request replacement of this invocation's durable recovery checkpoint. */ + checkpoint?: true; +} + +export type AgentHarnessToolUpdateCallback = ( + partialResult: AgentToolResult, + options?: AgentHarnessToolUpdateOptions, +) => void; +``` + +`AgentHarnessTool` uses this callback instead of the legacy `AgentToolUpdateCallback`. The harness always supplies it, even without a live listener, because a tool may request persistence through it. The legacy `AgentTool` and old agent loop remain unchanged because they cannot honor durable checkpoints. + +Every callback invocation remains an immediate live update. The callback is synchronous and returns `void`; `checkpoint: true` additionally requests persistence of that complete snapshot. It is not a durability acknowledgement. Internally, the harness retains the latest `events.emit(tool_update)` promise so existing listener delivery completes before `after_tool`; tools do not await or receive that promise. + +Every `checkpoint:true` call synchronously enqueues one scalar replacement on the Session mutation line, attaches the ordinary harness-fault observer to that promise, and replaces the process-local `latestCheckpointWrite` reference. Writes themselves are neither dropped nor coalesced, and replacing the reference never leaves an earlier rejection unobserved: + +- Session mutation FIFO preserves request order; +- each mutation verifies the same call remains `effect_pending`; +- completion of the latest promise implies completion of every earlier checkpoint write; +- tool-promise settlement stops accepting updates and awaits that latest promise before `after_tool`; +- a failed checkpoint commit faults the harness under the ordinary storage-fault rule. + +A tool that requests checkpoints faster than storage can queue work in memory. Under the trusted-tool contract, cadence is the tool's responsibility. The built-in bash policy bounds this queue in ordinary use. + +Tools should compare against their last requested checkpoint when duplicate suppression matters. Storage does not read and deep-compare the current scalar as part of every checkpoint. + +### Bash policy + +The built-in bash tool keeps its current 100 ms live update cadence. It requests a checkpoint at most once every two seconds and only when the complete bounded snapshot differs from its last requested checkpoint: + +```ts +const BASH_UPDATE_THROTTLE_MS = 100; +const BASH_CHECKPOINT_INTERVAL_MS = 2_000; +``` + +The current `ShellCaptureProgress` already supplies a bounded snapshot: the last 2,000 lines or 50 KiB, plus truncation metadata and the overflow-file path. The initial empty update is live-only. Output volume never accelerates checkpoint frequency. A short tool may settle without writing any checkpoint because its complete final result commits instead. + +## Tool-call state + +Extend the call union with `outcome_ready`: + +```ts +type ToolCall = + | { + status: "planned"; + sourceIndex: number; + resultEntryId: string; + } + | { + status: "effect_pending"; + sourceIndex: number; + resultEntryId: string; + replay: "never" | "safe"; + } + | { + status: "outcome_ready"; + sourceIndex: number; + resultEntryId: string; + terminate: boolean; + } + | { + status: "completed"; + sourceIndex: number; + resultEntryId: string; + terminate: boolean; + }; +``` + +`outcome_ready` means: + +- execution, error normalization, and `after_tool` have finished, or the harness has constructed a final synthetic result; +- the complete final `ToolResultMessage` exists at `pendingEntry(resultEntryId)`; +- invocation memos and partial-output storage are gone; +- the tool must never execute again; +- the immutable result entry may not exist yet because an earlier call has not materialized. + +The exact union may later carry small settlement metadata, but it must not duplicate the finalized result payload. + +## State transitions + +```text +planned + ├─ real effect cleared → effect_pending + └─ immediate/synthetic → outcome_ready + +effect_pending + ├─ live effect settles → outcome_ready + ├─ safe orphan replay settles→ outcome_ready + └─ unsafe orphan synthesis → outcome_ready + +outcome_ready + └─ source position eligible → completed +``` + +An implementation may fuse `outcome_ready → completed` into the same transaction that finalizes an immediately placeable head call, but the semantic checks and tests must still cover durable `outcome_ready` for out-of-order calls. Prefer implementing the explicit two-transaction form first. + +## Fresh execution + +### Clearance and intent + +Unchanged effect sandwich: + +```text +planned +→ prepare arguments, run before_tool, validate replacements +→ TX[ + set pi.op.tool_args, + set call = effect_pending(replay) + ] +→ post-commit tool_start +→ admit tool execution +``` + +The invocation-scoped capability becomes active only for this durable `effect_pending` call. + +### Partial output + +Every `onUpdate(partialResult, options)` publishes the live update through the existing event/snapshot path. When `options.checkpoint === true`, the harness additionally requests a scalar replacement: + +```text +TX[ + setValue(pendingToolOutput(operationId, invocationId), partialResult) +] +``` + +The mutation verifies the same operation, turn, source position, and invocation remain `effect_pending`. It does not rewrite `pi.op.state`. A late checkpoint after settlement returns without committing. + +The tool must checkpoint bounded complete snapshots, not growing unbounded values. Bash uses the same bounded `ShellCaptureProgress` snapshot it already sends live. Clients may render and locally retain live updates newer than the durable checkpoint, but those updates are explicitly process-local. + +### Finalization to `outcome_ready` + +When the tool promise settles: + +1. synchronously stop accepting updates and expire the invocation capability; +2. await the latest tracked `tool_update` delivery and latest checkpoint-write promise; each implies completion of its preceding queue; +3. run `after_tool` when this is a real fresh or safely replayed result and cancellation did not prevent the hook; +4. construct the complete final `ToolResultMessage`; +5. commit the result as `outcome_ready`; +6. emit and await `tool_end` from the committed staging transition. + +`setMemo()` returns a promise and tools must await it; `step.do` always does. An unawaited pre-return mutation is still enqueued before staging and is deleted by staging. Calls begun after capability expiry reject. No separate invocation-write drain exists. + +Transaction: + +```text +TX[ + setValue( + pendingEntry(resultEntryId), + { type: "message", payload: finalizedToolResultMessage }, + ), + deleteValue(pendingToolOutput(operationId, invocationId)), + deleteValue(memo.address) for every memo returned before commit by + scanValues(operationToolMemoPrefix(operationId, invocationId)), + setValue(operationState(operationId), call = outcome_ready(terminate)) +] +``` + +The transaction is the linearization point after which the invocation can never replay. Its post-commit `tool_end` is therefore durable evidence that the finalized outcome is ready; it is no longer a pre-commit effect observation. + +The staged message contains the final: + +- text/image content; +- provider tool-call ID and tool name; +- details; +- `isError`; +- usage snapshot, when reported; +- added tool names; +- timestamp. + +`terminate` remains orchestration state because it controls the batch continuation and is copied to the immutable entry's `terminate` field at placement. + +## Source-ordered materialization + +After any call becomes `outcome_ready`, find the contiguous ready prefix beginning at the first non-completed source position. + +Example: + +```text +[completed, outcome_ready, outcome_ready, effect_pending] + └──────── ready prefix ────────┘ +``` + +Before the placement transaction, emit and await each finalized result's `message_start` and `message_end` in source order. Materialize the prefix in one transaction when practical, then emit `entry_added` and reported usage events in the same source order: + +```text +TX[ + insert result entry i from pendingEntry(i), + deleteValue(pendingEntry(i)), + insert tool usage row i if reported, + + insert result entry i+1 with parent = result i, + deleteValue(pendingEntry(i+1)), + insert tool usage row i+1 if reported, + + setValue(branchTip(lane), newest result), + setValue(operationState(operationId), + calls i..i+1 = completed and, when complete, next checkpoint) +] +``` + +Each inserted entry uses its already-reserved `resultEntryId`. Writes construct the parent chain in source order inside the transaction. + +Tool-reported usage remains durable in the staged message until placement. The initial implementation writes its ledger row atomically with entry materialization, matching the current entry/usage ordering and avoiding a ledger row that references an entry not yet present. No usage ID reservation is needed because a failed placement transaction writes neither row nor completed state. + +When the final call materializes, the same transaction calls `scanValues(operationToolArgsPrefix(operationId, turnId))` and deletes every returned address, and transitions to the correct checkpoint: + +- every result terminates → `may_finish`, no final assistant required; +- otherwise → `need_assistant(false)`. + +## Parallel execution + +Outcome staging follows actual completion order. Entry materialization follows source order. + +```text +A, B, C start +B finishes → B outcome_ready +C finishes → C outcome_ready +A finishes → A outcome_ready + materialize A, B, C +``` + +Crash after B and C stage: + +```text +A effect_pending +B outcome_ready +C outcome_ready +``` + +Recovery applies unknown-outcome policy only to A. B and C require no tool registration or hook execution to become entries. + +The durable invariant changes from “completed calls form a source-ordered prefix” to: + +- completed calls form a source-ordered prefix; +- after that prefix, parallel calls may be `planned`, `effect_pending`, or `outcome_ready` in any mixture; +- only source-ordered materialization extends the completed prefix. + +Sequential execution constructs at most one non-planned call after the completed prefix. Committed call state is trusted on restore; the owning procedure enforces this shape while creating and consuming transitions rather than through a broad restore audit. + +## Unsafe recovery with partial output + +For orphaned `effect_pending` with `replay: "never"`: + +1. read `pendingToolOutput(operationId, invocationId)` when present; +2. preserve its bounded content and serializable details; +3. append a mandatory human-readable interruption marker; +4. construct a harness-owned `ToolResultMessage` with `isError: true`; +5. commit it as `outcome_ready` and clean invocation state. + +The marker must state that output is partial and the external outcome is unknown. `isError: true` describes the result delivered to the model; it does not assert that the external effect failed. + +Example final text suffix: + +```text +[Tool execution was interrupted. The preceding output is the latest durable progress snapshot; newer live output may be missing, and the external outcome is unknown.] +``` + +Rules: + +- do not run `after_tool` for this synthetic result; +- preserve checkpoint `usage` when present, but ignore checkpoint `terminate` because progress never has final-result authority; +- set `terminate: false` and add no tools; +- an absent checkpoint value is also valid and yields only the interruption result; +- never infer completion from an apparent success line in partial output; +- cleanup of partial output and invocation memos is atomic with staging the synthetic result. + +## Safe recovery + +For orphaned `effect_pending` where both the stored and current declarations are `replay: "safe"`: + +1. retain invocation memos; +2. atomically delete `pendingToolOutput(operationId, invocationId)`; +3. emit/reset process-local progress observation as needed; +4. rerun the tool with persisted arguments and the same `invocationId`; +5. completed `step.do` calls return their memoized values; +6. new partial output reconstructs a clean progress stream; +7. finalization follows the ordinary `outcome_ready` path. + +Deleting old progress prevents duplicate chunks when replayed code emits progress again. A crash after the delete but before replay admission remains `effect_pending`; the next recovery repeats the same safe procedure. + +If the current tool declaration is missing or no longer safe, use unsafe interruption recovery rather than suspending. + +## Invocation memos + +The harness-native tool call receives a purpose-built invocation capability conceptually equivalent to: + +```ts +interface AgentHarnessToolInvocation { + readonly invocationId: string; + readonly operationId: string; + readonly turnId: string; + + getMemo(key: string): Promise; + setMemo(key: string, value: JsonValue | undefined): Promise; +} +``` + +Each operation: + +1. validates the memo name and checks process-local capability expiry; +2. synchronously enqueues work on the Session mutation line before returning its promise; +3. verifies the operation, turn, source position, and invocation are still the same `effect_pending` call when that job executes; +4. constructs and reads or writes only `operationToolMemo(operationId, invocationId, name)`; +5. rejects after capability expiry or durable ownership loss. + +The durable check matters for authorized external finalization. In ordinary execution, a memo mutation initiated before the tool returns is FIFO-ordered before outcome staging; one initiated afterward fails the expired-capability check. + +A late zombie callback can neither recreate memos after `outcome_ready` nor write into a later operation. + +Memos are immediate durable replay state, not application-visible settlement state. They survive close/crash while the call remains `effect_pending` and are deleted when any real or synthetic outcome becomes ready. + +Terminal cleanup defensively scans and deletes the operation-owned families: + +```ts +scanValues(operationToolMemoPrefix(operationId)) +scanValues(pendingToolOutputPrefix(operationId)) +``` + +in addition to other operation-owned addresses. Each returned `StoredValue` supplies its exact bound address for `deleteValue`; no later operation receives a raw key. + +## Flue-style `step.do` + +Build `step.do` over invocation memos; it does not need its own harness state union: + +```ts +interface ToolSteps { + do( + name: string, + effect: () => T | Promise, + ): Promise; +} +``` + +Algorithm: + +```text +validate deterministic unique name +→ getMemo("step/" + name) +→ present: return stored value +→ absent: run effect +→ setMemo("step/" + name, value) +→ await durability +→ return value +``` + +Crash behavior: + +```text +before/during effect → effect may run on replay +effect returned, memo not committed → effect may run on replay +memo committed → replay returns memo +step A memoized, step B interrupted → rerun tool; A skips, B runs +``` + +This is exactly-once recorded and at-least-once executed. It does not make arbitrary external effects exactly once. An application may derive a stable external idempotency key from `(invocationId, stepName)` when the external API supports one. + +Errors are not memoized. A thrown effect either contributes to the current tool result or runs again after safe whole-tool recovery. + +Do not add per-step `replay: "never"` in this slice. Supporting it correctly requires a nested `planned → effect_pending → completed` state and an explicit unknown-outcome policy. Whole-tool replay policy is sufficient for the Flue use case already discussed. + +Within one live execution, calling the same step name twice is an invariant error. Names must be deterministic across safe replay. + +## Application persistent state + +Flue-style application state is distinct from invocation memos. + +Invocation memo: + +```text +step completed → memo becomes visible immediately +``` + +Application state: + +```text +tool stages state change +→ crash before outcome_ready: state must not appear committed +→ outcome_ready: state and finalized result become visible together +``` + +Do not implement application state by calling `Session.setValue()` directly during execution. + +Two valid implementation stages: + +1. Flue keeps its state externally and atomically stores application state plus a full result memo keyed by `invocationId`. +2. A later harness API accepts staged application value writes and promotes them in the `outcome_ready` transaction. + +The second option is required only if Flue's `usePersistentState` moves into harness-owned session values. Its public typing and conflict semantics remain an open design item for the harness-native tool discussion. + +## Cancellation + +Cancellation reconciliation never replays a restored tool. + +- `planned` calls receive a synthetic aborted result and become `outcome_ready`; +- a live started call may finalize its real local result under cancelled control, then become `outcome_ready` with `terminate: false`; +- restored `effect_pending` calls use an interrupted synthetic result, optionally including partial output, regardless of safe replay declaration; +- existing `outcome_ready` calls are preserved and materialized in source order; +- invocation memos and partial output are deleted with each staged cancellation outcome; +- no `before_tool` or `after_tool` starts during restored synthetic reconciliation. + +The aborted terminal transaction runs only after every call outcome has materialized and accepted deferred writes have drained under the existing cancellation rules. + +## Close and external finalization + +Close is still a controlled crash: + +- memo/checkpoint mutations already enqueued under the admission barrier may finish; +- live output newer than the latest committed tool-requested checkpoint may be lost; +- no synthetic outcome or cancellation marker is written; +- durable state remains at `effect_pending` or `outcome_ready`. + +External finalization deletes operation-owned arguments, invocation memos, partial output, staged pending outcomes, and other pending entries in its terminal transaction. A live task that later tries to stage an outcome fails the ownership fence and stops through `OperationEnded`. + +## Restore and consumption-time reads + +Base restore constructs the trusted lane/operation projection from required owner values. It does not hydrate or semantically audit tool arguments, invocation memos, progress checkpoints, staged outcomes, completed entries, completed-prefix shape, or captured execution-mode relationships. + +The procedure responsible for the current typed state performs only its exact consumption-time reads: + +### `planned` + +Clearance needs no auxiliary restore read. It prepares the call and writes arguments before effect admission. + +### `effect_pending` + +Activation reads `operationToolArgs(operationId, turnId, sourceIndex)` and optionally reads `pendingToolOutput(operationId, invocationId)`. Missing required arguments are an invariant defect at consumption. Invocation memos are read only through the scoped capability. Safe replay, unsafe interruption, and snapshots use no broad prefix scan. + +### `outcome_ready` + +Materialization reads `pendingEntry(resultEntryId)`. Its absence or wrong trusted message relationship is an invariant defect when materialization consumes it. No tool identity or effect recovery is needed. + +### `completed` + +Ordinary dispatch performs no restore-time entry audit. Context/tree reads later consume the immutable entry through their normal typed paths. + +Every live mutation still verifies current operation, turn, source position, invocation, and status on the Session line. Those checks fence concurrent settlement, cancellation, and external finalization; they are not historical restore validation. Terminal prefix cleanup remains defensive and does not make orphan scans part of restore. + +## Snapshots and reconnect + +A reconnecting client may see: + +- live process-local progress newer than the latest durable checkpoint before disconnect; +- after process replacement, only the latest committed bounded checkpoint; +- `outcome_ready` calls as settled rows in `runningTools` until source-ordered materialization; +- completed calls in the transcript. + +`LaneSnapshot.operation.runningTools` is a discriminated union. An effect-pending tool has `status: "running"` and an optional `result` containing the latest complete progress snapshot, falling back to the durable checkpoint after reopen. An outcome-ready call has `status: "settled"`, its required complete final `result`, and `isError`; it remains there until its immutable result entry's `entry_added` removes the row and places the same presentation in the transcript. Planned and completed calls are omitted. + +## Events and hooks + +- `tool_start` begins public processing presentation for a fresh call; it is emitted from the commit that establishes effect intent or a synthetic staged outcome and does not by itself prove an external effect started. It carries effective arguments for an intended effect and source arguments for an immediate synthetic result. +- live progress events and durable progress checkpoints do not prove completion. +- The harness awaits the latest `tool_update` delivery before `after_tool`, preserving the existing listener ordering without making `onUpdate` async. +- `tool_end` carries the complete finalized result after its `outcome_ready` staging commit, in completion order. It is durable settlement evidence and does not repeat the arguments from `tool_start`. +- for a fresh blocked, invalid, truncated, or planned-cancellation synthetic outcome, the staging commit emits `tool_start` followed by `tool_end`; these paths still run no tool effect or post-effect hook. Cancellation after effect intent uses the earlier intent-bound start and a staging-bound end. +- an unsafe restored effect is already represented as running by the initial snapshot and may emit only a recovery-tagged `tool_end` when interruption synthesis stages. +- message lifecycle and `entry_added` occur when the staged result materializes, not when it first becomes `outcome_ready`; `entry_added` removes only that settled row. +- passive listeners cannot mutate invocation state reentrantly. + +Instrumented-storage tests assert `intent commit → tool_start → tool_update* → outcome staging → tool_end → source-ordered placement` for execution and `outcome staging → tool_start → tool_end → source-ordered placement` for fresh synthetic results. Historical events are not replayed; a safely replayed execution emits recovery `tool_start` from its checkpoint-clear commit and `tool_end` from outcome staging. + +## Races + +| Race | Required result | +|---|---| +| checkpoint vs tool settlement | every accepted checkpoint was enqueued first; settlement awaits the latest promise, then staging deletes the checkpoint value; a late update is ignored | +| memo write vs `outcome_ready` | an awaited or pre-return-enqueued write precedes staging and is then deleted; a post-return call rejects; external finalization first causes the durable ownership check to reject | +| B outcome vs earlier A settlement | B stages independently; placement waits for A | +| crash after outcome staging | tool never replays; pending result later materializes | +| crash during source-prefix placement | transaction exposes either none or all of that placement prefix | +| safe replay vs old partial output | the old bound checkpoint value is deleted before replay emits new progress | +| cancellation vs real settlement | Session mutation order chooses real cancelled-control result or synthetic reconciliation; at most one outcome stages | +| terminal finalization vs late result | terminal ownership wins or outcome stages first; late task never recreates operation data | +| external finalization vs memo/checkpoint mutation | mutation first is removed by terminal cleanup; finalization first makes the mutation's durable ownership check reject | + +## Invariants + +1. `invocationId` equals the reserved result entry ID and is stable across safe replay. +2. A call in `outcome_ready` or `completed` never executes again. +3. Every `outcome_ready` call has exactly one complete matching `pi.pending.entry` value. +4. Completed calls form a source-ordered prefix. +5. Only source-ordered materialization extends that prefix. +6. Parallel calls after the completed prefix may mix planned, effect-pending, and outcome-ready states. +7. Invocation memos exist only while their call is `effect_pending`. +8. Partial output is auxiliary and never establishes effect completion. +9. Unsafe synthetic results explicitly state that captured output is incomplete and the external outcome is unknown. +10. Staging an outcome atomically deletes its invocation memos and partial output. +11. Materialization atomically inserts the immutable entry and deletes its staged pending value. +12. A late invocation capability cannot write after outcome settlement or operation loss. +13. `step.do` values are memoized only after their memo write commits; effects remain at-least-once. +14. Operation terminal cleanup leaves no tool args, invocation memos, partial output, or staged outcomes. + +## Required tests + +### State and restore + +- every trusted planned/effect-pending/outcome-ready/completed projection restores without auxiliary reads; +- base restore does not audit completed-prefix or execution-mode relationships; +- effect-pending consumption reads exact required arguments and optional bounded checkpoint; +- outcome-ready consumption reads the exact staged result; +- missing required arguments or staged result fails at the consuming procedure, not base restore; +- invocation memos and partial output are absent after staging; +- snapshot/activation hydrate only the exact bounded checkpoint value when needed. + +### Parallel ordering + +- B and C stage while A remains pending; +- crash/reopen proves B and C never replay; +- A recovery followed by one source-ordered A/B/C placement transaction; +- mixed `[completed, outcome_ready, planned, effect_pending, outcome_ready]` state; +- immediate synthetic outcomes stage out of order; +- all-terminating batch transitions correctly after ordered placement. + +### Replay and interruption + +- safe replay uses persisted arguments and the same invocation ID; +- safe replay preserves step memos but clears old partial output; +- current declaration downgrade from safe to never interrupts; +- unsafe recovery with no checkpoint and with a complete bounded checkpoint; +- synthetic result is error/incomplete/unknown and never runs `after_tool`; +- cancellation never safely replays a restored call. + +### Invocation memos and `step.do` + +- set/get/delete memo with invocation-address scoping; +- completed step skips effect after reopen; +- crash before memo commit reruns effect; +- crash after memo commit returns memo; +- several completed steps followed by one interrupted step; +- duplicate live step names reject; +- memo write racing outcome staging; +- capability write after expiry, cancellation outcome staging, and external finalization rejects; +- terminal cleanup removes crash-leaked memos. + +### Partial output + +- ordinary updates stay live-only; +- `checkpoint: true` writes the complete bounded snapshot; +- tool-selected checkpoint cadence and duplicate suppression; +- bash emits live snapshots at 100 ms and requests distinct checkpoints at most every two seconds; +- every selected checkpoint enqueues one write, and awaiting the latest promise at tool settlement implies all earlier writes completed; +- a checkpoint after outcome staging cannot recreate the address's value; +- Memory, JSONL, and SQLite restore the same checkpoint value; +- JSONL growth follows checkpoint cadence rather than raw bash output volume; +- terminal compaction reclaims superseded/deleted checkpoint snapshots according to the existing dead-byte policy. + +### Atomicity and instrumentation + +- exact intent, synchronous update acceptance, asynchronous update-delivery, `after_tool`, outcome-ready staging, post-commit `tool_end`, source-ordered message lifecycle, and materialization order; +- outcome staging is atomic with memo/output cleanup; +- materialization is atomic with pending deletion, usage, tip, and state; +- crash at every boundary; +- no effect starts before intent; +- no effect or hook starts from `outcome_ready`; +- terminal transaction removes every operation-owned tool value. + +## Implementation map + +Expected runtime areas: + +- operation-state types in `packages/agent/src/harness/session/types.ts`; +- trusted restore projection and exact consumption-time address reads; +- tool-batch procedure and source-ordered materialization; +- terminal cleanup and cancellation reconciliation; +- harness-specific update options, snapshots, and events; +- invocation-scoped capability implementation on the Session mutation line; +- backend conformance through the bound value/list APIs; +- instrumented-storage transaction assertions. + +Concrete built-in address constructors in `session/values.ts`: + +```text +operationToolMemo(operationId, invocationId, name) → value("pi.op.tool_memo", ...) +pendingToolOutput(operationId, invocationId) → value("pi.pending.tool_output", ...) +pendingEntry(resultEntryId) → value("pi.pending.entry", ...) +``` + +Implement `outcome_ready` and invocation memos before progress checkpoints. The state solves incorrect parallel replay by itself; checkpoints improve reconnect observation and unsafe interruption diagnostics without becoming completion authority. diff --git a/packages/agent/docs/values.md b/packages/agent/docs/values.md new file mode 100644 index 000000000..c8a2c8100 --- /dev/null +++ b/packages/agent/docs/values.md @@ -0,0 +1,735 @@ +# Typed values and lists + +This document specifies the mutable storage primitive used by Session, the harness, and applications. + +The public abstraction is a **bound typed address**: + +- `value(namespace, key?)` names one replaceable durable value; +- `list(namespace, key?)` names one append-only durable list whose elements have type `T`. + +The namespace/key pair is bound once when the address is constructed. Every later operation receives only that address. Application code therefore writes: + +```ts +const state = value("my-app.state"); +const events = list("my-app.events"); + +await session.getValue(state, context); +await session.setValue(state, nextState, context); +await session.readList(events, { limit: 100 }, context); +await session.appendList(events, event, context); +``` + +It does **not** repeatedly pass a second unexplained key: + +```ts +// Not the API. +await session.readList(events, "another-key", { limit: 100 }, context); +``` + +When an application genuinely has keyed instances, it constructs the address for that instance: + +```ts +const workspaceEvents = (workspaceId: string) => + list("my-app.events", workspaceId); + +await session.readList(workspaceEvents("pi"), { limit: 100 }, context); +``` + +Storage may physically index the address as `(kind, namespace, key)`, but that representation does not leak into each read or write call. Storage, Session, harness code, and applications use the same address vocabulary. There is no global value-type map, dynamic registry, token catalog, or separate application-state storage mechanism. + +## Goals + +1. Give one exact durable address one compile-time value type. +2. Let applications define scalar values and lists without declaration merging or editing a core type map. +3. Use the same typed addresses and operation names from Storage through Session. +4. Preserve current scalar replacement semantics. +5. Append one list element without reading or rewriting existing elements. +6. Give every list element its own existing session-global transaction `seq`, used for ordering and pagination. +7. Commit values and list elements atomically with entries and usage. +8. Produce identical logical behavior on Memory, JSONL, and SQLite. +9. Keep list reads bounded and explicit. +10. Keep scalar operation state authoritative; auxiliary lists never select a recovery state. + +## Non-goals + +This slice does not define: + +- assistant-frame contents or reduction semantics; +- tool-progress semantics; +- runtime validation of trusted in-process values; +- per-element or per-list byte limits; +- list truncation or per-element deletion; +- a generic event log, journal, stream-resumption protocol, or operation reducer; +- globally registering address objects; +- exposing raw Session or transaction access to tools. + +Consumers own address construction, content limits, cleanup points, fork policy, migration policy, and consumption-time hydration. `assistant-durability.md` defines the first list consumer. + +## Bound address model + +```ts +declare const storedValueType: unique symbol; + +interface StoredAddressBase { + /** Stable persisted grouping name. */ + readonly namespace: string; + /** Exact member inside that grouping. Empty is legal. */ + readonly key: string; + readonly kind: "value" | "list"; +} + +export interface Value extends StoredAddressBase { + readonly kind: "value"; + /** Compile-time only and invariant in T. */ + readonly [storedValueType]?: (value: T) => T; +} + +export interface ValueList extends StoredAddressBase { + readonly kind: "list"; + /** T is one element, not the whole list. */ + readonly [storedValueType]?: (value: T) => T; +} + +export function value(namespace: string, key = ""): Value { + validateAddress(namespace, key); + return Object.freeze({ namespace, key, kind: "value" }); +} + +export function list(namespace: string, key = ""): ValueList { + validateAddress(namespace, key); + return Object.freeze({ namespace, key, kind: "list" }); +} +``` + +The phantom function makes `T` invariant: an address for one type cannot silently widen to another. It has no runtime field. + +Rules: + +- `namespace` must be non-empty; +- namespace `pi` and every `pi.*` namespace are reserved for built-ins by contract; +- applications that construct a reserved address are defective trusted in-process code; no runtime privilege split, registry, or catalog exists; +- neither component may contain the Memory backend's internal separator (`\u0000`); +- an empty key is valid and is the natural address for one application-wide value or list; +- object identity has no durable meaning; +- separately constructed addresses with the same `(kind, namespace, key)` identify the same durable location; +- constructing the same durable location with incompatible TypeScript types is a trusted-programming defect; +- scalar and list addresses may not share the same `(namespace, key)` in one storage version; violating this is a trusted-programming defect and storage performs no cross-kind collision check; +- changing an address's namespace, key, kind, or incompatible value shape requires migration. + +The two components remain separate rather than concatenated. Dynamic application keys and operation IDs therefore require no escaping convention beyond the storage separator rule. + +### Exact addresses, not families + +An address names one value or one list. Internal code uses small constructors when it has dynamic keys: + +```ts +export const branchTip = (lane: string) => + value("pi.branch.tip", lane); + +export const operationState = (operationId: string) => + value("pi.op.state", operationId); + +export const operationToolArgs = ( + operationId: string, + stepId: string, + sourceIndex: number, +) => value>( + "pi.op.tool_args", + `${operationId}:${stepId}:${sourceIndex}`, +); + +export const pendingAssistantFrames = ( + operationId: string, + responseEntryId: string, +) => list( + "pi.pending.assistant_frame", + `${operationId}:${responseEntryId}`, +); +``` + +This encapsulates each key grammar at its owner. Call sites receive an already-bound typed address: + +```ts +await reader.getValue(operationState(operationId)); +await reader.readList(pendingAssistantFrames(operationId, responseEntryId), options); +``` + +### No global value map + +Delete the existing global namespace-to-type maps: + +```ts +interface RegisterValues { /* delete */ } +interface ListRegisterValues { /* delete */ } +type RegisterNamespace = keyof RegisterValues; // delete +``` + +A type belongs to an address constructor instead: + +```ts +export const applicationState = value("my-app.state"); +export const applicationEvents = list("my-app.events"); +``` + +Applications should use a stable, collision-resistant namespace prefix. Namespace `pi` and the complete `pi.*` prefix are reserved for built-ins by contract; similar-looking names such as `pi2` remain legal. The same `value()` and `list()` constructors serve core and application code. Tests assert that every built-in address uses its reserved prefix. There is no runtime privilege split, registry, or catalog. + +## Built-in addresses + +Built-in constructors live together in `packages/agent/src/harness/session/values.ts` and are imported directly by consumers. Representative definitions: + +```ts +export const branchTip = (lane: string) => + value("pi.branch.tip", lane); +export const laneConfig = (lane: string) => + value("pi.lane.config", lane); +export const laneState = (lane: string) => + value("pi.lane.state", lane); +export const operationResult = (operationId: string) => + value("pi.result", operationId); + +/** Used only by scanValues() to enumerate Branch names. */ +export const branchTipInventoryPrefix = () => + value("pi.branch.tip"); + +export const operationMeta = (operationId: string) => + value("pi.op.meta", operationId); +export const operationState = (operationId: string) => + value("pi.op.state", operationId); +export const operationToolArgs = (operationId: string, stepId: string, sourceIndex: number) => + value>( + "pi.op.tool_args", + `${operationId}:${stepId}:${sourceIndex}`, + ); +export const operationToolMemo = (operationId: string, invocationId: string, name: string) => + value("pi.op.tool_memo", `${operationId}:${invocationId}:${name}`); +export const operationPreparation = (operationId: string, taskId: string) => + value( + "pi.op.preparation", + `${operationId}:${taskId}`, + ); + +/** Prefix addresses are exported only for namespace-scoped scanValues(). */ +export const operationToolArgsPrefix = (operationId: string, stepId?: string) => + value>( + "pi.op.tool_args", + stepId === undefined ? `${operationId}:` : `${operationId}:${stepId}:`, + ); +export const operationToolMemoPrefix = (operationId: string, invocationId?: string) => + value( + "pi.op.tool_memo", + invocationId === undefined ? `${operationId}:` : `${operationId}:${invocationId}:`, + ); +export const operationPreparationPrefix = (operationId: string) => + value("pi.op.preparation", `${operationId}:`); + +export const pendingEntry = (entryId: string) => + value("pi.pending.entry", entryId); +export const pendingToolOutput = (operationId: string, invocationId: string) => + value>( + "pi.pending.tool_output", + `${operationId}:${invocationId}`, + ); +export const pendingAssistantFrames = (operationId: string, responseEntryId: string) => + list( + "pi.pending.assistant_frame", + `${operationId}:${responseEntryId}`, + ); +export const pendingToolOutputPrefix = (operationId: string) => + value>("pi.pending.tool_output", `${operationId}:`); + +export const sessionName = value("pi.session.name"); +export const entryLabel = (entryId: string) => value("pi.entry.label", entryId); +``` + +`OperationMeta` is immutable acceptance metadata stored at `pi.op.meta`. The process-local `Operation` projection is `{ meta: OperationMeta, state: OperationState }`, assembled from the separate metadata and state values; it is never stored at one address. + +The five exported scan-prefix constructors are `branchTipInventoryPrefix`, `operationToolArgsPrefix`, `operationToolMemoPrefix`, `operationPreparationPrefix`, and `pendingToolOutputPrefix`. Their addresses are consumed only by `scanValues()`. + +Applications define their own `value()` and `list()` addresses directly; there is no built-in custom application-state namespace or custom-state API. `AgentHarnessToolInvocation.getMemo()` and `setMemo()` are invocation-fenced capabilities over `operationToolMemo(...)`, not raw Session access. Invocation memos remain operation-owned and are deleted when their tool outcome becomes durable. + +Tests assert that built-in constructors produce the documented kind, namespace, and key grammar. Because constructors may be dynamic, there is no runtime catalog that tries to enumerate every possible address. + +## Shared read API + +Storage, Session, SessionReader, and SessionMutator use the same read signatures: + +```ts +export interface StoredValue { + address: Value; + value: T; + seq: number; +} + +export interface ListElement { + /** Global transaction-write sequence assigned by storage. */ + seq: number; + value: T; +} + +export interface ListCursor { + seq: number; +} + +export interface ListReadOptions { + /** Exclusive cursor. */ + cursor?: ListCursor; + /** Default: asc. */ + order?: "asc" | "desc"; + /** Query-page size. Default: 1,000. Values above 10,000 clamp to 10,000. */ + limit?: number; +} + +interface ValueReader { + getValue(address: Value): Promise | undefined>; + + /** Internal bounded-prefix operation. The address key is interpreted as a prefix. */ + scanValues(prefix: Value): Promise[]>; + + readList( + address: ValueList, + options?: ListReadOptions, + ): Promise[]>; +} +``` + +`scanValues(prefixAddress)` scans scalar addresses with exactly that namespace and keys beginning with the bound key, returning them in key-ascending order. Core call sites use only the exported prefix constructors above so raw namespace/key grammar stays in `session/values.ts`. Prefix addresses are passed only to `scanValues()`, never to exact get/set/delete operations. There is no unrestricted cross-namespace dump. Ordinary application reads use exact addresses. + +`Session` exposes direct one-transition writes using the same addresses: + +```ts +interface Session extends ValueReader { + setValue( + address: Value, + next: NoInfer, + context: Context, + ): Promise; + deleteValue(address: Value, context: Context): Promise; + appendList( + address: ValueList, + element: NoInfer, + context: Context, + ): Promise; + deleteList(address: ValueList, context: Context): Promise; +} +``` + +Purpose-specific helpers such as `getName()`, `setName()`, `getLabel()`, and `setLabel()` may remain thin wrappers over built-in addresses. Applications define and use their own scalar/list addresses directly. + +`SessionMutator` remains a read capability plus one atomic `commit(writes)`. It does not expose direct `setValue()`/`appendList()` methods that would consume its only commit separately; callers construct a typed write array and commit it together. + +## Typed transaction writes + +Writes are constructed through typed helpers. Entry and usage constructors hide their storage discriminants; value/list erasure happens only after the helper has checked the address/value type relationship: + +```ts +interface EntryWrite { + kind: "entry"; + entry: NewEntry; +} + +interface UsageWrite { + kind: "usage"; + row: Omit; +} + +interface ValueSetWrite { + kind: "value"; + op: "set"; + namespace: string; + key: string; + value: unknown; +} + +interface ValueDeleteWrite { + kind: "value"; + op: "delete"; + namespace: string; + key: string; +} + +interface ListAppendWrite { + kind: "list"; + op: "append"; + namespace: string; + key: string; + value: unknown; +} + +interface ListDeleteWrite { + kind: "list"; + op: "delete"; + namespace: string; + key: string; +} + +export function insertEntry(entry: NewEntry): EntryWrite; +export function insertUsage(row: Omit): UsageWrite; +export function setValue(address: Value, next: NoInfer): ValueSetWrite; +export function deleteValue(address: Value): ValueDeleteWrite; +export function appendList(address: ValueList, element: NoInfer): ListAppendWrite; +export function deleteList(address: ValueList): ListDeleteWrite; +``` + +`NoInfer` makes the address authoritative. TypeScript must not infer a wider `T` from an incompatible write value. + +`Write` includes all six helper return types. One transaction may mix every write kind atomically. Harness and application code use the helpers rather than manually constructing storage write shapes. + +The direct Session methods and transaction helpers intentionally use the same operation names. One performs and commits a single Session mutation; the other constructs a write for an explicitly composed transaction. + +## Scalar semantics + +For one `Value` address: + +- `setValue` replaces the current value; +- `deleteValue` removes it; +- deleting an absent value is a no-op; +- set after delete recreates it; +- there is no retained value history; +- the current value records the `seq` of its latest set; +- a failed transaction exposes neither the scalar write nor any sibling write. + +## List semantics + +One append write carries one immutable element. A transaction appending several elements contains several append writes. Every write receives its existing globally increasing transaction sequence: + +```text +TX[ + appendList(frames, A), // seq 41 + setValue(operationState, X), // seq 42 + appendList(frames, B), // seq 43 +] +``` + +Reading `frames` returns `A`, then `B`. Gaps from unrelated writes are expected. A list element's `seq` is session-global and unique to that committed write; it is an ordering/cursor identity, not an application domain ID. Applications that need domain identity include it in `T`. + +Rules: + +- append never reads existing elements; +- an element is immutable after commit; +- `deleteList(address)` removes every element at that exact address; +- deleting an absent list is a no-op; +- delete followed by append in one transaction creates a fresh list atomically; +- there is no per-element update, delete, insertion, or truncation; +- all validation and serialization required to admit a transaction completes before Memory state changes; +- a failed transaction exposes none of its list or non-list writes. + +“Append-only” describes elements while the list exists. Whole-list deletion is lifecycle cleanup, not element mutation. + +### List reads + +- ascending reads return `seq > cursor.seq`; +- descending reads return `seq < cursor.seq`; +- results are ordered according to `order` before `limit` is applied; +- absent and empty both return `[]`; +- callers continue with the last returned element's `seq`; +- an empty page ends iteration; +- `limit` is only the query-page size: it must be a positive safe integer, defaults to 1,000, and values above 10,000 clamp to 10,000; it never limits total list length or bytes. + +```ts +let cursor: ListCursor | undefined; +while (true) { + const page = await reader.readList(events, { cursor, order: "asc", limit: 100 }); + if (page.length === 0) break; + consume(page); + cursor = { seq: page[page.length - 1]!.seq }; +} +``` + +A cursor is a sequence filter, not a snapshot or list-incarnation token. Concurrent later appends may appear on later ascending pages. Whole-list deletion may make a cursor stale; reads simply apply its sequence comparison to currently surviving elements. + +Do not add an unbounded “read the whole list” helper. + +## Assistant partial frames + +Assistant partial durability is the first built-in list consumer: + +```ts +const frames = pendingAssistantFrames(operationId, responseEntryId); +``` + +`AssistantMessageFrame`, `AssistantMessageFrameEncoder`, and `reduceAssistantMessageFrames()` come from `@earendil-works/pi-ai`. Do not define a second frame codec or reducer. + +For every convertible non-terminal provider event, the assistant procedure: + +```text +convert event to frame +→ synchronously enqueue appendList(frames, frame) on the Session mutation line +→ attach the ordinary harness-fault observer to that returned promise +→ replace the process-local latestFrameWrite reference +→ emit and await the existing message event +→ consume the next provider event +``` + +The provider loop does not await storage for every frame. Synchronous enqueue preserves provider-event order. Replacing the latest-promise reference never leaves an earlier rejection unobserved because every promise receives the fault observer. Bounded output bounds queued work. On stream settlement, the procedure stops frame admission and awaits the latest append promise before `after_response`; Session mutation FIFO means that completion implies every earlier append completed. There is no timer, batcher, coalescer, or flush API. + +Scalar assistant `effect_pending` remains authoritative. Each append verifies that the same operation, attempt, and response ID still own the lane when its mutation executes. Frames never prove request admission, completion, success, or failure. + +Final or synthetic assistant settlement deletes the exact list atomically with its immutable response, usage, Branch tip, and next scalar state: + +```text +TX[ + insert final assistant entry, + insert usage, + deleteList(frames), + setValue(operationState(operationId), nextState), +] +``` + +`assistant-durability.md` defines frame conversion, unknown-outcome synthesis, cancellation, deferred polling, snapshots, and event ordering. + +## Restore policy + +Scalar operation state remains the sole restart authority: + +1. construct the trusted lane/operation projection from required scalar values; +2. trust committed typed values rather than auditing every referenced payload or phase relationship; +3. when a procedure or snapshot consumes auxiliary state, derive its exact bound address from current typed scalar state; +4. hydrate only the bounded scalar values or list pages that consumer requires. + +A missing auxiliary list is legal unless its consumer explicitly requires an element. List contents never prove that an external effect completed. Live mutations still verify current operation, phase, attempt, and reserved identity as concurrency fencing; that is not restore validation. + +Base restore does not enumerate lists. For assistant frames, snapshot or recovery derives `pendingAssistantFrames(operationId, responseEntryId)` only while consuming a typed assistant/deferred `effect_pending` state. + +Each list consumer defines: + +- address grammar; +- element and total-byte bounds; +- page/hydration budget; +- cleanup transitions; +- fork and migration policy. + +## Memory backend + +Memory may keep separate maps for current values and list elements: + +```ts +const scalarValues = new Map>(); +const listValues = new Map[]>(); + +function physicalKey(address: StoredAddressBase): string { + return `${address.namespace}\u0000${address.key}`; +} +``` + +- scalar set replaces one map value; +- scalar delete removes it; +- list append pushes the already-sequenced element; +- list delete removes the complete array; +- list read filters by exclusive cursor and slices to the validated limit; +- transaction preparation completes before entries, values, lists, usage, or stats mutate. + +Storage snapshots used by JSONL/fork tooling include current scalar values and surviving list elements with original sequence numbers. + +## SQLite backend + +The logical schema has one current-value table and one list-element table: + +```sql +CREATE TABLE scalar_values ( + namespace TEXT NOT NULL, + key TEXT NOT NULL, + seq INTEGER NOT NULL, + value TEXT NOT NULL, + PRIMARY KEY (namespace, key) +) WITHOUT ROWID; + +CREATE TABLE list_values ( + namespace TEXT NOT NULL, + key TEXT NOT NULL, + seq INTEGER NOT NULL, + value TEXT NOT NULL, + PRIMARY KEY (namespace, key, seq) +) WITHOUT ROWID; +``` + +WP01 replaces the unfinished format-4 schema in place: edit `sqlite/migrations/001_initial.sql`, rename the physical `registers` table to `scalar_values`, add `list_values`, and keep `SQLITE_STORAGE_VERSION = 1`. There is no migration runner in this WIP implementation, and pre-WP01 SQLite files are unsupported. Do not add migration machinery in this package. + +List operations: + +```sql +INSERT INTO list_values(namespace, key, seq, value) VALUES (?, ?, ?, ?); + +SELECT seq, value FROM list_values +WHERE namespace = ? AND key = ? AND seq > ? +ORDER BY seq ASC LIMIT ?; + +SELECT seq, value FROM list_values +WHERE namespace = ? AND key = ? AND seq < ? +ORDER BY seq DESC LIMIT ?; + +DELETE FROM list_values WHERE namespace = ? AND key = ?; +``` + +For a missing cursor, omit the sequence predicate. Every write participates in the existing `BEGIN IMMEDIATE` transaction; writable Session ownership belongs to the host lifecycle, not SQLite storage. Assert with `EXPLAIN QUERY PLAN` that paging uses the primary key and no temporary sort. + +## JSONL backend + +Logical records carry the bound address's physical components: + +```jsonl +{"kind":"list","op":"append","seq":41,"namespace":"pi.pending.assistant_frame","key":"O:R","value":{"type":"text_delta","contentIndex":0,"delta":"hi"}} +{"kind":"list","op":"delete","seq":52,"namespace":"pi.pending.assistant_frame","key":"O:R"} +``` + +Scalar records use `kind:"value"` with `op:"set"|"delete"`. WP01 keeps JSONL format 4 and storage version 1 but replaces the unfinished record spelling in place; pre-WP01 format-4 files are unsupported and no legacy `kind:"register"` decoder remains. + +Replay folds records into the Memory state: + +- scalar set replaces the current address; +- scalar delete removes it; +- list append adds `{ seq, value }`; +- list delete removes the complete list. + +A transaction remains one physical JSONL line, using an array for multiple writes. Torn-tail handling therefore remains atomic without new framing. + +### Snapshot compaction + +Compaction writes every surviving list element with its original `seq`, merged in sequence order with surviving entries, scalar values, and usage rows. Do not collapse one live list into a synthetic element or assign new sequence numbers; either change breaks cursors and backend equivalence. + +Deleted lists produce no snapshot records. Snapshot rewrites persist `nextSeq` in the format-4 header so dropping the latest delete cannot permit sequence reuse; ordinary append-only files may omit that field and derive it from replayed writes. + +## Forks and rewrites + +Fork and precise-rewrite code decides policy per concrete address grammar: + +- operation-owned `pi.op.*` scalar values are not copied into an idle fork; +- immutable `pi.result` operation records are not copied by forks; +- `pi.pending.entry`, `pi.pending.tool_output`, and `pi.pending.assistant_frame` values/lists are not copied; +- lane and semantic session values follow their existing scope rules; +- application-defined values/lists are not copied by the generic fork; a consuming feature must add an explicit address-specific policy before relying on copied application state. + +A precise rewrite retaining list elements preserves their `seq` values unless it explicitly remaps the entire destination sequence space. + +## Schema evolution + +A bound address's namespace, key grammar, kind, and value type are durable schema: + +- changing namespace or key grammar requires explicit address migration; +- changing scalar to list or list to scalar requires explicit migration; +- storage never infers or coerces kind from observed records; +- changing TypeScript value shape requires total value migration when old stored values are incompatible; +- a list migration pages elements in sequence order and either maps them while preserving `seq` or deletes the complete list; +- a migration must not load an unbounded logical list at once. + +Adding generic list storage replaces the current WIP backend schema in place. Constructing a new application address with no persisted value requires no migration. + +## Instrumentation and telemetry + +The instrumented storage decorator exposes the address-based read API and records committed erased writes in exact transaction order. + +Telemetry session-write item kinds distinguish scalar-value writes from list writes. Namespace/key names may be attributes when the telemetry schema permits them, but values, assistant frames, prompts, and tool output never enter telemetry. + +Append-path tests prove that no `readList` call occurs before append commit. Frame-persistence promises always receive the harness fault observer, even when an earlier promise is no longer the latest settlement-order reference. + +## Invariants + +1. One bound address has one stable namespace/key/kind and one trusted value type in a storage version. +2. Address object identity has no durable meaning. +3. Namespace `pi` and every `pi.*` are reserved by contract; every built-in namespace starts with `pi.`, and application use is a trusted-programming defect. +4. Exactly five built-in prefix constructors encapsulate Branch inventory and operation cleanup grammar; their results are consumed only by namespace-scoped `scanValues()`. +5. Scalar and list addresses must not occupy the same physical location; this is a trusted-programming rule, not a runtime cross-kind collision check. +6. Typed reads and helper-constructed writes preserve `T`. +7. Scalar helpers reject list addresses; list helpers reject scalar addresses. +8. A Session/Storage operation never requires a second key after address construction. +9. Every list element is immutable and carries its globally unique committed write `seq`. +10. Elements at one list address are returned in sequence order on every backend. +11. Append performs no read of the target list. +12. Scalar/list writes are atomic with entries and usage in the same transaction. +13. Whole-list delete leaves no elements at that address. +14. Missing and empty lists both read as `[]`. +15. Base restore depends only on required scalar state and never enumerates auxiliary lists. +16. Auxiliary lists never establish effect completion or select a restart state. +17. JSONL compaction preserves surviving element sequences. +18. Terminal cleanup leaves no operation-owned scalar values or lists. + +## Required tests + +### Address typing and identity + +- `value()` and `list()` preserve their declared `T` invariantly; +- scalar reads infer the bound address's value type; +- list reads infer its element type; +- `setValue` rejects an incompatible value at compile time; +- `appendList` rejects an incompatible element at compile time; +- scalar helpers reject list addresses and list helpers reject scalar addresses; +- independently constructed equal addresses access the same durable location; +- incompatible definitions of one physical address are documented/tested as a programming defect; +- empty keys work, while empty namespaces and separator-containing components reject; +- core and application code use the same `value()` and `list()` constructors, with no private constructor, privilege token, registry, or catalog; +- built-in address constructors produce exact `pi.branch.tip`, `pi.lane.*`, `pi.op.*`, `pi.pending.*`, `pi.session.name`, and `pi.entry.label` namespace/key/kind triples; +- every built-in namespace starts with `pi.`, while application fixtures use non-reserved namespaces; +- `branchTipInventoryPrefix()` binds the empty-key `pi.branch.tip` inventory prefix and is used only to enumerate Branches through `scanValues`; +- tool-args prefixes cover one operation and optionally one step, tool-memo prefixes cover one operation and optionally one invocation, preparation and tool-output prefixes cover exactly one operation; +- each prefix constructor result is used only by `scanValues`, and no inventory or cleanup call constructs a raw reserved namespace; +- application addresses work without declaration merging or core catalogs; +- no Storage or Session operation accepts an additional key argument. + +### Scalar regression + +- set/get/delete/recreate behavior is unchanged; +- replacement retains only the latest logical value and latest set `seq`; +- typed write helpers preserve mixed transaction order; +- prefix scans interpret the bound address key as a prefix and remain namespace-scoped; +- new scalar JSONL/SQLite files use only the value/list schema; pre-WP01 WIP files are explicitly unsupported. + +### List conformance + +Extend the shared backend conformance suite: + +- append one element and page it; +- multiple appends to one address in one transaction; +- appends separated by unrelated writes preserve per-list order; +- every element receives its own global write `seq`; +- ascending and descending exclusive cursors; +- default, explicit, invalid, and capped limits; +- absent list returns `[]`; +- whole-list delete and delete of absent list; +- delete followed by append in one transaction; +- rollback when a later write is invalid; +- atomic list + entry + usage + scalar transaction; +- identical pages and cursors on Memory, JSONL, and SQLite; +- JSONL torn multi-write transaction exposes no list element; +- JSONL replay and compaction preserve cursors; +- SQLite paging uses the primary key without temporary sorting; +- append performs no list read; +- base restore constructs trusted scalar projection without list reads, followed by bounded consumption-time hydration; +- close rejects later reads and honors already-admitted commits. + +### Application surface + +- an application-wide scalar value requires no extra key at get/set; +- an application-wide list requires no extra key at read/append; +- an application can construct dynamic per-workspace addresses explicitly; +- Storage and Session accept the same address objects and infer the same types; +- direct Session writes serialize and commit once; +- explicit `Session.mutate()` can atomically combine typed value/list writes with entries and usage. + +### Assistant-frame integration — deferred beyond WP01 + +- every converted non-terminal frame appends under the exact bound effect-pending response address; +- terminal `done`/`error` events append nothing; +- appends enqueue synchronously without provider backpressure; +- every frame-write promise has an observed fault path; +- only the latest promise reference is retained for settlement ordering; +- awaiting the latest promise implies every earlier append completed; +- reduced pages reconstruct the same partial message as uninterrupted streaming; +- missing list restores as no durable partial; +- final/synthetic settlement atomically deletes the frame list; +- unknown-effect recovery reads only the bounded list derived from current scalar state; +- external finalization deletes the operation-owned list; +- idle forks contain no frame list; +- backend byte growth is append-linear rather than repeated-snapshot growth. + +## Implementation map + +Expected primary changes: + +- replace `session/registers.ts` with `packages/agent/src/harness/session/values.ts` containing addresses, constructors, typed write helpers, and built-in address constructors; +- remove `RegisterValues`, namespace unions, register token types, and raw namespace/key read signatures from `session/types.ts`; +- expose `ValueReader` through Storage, SessionReader, SessionMutator, and Session; +- expose direct application scalar/list methods on Session using bound addresses; +- update Memory state, JSONL codec/storage, snapshots, fork/rewrite code, instrumentation, and conformance suites; +- replace SQLite's unfinished initial schema in place with `scalar_values` and `list_values`; keep storage version 1 and add no migration runner; +- update telemetry schema sources and regenerate `telemetry-schema.md`; do not edit that generated file manually. + +WP01 stops after generic addresses/storage and projection-only restore coverage. `assistant-durability.md` specifies the later consuming lifecycle; assistant execution, deferred polling, recovery, snapshot hydration, memo/checkpoint capabilities, and operation cleanup land only with their runtime work packages. diff --git a/packages/agent/docs/work-packages/00-runtime1-removal.md b/packages/agent/docs/work-packages/00-runtime1-removal.md new file mode 100644 index 000000000..49bb903e5 --- /dev/null +++ b/packages/agent/docs/work-packages/00-runtime1-removal.md @@ -0,0 +1,83 @@ +# WP00 — Runtime1 removal + +## Status + +Complete. No tag. Runtime2 is the sole public implementation. Do not release while its execution path is incomplete. + +## Goal + +Make runtime2 the sole public harness implementation, delete runtime1 and its obsolete tests, then stop before adding runtime2 behavior. + +## Prerequisites + +- The approved acceptance/hook redesign and durability handoffs are present in `harness.md`, `values.md`, `assistant-durability.md`, and `tool-durability.md`. +- Existing tests are evidence, not authority. + +## Work, in order + +1. **Reconcile the contract.** Fold the approved acceptance/hook redesign into `harness.md`, including durable `starting`, atomic hook-free acceptance, driver-owned `before_run`, `before_drive`, request-local system-prompt transformation, trusted restore, and removal of process-origin activation semantics. Audit §§0.4, 1.2, 3.1–3.6, 4.1–4.2, 4.5, 5.1–5.2, 5.5–5.6, and Parts 8–9; remove every stale `BeforeResumePrepared`, `before_resume`, `resumeData`, `systemPromptOverride`, stable-ID routing, reservation, and `fresh | continue | resume` activation reference. Delete obsolete runtime planning documents once `harness.md` and linked handoffs own the active contract. +2. **Harvest before deletion.** Inspect `agent-harness-runtime.test.ts`, `agent-harness-r2/r3/r4.test.ts`, and old `restore.test.ts`. Preserve unique scenarios in the detailed future rows or a temporary categorized inventory; explicitly discard old reservation, `before_resume`, `resumeData`, persisted hook prompt override, semantic restore audit, and pre-`outcome_ready` tool-crash behavior. +3. **Remove runtime1-only public members.** Delete `before_resume`, `BeforeResumePrepared`, `resumeData`, `systemPromptOverride`, and stable hook-ID routing. Add the approved `before_drive` and `transform_context` shapes. Update telemetry schema source and regenerate its document. Do not implement `starting` or acceptance behavior here. +4. **Switch the factory.** Add `packages/agent/src/harness/runtime2/index.ts`, point `agent-harness.ts` at it, and add a constructor-selection regression. Verify the experimental coding-agent worker still creates the harness, subscribes to events, and closes it. +5. **Delete runtime1 source and tests** listed below. +6. **Update `[Unreleased]` on `main` or a pull-request branch** for the public breaking removals and temporarily incomplete factory. Repository policy forbids changelog edits on `dev`, so WP00 records but does not perform that release-facing step here. +7. Run the retained tests and checks. Fix every failure; do not restore compatibility shims. + +## Delete + +```text +packages/agent/src/harness/runtime/** +packages/agent/src/harness/restore.ts +packages/agent/test/harness/agent-harness-runtime.test.ts +packages/agent/test/harness/agent-harness-r2.test.ts +packages/agent/test/harness/agent-harness-r3.test.ts +packages/agent/test/harness/agent-harness-r4.test.ts +packages/agent/test/harness/restore.test.ts +packages/agent/test/harness/scratch/r1.ts +packages/agent/test/harness/scratch/r2.ts +packages/agent/test/harness/scratch/r3.ts +packages/agent/test/harness/scratch/r4.ts +``` + +## Retain + +- all `test/harness/runtime2/**` tests; +- Session, Branch, storage, repository, backend-conformance, and instrumentation tests; +- execution assistant/tool/primitives tests; +- config, hooks, events, telemetry, compaction, and branch-summary code/tests; +- `types.test.ts`, updated for the reduced public contract; +- `packages/agent/src/agent-loop.ts` unchanged. + +Do not parameterize obsolete runtime1 suites against runtime2 and do not keep a runtime1 smoke suite. + +## Acceptance + +- No source import references `harness/runtime/*`. +- Public `AgentHarness.create()` selects runtime2. +- Runtime2 creation, events, inspection, close, and fault tests pass. +- These coding-agent tests pass: + - `experimental-remote-runtime.test.ts` + - `experimental-session-worker-manager.test.ts` (the upstream replacement for the removed `experimental-session-worker.test.ts`) + - `experimental-session-worker-lifecycle.test.ts` +- Every modified test passes individually. +- Agent and root TypeScript pass. +- `git diff --check` and `npm run check` pass. + +## Outcome + +- The accepted hook/drive contract is normative in `harness.md`; stale acceptance/resume contracts are absent. +- Runtime1 source, validating restore, obsolete suites, and R1–R4 scratch scenarios are deleted. +- `AgentHarness.create()` resolves through `runtime2/index.ts`; a constructor-selection regression proves it. +- The scenario harvest added missing tool-close, identity-preflight, recovery-ordering, turn-bracket, and telemetry cases to future rows. +- Upstream added two real remote prompt tests after this handoff was drafted. They remain present but skipped with an R2 re-enable requirement because runtime2 execution is intentionally incomplete. Worker creation, attachment, lifecycle, operation correlation, and close coverage passes. + +## Non-goals + +- No bound-value/list implementation. +- No implementation of `starting` or atomic acceptance. +- No drive owner, provider, retry, deferred, or tool execution. +- No runtime1 parity work, compatibility layer, archaeology tag, or release. + +## Stop condition + +Stop once runtime1 is absent, runtime2 is the public factory, retained coverage is green, and all checks pass. Report the deletion and harvested scenarios; do not begin another work package. diff --git a/packages/agent/docs/work-packages/01-bound-values-lists.md b/packages/agent/docs/work-packages/01-bound-values-lists.md new file mode 100644 index 000000000..d52f3c349 --- /dev/null +++ b/packages/agent/docs/work-packages/01-bound-values-lists.md @@ -0,0 +1,327 @@ +# WP01 — Bound values and lists + +## Status + +Complete. `harness.md` is normative. [`values.md`](../values.md) supplies the detailed address, backend, and conformance design. + +## Goal + +Replace the retained register/custom-state storage surface with bound `Value` and `ValueList` addresses across Session, Memory, JSONL, SQLite, instrumentation, tests, and public application access. Stop before runtime execution consumers. + +## Decisions fixed for this package + +1. **Core and application namespaces.** Every core address uses the exact documented `pi.*` namespace. Applications use their own non-reserved namespaces through the same public `value()` / `list()` constructors. `fact.custom` and its API are deleted, not renamed to a built-in custom namespace. +2. **Forks.** Generic forks copy only explicitly handled core addresses: Branch tip/lane configuration plus fresh lane state, session name, and labels whose targets copy. They copy no `pi.op.*`, `pi.pending.*`, list, ledger, or arbitrary application address. A later application feature must add an address-specific fork policy before relying on copied application state. +3. **Trusted kind discipline.** Using one `(namespace, key)` as both a value and a list is a trusted-programming defect. Backends do not add cross-kind collision checks, triggers, registries, or catalogs. +4. **Operation names.** Keep source `OperationMeta` for immutable acceptance metadata and `Operation` for the process-local `{ meta: OperationMeta, state: OperationState }` projection. `operationMeta(id)` binds `Value`; the composite is never persisted as one value. +5. **Query ordering and bounds.** `scanValues()` returns key-ascending results. `readList()` limits only one query page, never total list length or bytes: reject non-positive or non-safe limits, default to 1,000, and clamp larger values to 10,000. +6. **No WIP compatibility.** Replace the unfinished format-4 storage schema in place. JSONL remains format 4/storage version 1 but accepts only the new value/list records. SQLite keeps `SQLITE_STORAGE_VERSION = 1`, edits `001_initial.sql` in place, renames `registers` to `scalar_values`, and adds `list_values`. Pre-WP01 format-4 JSONL and SQLite files are unsupported. Add no migration runner or legacy decoder. +7. **Generic infrastructure only.** WP01 defines every built-in value/list constructor, including future assistant/tool addresses, but does not implement their runtime consumers. + +## Public and storage contract + +Add `packages/agent/src/harness/session/values.ts` with: + +- invariant `Value` and `ValueList` address types; +- universal `value(namespace, key?)` and `list(namespace, key?)` constructors; +- namespace/key validation only: non-empty namespace and no `\u0000` component; +- `StoredValue`, `ListElement`, `ListCursor`, and `ListReadOptions`; +- typed `setValue`, `deleteValue`, `appendList`, and `deleteList` write helpers using `NoInfer`; +- every exact built-in constructor and the five scan-prefix constructors from `values.md`. + +Replace the old API throughout: + +```ts +getRegister(namespace, key) -> getValue(address) +listRegisters(namespace, prefix) -> scanValues(prefixAddress) +register set/delete writes -> typed value helpers +``` + +Storage and the historical Session reader, mutator, tree-view, and repository surfaces expose the same bound-address reads: + +```ts +getValue(address: Value): Promise | undefined>; +scanValues(prefix: Value): Promise[]>; +readList(address: ValueList, options?: ListReadOptions): Promise[]>; +``` + +The historical tree-view and Session surfaces additionally expose one-commit direct writes: + +```ts +setValue(address: Value, next: NoInfer): Promise; +deleteValue(address: Value): Promise; +appendList(address: ValueList, element: NoInfer): Promise; +deleteList(address: ValueList): Promise; +``` + +`SessionMutator` retains one explicit `commit(writes)` and does not gain direct committing methods. Each write array composes helper-constructed value/list writes with entries and usage. + +Keep `getName` / `setName` and `getLabel` / `setLabel` as wrappers over `sessionName` and `entryLabel(id)`. Delete `getCustomFact` / `setCustomFact`. Rename the public passive metadata event from `fact_update` to the `value_update` shape already specified by `harness.md`; it covers only session-name and entry-label wrappers, not arbitrary application writes. + +## Files + +### Add + +- `packages/agent/src/harness/session/values.ts` +- `packages/agent/test/harness/values.test.ts` +- `packages/session-backends/sqlite-node/src/sqlite/session/values.ts` + +### Delete or rename + +- delete `packages/session-backends/sqlite-node/src/sqlite/session/registers.ts` after moving its scalar behavior to `values.ts`; +- remove all register/global-map/custom-fact declarations from `packages/agent/src/harness/session/types.ts`. + +### Agent source + +- `packages/agent/src/harness/agent-harness.ts` +- `packages/agent/src/harness/session/types.ts` +- `packages/agent/src/harness/session/commit.ts` +- `packages/agent/src/harness/session/storage-state.ts` +- `packages/agent/src/harness/session/memory.ts` +- `packages/agent/src/harness/session/session.ts` +- `packages/agent/src/harness/session/fork.ts` +- `packages/agent/src/harness/session/index.ts` +- `packages/agent/src/harness/session/jsonl/storage.ts` +- `packages/agent/src/harness/session/jsonl/repo.ts` +- `packages/agent/src/harness/session/testing/storage-decorator.ts` +- `packages/agent/src/harness/session/testing/instrumented-storage.ts` +- `packages/agent/src/harness/session/testing/types.ts` +- `packages/agent/src/harness/session/testing/conformance/storage.ts` +- `packages/agent/src/harness/session/testing/conformance/session-repo.ts` +- `packages/agent/src/harness/session/testing/benchmark/storage.ts` +- `packages/agent/src/harness/session/testing/benchmark/session-repo.ts` +- `packages/agent/src/harness/session/testing/index.ts` +- `packages/agent/src/harness/runtime2/restore.ts` +- `packages/agent/src/harness/runtime2/harness.ts` +- `packages/agent/src/harness/runtime2/lane.ts` +- `packages/agent/src/harness/telemetry.ts` +- `packages/agent/src/index.ts` and `packages/agent/src/node.ts` only as needed to verify the new public exports; do not add a second export path. + +### Agent tests and generated documentation + +- `packages/agent/test/harness/memory-storage.test.ts` +- `packages/agent/test/harness/memory-conformance.test.ts` +- `packages/agent/test/harness/memory-session-repo.test.ts` +- `packages/agent/test/harness/jsonl-storage.test.ts` +- `packages/agent/test/harness/jsonl-storage-conformance.test.ts` +- `packages/agent/test/harness/jsonl-session-repo.test.ts` +- `packages/agent/test/harness/jsonl-session-repo-conformance.test.ts` +- `packages/agent/test/harness/storage-backed-session.test.ts` +- `packages/agent/test/harness/session-tree.test.ts` +- `packages/agent/test/harness/session-create-lane.test.ts` +- `packages/agent/test/harness/instrumented-storage.test.ts` +- `packages/agent/test/harness/types.test.ts` +- `packages/agent/test/harness/telemetry.test.ts` +- `packages/agent/test/harness/runtime2/harness.test.ts` +- `packages/agent/test/harness/runtime2/lane.test.ts` +- `packages/agent/test/harness/runtime2/restore.test.ts` +- regenerated `packages/agent/docs/telemetry-schema.md` + +### SQLite backend + +- `packages/session-backends/sqlite-node/src/sqlite/migrations/001_initial.sql` +- `packages/session-backends/sqlite-node/src/sqlite/repo.ts` +- `packages/session-backends/sqlite-node/src/sqlite/session.ts` +- `packages/session-backends/sqlite-node/src/sqlite/storage.ts` +- `packages/session-backends/sqlite-node/src/sqlite/index.ts` if needed for the renamed module +- `packages/session-backends/sqlite-node/test/storage.test.ts` +- `packages/session-backends/sqlite-node/test/storage-conformance.test.ts` +- `packages/session-backends/sqlite-node/test/repo.test.ts` +- `packages/session-backends/sqlite-node/test/adapter.test.ts` +- `packages/session-backends/sqlite-node/test/sql.test.ts` + +### Coding-agent consumer + +- `packages/coding-agent/test/experimental-session-support.ts` +- verify `packages/coding-agent/test/experimental-remote-runtime.test.ts` +- verify `packages/coding-agent/test/experimental-server-replacement.test.ts` + +If the final old-API grep identifies another retained source/test call site, it belongs to WP01; do not add a compatibility shim to avoid touching it. + +## Work, in order + +1. **Add the address vocabulary.** Implement `values.ts`, export it through the existing session/root path, and add focused compile-time/runtime address tests. Include exact built-in namespace/key/kind tests and prefix-constructor tests before migrating callers. +2. **Cut the shared API once.** Replace register types and writes in `types.ts`/`commit.ts`; split `StorageState` into current scalar values and surviving list elements; implement Memory reads/writes, ordered prefix scans, paged list reads, transaction validation/application, snapshots, and direct Session methods. Remove custom-fact APIs and migrate name/label wrappers. +3. **Migrate JSONL and generic fork/snapshot code.** Encode only `kind:"value"` and `kind:"list"`; replay set/delete/append/delete; preserve transaction-line torn-tail atomicity; serialize surviving list elements with original `seq` merged in global sequence order; preserve the sequence high-water mark. This extends existing snapshot serialization only—do not add a new compaction trigger or precise-rewrite feature. Forks copy the fixed core set from Decisions item 2, re-sequence destination scalar values after copied entries as today, and copy no lists. +4. **Migrate instrumentation, conformance, and benchmarks.** The storage decorator exposes all three reads; instrumented storage records erased value/list writes in exact order without content telemetry. Extend shared conformance before backend-specific assertions. +5. **Migrate runtime2 shell call sites.** Replace lane/harness raw writes with built-in constructors/helpers. `restore.ts` uses `scanValues(branchTipInventoryPrefix())` plus exact `getValue` lookups and performs no `readList()` call. Do not add acceptance, drive, hydration, or cleanup behavior. +6. **Replace the SQLite WIP schema and adapter.** Edit `001_initial.sql` in place, implement scalar operations and indexed list append/delete/paging in `session/values.ts`, keep every write inside the existing `BEGIN IMMEDIATE` writer-lease transaction, update both fork snapshot paths, and retain all entry/usage/branch/lease behavior from current `dev`. +7. **Migrate public events, tests, and the coding-agent helper.** Remove old type assertions and raw namespaces. Change `fact_update` to `value_update`. Keep the two remote prompt tests skipped for the existing WP00 reason; WP01 must not alter runtime execution. +8. **Update telemetry and documentation.** Change `pi.session.write` item kinds from `register` to `value` and `list`, regenerate `telemetry-schema.md`, run the old-API sweeps, and record any branch-policy-deferred changelog requirement. Do not edit a changelog on `gramps` unless it becomes a pull-request branch or the user requests it. + +## Backend requirements + +### Memory + +- prepare and validate the complete transaction before mutating entries, values, lists, usage, or stats; +- current scalar replacement stores only the latest value and set `seq`; +- list append performs no list read and stores each global write `seq`; +- list delete removes the whole exact key; +- snapshots include current scalar values and surviving list elements. + +### JSONL + +- keep format 4/storage version 1 with no legacy register decode; +- one single-write object or multi-write array remains one atomic line; +- replay produces the same logical state as Memory; +- torn final lines expose no partial transaction; +- snapshot serialization preserves surviving list-element sequences and the next-sequence high-water mark. + +### SQLite + +Use: + +```sql +CREATE TABLE scalar_values ( + namespace TEXT NOT NULL, + key TEXT NOT NULL, + seq INTEGER NOT NULL, + value TEXT NOT NULL, + PRIMARY KEY (namespace, key) +) WITHOUT ROWID; + +CREATE TABLE list_values ( + namespace TEXT NOT NULL, + key TEXT NOT NULL, + seq INTEGER NOT NULL, + value TEXT NOT NULL, + PRIMARY KEY (namespace, key, seq) +) WITHOUT ROWID; +``` + +Ascending and descending list queries use the primary key with an exclusive sequence predicate and `LIMIT`. Add `EXPLAIN QUERY PLAN` assertions proving no table scan or temporary ordering b-tree. Do not change storage version, add migrations, or weaken the current lease/fence/fork behavior. + +## Required coverage + +### Address and type tests + +- invariant address typing and inferred scalar/list result types; +- `NoInfer` rejects incompatible set/append values; +- scalar helpers reject list addresses and list helpers reject scalar addresses; +- independently constructed equal addresses resolve the same location; +- empty key works; empty namespace and `\u0000` components reject; +- exact built-in namespaces/key grammars and exactly five prefix constructors; +- application-wide and dynamic non-reserved addresses require no second operation-time key; +- no registry, catalog, privilege constructor, global value map, or runtime `pi.*` gate. + +### Shared scalar/list conformance + +- scalar set/get/delete/delete-absent/recreate and latest set `seq`; +- namespace-scoped, key-ascending prefix scans; +- append one and several elements, including several appends in one transaction; +- appends separated by unrelated writes preserve per-list order and global element sequences; +- ascending/descending exclusive cursors; +- default, explicit, invalid, and clamped query-page limits; +- absent list, whole-list delete, delete-absent, and delete-then-append; +- atomic entry + usage + value + list transactions; +- rollback when any sibling write is invalid; +- no list read on append; +- close rejects later reads while already-admitted commits drain. + +Do not add a value/list collision test: cross-kind misuse is intentionally an unenforced trusted-programming defect. + +### Backend and repository tests + +- identical Memory/JSONL/SQLite pages and cursors; +- JSONL single/multi-write replay, torn-tail behavior, and sequence-preserving snapshot output; +- SQLite query plans and writer-lease transaction behavior; +- branch/tree forks copy session name, eligible labels, and lane configuration/Branch tip with fresh lane state; +- forks exclude operation/pending values, all lists, application addresses, last results, queues, and ledger rows; +- repository parent metadata, entry IDs, stats, branch indexes, v3 normalization, UUIDv7/follower IDs, and current SQLite lease/fork scenarios remain unchanged; +- runtime2 restore enumerates lanes through the one prefix constructor and reads no list. + +## Deferred consumers + +The following `values.md` requirements are explicitly not WP01 coverage: + +- assistant frame conversion, append scheduling, settlement, recovery, cancellation, snapshot hydration, and byte-growth tests (R2/R3/R6/R12); +- invocation `getMemo` / `setMemo`, tool-output checkpoint writes, outcome cleanup, and prefix-driven operation cleanup (R4/R6); +- any consumption-time list hydration beyond proving base restore reads no list; +- runtime acceptance, driving, provider/tool effects, or operation-state redesign. + +WP01 still exports `pendingAssistantFrames`, `operationToolMemo`, `pendingToolOutput`, and every cleanup prefix so later packages do not redesign storage. + +## Removal checks + +These must have zero matches in retained source/tests, excluding immutable released changelog history and archived prose: + +```bash +rg -n 'getRegister\(|listRegisters\(|RegisterValues|RegisterNamespace|RegisterSetWrite|\bRegister<' \ + packages/agent packages/session-backends/sqlite-node packages/coding-agent \ + --glob '!**/dist/**' --glob '!**/CHANGELOG.md' --glob '!**/docs/**' + +rg -n 'kind: "register"|getCustomFact\(|setCustomFact\(|fact\.(name|label|custom)|fact_update' \ + packages/agent packages/session-backends/sqlite-node packages/coding-agent \ + --glob '!**/dist/**' --glob '!**/CHANGELOG.md' --glob '!**/docs/**' + +rg -n '"(branch\.tip|lane\.(config|state|lastResult)|op\.(meta|state|tool_args|preparation)|pending\.entry)"' \ + packages/agent packages/session-backends/sqlite-node packages/coding-agent \ + --glob '!**/dist/**' --glob '!**/CHANGELOG.md' --glob '!**/docs/**' + +rg -n '\bregisters\b' packages/agent/src/harness/session packages/session-backends/sqlite-node/src \ + --glob '*.ts' --glob '*.sql' + +rg -n '"register"' \ + packages/agent/src/harness/telemetry.ts \ + packages/agent/test/harness/telemetry.test.ts \ + packages/agent/docs/telemetry-schema.md +``` + +Do not treat unrelated model/provider/hook registration terminology as storage API residue. + +## Validation + +Run each created or modified test file directly and iterate until green. At minimum: + +```bash +# From packages/agent +node "$(git rev-parse --show-toplevel)/node_modules/vitest/dist/cli.js" --run \ + test/harness/values.test.ts \ + test/harness/memory-storage.test.ts \ + test/harness/memory-conformance.test.ts \ + test/harness/memory-session-repo.test.ts \ + test/harness/jsonl-storage.test.ts \ + test/harness/jsonl-storage-conformance.test.ts \ + test/harness/jsonl-session-repo.test.ts \ + test/harness/jsonl-session-repo-conformance.test.ts \ + test/harness/storage-backed-session.test.ts \ + test/harness/session-tree.test.ts \ + test/harness/session-create-lane.test.ts \ + test/harness/instrumented-storage.test.ts \ + test/harness/types.test.ts \ + test/harness/telemetry.test.ts \ + test/harness/runtime2/harness.test.ts \ + test/harness/runtime2/lane.test.ts \ + test/harness/runtime2/restore.test.ts + +# From packages/session-backends/sqlite-node +node "$(git rev-parse --show-toplevel)/node_modules/vitest/dist/cli.js" --run \ + test/adapter.test.ts \ + test/repo.test.ts \ + test/sql.test.ts \ + test/storage.test.ts \ + test/storage-conformance.test.ts + +# From packages/coding-agent +node "$(git rev-parse --show-toplevel)/node_modules/vitest/dist/cli.js" --run \ + test/experimental-remote-runtime.test.ts \ + test/experimental-server-replacement.test.ts +``` + +Then run from the repository root: + +```bash +cd packages/agent && npm run check:telemetry-docs +cd "$(git rev-parse --show-toplevel)" +node_modules/.bin/tsgo --noEmit -p packages/agent/tsconfig.build.json +node_modules/.bin/tsgo --noEmit +git diff --check +npm run check +./test.sh +``` + +Never run unrestricted Vitest, `npm test`, paid-provider tests, or `npm run build`. + +## Stop condition + +Stop when every retained backend and Session surface uses bound values/lists; all core addresses use the exact `pi.*` grammar; arbitrary application addresses work but generic forks exclude them; old register/fact/custom-state APIs and physical names are absent; base restore performs no list read; schema/compatibility decisions above are implemented; focused, conformance, TypeScript, telemetry-doc, diff, and repository checks pass. Report the final schema and fork behavior. Do not begin runtime acceptance, assistant/tool consumers, or any later work package. diff --git a/packages/agent/docs/work-packages/02-atomic-run-acceptance.md b/packages/agent/docs/work-packages/02-atomic-run-acceptance.md new file mode 100644 index 000000000..83b1e4161 --- /dev/null +++ b/packages/agent/docs/work-packages/02-atomic-run-acceptance.md @@ -0,0 +1,542 @@ +# WP02 — Atomic acceptance and coherent lane observation + +## Status + +Complete. Phase A established minimal attachment, open-operation inventory, Session-line inspection/watch capture, commit-continuation recipient binding, and in-band identity-failure vocabulary. Phase B landed in `beac75ecc` with focused, monorepo-check, full-suite, and final Fable review passing. + +Implementation also updated downstream protocol/coding-agent wire projections required by the changed public contract. No execution owner, provider/tool effect, timer, retry, polling, cancellation procedure, manual action, or terminal settlement was added. + +## Goal + +Deliver two effect-free boundaries: + +```text +idle lane +→ atomic prompt/skill/template acceptance +→ durable open operation in payload-free starting + +open or running lane +→ Session-line watch capture +→ complete snapshot plus gap-free subsequent events +``` + +After `AgentHarness.create(options, context)` succeeds: + +- every configured lane has a complete small process-local projection; +- `open` inventories every durable current operation without predicting model/tool availability; +- attachment starts no hook, provider, tool, timer, breakpoint, drive owner, or application callback; +- `inspectExecution(context)` observes the small projection and local owner on the Session line; +- `watch(context)` registers buffering, clones live presentation, and performs bounded snapshot reads in one no-write lane job; +- every committing lane job publishes memory and synchronously binds its event batch in the commit continuation; +- the mutation does not await listener delivery, while the public operation does. + +WP02 does not implement drive, provider generation, hooks, tools, retries, deferred polling, cancellation procedures, manual action execution, or terminal settlement. + +## Decisions + +### 1. Attachment restores projection, not presentation + +Passing the open Session to `AgentHarness.create()` transfers orchestration ownership until create rejects or the harness closes. Direct `Session.mutate`, `Session.createLane`, reserved-address writes, and second-harness construction are prohibited during that interval, so lane inventory cannot race an out-of-band lane creation. + +Attachment reads only: + +- `branchTip`, `laneConfig`, `laneState`, optional `laneLastResult`; +- `operationMeta` and `operationState` for a current operation. + +It validates required existence, operation id/lane ownership, and intent/state kind compatibility. Projection corruption faults `create()`. + +Attachment does not read transcript, queues, pending writes, drained payloads, deferred sources, frames, tool calls, arguments, checkpoints, preparations, memos, or staged outcomes. Those references are checked by `watch()` or drive when consumed. Missing or contradictory required payload data is terminal storage corruption and faults that consumer. Optional frame/checkpoint absence is legal. + +### 2. Watch owns detailed snapshot reads + +One no-write Session mutation job defines the watch boundary: + +```text +enter after all earlier lane jobs +→ synchronously register buffering watcher +→ synchronously clone live presentation state +→ perform bounded durable reads while later lane jobs are excluded +→ assemble snapshot +→ release Session line +→ return handle +``` + +There is no special first-watch cache. The same path handles immediate post-attachment watch, reconnect, and watch during live execution. + +The bounded read set is: + +- one compaction-bounded branch scan from the current tip; +- exact `pendingEntry(id)` reads for next-run, steer, follow-up, writes, and abort drains; +- exact deferred source entry when represented; +- tools-phase assistant entry and exact args for represented `effect_pending` calls, using `batch.turnId` as the args step id; +- bounded exact assistant-frame pages and exact optional tool checkpoints. + +`ToolCall.sourceIndex` is the index in the assistant message's full content array, not a filtered tool-call ordinal. A represented call must index a tool-call block. + +### 3. Recipient binding belongs in the commit continuation + +A successful committing lane job performs: + +```text +commit +→ publish small owned projection +→ synchronously bind recipients and append the complete `{ event, context }` batch +→ return from the mutation without awaiting delivery +→ await delivery before the public operation resolves +``` + +WP02 initially implemented this with `enqueue()` plus caller-operated `start()`. WP04 replaces that gate with one immediate `emitBatch()` call in the same commit-observation continuation. Recipient binding and delivery awaiting remain unchanged. A listener or watcher registered after `emitBatch()` cannot receive that historical event. + +The only watcher/publication orders are: + +```text +watcher first +→ snapshot-before + complete buffered event batch + +publication/`emitBatch` first +→ snapshot-after + no old event +``` + +A live provider/tool presentation update follows the same synchronous publish-plus-`emitBatch` discipline. Frame/checkpoint commits are lane jobs and queue behind watch capture. + +### 4. Inspection is a no-write Session-line observation + +`inspectExecution(context)` observes: + +- lane and tip; +- configured model identity as unresolved `{ provider, modelId }` strings; +- current operation id/kind/start time; +- process status `running`, `open`, or durable `aborting`; +- captured model identity when the current durable phase contains one; +- optional latest result. + +It does not resolve model/tool registries and does not read transcript or presentation payloads. + +### 5. Missing implementations are in-band outcomes + +There is no `blocked`, missing-identity suspension, predictive classifier, or acceptance registry preflight. + +At the actual execution boundary: + +- unavailable captured model or configured active-tool definition before provider intent becomes a non-retryable configuration failure; +- pre-intent configuration failure reserves no response/usage ids and fabricates no assistant response or usage row; +- restored `effect_pending` settles uncertainty under its existing reserved ids before any later configuration failure; +- unavailable deferred model durably abandons redemption through configuration failure; R7 implements and tests restored `effect_pending` abandonment, including deletion of its exact old assistant-frame list while dropping the reserved response/usage strings without fabricating settlement; +- missing requested tool stages a direct `isError` `ToolResultMessage` and continues; +- missing/no-longer-safe replay implementation synthesizes interruption instead of waiting. + +Synthetic harness tool results omit `details`. A tool owns the type of its details contract; the harness must not invent `{}` or a diagnostic object. `isError` and human-readable content carry the tool-level diagnosis. Run-level configuration failure remains machine-readable through `OperationError` and `laneLastResult`. + +Stable configuration error codes: + +- `model_unavailable`, details `{ provider, modelId }`; +- `configured_tools_unavailable`, details `{ tools: string[] }`. + +`failure_drain` gains `{ kind: "configuration" }` provenance. Actual transitions land with their owning execution packages; WP02 lands the normative/source vocabulary only. + +### 6. Acceptance is independent of process registries + +Acceptance validates durable caller input and lane state, not current model/tool registrations. This avoids a time-of-check/time-of-use check and permits acceptance in one process followed by execution in another. + +A misconfigured convenience prompt eventually returns a durable failed run from drive. Explicit hosted acceptance remains durable even before an execution worker loads implementations. + +### 7. Invocation Context remains explicit + +Every current public harness/lane operation receives trailing `context: Context`. Acceptance, attachment, watch capture, Session reads/commit, faults, and event publication preserve it. Shared harness/lane/Session receivers retain no caller Context. Context and its signal/telemetry values are never durable business data. + +Buffered events retain the exact emitting Context. Invocation cancellation remains distinct from durable `requestAbort()`. + +## End-state public contract + +```ts +export interface ModelIdentity { + provider: string; + modelId: string; +} + +export type OperationStatus = "running" | "open" | "aborting"; + +export interface OpenOperation { + lane: string; + operationId: string; + kind: "run" | "compaction" | "navigation"; + startedAt: number; + aborting?: true; +} + +export interface CurrentOperationInfo { + id: string; + kind: "run" | "compaction" | "navigation"; + startedAt: number; + status: OperationStatus; + capturedModel?: ModelIdentity; +} + +export interface LaneExecutionInfo { + lane: string; + tipId: string | null; + configuredModel: ModelIdentity; + current: CurrentOperationInfo | null; + lastResult?: LaneLastResult; +} + +export interface LaneInfo { + name: string; + tipId: string | null; + operation: CurrentOperationInfo | null; +} + +export interface AgentHarnessConstructor { + create( + options: AgentHarnessOptions, + context: Context, + ): Promise<{ harness: AgentHarness; open: OpenOperation[] }>; +} +``` + +Rules: + +- `open` has exactly one item per durable current operation and omits idle lanes; +- `aborting:true` comes only from durable `cancel_requested`; +- `open` is inventory, not scheduling or identity advice; +- normal applications establish watch and call `resume(context)`; +- hosted schedulers retain expected-id `drive` fencing; +- configured/captured identity fields are durable strings and may not resolve. + +### Outcomes + +Delete `MissingIdentitySuspension`, `MissingIdentities`, missing-identity drive waiting, and missing-identity suspension events. + +Keep deferred suspension as provider semantics: + +```ts +{ kind: "suspended"; reason: "deferred"; ... } +``` + +WP05 removes the withdrawn action outcome before execution is enabled. Convenience operation outcomes remain operation-tagged branches of `ResumeOutcome`. + +### Snapshot + +```ts +export interface LaneSnapshot { + lane: string; + transcript: Entry[]; + tipId: string | null; + lastResult?: LaneLastResult; + operation: null | { + id: string; + kind: "run" | "compaction" | "navigation"; + startedAt: number; + status: OperationStatus; + action?: ActionInfo; + retry?: { attempt: number; maxAttempts: number; nextAttemptAt: number }; + deferred?: { handle: DeferredHandle; poll: number }; + drained?: { steer: QueuedItem[]; followUp: QueuedItem[] }; + streamingMessage?: AssistantMessage; + runningTools: { + toolCallId: string; + toolName: string; + args: unknown; + partialResult?: AgentToolResult; + }[]; + }; + queues: { steer: QueuedItem[]; followUp: QueuedItem[]; nextRun: QueuedItem[] }; + pendingWrites: { + entryId: string; + type: EntryType; + customType?: string; + message?: AgentMessage; + data?: JsonValue; + }[]; + faulted: boolean; +} +``` + +Configuration is not duplicated in snapshots. `inspectExecution()` exposes configured/captured model identities; getters expose current configuration. + +## Atomic run acceptance + +WP02 implements `accept()` for prompt, skill, and prompt-template requests. Compaction/navigation acceptance remains with their execution packages. + +State-independent normalization occurs before `Lane.command(plan, context)`: + +- prompt strings/images; +- supplied message or message array; +- explicit skill formatting; +- prompt-template formatting; +- pending assistant rejection; +- unknown skill/template errors; +- supplied or minted operation and prompt-entry ids. + +Public prompt convenience overloads remain exactly `[text, images | undefined, context]` and `[messageOrMessages, context]`, but convenience implementations remain `SliceNotImplemented` until R2. + +Inside one lane command: + +1. reject busy; +2. capture current `pendingNextRun` ids; +3. read and validate captured pending messages; +4. reject zero placed messages; +5. parent captured next-run entries before request prompt entries; +6. commit exactly once; +7. publish the small owned projection; +8. synchronously call `emitBatch` with the acceptance event batch and accepting Context; +9. return from the mutation callback without awaiting delivery; +10. await event delivery before `accept` resolves. + +Exact writes: + +```text +insert captured nextRun message entries +insert request prompt entries +delete captured pendingEntry values +set branchTip +set operationMeta +set operationState(run starting) +set laneState(current operation, pendingNextRun=[]) +``` + +Exact event order: + +```text +run_start +for each placed message: + message_start + message_end + entry_added +queue_update if nextRun was captured +``` + +Acceptance starts no drive or effect and writes no Context. + +## Phase A — normative rewrite + +Update `harness.md` before runtime source: + +- replace eager attachment hydration with minimal projection restore; +- replace predictive status/classifier with open inventory; +- specify configured/captured identity inspection without resolution; +- move detailed reads to ad-hoc Session-line watch capture; +- require recipient binding in the exact commit-observation continuation; +- remove identity preflight/suspension/error/event types; +- specify in-band model/tool unavailability and configuration provenance; +- require direct detail-free synthetic tool-result messages; +- update invariants, races, roadmap, glossary, and Appendix C. + +Review stop: + +1. `git diff --check`; +2. fresh Terra contradiction/source-feasibility audit; +3. full-context Fable review against complete docs and source; +4. resolve all findings and repeat until no findings; +5. obtain user approval before runtime source. + +## Phase B — implementation + +### Public and durable types + +Modify `agent-harness.ts`: + +- remove `SuspendedOperation`, `MissingIdentityInfo`, `MissingIdentities`, and missing-identity outcome/event branches; +- add `ModelIdentity`, `OperationStatus`, `OpenOperation`, and corrected inspection/snapshot types; +- change create result from `suspended` to `open`; +- land operation-tagged action-required outcome families; +- correct `executeAction`/`runToCompletion` signatures; +- retain trailing Context everywhere. + +Modify session types: + +- add `failure_drain` configuration provenance; +- add the already-normative `ToolCall.outcome_ready` vocabulary without producers; +- document `sourceIndex` as full assistant-content index; +- add callback-scoped `scanBranch` to `SessionReader`; inheritance by `SessionMutator` and `Session` is intentional; +- implement it in `StorageBackedSession` and its mutator and `MemorySessionFacade`; Session mutation authority remains process-local and has no remote Session facade. + +### Event publication + +WP02 initially added synchronous recipient binding with a reserved delivery gate. WP04 supersedes only that mechanism: + +- `HarnessEventBus.emitBatch()` snapshots ordinary and watcher recipients synchronously; +- delivery uses only that bound list; +- watcher buffers retain `{ event, context }`; +- a watcher registered after `emitBatch` receives nothing from that event. + +`LaneCommand` commit decisions retain a synchronous post-commit event batch. After commit succeeds, `Lane.command` publishes `next`, calls `emitBatch` as its final mutation action, and carries the delivery promise outside `Session.mutate` before awaiting it. Every existing event-producing commit uses this path, including direct idle/pending appends, lane configuration setters, and session-name/entry-label setters. + +WP04 also moves harness lane publication into `Session.createLane`'s committed-publication callback. Session commits, Harness publishes `lanesByName` and calls `emitBatch(lane_created)` in that continuation, and Session awaits the retained delivery promise after releasing the line. + +Do not execute listeners on the line. + +### Attachment + +Keep `restore.ts` projection-only. Validate lane/operation ownership and kind compatibility. Remove `describeSuspension` and all payload hydration from `createAgentHarness`. Construct lanes and return `open` inventory without resolving registries. + +### Inspection + +Implement `inspectExecution(context)` as a no-write `Lane.command` observation. Derive captured model identity from the current durable phase. Read no storage payloads and resolve no registry identities. + +### Watch + +Implement `watch(context)` as one no-write lane job: + +- register watcher and clone live presentation synchronously; +- perform the bounded read matrix above through callback-scoped readers; +- assemble isolated snapshot payloads; +- fault and unsubscribe on required corruption; +- retain legal optional absence; +- release the line before returning. + +A focused internal snapshot helper is allowed. Do not create a persistent hydrated presentation cache or generic reducer. + +### Out of scope + +`drive`, `resume`, prompt convenience, compaction/navigation acceptance, abort/queues, `executeAction`, and `runToCompletion` remain `SliceNotImplemented` where not already implemented. + +Provider/tool/configuration-failure transitions are specified now but implemented by R2/R3/R4/R7/R8. WP02 adds no effect, active operation, timer, hook, provider request, tool execution, retry, deferred fetch, cancellation reconciliation, or terminal transaction. + +## Required tests + +### Public types + +- `SuspendedOperation`, `MissingIdentities`, and missing-identity status/outcomes/events are absent; +- open/current/status unions narrow exhaustively; +- configured and captured model identities are unresolved strings; +- action-required outcome families and method signatures match; +- prompt overload tuples remain exact; +- `AgentHarnessOptions` has no receiver telemetry default. + +### Acceptance + +- text-only, images-only, text-plus-images, supplied arrays; +- pending assistant rejection; +- skills/templates and unknown resources; +- empty input writes nothing unless captured nextRun supplies input; +- no identity registry preflight; +- supplied/minted operation ids; +- exact writes, parent chain, metadata, starting state, settings, commit materialization; +- pending-next-run capture/deletion; +- busy run/structural operation; +- one concurrent accept winner; +- commit failure and close races; +- exact event order and object-identical accepting Context; +- no hook, provider, tool, timer, drive-owner, or option callback invocation; passive event-listener delivery remains required. + +### Attachment and inspection + +- idle omitted and every open operation inventoried; +- durable cancellation sets only `aborting:true`; +- no transcript/pending/frame/tool reads at create; +- configured and captured model identities may differ and remain visible when unresolved; +- inspection runs on the Session line and performs no payload reads; +- projection corruption faults create; +- no option callback/effect starts. + +### Event publication and watch + +- recipient set binds at `emitBatch`: a watcher registered after publication but before delivery receives nothing; +- state/event batch publication happens before Session-line release; +- direct append, lane configuration, session-name/entry-label, acceptance, and lane-creation commits all use that publication path; +- `value_update` and `lane_created` bind recipients in their commit continuation, so later listeners receive neither historical event; +- watcher-first gives snapshot-before plus complete events; +- publication-first gives snapshot-after without old events; +- live update during awaited capture appears only as a buffered event after the cloned live snapshot; +- frame/checkpoint mutations queue behind capture; +- first watch and reconnect use the same path; +- exact transcript, queues, writes, drain, deferred, frame, tool args/checkpoint fields; +- required payload corruption faults watch and removes watcher; +- absent frames/checkpoints omit optional partials; +- pre-registration lifecycle is not replayed; +- buffered events retain object-identical emitting Context; +- payload mutation cannot affect later state/listeners; +- close/fault lifecycle matches the event bus. + +### In-band identity vocabulary + +Type/direct-state tests prove: + +- acceptance has no `MissingIdentities` path; +- configuration failure provenance and stable error codes are representable; +- deferred configuration abandonment remains assigned to R7 rather than adding an execution transition here; +- missing-tool synthetic `ToolResultMessage` may omit `details`; +- sourceIndex uses full assistant-content indexing; +- outcome-ready calls are not rendered as running. + +No execution transition is added in WP02. + +## Files + +### Add + +- `packages/agent/test/harness/runtime2/accept.test.ts` +- focused watch tests if existing files become oversized. + +### Modify + +- `packages/agent/docs/harness.md` +- `packages/agent/docs/work-packages/02-atomic-run-acceptance.md` +- `packages/agent/src/harness/agent-harness.ts` +- `packages/agent/src/harness/events.ts` +- `packages/agent/src/harness/session/types.ts` +- `packages/agent/src/harness/session/session.ts` +- `packages/agent/src/harness/session/memory.ts` +- `packages/agent/src/harness/session/remote.ts` +- `packages/server/src/remote-session-manager.ts` +- `packages/agent/src/harness/runtime2/harness.ts` +- `packages/agent/src/harness/runtime2/lane.ts` +- `packages/agent/src/harness/runtime2/restore.ts` +- `packages/agent/src/harness/runtime2/types.ts` +- `packages/agent/test/harness/runtime2/harness.test.ts` +- `packages/agent/test/harness/runtime2/lane.test.ts` +- `packages/agent/test/harness/runtime2/restore.test.ts` +- `packages/agent/test/harness/types.test.ts` +- `packages/agent/test/harness/storage-backed-session.test.ts` +- `packages/agent/test/harness/memory-session-repo.test.ts` +- `packages/server/test/conformance.test.ts` +- event/session test files required by recipient binding and callback-scoped branch reads. + +No backend schema, telemetry schema, coding-agent, or changelog change is expected. Stop for boundary review if one becomes necessary. On `dev`, defer changelog entries. + +## Validation + +After Phase A: + +```bash +git diff --check -- \ + packages/agent/docs/harness.md \ + packages/agent/docs/work-packages/02-atomic-run-acceptance.md +``` + +After Phase B: + +```bash +cd packages/agent +node "$(git rev-parse --show-toplevel)/node_modules/vitest/dist/cli.js" --run \ + test/harness/runtime2/accept.test.ts \ + test/harness/runtime2/harness.test.ts \ + test/harness/runtime2/lane.test.ts \ + test/harness/runtime2/restore.test.ts \ + test/harness/types.test.ts + +cd "$(git rev-parse --show-toplevel)" +git diff --check +npm run check +./test.sh +``` + +Report runtime2 source line counts. The synchronized pre-WP02 runtime2 baseline is 967 lines; treat growth above 1,900 source lines as a design review trigger, not a target. + +## Stop condition + +Stop when: + +- acceptance commits exactly once into payload-free `starting` without registry preflight; +- attachment returns minimal complete projections and open inventory; +- inspection is a coherent Session-line no-write observation; +- watch captures detailed state ad hoc on the Session line; +- state publication and `emitBatch` recipient binding occur in the commit continuation while the mutation never awaits delivery; +- snapshots and buffered events have no gap or duplicate; +- required payload corruption faults its consumer; +- no execution effect or owner is introduced; +- focused tests, `npm run check`, and full tests pass; +- final Fable review has no findings. + +Do not begin the first real drive package. diff --git a/packages/agent/docs/work-packages/03-remove-drive-deadlines.md b/packages/agent/docs/work-packages/03-remove-drive-deadlines.md new file mode 100644 index 000000000..e584103f3 --- /dev/null +++ b/packages/agent/docs/work-packages/03-remove-drive-deadlines.md @@ -0,0 +1,183 @@ +# WP03 — Remove drive deadlines + +## Status + +Complete. `DriveOptions.deadline` and the `DriveOutcome` `yielded` branch are removed from public types, active documentation, future package requirements, invariants, races, and tests. Obsolete `runtime2.md` is deleted. Focused tests, `npm run check`, and `./test.sh` pass. + +WP02 is complete at `beac75ecc`. Preserve unrelated concurrent source work, especially current `packages/agent/src/harness/runtime2/lane.ts` changes, JSONL/fork work, plugins, RPC, and experimental directories. + +## Problem + +`DriveOptions.deadline` and the corresponding `DriveOutcome { kind: "yielded" }` do not provide a correctness boundary. + +A deadline is checked only before starting another transition or effect. An admitted provider/tool/hook may run beyond it and the host may terminate the process anyway: + +```text +check deadline +→ admit provider or tool +→ host limit expires while the effect is running +→ process dies with durable effect_pending +``` + +Unknown-outcome recovery remains mandatory. The deadline therefore does not prevent process loss, bound admitted work, make effects exactly once, or simplify recovery. Flue-style tool memoization depends on stable invocation ids and durable memos, not drive deadlines. + +Deadline handling instead adds wall-clock policy to the durable core: + +- a `yielded` public outcome unrelated to durable state; +- safe-boundary checks before hooks/effects/transitions; +- deadline-versus-retry-timer arbitration; +- deadline-versus-effect-admission races; +- convenience loops and event-bracket behavior for yields. + +The host already owns scheduling and termination. Process loss is a controlled crash boundary recovered from durable operation state. + +## Decision + +Remove drive deadlines completely: + +```ts +interface DriveOptions { + operationId: string; + waitForRetry?: boolean; + pollDeferred?: boolean; +} + +type DriveOutcome = + | { kind: "settled"; operationId: string; outcome: TerminalOperationOutcome } + | { kind: "waiting"; operationId: string; reason: "retry"; notBefore: number } + | { kind: "waiting"; operationId: string; reason: "deferred"; deferred: DeferredHandle } + | { kind: "action_required"; operationId: string; action: ActionInfo }; +``` + +There is no deprecated alias, ignored `deadline` field, compatibility overload, alternate timestamp option, or replacement pause flag in WP03. + +The next drive package uses direct durable transitions. Deterministic tests gate commits and control hooks, providers, tools, and timers without adding production execution barriers. + +## Host behavior + +Hosts own execution budgets: + +```text +invoke drive +→ terminal or durable waiting result: schedule normally +→ planned shutdown: stop routing/releasing work and close session processes +→ forced termination: replacement attaches and recovers durable open operations +``` + +A process-per-session host may stop routing work and close before exit, and use process termination as the hard fence for non-cooperative providers, tools, hooks, storage, or event listeners. This operational policy does not require a wall-clock field in `DriveOptions`. + +WP03 does not add `stopAfterCheckpoint`, `pause`, `quiesce`, or in-process crash simulation. Those ideas remain outside the direct durable-drive design. In-process process-loss simulation is not a public core primitive: an old continuation requires fencing, for which process isolation is the reliable mechanism. + +## Work + +### Public types + +In `packages/agent/src/harness/agent-harness.ts`: + +- delete `DriveOptions.deadline`; +- delete the `DriveOutcome` `yielded` branch; +- preserve expected-id fencing, retry waiting, deferred waiting, and manual action branches unchanged. + +No current runtime2 drive implementation exists, so this package adds no execution behavior or owner. + +### Normative documentation + +Update `packages/agent/docs/harness.md` completely: + +- remove safe-yield scheduling language from non-goals/orientation; +- remove only the deadline half of cancellation/deadline prerequisite wording in §3.6, §5.6's `before_drive` row, and invariant 22; the cancellation prerequisite remains; +- remove deadline checks and yielded returns from the drive-pass pseudocode; +- remove deadline policy from pass joining, retry waiting, convenience composition, recovery, and public method prose; +- remove `deadline` from `DriveOptions` and `yielded` from `DriveOutcome`; +- remove deadline-specific event/turn requirements; +- remove deadline language from `before_drive` hook timing; +- remove deadline/yield requirements from future work rows; +- replace invariant 25 with the explicit no-wall-clock-policy invariant while retaining the rule that an admitted effect settles normally or is recovered after task loss; +- remove deadline races from the race catalog; +- update the drive-pass glossary. + +Do not alter generic RPC timeout/deadline documentation or unrelated process/model-catalog timeouts. Those are invocation/transport policies, not `AgentLane.drive`. + +Delete obsolete `packages/agent/docs/runtime2.md`; `harness.md` plus linked handoffs are the sole implementation plan and history needed for active work. + +Mark WP02 complete in its handoff and Part 8. Add WP03 as the concrete cleanup package. Leave the former R2/R3 drive rows as historical future candidates, minus deadline/yield requirements, until the reviewed direct-drive handoff replaces them. + +### Delete + +- `packages/agent/docs/runtime2.md` + +### Type tests + +Extend `packages/agent/test/harness/types.test.ts`: + +- assert `keyof DriveOptions` is exactly `"operationId" | "waitForRetry" | "pollDeferred"`; +- assert `DriveOutcome["kind"]` excludes `"yielded"`; +- add `@ts-expect-error` coverage proving callers cannot supply `deadline`; +- retain existing drive/result signatures. + +Run the focused type test after editing it. + +### Downstream compatibility + +Search protocol, server, coding-agent, examples, and tests for structural mirrors or exhaustive `DriveOutcome` switches. Update only compile-time/type compatibility forced by the public removal. Coding-agent experimental worker/remote-runtime behavior and tests are out of scope. + +The current protocol harness schemas expose prompt/run/watch DTOs, not `DriveOptions` or `DriveOutcome`; no protocol edit is expected unless the final search proves otherwise. + +## Non-goals + +WP03 does not implement or redesign: + +- `drive`, `resume`, prompt conveniences, operation ownership, or latest-result lookup; +- manual actions or automatic barrier release; +- provider/tool execution, recovery, retry timers, deferred polling, abort, or terminal settlement; +- checkpoint pause/quiesce; +- close admission changes; +- worker/RPC cancellation or experimental remote prompting; +- storage/schema/migration behavior. + +## Required checks + +```bash +# No drive deadline/yield contract remains in active harness docs or source. +rg -n 'deadline|yield' \ + packages/agent/src/harness \ + packages/agent/test/harness \ + packages/agent/docs/harness.md \ + packages/agent/docs/work-packages + +cd packages/agent +node "$(git rev-parse --show-toplevel)/node_modules/vitest/dist/cli.js" --run \ + test/harness/types.test.ts + +cd "$(git rev-parse --show-toplevel)" +git diff --check +npm run check +./test.sh +``` + +The removal grep may still match this handoff's historical problem explanation. Every remaining match must be reviewed; no active API, normative behavior, future acceptance criterion, or test expectation may retain the removed contract. + +## Review + +Before implementation: + +1. Fable reviews this handoff against complete docs/source. +2. As explicitly requested by the user, `openai-codex/gpt-5.6-sol` reviews it with thinking level high. +3. Resolve all findings and repeat until no findings. + +After implementation, repeat both reviews over the final documentation/type diff. + +## Stop condition + +Stop when: + +- `DriveOptions` has no wall-clock budget; +- `DriveOutcome` has no `yielded` branch; +- active normative docs contain no deadline/yield behavior; +- future drive rows contain no hidden deadline requirements; +- type tests prove removal; +- unrelated timeout/deadline APIs remain untouched; +- focused tests, `npm run check`, and `./test.sh` pass; +- final Fable and Codex reviews have no findings. + +Do not begin the direct durable-drive implementation in this package. diff --git a/packages/agent/docs/work-packages/04-mutation-publication.md b/packages/agent/docs/work-packages/04-mutation-publication.md new file mode 100644 index 000000000..6cfc6bfba --- /dev/null +++ b/packages/agent/docs/work-packages/04-mutation-publication.md @@ -0,0 +1,337 @@ +# WP04 — Mutation publication and event delivery + +## Status + +Complete. Phase A and the final implementation rereview passed Fable with no findings. Focused agent/server/SQLite tests, `npm run build`, `npm run check`, and `./test.sh` pass. `packages/agent/docs/harness.md` remains normative. + +> Historical note: WP06 later replaced the lane-creation API and keyed line described in this completed handoff with atomic `AgentHarness.lane()` acquisition on one Session line. The event-publication guarantees remain current. + +WP02 established atomic acceptance, recipient binding, and coherent lane watches. WP03 removed drive deadlines. WP04 removes the caller-operated event-delivery gate without weakening those guarantees and makes the historical `Session.createLane()` own lane creation end to end. The direct durable-drive package follows as WP05. + +## Problem + +A committing lane job currently uses a two-part event API: + +```text +inside Session.mutate: + commit + publish process-local state + delivery = events.enqueue(batch, context) + +outside Session.mutate: + await delivery.start() +``` + +The split preserves the correct boundary, but it is a footgun: calling `emit()` too early, calling `start()` too early, or dropping `start()` can violate observation semantics or stall the global event tail. The historical `Harness.createLane()` repeats the choreography manually. + +Lane creation has a second one-off boundary. The historical `Session.createLane()` owns validation and the durable transaction, but Harness cannot publish its process-local `Lane` and bind `lane_created` recipients from that same commit continuation. Harness therefore opens `Session.mutate()` itself and calls exported `createLaneWithMutator()`. + +WP04 removes both caller-operated seams while preserving current direct-listener and hook barriers. + +## Required semantics + +### Direct events remain awaited observations + +Direct `events.on()` listeners remain passive but causally ordered: + +```text +hook or preparation +→ commit +→ publish process-local state +→ bind and append event batch +→ release lane mutation line +→ await direct listeners +→ resolve operation +→ later hook or transition +``` + +Passive means a listener cannot transform the in-flight operation and listener failures are isolated as `handler_error`. It does not mean fire-and-forget. An extension may update process-local state in an event listener and inspect it from a later hook. Awaiting also supplies producer backpressure. + +A direct listener must not call a state-mutating harness API: an emitted mutation would queue a later event behind the event currently awaiting that listener. Read-only lane calls remain legal. + +Deliberate exceptions remain unchanged: + +- watchers and RPC/watch consumers use their own buffered FIFO and are not awaited by operations; +- fault publication is fire-and-forget, and close never waits for listener completion; +- high-frequency tool updates enqueue every event and retain only the latest delivery promise; tool settlement awaits that promise before `after_tool` and outcome publication, which drains all earlier updates through global FIFO without per-update backpressure. + +### Commit and recipient binding remain one continuation + +Every event-producing committing lane job performs, in the exact continuation that observes successful commit: + +```text +commit succeeds +→ publish complete process-local state +→ synchronously call emitBatch(batch, context) + - clone payloads + - bind current ordinary listeners and watchers + - append the complete batch to the global delivery tail +→ return from the mutation callback +``` + +There is no scheduler-owned after-release publication phase. Moving recipient binding into a later promise continuation creates a gap in which another task may observe committed state and register a listener before the old event binds recipients. + +`emitBatch()` starts asynchronous delivery immediately and returns its completion promise. The mutation callback never awaits that promise. Listener code may begin after publication/binding but before the mutation line technically releases; a reentrant lane read queues behind the current job. The command carries the promise out of `Session.mutate()` and awaits it before resolving publicly. + +This preserves the two legal watcher races: + +```text +watcher registration first +→ snapshot-before + complete buffered batch + +commit publication first +→ snapshot-after + no old event +``` + +### Ordering + +- One non-empty `emitBatch()` call publishes one contiguous batch. +- Batches enter the existing global event tail in `emitBatch()` invocation order, including across lanes. +- Events within a batch retain source order. +- Direct listeners run serially in registration order. +- Same-lane committing jobs bind batches in lane mutation order. +- A lane procedure awaits each command's delivery before invoking its next hook or transition. +- Hooks on unrelated lanes may overlap; there is no total cross-lane hook order. +- Context remains the exact emitting invocation Context and is always the final parameter. + +## Event bus contract + +Replace the public/internal delivery split with: + +```ts +class HarnessEventBus implements Events { + emit(event: HarnessEvent, context: Context): Promise; + emitBatch(events: readonly HarnessEvent[], context: Context): Promise; +} +``` + +`emitBatch()`: + +1. returns an already-resolved promise for an empty batch or a closed bus; +2. synchronously structured-clones every payload and binds its current recipients; +3. appends one contiguous delivery to the existing global tail; +4. delivers cloned payloads serially with the emitting Context; +5. isolates listener failures and publishes non-recursive `handler_error` as today; +6. returns a promise that resolves after eligible direct listeners settle and never rejects because a listener failed. + +Synchronous publication defects such as an uncloneable internal payload still throw in the caller's commit continuation and follow the existing harness-fault path. + +Delete: + +- `HarnessEventDelivery`; +- `HarnessEventBus.enqueue()`; +- caller-operated `start()`; +- delivery gates and `pendingStarts`; +- close-time forced gate release. + +`close(error)` seals listener/watch registration and future publication immediately. Already appended batches retain their bound recipients and drain through the existing tail; listener completion still does not block Harness close. + +## Lane command integration + +The commit branch of `Lane.command()` returns a plain internal outcome containing the caller result and optional delivery promise: + +```ts +const events = decision.events?.(commit) ?? []; +const delivery = events.length === 0 + ? undefined + : this.onEvent(events, context); + +return { + kind: "return", + result, + ...(delivery === undefined ? {} : { delivery }), +}; +``` + +`onEvent` returns `Promise` and delegates to `emitBatch()`. It is called only after commit, complete process-local state publication, and synchronous result materialization, as the final action before returning the mutation outcome. + +After `Session.mutate()` returns: + +```ts +if (outcome.kind === "reject") throw outcome.error; +await outcome.delivery; +return outcome.result; +``` + +Expected no-commit rejections publish no events. Commit/materialization/publication errors retain the existing harness-fault semantics. + +## Session lane creation contract + +The historical `Session.createLane()` owns validation, commit, and the synchronous committed-publication callback. Context remains last: + +```ts +createLane( + name: string, + at: string | null, + configuration: LaneConfiguration, + onCommitted: ((context: Context) => void | Promise) | undefined, + context: Context, +): Promise; +``` + +The implementation performs: + +```text +enter the prospective lane's mutation line +→ validate name, absence, complete lane shape, and target +→ commit lane configuration + leaf + idle lane state +→ synchronously invoke onCommitted(context) in that same commit continuation +→ retain its returned promise inside a non-thenable outcome object +→ return from the mutation callback and release the line +→ await the retained promise +→ return Session.view(name) +``` + +The callback receives neither `SessionTree` nor `SessionMutator`. It is only the process-local publication point. Its synchronous prefix must complete the publication needed before another same-lane job can run. Ordinary Session callers pass `undefined`. + +If validation or commit fails, the callback is not invoked. If the callback throws after commit or its retained promise rejects after line release, the durable lane remains and the caller rejects; Harness converts that committed-publication defect to its existing fault path. A promise returned by the Harness callback is event delivery, whose listener failures are isolated by the bus. + +The current lane validation/transaction implementation becomes private to Session. Delete exported `createLaneWithMutator()` and its direct tests. + +Harness preconstructs a detached `Lane`, then calls Session: + +```ts +const lane = this.buildLane(name, state); + +await this.session.createLane( + name, + at, + this.seed, + (context) => { + if (this.closedError !== undefined) lane.seal(this.closedError); + this.lanesByName.set(name, lane); + return this.events.emitBatch( + [{ type: "lane_created", lane: name, at }], + context, + ); + }, + context, +); + +return Result.ok(lane); +``` + +The callback's synchronous prefix publishes `lanesByName` and binds `lane_created` before the first creation job releases its line. A queued duplicate therefore cannot report `LaneExists` before the winner is visible through `harness.lane(name)`. + +If close or fault wins while commit is admitted, the callback publishes the new Lane sealed. Emission on the already-closed bus is a resolved no-op, matching the existing admitted-creation race. The successful admitted creation still returns its sealed Lane. + +## Scope + +### Source + +Modify: + +- `packages/agent/src/harness/events.ts`; +- `packages/agent/src/harness/runtime2/lane.ts`; +- `packages/agent/src/harness/runtime2/harness.ts`; +- `packages/agent/src/harness/session/types.ts`; +- local Session implementations and tests required by the historical `createLane` signature; +- direct event primitive, acceptance, watch, lane, and harness tests. + +Remote/experimental runtime behavior is not a design constraint for WP04. Session mutation authority remains process-local; do not add remote Session callback transport, protocol machinery, compatibility abstractions, or boundary tests. + +### Documentation + +Update normative `harness.md`: + +- replace enqueue/start language with synchronous `emitBatch` binding and post-mutation awaiting; +- state accurately that listener execution may begin after publication/binding but before technical line release; +- retain awaited direct-listener, event/hook, watcher, Context, close, and ordering semantics; +- replace shared exported mutator-procedure lane creation with the historical `Session.createLane(onCommitted, context)`; +- update invariants, races, tests, glossary, and Part 8; +- link WP04 and move direct durable drive to WP05. + +Update the historical WP02 handoff only where it points forward or claims the old mechanism remains current. Do not rewrite its completed-package record as though WP04 behavior landed in WP02. + +## Non-goals + +- fire-and-forget direct events or a public flush API; +- sequence/watermark delivery redesign; +- per-extension event queues; +- changing hook aggregation or event/hook causal barriers; +- changing watcher/RPC buffering; +- changing tool-update settlement barriers; +- making event-listener mutation safe; +- drive, breakpoints, providers, tools, recovery, retries, polling, abort, or terminal settlement; +- generic Session post-commit/after-release task APIs; +- remote callback execution or remote-runtime redesign. + +## Required tests + +### Event primitives + +- `emitBatch` binds ordinary listeners and watchers synchronously; +- a listener registered after `emitBatch` but before delayed delivery receives nothing; +- a complete batch is contiguous and preserves event order; +- concurrent batch publication preserves invocation order; +- listener payload mutation remains isolated; +- listener rejection emits one non-recursive `handler_error` and does not reject delivery; +- empty and post-close batches resolve without delivery; +- already appended batches drain after close; +- no gate/start tests remain. + +### Lane commands and watch + +- direct listeners may perform reentrant read-only lane inspection without deadlock; +- a command does not resolve until its direct listeners finish; +- committed memory is visible to listeners; +- commit failure publishes nothing; +- a late listener that observes durable state receives no historical event; +- watcher-first yields snapshot-before plus the complete buffered batch; +- publication-first yields snapshot-after without replay; +- source Context identity survives delayed and buffered delivery. + +### Lane creation + +- Session callback runs exactly once after successful commit and never on validation/commit failure; +- callback synchronous publication occurs before a queued duplicate reports `LaneExists`; +- Harness Lane and durable configuration are visible to `lane_created` listeners; +- Harness waits for asynchronous `lane_created` listeners before resolving; +- admitted creation racing close publishes a sealed Lane and does not require event delivery; +- callback throw and retained-promise rejection after commit follow the documented committed-publication failure path; +- ordinary Session creation with `undefined` returns its view; +- no exported `createLaneWithMutator` remains. + +### Ordering barriers + +- acceptance events finish before `accept()` resolves; +- command resolution cannot overtake its awaited direct event delivery; WP05 tests the first subsequent procedure hook; +- event batches are globally ordered across concurrent lane publication; +- tool-update latest-delivery settlement behavior remains unchanged where implemented. + +## Validation + +After documentation: + +```bash +git diff --check -- \ + packages/agent/docs/harness.md \ + packages/agent/docs/work-packages/02-atomic-run-acceptance.md \ + packages/agent/docs/work-packages/04-mutation-publication.md \ + packages/agent/docs/work-packages/05-direct-durable-drive.md +``` + +After implementation, run every modified focused test, then: + +```bash +git diff --check +npm run check +./test.sh +``` + +Review the final implementation with Fable before declaring WP04 complete. Do not commit without explicit user approval. + +## Stop condition + +WP04 is complete when: + +- event publication has one `emitBatch()` operation and no caller-operated gate; +- recipient binding remains in the exact commit-observation continuation; +- direct event delivery remains globally FIFO and awaited before public operation resolution; +- existing event/hook causal barriers remain intact; +- lane watches retain exactly the two coherent race outcomes; +- Session owns lane creation and invokes Harness publication before releasing the creation job; +- `createLaneWithMutator()` is gone; +- Context is trailing and source-identical throughout; +- focused tests, `npm run check`, and `./test.sh` pass; +- final Fable review reports no findings. diff --git a/packages/agent/docs/work-packages/05-direct-durable-drive.md b/packages/agent/docs/work-packages/05-direct-durable-drive.md new file mode 100644 index 000000000..b1deaffa7 --- /dev/null +++ b/packages/agent/docs/work-packages/05-direct-durable-drive.md @@ -0,0 +1,606 @@ +# WP05 — Direct durable drive + +**Status: WP05 complete through M10: lane-owned inbox, immutable result records, 13 family-neutral leaves, atomic boundary planning, total cancellation/dispatch, public and replicated lane surfaces, documentation reconciliation, and lane-safe provider cache identity. Remaining assistant-output work is owned by the [mobile assistant-output handoff](../mobile-handoff/01-harness/05-assistant-output/message-update.md), outside the public-drive gate.** + +WP06's Session/Branch/Lane separation is part of the foundation. Public drive is enabled; `watchSession` is the sole deferred Harness method. + +Format 4 remains work in progress. Every durable type replacement in this package requires no migration and no compatibility decoder for the pre-redesign shapes. + +The standalone-compaction inbox/promotion design previously in this document is **withdrawn** (§5). R1–R3 and M7/M8 replace it. M9 reconciled `docs/harness.md`, which is normative on its own again. + +## 0. Mandatory reading + +Read completely before implementation work: + +1. `packages/agent/docs/harness.md` (normative; reconciled by M9) +2. `packages/agent/docs/runtime-simplification.md` (its 22-leaf list is replaced by R3) +3. `packages/agent/src/harness/session/types.ts` +4. `packages/agent/src/harness/session/values.ts` +5. `packages/agent/src/harness/runtime/lane.ts` +6. `packages/agent/src/harness/runtime/types.ts` +7. every existing `packages/agent/src/harness/runtime/drive/*.ts` +8. `packages/agent/src/harness/runtime/progress.ts` +9. `packages/agent/src/harness/runtime/restore.ts` +10. `packages/agent/src/harness/execution/{effect-gate,assistant,tools}.ts` +11. relevant focused runtime tests + +Do not inspect Git history or removed runtime implementations. The current source and these documents are the only implementation inputs. + +## 1. Runtime model (target) + +### Lane authority + +`Lane.state` is authoritative while a Harness owns its Session. It contains all orchestration state required to dispatch: + +- tip; +- lane configuration; +- the lane inbox: tagged queued-input ids (R1); +- last operation id (R2); +- operation metadata; +- flat operation state, including control. + +Every supported mutation commits on the one Session mutation line and publishes the matching `Lane.state` before releasing the line. Drive procedures never reread `laneState`, `operationMeta`, `operationState`, `branchTip`, `laneConfig`, or `operationResult` from storage. + +`SessionReader` is used only to dereference content named by in-memory state and to enumerate operation-owned cleanup addresses: + +- tree entries and branch context; +- pending entry payloads; +- assistant frame lists; +- tool arguments, checkpoints, and memos; +- structural preparations; +- staged tool outcomes; +- cleanup-prefix scans. + +### One lane-owned inbox (R1) + +Queued input is lane-owned, never operation-owned. `LaneState` (durable and process-local) carries one ordered inbox of `{ entryId, kind }` items with `kind: "steer" | "followUp" | "nextRun" | "write"`. Payload staging is unchanged: enqueue mints the entry id and writes `pendingEntry(id)`; the inbox holds ids only. Item lifecycle is admission → consumption | cancellation; terminal cleanup never touches queue payloads. + +Tags are consumption-eligibility markers, not ownership. Enqueue always succeeds — during runs, structural operations, cancellation, and idle. Every drain point selects eligible items per tag, applying queue modes during selection, but places all selected items in the inbox's single global admission order. Tag grouping must never reorder user input. At acceptance, request prompt entries follow the selected inbox items because the request is the newest admission. + +| Drain point | Eligibility and decision order | +| --- | --- | +| acceptance (idle lane) | write + nextRun (all), steer (`steeringMode`: all or oldest), and followUp (`followUpMode`) are eligible. Place selected items in global admission order, then request prompt entries. The acceptance transaction places them and removes their ids; `starting` drains nothing. followUp is eligible because an idle lane vacuously satisfies its "after current work" condition. | +| turn-end boundary pass (run) | write + steer are eligible before threshold/continuation planning and are placed in global admission order. followUp becomes eligible only at `may_finish`, before `before_run_end` and finish. nextRun is never eligible mid-run and never blocks finish. | +| idle direct append | queued write items are placed in admission order, then the new entry, in one commit | +| abort (M7) | steer + followUp are removed from the inbox, payload values deleted, and payloads returned; nextRun and write items stay | + +**One decision, at most one commit (R1b).** A boundary pass may enter the mutation line several times for bounded reads and for `before_run_end` mediation, but it performs **at most one commit**, and that commit always lands in a state that neither drains nor rechecks (`assistant.ready`, `summary.deciding`, placed entries + `assistant.ready`, or a terminal transaction). No boundary decision ever commits back into `checkpoint`. A crash before the commit re-runs the whole decision with nothing consumed; a crash after it lands past the decision point. Consequently `skipInboxOnce` and `thresholdCheckedTriggerEntryId` are deleted: drains cannot re-fire because their target states do not drain, and the threshold marker is replaced by a guard derived from the branch itself — threshold fires only when `shouldCompact` holds **and** the branch's newest compaction entry is older than the trigger entry, so a committed threshold compaction is its own durable marker. Mode remainders stay queued and are consumed at later boundaries, giving the per-turn steer cadence and per-run-end followUp cadence. + +There is no terminal drain. `cancelQueued` triage is unchanged and single-location. Queued input survives operation termination (including abort, for nextRun/write) until consumed or cancelled; this is a deliberate product decision. + +### Flat state (R3) + +`OperationState` has one `at` discriminator with **13** direct leaves: + +- `starting` +- `checkpoint` +- `assistant.ready` +- `assistant.effect_pending` +- `assistant.retry_wait` +- `tools` +- `deferred.suspended` +- `deferred.effect_pending` +- `summary.deciding` +- `summary.ready` +- `summary.effect_pending` +- `summary.retry_wait` +- `navigation.ready_to_commit` + +Every leaf carries one uniform scope `{ control, settings, latestAssistantEntryId }`. There are no `run.`/`compaction.`/`navigation.` leaf prefixes, no per-family scope types, and no intersection zoo. The four `summary.*` leaves carry a `SummaryTask` whose `boundary` datum (a closed three-arm union, §8) decides what happens at the result boundary. `starting` and `navigation.ready_to_commit` are intent-locked entry leaves. `ToolBatch`/`ToolCall` remain the nested child collection state machine. `Control` stays orthogonal and loses its drained fields when M7 installs drain-and-return abort. + +### Durable operation results (R2) + +Every terminal transaction writes one small immutable result record at `operationResult(operationId)` (namespace `pi.result`, operation id key). The record is lane-lived; keeping it outside `pi.op.*` keeps that namespace's "deleted no later than the terminal transaction" grammar total and exception-free. `LaneState.lastOperationId` points at the newest record. `laneLastResult` (value, address constructor, and type union) is deleted. + +- `drive(id)` becomes total for settled operations: current id → install/join; record exists → return the record itself; neither → `OperationMismatch`. +- Recovery and attachment never read result records; they are observation only. +- Terminal-control invariant: a terminal transaction that executes under durable `cancel_requested` always records `status: "aborted"`; equivalently, any other status implies running terminal control. (Every non-aborted terminal path goes through `continueOperation`, which diverts under cancellation, and the mutation line serializes the marker against the terminal commit.) +- There is no listing, filtering, pagination, or retention API for records — and no hydration layer. `getResult(operationId)` and the drive settled arm are the entire read surface; both return the record with no entry dereference, so observation can never fault on missing entries. + +### One lane-owned Drive + +Unchanged: one process-local Drive pass per lane; first matching caller installs, later matching callers observe the same completion; no caller owns it after installation; invocation cancellation rejects only that caller's observation; an installed Drive is never abandoned or replaced in-process; `requestAbort(operationId)` is the only durable operation cancellation; a stale operation id returns `OperationMismatch` and affects nothing. `Drive.context` removes the installing invocation signal. + +### Close + +Unchanged: close is not abort. It seals mutation admission, rejects local observations with `HarnessClosed`, observes detached pass failures, drains mutations admitted before the seal, and closes the Session. It writes no cancellation marker or synthetic terminal state and never replaces a Drive. + +### No external finalization + +Unchanged: no live-process external-finalization path, `OperationEnded`, `finalizedOutcome`, ownership-loss result, or exact-Drive ABA fence. + +## 2. Procedure shape + +Live procedures are ordinary straight-line async functions: + +```text +prepare +→ commit intent +→ admit and await effect +→ commit settlement +``` + +A procedure is the sole writer that changes its top-level `at` leaf. `requestAbort` changes only `control` (and the lane inbox); inbox admission changes only the lane inbox. Operation state has exactly two supported writers: the procedure and `requestAbort`. + +The Lane supplies `continueOperation` and `settleOperation` as today. A terminal decision appends the universal suffix: + +1. procedure-specific publication and cleanup writes; +2. `setValue(operationResult(operationId), record)`; +3. idle `laneState` with `currentOperationId: null`, `lastOperationId: operationId`, and the preserved inbox; +4. idle process-local projection; +5. terminal result and event materialization. + +The state argument is a type capability established by dispatcher control flow, never compared with current state at runtime. + +## 3. Remaining concurrency checks + +Keep checks only where another supported writer can change the relevant fact: + +1. `requestAbort` versus effect admission and settlement; +2. lane-inbox arrival before a drain point, a summary result boundary, or terminal finish; +3. parallel tool-call statuses and source-ready placement; +4. queued frame/checkpoint writes versus settlement; +5. invocation memo/checkpoint calls versus effect completion; +6. retry timers versus cancellation/close; +7. deferred permit consumption; +8. accept/claim serialization; +9. external/provider/content validation. + +Do not reintroduce operation identity, expected-`at`, exact-Drive, or ownership-loss checks into ordinary transitions. + +`requestAbort` keeps the two-step gate order: `beginAbort` before the cancellation mutation, commit `cancel_requested`, `signalAbort` after the commit. + +## 4. Completed milestones + +### M0 — Withdrawn execution-step controls + +Complete. Breakpoints, manual drive, and drive deadlines are absent. + +### M1–M2 — Foundations and terminal mechanics + +Complete. Includes the split effect gate, deterministic gated storage, authoritative process-local Lane projection, commit-result usage totals, progress channels, terminal cleanup, and immutable operation-result observation. + +### M3 — Assistant generation and recovery + +Complete. Includes `run.starting → run.checkpoint`, assistant ready/intent/effect/settlement, frame persistence and cleanup, configuration failure, unknown-outcome assistant recovery, and response/usage/state atomicity. + +### M4 — Retry and deferred + +Complete. Includes durable retry waits, local wait policy, one deferred poll permit per pass, unknown-poll replacement under fresh ids, and event-stream-preserving `Models.streamDeferred`. + +### M5 — Durable tools + +Complete. Includes planned/effect-pending/outcome-ready/completed calls, safe replay, unsafe interruption, invocation memos, bounded checkpoints, completion-order staging, source-order placement, sequential and parallel modes, and tool-reported usage. + +### Pre-M6 simplification + +Complete. Canonical flat `at` types, `continueOperation`/`settleOperation`, removal of the installer-owned Drive model, and conversion of M3–M5 procedures. + +### M6 structural foundation + +Committed (`9b23c6583`): `runtime/drive/structural.ts`, `test/harness/runtime/drive-structural.test.ts`, the one-provider-request seams in the compaction modules, threshold routing, overflow preparation, per-request structural intent/usage settlement, hook decisions, structural retry/recovery, and unsummarized navigation commit. Its family cross-product layer (triplicated quadruples, three-armed publishers) is rebuilt by R3; its effect seam, preparation plumbing, threshold/overflow logic, and navigation commit are retained. + +## 5. Withdrawn: standalone-compaction inbox and run-continuation promotion + +The design that gave `compaction.*` leaves a `RunScope` inbox and a one-way cross-family move into `run.*` at the result boundary is withdrawn. Its problems: a relaxed intent/state invariant with bespoke restore rules, `compact()` resolving with a run outcome that does not fit `CompactionResult`, suppressed `run_start` producing unbalanced event brackets, fused-consumption rules to avoid `cancelQueued` stranding, and promotion-specific arms in reconciliation, cleanup, watch, and admission. + +Replacement, spread across this package: + +- queued input during any operation is lane-owned (R1) — no operation inbox exists to promote; +- continuation after standalone compaction/navigation is a **second ordinary run operation** with a fresh id, composed in the convenience layer (M8); +- each operation delivers exactly one result; both results are returned by the convenience call and both are durable records (R2). + +No code implements promotion. M9 removed the withdrawn design from `harness.md`; this section remains only as the historical decision record. + +## 6. R1 — Lane-owned tagged inbox + +### Goal + +Move all queued input to one lane-owned tagged inbox first (R1a, implemented), then — after R3 — replace the transitional stepwise checkpoint drains with one shared atomic boundary planner (R1b). `Control.drained*` remains until M7 introduces drain-and-return abort; it is not part of inbox ownership. + +### R1a — Ownership relocation + +- `src/harness/session/types.ts` — durable `LaneState { currentOperationId, inbox }` with `InboxItem { entryId, kind }` replaces `pendingNextRun`; delete `Inbox` and `RunScope.inbox`. Keep `Control.drained*`, `skipInboxOnce`, and `thresholdCheckedTriggerEntryId` temporarily. +- `src/harness/runtime/types.ts` — process-local `LaneState.inbox` replaces `pendingNextRun`; `LanePatch.inbox` pairs durable lane-state replacement with process-local publication. +- `src/harness/runtime/lane.ts` — acceptance selects per tag and mode but places selected items in global admission order, then request prompt entries; it deletes selected pending values and removes their ids atomically with the `LaneBusy` check. A lone queued write never validates acceptance. `append` during any operation enqueues a write-tagged lane item without touching operation state; idle `append` places queued writes first, then the new entry, in one commit. `watch` derives queues and pending writes from the lane inbox. +- `src/harness/runtime/drive/checkpoint.ts` — transitional stepwise drains read and replace the lane inbox rather than operation state. `skipInboxOnce` and `thresholdCheckedTriggerEntryId` continue to protect those temporary multi-commit boundaries. +- `src/harness/runtime/drive/terminal.ts` — operation cleanup never deletes lane-inbox payloads. Staged `outcome_ready` results, drained abort payloads, and other operation-owned addresses are unchanged. +- `src/harness/runtime/drive/{generation,response,deferred,tools,structural}.ts` — `runScopeOf` and scope copies drop `inbox`; ordinary operation-state transitions cannot clobber concurrent input. +- `src/harness/runtime/restore.ts`, fork/legacy normalization, and focused tests use the tagged lane inbox. + +### R1b — Shared atomic boundary planner + +**Lands after R3**, because pre-R3 the structural result boundary is spread across the triplicated publishers: fusing the planner into three arms and then collapsing them in R3 would be double work, while post-R3 the single `ResultBoundary` switch is exactly one planner call site. The two `CheckpointData` patch fields ride through R3's mechanical rename as inert baggage. + +**Core invariant: no boundary decision ever commits back into `checkpoint`.** `checkpoint` is the durable "boundary pending" resting leaf, entered only from settlements, run-start consumption, and deferred redemption. Every exit of its pass is `assistant.ready` (with any placed entries), `summary.deciding`, a placed follow-up + `assistant.ready`, or the terminal transaction. + +One shared boundary planner with a process-local `threshold: "check" | "skip"` parameter has exactly two callers: + +- the `checkpoint` pass (`threshold: "check"`): select write + steer (modes, global admission order) → threshold guard → continuation → followUp (at `may_finish`) → `before_run_end` → finish; one commit; +- the structural result boundary for `resume_checkpoint` (`threshold: "skip"`): the **same planner runs inside the publication commit** — compaction entry (on success) first, then selected write/steer items, then the routed continuation, all in one transaction. Any placed conversational item routes to `assistant.ready` with that item as trigger, overriding a `may_finish` resume continuation (steer semantics). With nothing queued: `need_assistant` resumes commit `assistant.ready` directly; `may_finish` resumes commit `checkpoint{may_finish}` as the resting leaf and the live pass continues into the finish phases. + +**Threshold without the durable marker.** The guard is derived from the branch: threshold fires only when `shouldCompact` holds and the newest compaction entry is older than the trigger entry. A committed threshold compaction is therefore its own marker — any crash re-entry sees the newer compaction and skips. A **declined** threshold compaction publishes no entry and needs no marker: decline never lands on `checkpoint`. The decline decision stays process-local at `summary.deciding` until its single continuation commit (`assistant.ready`, follow-up + ready, or the terminal after `before_run_end`); a crash in that window restores `deciding` and re-runs `before_compaction` under its documented repetition contract. A live decline/recheck loop is structurally impossible because no decline path re-enters the threshold-checking pass. + +**`before_run_end` without hooks on the mutation line.** The finish arm is the one multi-phase case: (1) on-line verdict, no commit — reached only at `may_finish` with no eligible write/steer/followUp; (2) hook runs off the line; (3) on-line **replan from current state** — if the inbox gained eligible items or control changed, the stale hook result is dropped and the planner takes the new decision; otherwise the single commit is the follow-up injection (entry + `assistant.ready`) or the terminal transaction. Stale-dropping is replanning, not a version check. + +Concrete control trace (post-R3 names; threshold compaction with `may_finish` resume and a steer arriving mid-compaction): + +```text +TX1 settlement: response n7 + usage + tip, S=checkpoint{may_finish, trigger n7} +pass (check): shouldCompact && newestCompaction < n7 → prepare +TX2: preparation, S=summary.deciding{boundary: resume_checkpoint{may_finish, n7}} +TX3 (any time): steer s1 → pendingEntry(s1), laneState.inbox += s1 +TX4..n: deciding → ready → effect_pending → nested request/usage settlements +TX-pub (skip): insert compaction c1, tip=c1, insert s1 entry (parent c1), + delete pendingEntry(s1), laneState.inbox -= s1, + S=assistant.ready{trigger s1} ← one commit, checkpoint never re-entered +…turn answers s1; next settlement → checkpoint{may_finish, trigger n9} +pass (check): newestCompaction c1 > … tokens small → no threshold; inbox empty; verdict finish + → before_run_end off-line → replan: still empty, still running +TX-final: record + cleanup + laneState{current: null, last: op} +``` + +Crash between TX4..n and TX-pub → structural recovery (attempt unknown, per existing rules); crash during the hook window → `checkpoint{may_finish}` restored, pass re-runs (threshold skipped by the recency guard), hook re-runs per contract. + +Slices, in order (the order is load-bearing — deleting the marker while decline still re-entered `checkpoint` would live-loop): + +1. **R1b-1**: fuse the future shared planner into the structural `resume_checkpoint` boundary (`threshold: "skip"`); structural success/decline route directly except for a `may_finish` resting checkpoint, which remains safe for either outcome while the durable marker still exists. Keep the route inline until R1b-2 creates its second caller. Both patch fields are still written and checked by the old checkpoint pass — harmless. +2. **R1b-2**: convert the `checkpoint` pass to single-commit exits with the recency guard, implement the three-phase finish arm, delete `skipInboxOnce` and `thresholdCheckedTriggerEntryId`. + +### Rules + +- The §1 eligibility table, global placement order, and one-decision-one-commit target are normative. +- Queue modes apply at selection time. Remainders preserve global admission order and are consumed only at later eligible boundaries. +- nextRun items are never consumed mid-run and never block run finish. +- Deferred-suspension enqueue always admits; consumption happens at the post-redemption boundary. + +### Focused validation + +Enqueue during idle/run/structural/cancelled states always admits; acceptance preserves global order with modes applied and remainders retained; `one-at-a-time` cadence gives each remaining steer its own later boundary; followUp is captured at idle acceptance and only at `may_finish` during a run; a boundary crash before its commit consumes nothing; threshold is checked at most once per boundary by construction; idle append places queued writes first; items survive every terminal status and process loss; `cancelQueued` works at every boundary; watch groups by tag without changing order; no terminal transaction deletes a lane-inbox payload; a lone queued write never validates acceptance. + +## 7. R2 — Neutral operation outcome and durable result records + +### Goal + +Replace the three per-family outcome unions and `laneLastResult` with one immutable per-operation result record. The record **is** the public settled outcome; there is no separate outcome type, no embedded entries, and no hydration. + +### Types + +```ts +type TerminalStatus = "completed" | "declined" | "aborted" | "failed"; + +/** Stored at operationResult(operationId) — namespace "pi.result" — by the terminal transaction. Immutable, lane-lived. */ +interface OperationResultRecord { + operationId: string; + kind: "run" | "compaction" | "navigation"; // meta.intent.kind; matches OperationAdmission.kind + status: TerminalStatus; + error?: OperationError; // status "failed" + fromTipId: string | null; // meta.sourceTipId — start of the transcript segment + tipId: string | null; // lane tip at terminal — end of the segment + startedAt: number; // Unix ms, from meta + endedAt: number; // Unix ms, Date.now() at terminal planning +} + +/** Convenience-only suspension observation for prompt()/resume() (M8). Never stored. */ +interface SuspendedRun { operationId: string; status: "suspended"; deferred: DeferredHandle } +``` + +The record is a disposition plus a pointer to the transcript segment `(fromTipId, tipId]`. It never lists intermediate work (compactions, turns, tools) and never embeds entries. There is **no `tipEntry` and no `OperationOutcome` union**: a caller that wants the payload dereferences `tipId` with the already-public `getEntry`/`findEntry` (for runs, the final message was additionally delivered by `message_end`/`entry_added`). This buys three things: terminal planners perform zero outcome reads, observation cannot fault on a missing entry (a `getEntries` throw inside hydration was a harness-fault path for a read-only convenience), and the future protocol never serializes `Entry` inside results — a result frame is eight flat fields. `SuspendedRun` is the one non-terminal observation, exists only on convenience returns, and carries nothing derivable elsewhere. + +### Files + +- `src/harness/session/types.ts` — add the record; delete the `LaneLastResult` union; `LaneState` gains `lastOperationId: string | null`; delete `failure_drain`, `RunFailureDrainOperation`, and `FailureProvenance`. +- `src/harness/session/values.ts` — `operationResult(operationId) = value("pi.result", operationId)`; delete `laneLastResult`. The `pi.op.*` lifetime grammar stays total: no operation-lived namespace survives its terminal transaction. +- `src/harness/agent-harness.ts` — delete `RunOutcome`, `CompactionOutcome`, `NavigationOutcome`, `OptionalFinalAssistant`, `TerminalOperationOutcome`, `ResumeOutcome`, and the transitional `OperationOutcome` union; `DriveOutcome.settled` carries `OperationResultRecord` directly (no duplicate `operationId` field); result aliases become + `RunResult = Result`, + `CompactionResult = Result<{ compaction: OperationResultRecord; run?: OperationResultRecord | SuspendedRun }, …>`, + `NavigationResult = Result<{ navigation: OperationResultRecord; run?: OperationResultRecord | SuspendedRun }, …>`, + `ResumeResult = Result`; + `QueueResult` loses `NoActiveRun` and absorbs `NextRunResult`. `SuspendedRun` is declared here but constructed only by M8 convenience paths. +- End-event payloads, one shape per event type: `run_end` and `navigation_end` carry `{ runId, status, error?, fromTipId, tipId }`. `compaction_end` is a segment event, not a terminal event — it also closes in-run threshold/overflow brackets — and always carries `{ runId, reason, status: "completed" | "declined" | "failed" | "aborted", error?, entryId? }`, where `entryId` names the compaction entry on success. Embedded `entry`/`summaryEntry`/final-assistant fields are dropped everywhere (`entry_added`/`message_end` already delivered payloads). +- `src/harness/runtime/types.ts` — `FinishDecision` carries the record instead of `lastResult`. +- `src/harness/runtime/lane.ts` — terminal suffix per §2; `getLastResult` is replaced by `getResult(operationId, context): Promise` — one `getValue`, nothing else; `inspectExecution` reports `lastOperationId` only. +- `src/harness/runtime/drive/terminal.ts` — `hydrateTerminalOutcome`/`hydrateOperationOutcome` are deleted outright; the file keeps only `operationCleanupWrites` and the `operationResultRecord` constructor. Finish decisions materialize `{ kind: "settled", outcome: record }` from the record they already built — no reader dereference inside any terminal planner. +- `src/harness/runtime/drive/{checkpoint,response,structural,recovery,deferred,tools}.ts` — every finish decision constructs the record; `runCompletion`, per-family last-result construction, and final-assistant plumbing in results are deleted. Every former `failure_drain` producer instead commits terminal failure directly: response publication and cleanup stay in the same transaction, in-run structural failure closes `compaction_end` then `run_end`, and queued lane input remains untouched for a later ordinary run. (`latestAssistantEntryId` stays in scope for cancellation classification and checkpoint logic; it just no longer feeds results.) +- `src/harness/runtime/restore.ts` — restore `lastOperationId`; attachment never reads records. +- Focused tests. + +### Rules + +- Records are written exactly once, by the terminal transaction, and never deleted, updated, or read by recovery. +- Terminal-control invariant (§1): a terminal transaction under durable `cancel_requested` records `aborted`; no path commits any other status under cancellation. M7 adds the enforcing test; M9 adds the Part 9 invariant. +- Forks exclude `pi.result` values. +- `drive(id)` arms: current → install/join; record → returned directly; neither → `OperationMismatch`. + +### Focused validation + +Record written for every terminal path of every operation kind and status; `drive(id)` returns records for arbitrarily old ids across reopen; `getResult` is a plain value read returning the record or `undefined`; no terminal planner or observation path reads entries for the result; `lastOperationId` restore; suspension never stored; fork exclusion; segment pointers correct (`fromTipId` = pre-acceptance tip, including navigation); `compaction_end` closes both standalone and in-run brackets, including `aborted` via reconciliation; each former failure-drain producer finishes in one transaction and preserves every lane-inbox item for a later run. + +## 8. R3 — Family-neutral leaves and structural rebuild + +### Goal + +Collapse 22 leaves to the 13 in §1; rebuild `structural.ts`'s state-shape layer on one summary quadruple with an explicit result-boundary datum. + +### Types + +```ts +interface OperationScope { control: Control; settings: RunSettings; latestAssistantEntryId: string | null } + +type ResultBoundary = // closed; do not extend + | { kind: "resume_checkpoint"; resumeAfter: CheckpointData } // in-run threshold/overflow + | { kind: "finish" } // standalone compaction + | { kind: "commit_navigation"; targetId: string; label?: string }; + +interface SummaryTask { + taskId: string; + reason?: "manual" | "threshold" | "overflow"; + customInstructions?: string; + boundary: ResultBoundary; +} +``` + +The summary algorithm kind is **derived from the boundary** (`resume_checkpoint`/`finish` → compaction; `commit_navigation` → branch summary); it is not stored, so no contradictory kind/boundary combination is representable. `summary.ready/effect_pending/retry_wait` additionally carry the generation snapshot (result entry id, configuration, stream options, retry policy, attempt counters); the snapshot drops the `taskId`/`kind`/`reason` fields it previously duplicated. Canonical declarations land in `session/types.ts`. Uniform scope means compaction/navigation acceptance captures `settings` exactly like run acceptance; `latestAssistantEntryId` stays null for structural intents. + +### Reachability (restore check; replaces the intent-prefix check) + +| Intent | Admissible leaves | +| --- | --- | +| run | `starting`, `checkpoint`, `assistant.*`, `tools`, `deferred.*`, `summary.*` with boundary `resume_checkpoint` | +| compaction | `summary.*` with boundary `finish` | +| navigation | `navigation.ready_to_commit`; `summary.*` with boundary `commit_navigation` | + +Forbidden and unreachable by construction (assert in tests, never implement): any edge into `starting` or `navigation.ready_to_commit` other than acceptance; `summary.*` → `tools`/`assistant.*` directly; `navigation.ready_to_commit` → `summary.*`; terminal → anything. + +### Result-boundary rule + +Every summary boundary (hook decline, hook-supplied result, generated success, terminal generation failure, model unavailability) plans on the mutation line and switches on `boundary` in one visible place: + +- `resume_checkpoint` → publish result and restore the marked checkpoint on success or threshold decline; overflow decline or structural failure closes the compaction bracket and terminal-fails the run in the same commit; +- `finish` → publish the compaction entry and terminal-complete (or terminal declined/failed) in one commit; +- `commit_navigation` → the single move/summary/label/terminal commit (or terminal declined/failed). + +A `cancel_requested` boundary defers to reconciliation and never takes its continuation. + +### Files + +- `src/harness/session/types.ts` — the 13-leaf union, `OperationScope`, `SummaryTask`, `ResultBoundary`; delete `RunScope`, `CompactionScope`, `NavigationScope`, `NavigationSummaryScope`, `RunCompactionScope`, `StructuralTask`, per-family `Extract` aliases, `isRunOperationState`. +- `src/harness/runtime/drive/structural.ts` — rebuild: keep `durableCompactionPreparation`/readers, the nested request seam, `performStructuralAttempt`, `runCompactionThreshold`, `prepareOverflowCompaction`, `commitNavigation`; delete the seven union aliases, the `startsWith` guards, and the three-armed `effectPendingFromReady`/`retryWaitFromEffect`/`readyFromRetryWait`/`publishStructuralReady`/`publishStructuralDecline`/`publishStructuralFailure`/`publishCompactionResult`-vs-`publishNavigationSummary` split; one quadruple of converters plus one boundary switch replace them. +- `src/harness/runtime/drive/{checkpoint,generation,response,recovery,deferred,tools,tool-placement,terminal}.ts` — leaf literal renames (`run.checkpoint` → `checkpoint`, …); `response.ts` overflow arm constructs `summary.deciding` with `resume_checkpoint`; no other behavior change. +- `src/harness/runtime/progress.ts` — frame/checkpoint fences currently match the leaf literals `"run.assistant.effect_pending"`, `"run.deferred.effect_pending"`, and `"run.tools"`; retarget them to the neutral leaves. +- `src/harness/runtime/restore.ts` — reachability predicate above. +- `src/harness/runtime/lane.ts` — `capturedModel` and `watch` switch over the neutral leaves; acceptance writes neutral initial leaves. +- `docs/runtime-simplification.md` — replace the 22-leaf list and status. +- `test/harness/runtime/drive-structural.test.ts` and other focused tests — retarget; behavioral assertions survive. + +### Focused validation + +Everything the M6 foundation covered, re-expressed over neutral leaves, plus: reachability accept/reject matrix (including corrupted boundary/intent combinations faulting restore); each boundary arm × {success, hook result, decline, failure, model absence} × {running, cancelled}; a post-terminal leak scan asserting every `pi.op.*` address is gone for every leaf while the `pi.result` record is present; crash/reopen at every leaf under every intent that can reach it. + +## 9. M7 — Cancellation reconciliation and total switch + +### Goal + +**Implemented and reviewed.** Make the internal graph total without public wiring. + +Create `src/harness/runtime/drive/reconcile.ts`, `src/harness/runtime/drive.ts`, and focused reconciliation and switch tests. + +### `requestOperationAbort` + +Package-private expected-id primitive: + +1. reject with `OperationMismatch` when the expected id is not the current durable operation, including when that id already has a settled result record; +2. synchronously `beginAbort` on a matching live Drive; +3. one commit on the Session line: `control = cancel_requested` (with no drained fields) **plus** removal of steer/followUp-tagged ids from the lane inbox and deletion of their `pendingEntry` values; payloads are read in the same mutation and returned; nextRun/write items stay; +4. `signalAbort` after the commit; +5. after a newly requested abort commits, publish `{ type: "operation_abort", operationId, steer, followUp }` and return once cancellation is durable. This family-neutral event replaces `run_abort`/`runId`; the lane snapshot already identifies the operation kind. Repeat calls against the same current cancelled operation publish nothing, drain nothing further, and report `newlyRequested: false`. + +With no Drive but a matching current durable operation, it commits the same marker and starts no pass. There is no `control.drained*`; the drained payloads exist only in the returned result (accepted loss: a crash inside the abort window loses their content). + +### Reconciliation + +Checked before `before_drive` and ordinary dispatch; a single switch over the 13 leaves; never starts new ordinary work. It must handle: + +- assistant and deferred effects with live or reconstructed results; +- planned/effect-pending/outcome-ready tool calls; +- structural process-local results (discarded unless already atomically published); +- cancelled summary boundaries finishing `aborted` without taking their `ResultBoundary` continuation; +- retry waits, checkpoints, and suspended deferred work; +- best-effort deferred-provider cancellation using the Drive's close-only signal, never its already-triggered operation-abort gate; +- the aborted terminal transaction (result record `status: "aborted"`); +- the terminal-control invariant test: no terminal path commits a non-aborted status under cancelled control (this keeps §10's continuation rule decidable from the record alone). + +Queued lane-inbox items are **not** applied or deleted by reconciliation; nextRun/write items simply remain queued. + +Each Drive owns a private close controller and exposes its signal to mandatory cleanup. `closeGate()` aborts that controller on harness close or fault; operation abort does not. Deferred-provider cancellation uses this close-only signal, so the cleanup request can start after durable operation cancellation but cannot outlive harness shutdown. Do not add a harness-global cancellation controller. + +### Total switch + +`runtime/drive.ts` owns one direct `state.at` switch over 13 leaves. It imports only complete procedure modules. No graph table, action interpreter, ownership-loss arm, external-finalization arm, or storage-state reload. A `continue` result must correspond to a replaced `Lane.state` projection **or** to currently observable `cancel_requested` control, which the switch routes to reconciliation before ordinary dispatch; any other unchanged continuation is an invariant defect. + +Public methods remain guarded through M7. + +## 10. M8 — Public surfaces + +**Implemented.** Execution guards were removed only after every leaf and reconciliation path became total. + +Order: + +1. admit compaction/navigation requests (acceptance captures `settings`; writes the R3 entry leaves); +2. implement `drive` install/join/record lookup (three arms, §7); +3. expose `requestAbort`; +4. add convenience compositions; +5. add queues/configuration/usage/idle surfaces; +6. retain `watchSession` as the sole `SliceNotImplemented` method. + +### Queues + +`steer`/`followUp`/`nextRun` are tag sugar over one enqueue and always admit. `queue_update` and `LaneSnapshot.queues` expose the same tagged ordered inbox, including pending writes; clients group by tag without reordering it. + +### Client replication surface + +Findings from a working RPC-shaped client replica (mini TUI exercise) fold in here. The goal is one testable contract: a remote client renders exclusively from a replicated `LaneSnapshot` and the event stream, with no side-channel getters. + +- **Normative reducer.** Export `reduceLaneSnapshot(snapshot, event): LaneSnapshot | { rebase: true }` from `packages/agent`. It is the single supported event fold: in-run `compaction_start`/`compaction_end` are segments and must not clear `operation` (the fold knows the open operation's kind, so no event field is needed); `run_suspend` keeps `operation` non-null with `deferred` set; only `run_end`, `navigation_end`, and `compaction_end` under a compaction-kind operation are operation-terminal. `navigation_end` returns `{ rebase: true }` because the moved tip may lie outside the replica. Conformance assertion: for every non-navigation flow, folding a `watch()` snapshot over its own event stream equals a later `watch()` snapshot — this equivalence test is what keeps the event vocabulary complete. +- **Snapshot completeness.** `LaneSnapshot` gains `configuration: LaneConfiguration`, `lastResult?: OperationResultRecord` (replacing the bare `lastOperationId` field), a real `faulted` flag (the current hardcoded `false` is a defect), and session `stats: SessionStats` as the usage baseline (usage events already carry `totals`; the snapshot supplies the value before the first event). +- **Replicable configuration events.** Every `config_update` variant whose value is data carries `value`/`previous` (`streamOptions`, `retryPolicy`, `compactionSettings`, `steeringMode`, `followUpMode`, plus the existing lane variants); `tools`/`resources` remain notification-only because registries are code — clients re-fetch names. +- **Identity-based `setModel`.** `setModel` accepts `ModelIdentity` (`{ provider, modelId }`); a live `Model` object is a process-local registry concern, and an unregistered identity fails in-band at generation exactly like any registry absence. +- **Re-basing.** `WatchHandle.resnapshot(context): Promise` captures a fresh snapshot on the mutation line, serialized against the same stream — the recovery path after `{ rebase: true }` with no teardown/re-subscribe choreography. +- **Start-event timestamps.** `run_start`/`compaction_start`/`navigation_start` carry `startedAt` (operation starts: `meta.startedAt`; in-run segment starts: the commit timestamp), so folds never invent times. + +### Convenience compositions and continuation + +- `prompt`/`skill`/`promptFromTemplate`: accept + drive; return `Result` — the record for terminal outcomes, `SuspendedRun` when the run defers. +- `compact`/`navigateTree`: accept A + drive A. If A settled with status `completed`, `declined`, or `failed` and eligible steer/followUp/nextRun items exist, accept a continuation run B with the **public empty-prompt request** (`{ kind: "prompt", prompt: "" }`): acceptance places no request messages and is legal exactly when its capture places at least one conversational item; `OperationMeta.intent` is an ordinary run intent with empty `promptEntryIds`. Then drive B and return `{ compaction|navigation: A, run?: B }`. Never after `aborted`: abort already drained steer/followUp, and the terminal-control invariant (§7) makes `status ∈ {completed, declined, failed}` imply the operation was never durably cancelled, so the rule is decidable from the record alone. B is an ordinary run — it emits `run_start`, runs `before_run` (with `prompt: []`, the existing captured-only acceptance shape), and owns the full run graph with no continuation-aware special case. +- Continuation races: a competing accept that wins the idle window captures the queued input into its own run; the continuation accept then fails empty (`InvalidMessage`) or `LaneBusy`, and the convenience returns A without `run`. Both histories are valid. A crash between A's terminal and B's acceptance leaves the items queued; no auto-start on reopen. +- `resume`: inspect + drive current id, one deferred-poll permit; returns `OperationResultRecord | SuspendedRun`. +- `abort`: inspect + `requestAbort` + ensure a reconciliation pass; returns the drained steer/followUp payloads. +- `getResult(operationId)`: public. +- Equivalence: every convenience call ≡ its primitive composition using only public request kinds (`compact()` with continuation ≡ `accept(A); drive(A); accept({ kind: "prompt", prompt: "" }); drive(B)`), byte-identical writes and events — externally reproducible by any scheduler. + +### Focused validation + +One install and same-id joins; stale-id isolation; record lookup for old ids; caller cancellation before/after installation; close/fault during cooperative and non-cooperative effects; accept/drive vs convenience equivalence including continuation via the public empty-prompt request; steer-only, followUp-only, and nextRun-only continuation each start run B; both continuation race orders; abort drain-and-return including the both-orders abort/settlement race; enqueue during every state through public surfaces; full crash matrix across all 13 leaves; no unhandled detached rejection; no `SliceNotImplemented` except `watchSession`; reducer fold-equivalence across every non-navigation flow (including in-run compaction segments, suspend/resume, retry waits, and reattach mid-stream); `navigation_end` fold returns rebase and `resnapshot` restores equivalence; snapshot `configuration`/`lastResult`/`faulted`/`stats` replicate through events alone. + +## 11. M9 — Documentation reconciliation + +**Implemented.** `docs/harness.md` is reconciled to the runtime and normative on its own. Updated sections include: + +- §1.3 address table and lifetime grammar: new lane-lived `pi.result` namespace, `laneLastResult` removed, `LaneState.inbox`/`lastOperationId`; `pi.op.*` stays strictly operation-lived; +- §1.7 JSONL/SQLite examples referencing `pi.lane.lastResult`; state the bounded growth tradeoff — one small immutable record per operation, retained forever, carried through JSONL snapshot compaction; +- §2.9 precise rewrite: decide and document the policy for records whose `tipId` the rewrite removes (retain-dangling or delete); +- §3.1–§3.2 state shapes: 13 neutral leaves, `OperationScope`, `SummaryTask`/`ResultBoundary` (kind derived from boundary), `Control` without drained fields, `CheckpointData` without `skipInboxOnce`/`thresholdCheckedTriggerEntryId`; +- §3.5 graph (one summary quadruple, boundary arms); +- §3.6 acceptance: per-tag mode-respecting selection, global admission-order placement, prompt entries last, idle followUp eligibility, and empty-prompt continuation acceptance; +- §3.9/§3.10 summary machinery expressed once over the boundary datum; +- §3.11 complete rewrite: one lane inbox, the selection table, one-decision-one-commit boundaries, silent deferral of late steer (an unconsumed steer becomes future-run input rather than an error), and unbounded write pendency during structural operations with the `waitForIdle`-then-append escape hatch; +- §3.12 checkpoint procedure: the one-commit boundary decision replaces the stepwise algorithm; failures terminalize while queued lane input remains for a later ordinary run; +- §3.13 terminal transactions: result records, universal suffix, observation contract (`drive(id)` total, `getResult`); +- §4.6 abort: drain-and-return, no drained control, and the client-visible consequence — drained steer/followUp payloads exist only in the returned result, so a crash or lost response in that window loses their content permanently; +- §5.1 lane surface and all result types; §5.5 `queue_update` and end-event payloads (`compaction_end` as a segment event with `aborted`); +- Part 9: invariants 12–16, 21, 26 (result-record lifetime, observation via records, continuation equivalence), the new terminal-control invariant (a non-aborted terminal status implies running terminal control), and the race catalog rows touching drained items, `cancelQueued`, and continuation; +- Appendix A glossary (Inbox, Result record, Continuation run, Boundary pass; remove Drained); +- §5.5 event taxonomy: state explicitly which events are operation-terminal (`run_end`, `navigation_end`, compaction-kind `compaction_end`) versus segment brackets (in-run `compaction_start`/`compaction_end`) versus non-terminal lifecycle (`run_suspend` leaves the operation durably open), and name `reduceLaneSnapshot` as the normative fold; +- §2.5 branch-scan sharp edge: `stopAtType` applies after ordering, so `oldestFirst` + `stopAtType: "compaction"` returns the oldest segment; document the canonical context read as `newestFirst` + reverse; +- serving-layer boundary statement: branch-relative reads/appends remain on `AgentLane`, while whole-tree browsing, forks, label inventory, and session listing live beside it in Session/repository services composed by an RPC facade. + +Also update `docs/runtime-simplification.md` status and remove every remaining promotion reference in the repository's docs. Grep gate: + +```bash +rg -i 'promotion|drainedSteer|drainedFollowUp|laneLastResult|lastResult|skipInboxOnce|thresholdChecked' packages/agent/src packages/agent/docs +``` + +(matches must be zero in `src`; docs may retain historical notes only inside completed-milestone records.) + +## 12. M10 — Provider KV-cache identity review + +### Goal + +**Implemented and reviewed.** Core Session identity alone is not a valid provider cache lineage: several lanes may issue concurrent requests over divergent transcripts. Ordinary assistant requests therefore derive identity from Session metadata id plus lane name; structural requests remain isolated. + +### Review scope + +- Inventory every harness and pi-ai path that sets, preserves, derives, or consumes `SimpleStreamOptions.sessionId`, including prompt-cache keys, affinity headers, WebSocket/session-resource caches, deferred requests, and structural summary requests. Ordinary harness generation now forwards the derived lane identity; legacy `src/agent.ts` keeps its independent conversation id; structural summary requests mint fresh ids. +- Define a provider-request cache lineage distinct from the durable Session id and operation id. Concurrent lanes must never share one lineage merely because they belong to the same Session. +- Derive ordinary assistant lineage as `Session metadata id + ":" + lane name`; do not store another durable identifier. It remains stable for the lane's lifetime and differs across lanes in one Session. +- Preserve same-lane reuse across ordinary assistant turns and retries. Compaction, navigation, branch replacement, and model changes may miss an old prefix, but cannot incorrectly reuse it; do not add rotation machinery. +- Keep structural summary requests isolated: `cacheRetention: "none"` and fresh request identity per nested request remain the baseline unless the review proves a safe, useful alternative. +- Distinguish cache identity from observability correlation. Session, lane, operation, task, and request ids remain available to telemetry without being reused blindly as provider affinity/cache keys. +- Review provider-specific semantics rather than assuming `sessionId` means only KV caching; some adapters also use it for headers, WebSocket reuse, fallback state, or resource cleanup. + +### Focused validation + +- Two active lanes in one Session issue concurrent, divergent prompts and receive distinct provider cache/affinity identities. +- One lane keeps a stable lineage across append-only assistant/tool turns where reuse is valid. +- Context-prefix discontinuities safely miss old cached prefixes without lineage rotation. +- Assistant retries retain the lane identity; deferred handle polling needs no cache identity. +- Structural split-turn requests remain isolated from transcript assistant caching. +- Faux-provider tests assert stable same-lane and distinct cross-lane request identities. + +### Exit condition + +Ordinary assistant requests use the derived lane identity, structural requests keep fresh identities with `cacheRetention: "none"`, and deferred handle polling sends no cache identity. The policy is documented in `docs/harness.md` and independently reviewed before WP05 completion. + +## 13. Mobile assistant-output handoff + +**Tracked follow-up; not part of the M8/M10 public-drive gate.** A trivial mini coding-agent Session produced an approximately 300 KB JSONL file because live assistant streaming persists many `pi.pending.assistant_frame` list appends. Logical frame cleanup does not reclaim bytes already appended to the JSONL history, so short conversations can have disproportionate durable storage and replay cost. + +The authoritative follow-up is the [mobile assistant-output handoff](../mobile-handoff/01-harness/05-assistant-output/message-update.md) and its numbered prerequisites in the [mobile handoff README](../mobile-handoff/README.md). It addresses the complete path rather than only batching frames: Chord op tracking, scoped pending-output durability, tool/assistant output reduction, and `message_update` replication amplification. Preserve the existing crash contract: an admitted assistant effect remains reconstructible, progress observation remains useful, and settlement retires operation-owned pending state. + +Exit checks: + +- representative short streamed responses do not create hundreds of kilobytes of durable frame history; +- crash/reopen at every assistant and deferred effect boundary reconstructs the same pending message; +- frame/checkpoint writers remain fenced to the owning durable effect; +- backend conformance covers both reconstruction and bounded write amplification. + +## 14. Module boundaries + +```text +session/** durable storage; imports no runtime module +execution/** neutral provider/tool/gate mechanics +runtime/types.ts LaneState, Drive, command decisions +runtime/progress.ts frame/tool progress channels +runtime/drive/*.ts direct procedures +runtime/drive.ts total flat switch, created only in M7 +runtime/lane.ts Lane actor and public surfaces +runtime/harness.ts Harness lifecycle and Lane composition +``` + +Procedure modules import concrete `Lane` type-only. `TContext` remains `object | undefined` invariant. No `any`, `Lane`, `as unknown as`, `@ts-expect-error`, inline imports, parameter properties, enums, or other non-erasable TypeScript syntax. + +## 15. Exclusions + +Do not introduce: + +- generic scheduler, graph, action interpreter, or effect-plan DSL — `ResultBoundary` stays a closed three-arm data union switched in one visible place; +- per-family outcome unions, `OptionalFinalAssistant`, or embedded result entries in stored records; +- result-record listing, filtering, pagination, status queries, or retention machinery; +- operation-owned queues, drained-control fields, or terminal queue cleanup; +- automatic continuation anywhere below the convenience layer (never inside `drive`); +- a second mutation line or transaction framework; +- expected-`at` runtime checks for the sole top-level writer; +- process-local Drive replacement or caller-owned lifetime; +- external finalization; +- storage rereads of authoritative control state; +- read caches, read budgets, or generic `getValues` batching; +- compatibility aliases for any pre-redesign durable shape; +- a `phase`/segment discriminator field on structural events — the reducer derives segment-vs-terminal from the open operation's kind; +- whole-tree, fork, label-inventory, or repository methods on `AgentLane`; branch-relative reads/appends remain part of the lane facade, while broader administration lives beside it. + +Procedure-specific writes, effect admission, settlement classification, and event construction remain visible at their call sites. + +## 16. Validation and reviews + +After every code stage: + +```bash +npm run check +``` + +Run each modified focused test file from the package root. Do not invoke the full Vitest suite directly. Run `./test.sh` only for final package validation or when explicitly requested. + +Review checkpoints are mandatory at the end of R3, R1b, M7, M10, and final completion. Delegated reviews use provider `anthropic` and model `claude-fable-5`. + +Final greps: + +```bash +rg 'lost_ownership|LostOwnership|DriveAbandoned|commandDriveOwned|installerSignal|finalizedOutcome|OperationEnded' \ + packages/agent/src/harness packages/agent/test/harness +rg 'drainedSteer|drainedFollowUp|laneLastResult|RunOutcome|CompactionOutcome|NavigationOutcome|TerminalOperationOutcome|skipInboxOnce|thresholdCheckedTriggerEntryId' \ + packages/agent/src/harness +rg 'SliceNotImplemented' packages/agent/src/harness/runtime +rg ': any\b||as unknown as|@ts-expect-error' packages/agent/src/harness/runtime +``` + +Final exit conditions: + +- every flat leaf is driveable and reconcilable; +- public primitive/convenience behavior is equivalent, including continuation; +- every focused test and backend conformance path passes; +- public drive exposes no partial graph; +- `watchSession` is the only deferred public method; +- `drive(id)` answers every settled operation id in the session; +- harness.md is self-consistent with the implementation (M9); +- provider cache/affinity identity is lane-safe across concurrent and context-reset histories (M10); +- independent final review reports no blocker. diff --git a/packages/agent/docs/work-packages/06-session-branch-lane-separation.md b/packages/agent/docs/work-packages/06-session-branch-lane-separation.md new file mode 100644 index 000000000..1e22e89ea --- /dev/null +++ b/packages/agent/docs/work-packages/06-session-branch-lane-separation.md @@ -0,0 +1,734 @@ +# WP06 — Session, Branch, Lane separation + +**Status: implemented before WP05 M4.** WP05 M3 remains complete. Retry/deferred work resumes on the composition, mutation, and ownership boundaries landed here. + +This package replaces the mixed `SessionTree`/implicit-main inheritance design with four explicit concepts: + +```text +Session global durable data + one mutation line +Branch one path through the entry tree, with a movable tip +AgentLane Branch data surface + agent operations/configuration +AgentHarness manager of AgentLanes; never a lane itself +``` + +--- + +## 0. Mandatory reading + +Read completely before editing: + +1. `packages/agent/docs/harness.md`. +2. `packages/agent/docs/work-packages/05-direct-durable-drive.md`. +3. `packages/agent/src/harness/session/types.ts`. +4. `packages/agent/src/harness/session/session.ts`. +5. `packages/agent/src/harness/session/memory.ts`. +6. `packages/agent/src/harness/session/jsonl/repo.ts` and `jsonl/storage.ts`. +7. `packages/session-backends/sqlite-node/src/sqlite/session.ts`, `storage.ts`, and repo implementation. +8. `packages/agent/src/harness/runtime/lane.ts`, `harness.ts`, `restore.ts`, and `types.ts`. +9. `packages/agent/src/harness/agent-harness.ts`. +10. `packages/agent/src/harness/session/fork.ts` and repository conformance tests. +11. Every test named in §8 before modifying it. + +Do not inspect deleted runtime implementations or Git history. Current source, `harness.md`, WP05, and this package are the only sources of truth. + +--- + +## 1. Problem + +### 1.1 `SessionTree` mixes unrelated ownership + +`SessionTree` currently contains: + +```ts +interface SessionTree { + // Lane/path data. + getLeafId(...): Promise; + findEntriesOnBranch(...): Promise; + findEntryOnBranch(...): Promise; + appendMessage(...): Promise; + appendCustomEntry(...): Promise; + + // Session-global data that is not lane- or branch-owned. + getEntry(...): Promise; + getStats(...): Promise; + findEntries(...): Promise; + findEntry(...): Promise; + getValue(...): Promise | undefined>; + setValue(...): Promise; + readList(...): Promise[]>; + appendList(...): Promise; + getName(...): Promise; + setName(...): Promise; + getLabel(...): Promise; + setLabel(...): Promise; +} +``` + +A selected tree view changes only which mutation queue a global write enters. It does not change the durable address. Two views can therefore read the same global value under different lane lines and commit a lost update. + +### 1.2 `Session` silently means `main` + +`Session extends SessionTree`. Its inherited branch methods and high-level writes silently delegate to the `main` lane. `session.setValue(...)` is actually `setValueForLane("main", ...)`. A fresh repository session creates a partial implicit main lane before any harness exists. + +### 1.3 `AgentHarness` silently means `main` + +`AgentHarness extends AgentLane`, and the runtime `Harness extends Lane`. The manager object is inserted into its own lane map as `main`. Calls such as `harness.prompt(...)`, `harness.watch(...)`, or `harness.getModel(...)` silently target main. + +### 1.4 Per-lane mutation queues solve the wrong problem + +Storage already serializes atomic commits and assigns one session-wide `seq` per write. Per-lane mutation queues allow useful preparation overlap, but this runtime does not need that complexity now: all harness mutation callbacks perform bounded storage reads, prepare one write set, commit at most once, publish process-local state, and return. Providers, tools, hooks, timers, and asynchronous event delivery are outside mutation callbacks. + +The approved first implementation uses one Session mutation line. Keyed lines may be added later if profiling proves the global line is a bottleneck; that future change requires a mutable-ownership audit but no public API redesign. + +--- + +## 2. Approved terminology and ownership + +### Session + +Owns: + +- session metadata; +- global entry and usage queries; +- application values and lists; +- session name and entry labels; +- Branch discovery/creation; +- one process-local mutation line for every supported mutation; +- one open storage/backend lifecycle. + +A Session does not implement Branch and has no implicit branch. + +### Branch + +Data-only capability describing one named path through the immutable entry tree. + +Owns only: + +- its current tip; +- branch-relative entry queries; +- direct extension of its tip with message or custom entries. + +A Branch has no model configuration, queues, operation state, drive, hooks, or agent policy. + +### AgentLane + +One Branch plus agent configuration and operations. It exposes Branch methods directly rather than exposing a nested `Branch`, tree, store, view, or access object. + +When idle, `AgentLane.appendMessage` / `appendCustomEntry` extend the tip directly. During an active run, they retain the existing deferred-write semantics: reserve the entry id, persist `pendingEntry(id)`, and enqueue the id in the operation inbox for checkpoint placement. A raw Branch append is always a direct data append; raw Branch mutation while a Harness owns the corresponding lane is a trusted-programming defect. + +### AgentHarness + +Owns global registries/configuration, hooks, events, lifecycle, and a map of AgentLanes. It is not an AgentLane and exposes no implicit-main operation methods. + +--- + +## 3. Target public types + +### 3.1 Session reader and mutation + +```ts +export interface SessionReader { + getEntries(ids: string[], context: Context): Promise>; + getValue( + address: Value, + context: Context, + ): Promise | undefined>; + scanValues(prefix: Value, context: Context): Promise[]>; + readList( + address: ValueList, + options: ListReadOptions | undefined, + context: Context, + ): Promise[]>; + scanBranch(query: StorageBranchScan, context: Context): Promise; +} + +export interface SessionMutation extends SessionReader { + /** Exactly zero or one attempt. A second call rejects, including after failure. */ + commit(writes: Write[], context: Context): Promise; + /** Wait for any admitted commit, invalidate the capability, and release the Session line. */ + end(context: Context): Promise; +} + +export type SessionMutator = Omit; + +export type SessionMutationCallback = ( + mutator: SessionMutator, + context: Context, +) => TResult | Promise; +``` + +`beginMutation()`/`SessionMutation.end()` remain the transportable scope used by the current remote Session protocol. They are keyless and carry no lane field. Normal local/harness code uses `mutate()`; no caller selects a mutation key. + +### 3.2 Branch + +```ts +export interface Branch { + readonly name: string; + getTipId(context: Context): Promise; + findEntries( + query: BranchScan | undefined, + context: Context, + ): Promise; + findEntry( + query: BranchScan | undefined, + context: Context, + ): Promise; + appendMessage(message: AgentMessage, context: Context): Promise; + appendCustomEntry( + customType: string, + data: JsonValue | undefined, + context: Context, + ): Promise; +} +``` + +Because the receiver is already a Branch, the public names are `findEntries` and `findEntry`, not `findEntriesOnBranch` and `findEntryOnBranch`. + +### 3.3 Session + +```ts +export interface Session< + TMetadata extends SessionMetadata = SessionMetadata, +> extends SessionReader { + readonly metadata: TMetadata; + readonly idGenerator: IdGenerator; + + // Direct reads. No mutation-line acquisition. + getEntry(id: string, context: Context): Promise; + getStats(context: Context): Promise; + findEntries( + query: EntryQuery | undefined, + context: Context, + ): Promise; + findEntry( + query: EntryQuery | undefined, + context: Context, + ): Promise; + getName(context: Context): Promise; + getLabel(targetId: string, context: Context): Promise; + + // Existing Branch acquisition performs durable I/O and therefore receives Context. + branch(name: string, context: Context): Promise; + createBranch( + name: string, + at: string | null, + context: Context, + ): Promise; + + // Transportable explicit scope; RemoteSession maps begin/read/commit/end over RPC. + beginMutation(context: Context): Promise; + + // Trusted sharp edge. The callback holds the sole Session mutation line. + mutate( + mutation: SessionMutationCallback, + context: Context, + ): Promise; + + // One-write conveniences implemented through mutate(). + setValue( + address: Value, + next: NoInfer, + context: Context, + ): Promise; + deleteValue(address: Value, context: Context): Promise; + appendList( + address: ValueList, + element: NoInfer, + context: Context, + ): Promise; + deleteList(address: ValueList, context: Context): Promise; + setName(name: string | undefined, context: Context): Promise; + setLabel( + targetId: string, + label: string | undefined, + context: Context, + ): Promise; + + close(context: Context): Promise; +} +``` + +`mutate()` remains public. Plugins are trusted not to retain the mutator, invoke nested public writers, perform effects, or hold the line across unbounded work. Misuse may block every mutation in that Session and is a plugin defect. `beginMutation()` exists for transport/lifecycle integration rather than ordinary plugin work; every direct caller must call `end()` in `finally`. + +### 3.4 AgentLane + +`AgentLane` keeps its operation/configuration/observation methods and directly adds the Branch surface: + +```ts +export interface AgentLane { + readonly name: string; + + getTipId(context: Context): Promise; + findEntries( + query: BranchScan | undefined, + context: Context, + ): Promise; + findEntry( + query: BranchScan | undefined, + context: Context, + ): Promise; + appendMessage(message: AgentMessage, context: Context): Promise; + appendCustomEntry( + customType: string, + data: JsonValue | undefined, + context: Context, + ): Promise; + + getLastResult(context: Context): Promise; + accept( + request: OperationRequest, + context: Context, + ): Promise; + drive(options: DriveOptions, context: Context): Promise; + requestAbort( + operationId: string, + context: Context, + ): Promise; + inspectExecution(context: Context): Promise; + // Existing convenience, queue, configuration, idle, and watch methods remain. +} +``` + +Delete `AgentLane.sessionTree`. + +### 3.5 AgentHarness + +```ts +export interface AcquireLaneOptions { + /** Used only when the AgentLane does not exist. Defaults to null. */ + createAt?: string | null; +} + +export interface AgentHarness< + TContext extends object | undefined = object | undefined, +> { + lane(name: string, context: Context): Promise; + lane( + name: string, + options: AcquireLaneOptions, + context: Context, + ): Promise; + lanes(context: Context): Promise; + + // Session-global metadata wrappers preserve existing value_update events. + getName(context: Context): Promise; + setName(name: string | undefined, context: Context): Promise; + getLabel(targetId: string, context: Context): Promise; + setLabel( + targetId: string, + label: string | undefined, + context: Context, + ): Promise; + + // Existing global tools/resources/options/settings/hooks/events/watchSession/close surface. +} +``` + +`AgentHarness` does not extend `AgentLane`. Delete `createLane`; `lane()` is atomic get-or-create. Existing lanes ignore `createAt`. A missing lane uses `createAt ?? null`. Concurrent acquisitions return the same published AgentLane. Invalid names and unknown non-null targets reject with the existing tagged errors; close/fault reject with their existing lifecycle errors. + +A fresh Session and fresh Harness contain no implicit main Branch or AgentLane. `await harness.lane("main", context)` creates main completely. `lanes()` may return `[]`. + +--- + +## 4. Mutation and read semantics + +### 4.1 One Session mutation line + +Replace `LaneMutationLine` with a single `MutationLine`: + +```ts +export class MutationLine { + private tail: Promise = Promise.resolve(); + private sealedError: Error | undefined; + + run(operation: () => TResult | Promise): Promise; + seal(error: Error): Promise; +} +``` + +`StorageBackedSession.beginMutation()` acquires that line and returns one explicit keyless capability; only `end()` releases it. `mutate()` is the callback convenience built from begin/end and always ends in `finally`. The callback may read, prepare, commit once, publish process-local state, synchronously bind event recipients, and return. `close()` seals admission and waits for every acquired scope to end before closing Storage. + +High-level Session writes, Branch creation/appends, every `Lane.command`, progress writes, restore snapshots requiring coherence, and Harness lane acquisition all call the same `Session.mutate()`. + +### 4.2 Reads bypass the line + +Ordinary Session and Branch reads call Storage directly. They observe the latest fully applied atomic storage commit at the time each read executes: + +- a queued/planning/unapplied mutation is invisible; +- no partial commit is visible; +- once Storage commit resolves, direct reads may observe it even while the mutation callback is still publishing process-local state; +- multiple reads outside one `mutate()` are not a snapshot; +- coherent read-decide-write/CAS uses `Session.mutate()`. + +### 4.3 Effects stay outside + +A mutation callback must not perform or await: + +- providers or deferred fetch/cancel; +- tools; +- hooks; +- timers; +- asynchronous event delivery; +- idle callbacks or Drive completion; +- nested Session/Branch/AgentLane mutators. + +The callback may synchronously call `emitBatch` after publication to bind recipients and retain its delivery promise. The public operation awaits delivery after `mutate()` returns. + +### 4.4 Storage remains independently atomic + +Storage retains its one-session commit serializer and assigns one global `seq` per write. The Session mutation line protects read-decide-commit-publication procedures; the Storage queue protects atomic transaction application, sequence assignment, fork snapshots, and backend callers. Do not merge the two abstractions. + +--- + +## 5. Branch and lane durable shape + +### 5.1 No implicit main + +Repository `create()` writes only session metadata/header/catalog state. It writes no branch tip, lane configuration, or lane state. Remove main seeding from Memory and JSONL creation and from SQLite initialization. + +Legacy coding-agent v3 normalization may still produce a main Branch because the imported transcript has one selected path. + +### 5.2 Branch completeness + +A Branch exists exactly when its required tip value exists. `createBranch(name, at, context)` validates name, absence, and non-null target, then writes the tip in one mutation. It writes no model configuration or operation state. + +Use Branch terminology in source. This package renames the typed constructor and public concepts to `branchTip`/`tipId`. The persisted namespace and durable field spelling decision must be consistent across all backends and docs: + +- use `pi.branch.tip` and rename format-4 fields from leaf to tip rather than retain misleading new-code aliases; +- format 4 and the new harness are WIP, so replace their schema and field names in place: no storage-version bump, migration, compatibility decoder, or old-format rejection path; +- legacy coding-agent v3 import remains supported: it maps its selected main leaf to a main Branch tip and walks that selected physical ancestry to reconstruct the nearest `model_change`, `thinking_level_change`, and `active_tools_change` independently; unsupported nearest values do not fall back to older history; +- when the importer can reconstruct a total configuration, it writes ordinary `laneConfig("main")` plus fresh idle `laneState("main")` before returning the Session; missing active-tools history normalizes to `[]` because v3 did not persist the initial tool inventory; +- if required model or thinking configuration is missing or unsupported, the importer leaves a data-only main Branch rather than persisting partial compatibility state; +- update the legacy active-tools record type to read its encoded `activeToolNames` array; +- update protocol schemas and experimental adapters for public `tipId` fields in the same package. + +Inventory uses `branchTipInventoryPrefix()`. Legacy import emits only ordinary Branch/Lane values; there is no temporary compatibility address or attachment-time migration. + +### 5.3 AgentLane completeness + +An AgentLane adds total `laneConfig`, `laneState`, optional `laneLastResult`, and optional current operation values to an existing Branch. + +`harness.lane(name, options?, context)` executes one Session mutation and handles exactly these cases: + +| Durable state | Result | +| ----------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Branch absent; lane values absent | validate `createAt`, commit Branch tip + the immutable `AgentHarnessOptions` seed config + idle lane state, publish one AgentLane and `lane_created { at: createAt ?? null }` | +| Branch present; lane config/state absent and no last result | commit the immutable seed config + idle lane state at the existing tip, publish one AgentLane and `lane_created { at: existingTip }` | +| Branch and complete lane values present | return the restored/published AgentLane; no commit/event | +| Any partial or contradictory combination | fault as storage corruption | + +The committing callback publishes the new Branch/AgentLane into process-local maps and synchronously calls `emitBatch(lane_created, context)` before returning from `Session.mutate()`. Event delivery is awaited after line release. + +`AgentHarness.create()` restores every complete durable AgentLane and open operation, but creates nothing and requires no main. A data-only Branch remains a Branch until `harness.lane(name, ...)` attaches agent state. + +### 5.4 Append behavior + +`Branch.appendMessage` / `appendCustomEntry` always commit an immutable entry parented to the current tip and move the tip in the same transaction. + +`AgentLane.appendMessage` / `appendCustomEntry` retain harness semantics: + +- idle: append and move tip immediately; +- active run: stage `pendingEntry(id)` and enqueue `inbox.writes`; +- active structural operation: retain the existing wait/re-evaluate contract when that surface lands; +- pending assistant messages reject before commit. + +Both return the entry id reserved before their mutation. + +--- + +## 6. Runtime composition + +### 6.1 Harness + +Replace inheritance with composition: + +```ts +export class Harness< + TContext extends object | undefined, +> implements AgentHarness { + readonly session: Session; + readonly models: Models; + readonly hooks: HookRegistry; + readonly events: HarnessEventBus; + readonly lanesByName = new Map>(); + // global config/lifecycle fields +} +``` + +The constructor builds every restored Lane as an ordinary object. It never calls `super("main", ...)` and never inserts `this` into `lanesByName`. + +Fault and close iterate ordinary Lane objects. Global getters/setters live only on Harness. Coding-agent experimental services and workers must acquire/cache `main` explicitly before invoking lane operations. + +### 6.2 Lane + +`Lane.command` and `commandDriveOwned` call keyless `session.mutate(plan, context)`. The live `Lane.state` remains the authoritative process projection. Exact Drive fencing remains adjacent to commit admission. + +Lane directly implements Branch query/append names. It may hold a package-private Branch implementation for direct data reads, but no nested Branch is exposed publicly. + +### 6.3 Restore + +Restore no longer enters a named mutation line. `AgentHarness.create()` owns the Session attachment interval and performs one bounded keyless `Session.mutate()` callback to inventory/restore every complete AgentLane before publishing the Harness; it commits only when an intentional attachment normalization is required. Coherent live watch/inspection likewise uses `Session.mutate()` as a no-commit callback. + +Restore inventories complete AgentLanes separately from data-only Branches. Missing main is legal. + +--- + +## 7. Repository, backend, and fork requirements + +### Memory/JSONL/SQLite + +All Session facades: + +- remove lane arguments and lane fields from begin/mutate capabilities; +- retain explicit `beginMutation(context)` / `SessionMutation.end(context)` forwarding for local lifecycle and remote transport; +- keep an explicit scope admitted until `end()` and keep callback `mutate()` admitted through its implicit finally/end; +- expose Branch acquisition/creation; +- keep direct reads outside the line; +- keep Storage commit sequencing unchanged. + +SQLite and future SQL backends still serialize sequence-allocating commits in Storage. The Session mutation line intentionally serializes complete callbacks only within one open Session owner; separate sessions remain concurrent. + +### Forks + +Preserve one coherent source Storage snapshot. Update terms from lane leaf to Branch tip. + +- branch-scope fork creates destination Branch `main` at the copied path; +- if no explicit source entry is provided, source `main` must exist or fork rejects; +- branch scope copies source main's configuration and writes idle lane state **together iff** source main is a complete configured AgentLane; an unconfigured/data-only source main produces only a data-only destination main Branch; +- tree scope copies every Branch tip; each complete configured source AgentLane copies its configuration plus fresh idle lane state together, while each data-only Branch remains data-only; +- operation values, pending values/lists, last results, and usage rows remain excluded; +- destination sessions do not gain an unrelated implicit main; +- retain the explicit begin/commit/end fork-ordering seam: code that must admit a commit before starting a fork snapshot calls keyless `beginMutation()`, invokes `commit()`, starts the repository snapshot only after commit admission, and calls `end()` in `finally`; Storage queues the source snapshot against commits to choose one coherent boundary. + +--- + +## 8. Implementation phases and file manifest + +Public drive remains disabled. Complete this package before resuming WP05 M4. + +### Phase A — Session mutation and Branch + +**Rename** + +- `src/harness/session/lane-mutations.ts` → `mutation-line.ts`. + +**Modify** + +- `src/harness/session/types.ts` — `Branch`, keyless `Session.mutate` and keyless begin/end transport scope, no `SessionTree`, Session-global methods. +- `src/harness/session/session.ts` — one line, Branch implementation, no implicit-main delegates, direct Branch append; rename lane-creation validation errors to Branch terminology. +- Memory/JSONL/SQLite format-4 schema and codecs — replace WIP leaf/lane spellings in place; do not add a version gate or migration. +- `src/harness/session/memory.ts`. +- `src/harness/session/jsonl/repo.ts`, `jsonl/legacy-v3.ts`, and related open/create facade files. +- `packages/session-backends/sqlite-node/src/sqlite/session.ts` and repo creation. +- `src/harness/session/fork.ts`. +- `src/harness/session/index.ts` and package exports. +- storage/repository benchmarks and conformance call sites. + +**Rename tests** + +- `test/harness/session-tree.test.ts` → `branch.test.ts`. +- `test/harness/session-create-lane.test.ts` → `session-create-branch.test.ts`. + +**Modify tests** + +- `test/harness/storage-backed-session.test.ts`. +- `test/harness/memory-session-repo.test.ts`. +- `test/harness/jsonl-session-repo.test.ts`. +- `test/harness/memory-conformance.test.ts`. +- `src/harness/session/testing/conformance/session-repo.ts`. +- SQLite repo/storage tests. +- compaction/branch-summarization type fixtures. + +### Phase B — Harness/Lane composition + +**Modify** + +- `src/harness/agent-harness.ts`. +- `src/harness/runtime/harness.ts`. +- `src/harness/runtime/lane.ts`. +- `src/harness/runtime/restore.ts`. +- `src/harness/runtime/types.ts` where `leafId` becomes `tipId`. +- `src/harness/session/values.ts` and durable state types for Branch tip naming. +- all runtime drive modules already present (`checkpoint`, `generation`, `recovery`, `terminal`, `progress`) only where names/signatures change. +- `src/harness/compaction/branch-summarization.ts`. +- `packages/protocol/src/harness.ts`. +- `packages/coding-agent/src/experimental/services/agent-controller-provider.ts`. +- `packages/coding-agent/src/experimental/services/models-provider.ts`. +- `packages/coding-agent/src/experimental/session-worker.ts`. +- experimental harness wire/session worker tests. + +**Modify focused tests** + +- every `test/harness/runtime/*.test.ts` helper/call site using keyed mutate, implicit Harness-as-main, `sessionTree`, `leafId`, or `createLane`; +- `test/harness/types.test.ts`; +- `test/harness/branch-summarization.test.ts`; +- protocol and coding-agent experimental tests. + +Phases A and B are one atomic landing. Removing `SessionTree`, keyed mutate, and Harness inheritance cannot compile as separately committed compatibility phases, and this package intentionally adds no temporary aliases. + +### Phase C — Normative documentation + +Update `packages/agent/docs/harness.md` completely and consistently: + +- orientation/system model and worked examples; +- bound Branch tip addresses; +- Branch, Session metadata, queries, forks, and repository boundary; +- operation metadata/result/snapshot `tipId` terminology; +- one Session mutation line in Parts 3–5; +- attachment with no required main; +- public Branch, AgentLane, AgentHarness, and Session surfaces; +- event/watcher ordering under the Session line; +- remove the old second harness-settings-line lock-order narrative: harness-global settings and every durable lane mutation now serialize through the sole Session line when a coherent snapshot is required; pure synchronous registry reads remain direct; +- work-package table, invariants, races, backend conformance, and glossary. + +Update WP05 before M4: + +- replace `SessionTree`/lane-line/inheritance assumptions; +- replace `leafId` source examples where the new type names require it; +- keep all M0–M3 historical behavior and M4–M8 durable requirements; +- state that WP06 is the foundation between M3 and M4. + +Update every current supporting document whose public names or ordering statements change: + +- `docs/assistant-durability.md` and `docs/tool-durability.md` — Session-line FIFO and `branchTip` terminology; +- `docs/values.md` — Session-global value/list surface, Branch tip addresses, and no `SessionTree`; +- `docs/telemetry.md` — receiver inventory and Session mutation spans; +- `docs/plugins.md` — remove `sessionTree`; the plugin's AgentLane directly supplies Branch methods, while a scoped Session-data facet supplies global value/list/name/label/query methods and excludes raw `mutate`, `idGenerator`, close, and backend authority; +- `docs/extensions/pi-extensions-v2.md` and `docs/extensions/pi-server-artifact/index.md` where examples/types use the changed surfaces; +- completed WP00–WP04 only where a forward-looking/current-state statement would otherwise claim the removed API still exists. + +Retain the current remote Session mutation contract and update it from a named lane line to the sole Session line: worker `RemoteSession.mutate()` performs keyless begin RPC → local callback with remote reads/one remote commit → local post-commit publication → end RPC. The server holds the Session line through commit and publication until end acknowledgment. Disconnect/timeout terminates the scope under the existing hosting policy. Update the current remote protocol/vertical-slice documentation and every implementation/test present on dev; do not delete or defer this behavior. + +Do not rewrite released changelog sections. Add no changelog entry on a non-main/non-PR development branch. + +--- + +## 9. Required tests + +### Session mutation line + +- concurrent `mutate()` callbacks serialize globally, including callbacks invoked from different AgentLanes; +- a second callback does not enter until the first callback returns after commit/publication; +- direct reads do not wait for a callback that has not committed; +- direct reads observe a fully landed commit even while that mutation callback remains open after commit; +- no direct read observes a partial multi-write commit; +- two read-modify-write counter mutations produce `1`, then `2`; +- separate direct `getValue` + `setValue` calls remain deliberately non-atomic; +- zero-commit callback is legal; +- keyless `beginMutation()` excludes every other mutation until `end()`, commit does not release it, end-without-commit is legal, repeated end is idempotent, and close waits for end; +- RemoteSession begin/read/commit/publication/end preserves that same scope; +- second commit attempt rejects, including after a failed first attempt; +- nested public write from a mutation callback is documented as invalid and deterministically blocks/rejects according to the chosen guard; +- close-first rejects mutation; mutation-first completes and close waits; a trusted callback that never returns can block close; +- Storage commit `seq` and stats behavior is unchanged. + +### Session and Branch + +- fresh Session has zero Branches and no main tip value; +- no storage-version bump, migration, compatibility decoder, or rejection path is added for the replaced WIP format-4 schema; +- legacy coding-agent v3 import reconstructs ordinary total main-lane configuration plus idle state before returning when valid model/thinking history exists; otherwise it returns a data-only main Branch; +- legacy config tests cover complete and incomplete histories, invalid legacy fields, and branched histories where only changes on the selected main path apply; +- `branch(name, context)` returns undefined for absence and receives the exact Context on reads; +- `createBranch` validates name, target, and duplicate atomically; +- two concurrent creates have one winner; +- Branch queries default to its tip and preserve scan/filter/cursor behavior; +- direct Branch message/custom append extends its tip atomically; +- custom entry with absent data remains valid; +- pending assistant append rejects; +- Session global values/lists/name/labels/global queries no longer depend on a branch; +- all pre-close Branch objects reject after close; +- branch-scope fork copies config + idle state together iff source main is configured; tree scope does the same independently per configured lane; data-only Branches remain data-only; +- explicit keyless begin/commit/end preserves commit-before-fork-snapshot ordering and captures one Storage-serialized snapshot boundary. + +### AgentHarness and AgentLane + +- Harness has no AgentLane methods at type or runtime; +- Session has no Branch methods at type or runtime; +- AgentLane has Branch methods directly and no `sessionTree`/nested Branch property; +- fresh Harness `lanes()` is empty; +- `lane("main")` creates complete main atomically; +- missing named lane defaults to null tip; `createAt` anchors creation; +- existing data-only Branch becomes one complete AgentLane without moving its tip; +- existing complete AgentLane acquisition commits nothing and emits no creation event; +- concurrent acquisitions return the same object and emit exactly one `lane_created`; +- creation commit publishes the Lane and binds recipients before releasing `Session.mutate`; +- invalid name/unknown anchor commit nothing; +- restored complete lanes/open operations are inventoried without creating main; +- partial Branch/config/lane-state combinations fault; +- Harness global metadata wrappers retain commit → publication → `value_update` delivery; +- AgentLane idle append moves the tip; active-run append stages one deferred write and preserves authoritative state; +- close/fault seal every ordinary Lane without relying on Harness inheritance. + +### Regression + +- M2 exact-Drive ABA fence still checks immediately before commit admission; +- M3 generation intent/frame/settlement writes are byte-identical apart from renamed durable/public fields; +- frame FIFO remains correct under the one Session line; +- watch has only snapshot-first or publication-first outcomes; +- usage totals remain commit-boundary exact across lanes; +- Context remains trailing and source-identical throughout; +- Memory, JSONL, and SQLite repository/storage conformance pass. + +--- + +## 10. Exclusions + +Do not add: + +- keyed mutation lines, arbitrary lock names, resource locks, multi-lock ordering, versions, or optimistic retries; +- a scheduler, transaction framework, action interpreter, or generic post-commit task system; +- a nested Branch/tree/store/access property on AgentLane; +- compatibility aliases for `SessionTree`, `view`, implicit Session main methods, Harness lane methods, or `createLane`; +- keyed/named begin/end RPC scopes or removal of the current keyless RemoteSession mutation transport; +- effects inside `Session.mutate()`; +- a second Storage commit/sequence mechanism; +- automatic work start from `AgentHarness.create()` or `harness.lane()`; +- public drive before WP05 M8. + +Do not modify provider/tool behavior, durable execution phases, retry/deferred policy, or assistant-frame semantics beyond signature/name propagation required by this package. + +--- + +## 11. Validation + +Run every modified focused test, then: + +```bash +git diff --check +npm run check +./test.sh + +rg -n "SessionTree|sessionTree|\.view\(|LaneMutationLine|extends AgentLane|extends Lane" \ + packages/agent/src packages/agent/test packages/agent/docs \ + packages/session-backends packages/protocol packages/coding-agent/src/experimental \ + packages/coding-agent/test/experimental* + +rg -n "beginMutation\([^)]*,|mutate\([^)]*,[^)]*," \ + packages/agent/src packages/agent/test packages/session-backends packages/coding-agent/src/experimental + +rg -n "session\.mutate\([^)]*\"|\.mutate\(\s*[a-zA-Z_$][a-zA-Z0-9_$]*\s*," \ + packages/agent/src packages/agent/test packages/session-backends +``` + +Expected first grep matches only this package's problem statement, explicitly historical work-package API descriptions, negative type assertions, and unrelated coding-agent tree-widget names such as `SessionTreeNode`; no current harness `SessionTree` concept remains. The keyed-mutate grep must be empty after manually checking false positives. + +Review the complete implementation and normative doc update with Fable before committing. Do not commit without explicit user approval. + +--- + +## 12. Stop condition + +WP06 is complete when: + +- Session has one keyless mutation line; callback `mutate()` and explicit begin/commit/end remote transport share it, and neither accepts a lane key; +- direct reads bypass that line and expose only fully applied Storage commits; +- `SessionTree` and implicit-main Session behavior are gone; +- Branch is the data-only path/tip abstraction; +- AgentLane exposes Branch methods directly and retains operation-aware append semantics; +- AgentHarness is composition-only, has no AgentLane methods, and atomically gets/creates lanes; +- fresh Session/Harness require no main; +- all process-local publication and event-binding boundaries remain correct; +- all three backends and focused race tests pass; +- `harness.md` and WP05 describe the new model consistently; +- final Fable review reports no findings; +- WP05 M4 may resume. diff --git a/packages/agent/docs/work-packages/07-sqlite-host-ownership-live-forks.md b/packages/agent/docs/work-packages/07-sqlite-host-ownership-live-forks.md new file mode 100644 index 000000000..cc04353b8 --- /dev/null +++ b/packages/agent/docs/work-packages/07-sqlite-host-ownership-live-forks.md @@ -0,0 +1,372 @@ +# WP07 — SQLite host ownership and live-source forks + +**Status: implemented.** + +The delivered backend has no writer lease or replacement ownership primitive. It provides no-create read-write/read-only opens, queued same-repository snapshots, independent read-only WAL snapshots for live external sources, canonical physical identity, path-safe IDs, repository-local deletion reservation, and all-settled close. The tests cover both per-file and shared-container layouts, including a writer commit completed after a read snapshot boundary but before that reader closes. + +This package aligns `packages/session-backends/sqlite-node` with the product ownership model: the server owns Session records and worker lifecycle, and exactly one host-assigned process owns writable Session authority at a time. Normally that process is the Session worker. The server may temporarily own a newly created or forked destination, but it closes that Session before handing its metadata to a worker. + +Storage does not implement writer ownership. Remove the SQLite writer lease; do not repair or replace it. + +A server-side fork is intentionally different from a second writer: it may open a live worker-owned source concurrently for one coherent read-only snapshot while the worker keeps committing. Shared SQLite containers remain supported. + +## 0. Mandatory reading + +Read completely before editing: + +1. `packages/agent/docs/plugins.md` ownership, replacement, and removal sections. +2. `packages/server/README.md` and relevant Session routing/removal source and tests. +3. `packages/agent/docs/harness.md` §§0.6, 1.4–1.7, 2.7–2.8, 4.3, and Part 9. +4. `packages/agent/docs/post-wp05-roadmap.md`. +5. completed WP06 §7 and repository/fork conformance. +6. every source file under `packages/session-backends/sqlite-node/src`. +7. every test and benchmark under `packages/session-backends/sqlite-node`. +8. `packages/session-backends/sqlite-node/README.md` and `CHANGELOG.md`. + +Do not use stale files under `dist/` as implementation input. Completed WP01/WP06 documents are historical; do not rewrite them to hide the earlier lease implementation. + +## 1. Fixed architecture + +### 1.1 Writable authority belongs to the host + +Exactly one host-assigned process owns a writable Session. The Session worker is the normal owner. Worker replacement closes the old owner before the new worker opens the Session. Server management serializes creation, forking, removal, and attachment lifecycle around that ownership transfer. + +Memory, JSONL, and SQLite do not detect a second process opening the same Session for writes. Bypassing the server/worker lifecycle is a trusted-host defect, not a storage race to repair. A repository still rejects duplicate writable handles it owns in one process. + +Do not add a storage lease, filesystem lock, fencing token, heartbeat, timeout-based takeover, deletion tombstone, quarantine protocol, or generic lock manager. + +### 1.2 Read-only fork access may overlap the worker + +The server owns repository administration and may fork a source while its Session worker continues writing it. The source side of that fork: + +- opens the exact source container without creating it; +- is read-only; +- uses one deferred `BEGIN` transaction; +- reads the version-gated Session row, scalar values, and selected entries/branch index in that transaction; +- never upgrades to a write or claims writable authority; +- closes after `COMMIT`/`ROLLBACK`. + +SQLite WAL permits later worker commits while the read transaction remains open. The fork sees each source commit entirely before or entirely after its snapshot boundary, never a mixture. + +### 1.3 Same-repository fork ordering remains distinct + +A source already open in the same repository uses its active `SqliteStorage.snapshot()` path. That snapshot is queued on the source `commitQueue`, preserving WP06's admitted-commit ordering seam and the existing conformance case. + +Do not replace this path with an independent connection. Instead, bind active storage lookup to exact physical identity plus Session ID so metadata for another container cannot select it accidentally. + +### 1.4 Destination ownership does not overlap + +`SessionRepo.create()` and `fork()` continue returning an open Session. A server may temporarily own that new destination, capture its metadata, and close it before launching a worker. This is a valid ownership transfer, not a reason to redesign `SessionRepo` into a record-only API. + +## 2. Problems in current source + +### 2.1 SQLite duplicates host ownership + +Current source contains: + +- `writer_lease` schema state; +- claim, renew, and release helpers; +- lease claims in create/open/fork; +- an idle renewal timer and lease-loss path in `SqliteOpenSession`; +- a pre-commit renewal callback in `SqliteStorage`; +- lease-based deletion checks. + +This is a second, incomplete ownership system. Its pre-commit renewal is not atomic with the following data transaction, but the correct fix is deletion of the ownership mechanism, not transaction-local fencing. + +### 2.2 Non-creation access can create files + +The database factory exposes only `open(path)`, which creates a missing SQLite file. Metadata open, listing probes, deletion, and fork-source reads must not turn a removed path into an empty database. + +Fork-source reads also configure `PRAGMA journal_mode = WAL`, which is a write-oriented setup step and must not run on a read-only connection. + +### 2.3 Delete does not reserve its local critical section + +`delete()` checks `pendingIds` but does not reserve the ID. A same-repository create/open/fork destination can enter while asynchronous destructive work is in progress. Host lifecycle owns cross-process ordering; the repository still must serialize its own local operations. + +### 2.4 Physical identity, paths, and close need correction + +- `openStorages` is keyed only by Session ID, so the same ID at another physical path may select the wrong active source. +- `create()` makes `options.directory`, not the parent of an explicit `databasePath`. +- Per-session filenames interpolate arbitrary caller IDs directly; `/`, `\`, `..`, `%`, and platform separators must not escape `directory`. +- `repo.close()` uses fail-fast `Promise.all`, so it may return before every open Session has attempted to drain and close. + +Deterministic list ordering, bind-variable limits, branch-copy cost, fork scalar filtering, prepared statements, and VACUUM policy remain separate. + +## 3. Required result + +### 3.1 Remove storage-layer writer ownership + +Delete all runtime lease behavior: + +- delete `src/sqlite/session/writer-lease.ts`; +- remove `writer_lease` from WIP `001_initial.sql`; +- remove lease deletion from `deleteSessionRows()`; +- remove claim/renew/release code from `SqliteSessionRepo`; +- remove `beforeCommit` from `SqliteStorage`; +- remove renewal/release options, timer, and `leaseError` from `SqliteOpenSession`; +- remove lease-specific tests and replace them with host-authority and live-fork coverage. + +Keep: + +- the Storage `commitQueue`; +- one `BEGIN IMMEDIATE` transaction per commit; +- in-transaction `next_seq` allocation; +- entry/usage uniqueness and parent triggers; +- Session mutation admission and close draining; +- process-local duplicate-open rejection. + +Format 4 remains WIP. Remove the table from new schema in place; an old file containing an unused `writer_lease` table remains readable and the table and stale rows are ignored forever. Post-WP07 code does not delete them because doing so serves no runtime behavior. A pre-WP07 binary cannot open a new post-WP07 database without that table; backward compatibility for this WIP format is not required. Add no migration, compatibility path, or storage-version bump. + +### 3.2 Add explicit database open modes + +Extend `SqliteDatabaseFactory` with narrow operations: + +- `open(path)` — intentional creation or create-if-missing; +- `openExisting(path)` — read-write open that fails if the file does not exist; +- `openReadOnly(path)` — read-only open that fails if the file does not exist. + +The Node adapter uses `DatabaseSync(path, { readOnly: true })` for read-only access. Implement and test an actual no-create read-write mode for `openExisting`; do not rely on `access()` followed by a create-capable open. + +Split connection setup: + +- writable connections establish WAL mode and `busy_timeout`; +- read-only connections set only read-safe options such as `busy_timeout` and never attempt to change journal mode. + +Use no-create modes for metadata open, listing probes, deletion, and fork-source reads. + +### 3.3 Preserve both fork-source paths + +**Source open in this repository:** retain `SqliteStorage.snapshot()` and queue it after prior admitted commits. Replace the ID-only active map key with canonical `(containerPath, sessionId)` identity and use the same helper for publish, lookup, and removal. + +**Source not open in this repository:** this includes a closed source and a source currently owned by a worker in another process. Open the exact source through `openReadOnly`, then capture it in one deferred read transaction. Validate the Session row and storage version inside that transaction. Do not consult destination reservations, claim source ownership, or block the worker's later commits. + +The destination remains a normal writable create/fork transaction after source capture. In shared-container mode the source worker and destination transaction may use the same file; SQLite serializes destination writes while preserving Session row isolation. + +### 3.4 Make deletion locally exclusive + +The host must close the Session worker before calling `repo.delete()`. Direct cross-process deletion of a live Session is unsupported. + +Within one `SqliteSessionRepo`, deletion must reserve the Session ID from entry through completion and release it in `finally`: + +- an already open or reserved Session rejects deletion; +- create/open/fork destination for that ID rejects while deletion runs; +- shared-container deletion removes only the target Session's rows in one `BEGIN IMMEDIATE` transaction on one connection; +- per-file deletion removes the database and its WAL/SHM sidecars after a no-create open/existence check; +- a missing Session rejects without creating a file. + +Do not add a lease check, tombstone, quarantine rename, or stale-deleter protocol. Cross-process removal ordering is the server's responsibility. + +### 3.5 Bind metadata to physical identity and make paths safe + +- Canonical identity is `(canonical container path, sessionId)`. +- In per-file mode metadata must identify the repository-affine encoded path for its durable ID. +- In shared-container mode metadata must identify the configured canonical container and Session ID. +- A foreign or mismatched path must never alias a local active source. Pin in focused tests and the SQLite README whether foreign source metadata is rejected or read only from its exact path; do not silently substitute a local storage by ID. +- Create `dirname(databasePath)` when `databasePath` is configured. +- Encode arbitrary explicit IDs into safe per-session filenames without changing the durable ID. The encoding must prevent path escape and round-trip through metadata/list/open/fork. +- Two different Session IDs in one shared container remain independently addressable. + +### 3.6 Drain all repository-owned closes + +`SqliteSessionRepo.close(context)` must: + +1. seal repository admission once; +2. start close on every currently open Session; +3. wait for every close to settle; +4. resolve when all succeed; +5. otherwise reject only after all cleanup attempts, returning the one error or an `AggregateError` containing all failures; +6. return the same promise on repeated close. + +This is backend-local resource cleanup. Do not change the shared `SessionRepo` interface or JSONL lifecycle in this package. + +## 4. Implementation slices + +### Slice A — remove writer leases + +Files: + +- delete `src/sqlite/session/writer-lease.ts`; +- `src/sqlite/migrations/001_initial.sql`; +- `src/sqlite/session/session-row.ts`; +- `src/sqlite/storage.ts`; +- `src/sqlite/session.ts`; +- `src/sqlite/repo.ts`; +- lease-focused repository tests. + +Tasks: + +1. Remove schema/runtime lease state and timer behavior. +2. Preserve commit serialization, one write transaction, mutation draining, and process-local duplicate-open behavior. +3. Prove ordinary commits now use one write transaction rather than renewal plus write. + +### Slice B — no-create opens and deletion reservation + +Files: + +- `src/index.ts`; +- `src/sqlite/types.ts`; +- `src/sqlite/repo.ts`; +- focused adapter/repository tests. + +Tasks: + +1. Add `openExisting` and `openReadOnly` with tested no-create behavior. +2. Separate writable and read-only connection configuration. +3. Reserve deletion locally for its full critical section. +4. Delete one shared-container Session in one transaction; preserve unrelated Sessions. + +### Slice C — live-source read-only forks and identity + +Files: + +- `src/sqlite/repo.ts`; +- repository/conformance tests. + +Tasks: + +1. Key active storage by canonical container plus Session ID; keep same-repository queue ordering. +2. Use one independent read-only deferred transaction for non-open/live-worker sources. +3. Validate source metadata/version inside the snapshot. +4. Preserve shared-container destination behavior. + +### Slice D — paths and close draining + +Files: + +- `src/sqlite/repo.ts`; +- focused repository/conformance tests. + +Tasks: + +1. Create the actual custom container parent. +2. Encode arbitrary IDs safely. +3. Reject or exactly handle foreign metadata without active-source aliasing. +4. Make repository close all-settled and error-complete. + +### Slice E — documentation + +Files: + +- `packages/agent/docs/harness.md`; +- `packages/agent/docs/post-wp05-roadmap.md`; +- `packages/agent/docs/values.md`; +- `packages/session-backends/sqlite-node/README.md`; +- changelog only under normal branch rules. + +Document host-owned writable authority, the two fork-source paths, no-create opens, local deletion reservation, and absence of storage-layer ownership. + +## 5. Required tests + +Use real independent `node:sqlite` connections. Test-only wrappers may expose deterministic transaction boundaries; production code gets no sleeps or race flags. + +### Lease removal + +- new schema has no `writer_lease` table; +- create/open/fork/commit/close perform no lease reads or writes and start no renewal timer; +- same-repository duplicate writable open still rejects through process-local reservation; +- ordinary commit remains one `BEGIN IMMEDIATE` transaction. + +### Live fork source + +For both per-file and shared-container layouts: + +- a server repository forks a source held open by a separate repository/connection representing its worker; +- source capture uses a distinct read-only connection and claims no writable ownership; +- a source commit completed before the snapshot boundary appears wholly in the fork; +- a commit after the read snapshot is established can complete before the reader closes and is wholly absent from that fork; +- no fork contains an entry without the same commit's Branch tip/value/stats changes; +- a later fork includes the later commit; +- same-repository admitted-commit fork conformance remains unchanged. + +### Deletion + +- open/reserved Session first → same-repository delete rejects; +- delete reservation first → same-repository create/open/fork destination for that ID rejects; +- shared deletion removes only the target rows; +- per-file deletion removes database/WAL/SHM files; +- missing-path open/list/fork/delete does not create an empty database; +- tests state the host precondition: worker close precedes deletion; no cross-process bypass safety is promised. + +### Identity and paths + +- same Session ID at two physical paths cannot cross-select active source storage; +- two Session IDs in one shared container remain independent; +- `databasePath` succeeds when its parent does not exist; +- explicit IDs containing `../`, `/`, `\`, `%`, dots, and Unicode stay inside `directory` and preserve the metadata ID; +- metadata returned by create/list/open/fork names the actual container. + +### Close + +- one Session close failure does not prevent every other Session cleanup attempt; +- multiple failures are reported after all settle; +- repeated repository close returns the same promise; +- every successfully closed Session releases its connection. + +### Regression + +- existing storage and repository conformance remains semantically unchanged; +- fork destination reservation and same-repository source ordering remain intact; +- fork snapshots exclude operation/pending/result/usage/application state exactly as before; +- shared-container create/list/open/fork/delete remains supported; +- every write transaction still uses `BEGIN IMMEDIATE`; +- no migration, storage-version bump, compatibility layer, or replacement ownership primitive appears. + +## 6. Validation and review + +After each code slice: + +```bash +npm run check +``` + +Run each modified focused test from `packages/session-backends/sqlite-node` with the repository Vitest binary. Final validation: + +```bash +./test.sh +``` + +Review checkpoints: + +1. Fable after Slice A: verify no storage ownership mechanism remains and close draining is intact. +2. Fable after Slice C: verify same-repository ordering and live-worker read-only overlap both hold. +3. Final Fable review of source, tests, docs, and exclusions. + +Delegated reviews use provider `anthropic` and model `claude-fable-5`. + +## 7. Exclusions + +Do not include: + +- any storage lease, lock, fence, heartbeat, takeover, tombstone, or quarantine; +- a record-only `SessionRepo` redesign or compatibility facade; +- server/router/worker-manager redesign; record any backend-independent lifecycle race separately; +- SQLite branch-segment redesign or uncompacted-divergence optimization; +- fork scalar filtering/indexing; +- `getEntries` bind-limit chunking or general query-limit normalization; +- statement caches or stats aggregation optimization; +- catalog redesign, async database replacement, or VACUUM policy; +- search/FTS; +- R11 migration machinery or a storage-version bump; +- [mobile assistant-output handoff](../mobile-handoff/01-harness/05-assistant-output/message-update.md) changes or JSONL compaction; +- repository-wide `SessionRepo.close()` contract changes; +- removal of shared-container support; +- transaction DSLs, schedulers, generic lock managers, or compatibility layers. + +If implementation requires an excluded item, stop and revise the handoff rather than expanding silently. + +## 8. Exit condition + +WP07 is complete when: + +- SQLite contains no active or schema-defined writer lease and implements no replacement ownership mechanism; +- host ownership is the documented single-writer authority; +- same-repository active-source forks retain their queued commit boundary; +- live worker-owned sources fork through an independent read-only snapshot while later WAL commits continue; +- deletion reserves its same-repository critical section and assumes worker-first host removal; +- non-creation paths cannot create empty databases; +- active source identity includes physical container plus Session ID; +- explicit IDs cannot escape the directory and custom database parents are created; +- repository close waits for every cleanup attempt; +- shared-container mode remains fully covered; +- focused tests, `npm run check`, and `./test.sh` pass; +- final Fable review reports no blocker. diff --git a/packages/agent/docs/work-packages/08-named-branch-streaming-forks.md b/packages/agent/docs/work-packages/08-named-branch-streaming-forks.md new file mode 100644 index 000000000..3b48c066d --- /dev/null +++ b/packages/agent/docs/work-packages/08-named-branch-streaming-forks.md @@ -0,0 +1,246 @@ +# WP08 — Named-branch and tree forks with streaming copies + +**Status: in progress — implementing Slice C.** + +This package replaces the fork contract: `ForkOptions` gains a mandatory scope and a mandatory named source branch, branch forks validate a complete configured source AgentLane and ancestry membership, tree forks copy the complete immutable tree plus current application values/lists, and all three backends replace materialized source snapshot arrays with bounded-memory streaming copies. A JSONL fork never repairs or mutates its source. One closed core classifier owns every namespace's fork disposition. + +WP07 is a hard dependency and is preserved unchanged: no-create database modes, canonical `(containerPath, sessionId)` identity, independent read-only WAL reader for external/live-worker sources, repository-local deletion reservation, and all-settled close. This package supersedes only the roadmap's "SQLite fork cost" performance item. + +## 0. Mandatory reading + +Read completely before editing: + +1. `packages/agent/docs/harness.md` §§0.6, 1.3–1.7, 2.3, 2.7–2.9, Part 9 (invariants 3–5, 13, 16; ledger completeness). +2. `packages/agent/docs/values.md`, especially "Forks and rewrites" and the backend sections. +3. `packages/agent/docs/post-wp05-roadmap.md` (SQLite fork cost, repository lifecycle context). +4. Completed WP06 §7 and WP07 (historical; do not edit them). +5. `packages/agent/src/harness/session/fork.ts`, `types.ts` (`ForkOptions`, `SessionRepo`), `values.ts`, `in-memory-storage-state.ts`, `memory.ts`, `session/index.ts`. +6. `packages/agent/src/harness/session/jsonl/repo.ts`, `jsonl/storage.ts`, `jsonl/codec.ts`, `jsonl/legacy-v3.ts`, `jsonl/types.ts`. +7. `packages/session-backends/sqlite-node/src/sqlite/repo.ts`, `storage.ts`, `session/values.ts`, `session/entries.ts`, `session/branch-entries.ts`, `types.ts`. +8. `packages/agent/src/harness/session/testing/conformance/session-repo.ts` and every test named in §5. +9. `packages/agent/src/harness/session/testing/benchmark/session-repo.ts` and both `session-repo.bench.ts` files. + +Do not use `dist/` output as implementation input. WP00–WP07 documents and released changelog sections are immutable in this package. + +## 1. Fixed architecture + +### 1.1 Public contract + +```ts +export type ForkOptions = + | { scope: "branch"; branch: string; entryId?: string; position?: "before" | "at"; id?: string } + | { scope: "tree"; id?: string }; +``` + +`scope` is required. `branch` is required for branch scope. There is no default scope, no implicit `main`, and no compatibility alias. + +**Branch scope** requires a complete configured source AgentLane: `pi.branch.tip/{branch}`, `pi.lane.config/{branch}`, and `pi.lane.state/{branch}` must all exist. A missing tip rejects (`unknown branch`); a data-only Branch (tip without config/state) rejects. Forking does not audit unrelated lanes or malformed lane records: supported writers create lane state atomically, so branch validation checks only the selected Branch's required tip, config, and state. `entryId`, when supplied, must be an entry on that Branch's current tip ancestry (inclusive); an entry elsewhere in the tree rejects. Omitted `entryId` means the current tip. `position` defaults to `"at"`; `"before"` selects the target's parent and may yield a `null` destination tip (before the root entry, or a `null` source tip with no `entryId`) — legal. The destination contains exactly one Branch with the same name, the selected tip, the copied `LaneConfiguration`, and fresh idle lane state `{ currentOperationId: null, lastOperationId: null, inbox: [] }`. No other Branch or lane exists in the destination. + +This deliberately makes branch forks of data-only Branches — including legacy v3 imports whose main lacked reconstructable model/thinking history — unavailable. Tree scope remains available for those sources. + +**Tree scope** copies: every immutable entry, including entries unreachable from every current Branch tip; every Branch tip verbatim; every configured lane's config plus fresh idle lane state under the same name; data-only Branches (tip present, config/state absent) as data-only Branches. Forking does not perform a corruption audit of lane records; supported writers create valid lane state atomically. + +**Both scopes**: copy `pi.session.name`; copy `pi.entry.label` values only for copied entries; exclude the usage ledger, `pi.result`, all `pi.op.*`, all `pi.pending.*` (entries, tool checkpoints, assistant frames), and every trace of open-operation state. Destination usage totals start at zero; destination `messageCount` equals the number of copied message entries, exactly as current conformance already proves. Destination metadata records `parentSessionId = source.id`. Sequence preservation is not part of the fork contract: backends may allocate destination-local sequences for copied and transformed writes, provided destination sequence and list-cursor semantics remain internally valid. + +**Application values/lists (outside the reserved `pi`/`pi.*` namespaces)**: tree scope copies every current scalar value and every surviving list element; branch scope copies none. A `seq <= tipSeq` cutoff is forbidden as "historical reconstruction" — it is provably not one: + +```text +Scalar: TX[seq 10: set my-app.state = v1] · TX[seq 12: insert e1] · + TX[seq 50: set my-app.state = v2] (v2 describes work after e1) +Branch fork at e1, cutoff seq <= 12: only v2 exists (v1 was replaced, no +history is retained); 50 > 12 excludes it → state absent, though the app +demonstrably had state v1 at the fork point. The cutoff cannot recover v1. + +List: append seq 5 · append seq 20 · deleteList seq 40 · append seq 60 +Cutoff seq <= 30: elements 5 and 20 no longer exist (whole-list delete +destroyed them); only 60 survives and is excluded → empty list, though the +list held {5, 20} at seq-30 time. +``` + +Any cutoff filters *survivors*, silently mixing rewound intent with post-deletion reality. Branch scope therefore copies nothing application-owned, matching its fresh idle lane state and zero ledger; applications own their own re-derivation. + +### 1.2 One closed fork classifier + +All namespace fork knowledge lives in one core module beside `session/values.ts` and `session/fork.ts` (e.g. `session/fork-policy.ts`). It is a closed switch, not a registry, plugin policy, or DSL: + +- `pi.op.*`, `pi.pending.*`, `pi.result` → exclude, both scopes. +- `pi.session.name` → copy. +- `pi.entry.label` → copy iff the keyed entry is copied. +- `pi.branch.tip`, `pi.lane.config`, `pi.lane.state` → structured lane actions; the scope-specific rules (branch keeps only the named lane and rewrites its tip; lane state is always replaced with fresh idle state; tree keeps all) live in one shared driver consumed by all backends. +- The exact namespace `pi` and any other `pi.*` namespace → the fork **fails**, but only when current surviving state exists at fork time (a current scalar row or a surviving list element). Historical writes later replaced or deleted are absent from current state on every backend and must not alone fail a JSONL fork — behavior is backend-equivalent. Introducing a new built-in namespace without declaring its fork semantics must break fork tests, not silently copy or drop state. +- Neither `pi` nor `pi.*` → application: copy on tree, exclude on branch. The only built-in list namespace (`pi.pending.assistant_frame`) excludes; application lists follow the application rule. + +The driver exposes a streaming shape — accept one committed value/list write (or current row), emit zero or more destination writes, `finish()` emits fresh idle lane states and the rewritten branch tip. Entry-copy membership is a backend-supplied predicate so each backend uses its own index. `createForkSnapshot`, `forkSnapshotWrites`, `ForkSourceSnapshot`, `ForkDestinationSnapshot`, and the `entriesComplete` escape hatch are deleted. + +### 1.3 Streaming backend procedures + +Forks must stream source writes rather than materializing full source snapshot arrays. Source copy paths must not call array-returning full reads: no `snapshotEntriesAndValues()`, `captureForkSource()`, `readAllScalarValueRows()`, `readAllEntryRows()`, whole-file `readTextFile`, or SQLite `.all()` over unbounded row sets. JSONL may retain its structural index and legacy-v3 compaction tail in memory. + +**Memory.** At a source `commitQueue` boundary, build the destination `InMemoryStorageState` directly by iterating source maps once through the classifier/driver. Branch scope walks `parentId` from the branch tip toward the root to compute the ancestry id set and verify `entryId` membership; copy exactly that set. No intermediate snapshot arrays; `MemoryStorage.fromSnapshot` and `captureForkSource` are removed. + +**JSONL.** The source is read through a read-only path that captures the source's current `nextSeq` at the commit-queue boundary; both scans emit only complete writes below that sequence. The fork never writes the source: no torn-tail truncation/rewrite, no v3 normalization persistence, no `.tmp` beside the source. A torn or incomplete final line is discarded in memory. This relies on format-4 sources remaining append-only and not being replaced while the fork runs; compaction/replacement coordination is deferred with J1. + +- *Both scopes* use two scans up to the captured sequence boundary. Pass 1 streams every line and folds value/list writes into an in-memory structural index: per scalar address, the seq of the current surviving `set` (absent after a trailing `delete`); per list address, the set of surviving element seqs (whole-list delete clears it); plus entry `id → parentId` where branch scope needs ancestry. The fold applies the same current-state semantics replay does. +- Pass 2 streams again and emits a write only when the index proves it current and the classifier selects it: an entry write in the selected set; a scalar `set` whose seq equals the index's current-row seq for its address; a list `append` whose element seq is in the survivor set. Deletes, superseded sets, and dead application value/list history are never emitted as destination state. +- Transformed built-ins emit when pass 2 reaches the source's current corresponding row for a kept branch or configured lane. The destination writer may assign destination-local sequences. +- *Branch scope* walks tip→root through the in-memory index to materialize the ancestry set and verify `entryId` membership before pass 2; labels emit only for member entries. +- The destination is staged in a temp file and atomically renamed (existing `publishFileAtomically`); failure removes the temp file. Legacy v3 splits by source state. Forking an **open** legacy-v3 `JsonlStorage` **rejects with a clear error** until a normal non-empty commit has upgraded and persisted its normalized format-4 ids: v3 normalization mints nondeterministic UUIDv7 tails, so an independent disk reparse cannot reproduce the ids the open Session exposes or validate a caller `entryId`; the fork must not mutate/upgrade the source itself. A **closed** legacy-v3 source uses the read-only parser/normalizer and never touches the file: tree forks succeed; branch forks may use an omitted `entryId` (the default normalized tip) when the import reconstructs a complete configured lane, while any caller-supplied process-local id from an earlier open is not stable across reparses and rejects normally; a data-only reconstructed lane rejects as everywhere. The parser may retain structural metadata and compaction `retainedTail` context in memory. Ordinary writable `JsonlStorage.open` may retain in-memory normalization, but resident state is never a fork source. +- There is **no resident-state path**: an open same-repository JSONL source enqueues a short boundary callback on the source `commitQueue` whose only job is to capture `nextSeq`, then releases the queue; the fork runs the same two-scan procedure below that boundary while later source appends proceed. Committed writes are durable in the file before they are applied to resident state, so the file is authoritative at the boundary. + +**SQLite.** Iterator-based row transfer through a bounded-memory temporary on-disk SQLite **staging database**, used uniformly for per-file and shared-container layouts. The source reader never streams directly into the destination transaction — a WAL reader and one writer can coexist, but in shared-container mode the destination writer would hold the container's sole write lock and block the post-boundary source writer while capture is still streaming; staging into a separate file removes that coupling. Use prepared-statement iteration (`iterate`/stepwise), never source-sized `.all()` snapshot arrays: + +- *External/closed/live-worker source:* WP07 path — `openReadOnly` on the exact canonical path, one deferred read transaction, session row and storage version validated inside it. +- *Same-repository open source:* open the independent read-only connection **first**, then enqueue a short boundary callback on the source Storage `commitQueue` whose only job is to `BEGIN` and establish the independent reader's snapshot (issue a trivial read) before releasing the queue. Do not begin a read transaction on the source writer connection and then release its queue, and do not hold the queue for the copy duration. +- *Stage:* while the source reader remains open, stream selected entries `ORDER BY seq` and classifier-selected current scalar/list rows (SQL-level namespace prefilters matching the classifier — enumerated built-in namespaces plus the application predicate excluding exact `pi` and `pi.%` — with each row still passing the classifier) into the temporary staging database in bounded batches. Branch scope enumerates the ancestry through the `branch_entries` segment chain up to the selected entry and answers `entryId` and label membership through that index, not an in-RAM id set. Later source commits use the original writer connection and may complete while staging streams, in **both** layouts, because stage writes target another file. +- *Publish:* close/commit the source read transaction, then stream the stage into one destination `BEGIN IMMEDIATE` transaction — entries in source order maintaining the destination branch index and `message_count` incrementally, then values/list elements — and delete the staging database in `finally` on success and failure. The destination allocates its own valid sequence range. Shared-container destinations write only the new session's rows. + +### 1.4 Preserved WP07 behavior + +Destination id reservation across create/open/fork/delete, no-create opens, foreign-metadata rejection, per-file and shared-container layouts, WAL commit-boundary wholeness (a source commit is wholly inside or wholly outside one fork), and all-settled repository close are unchanged. + +### 1.5 Coding-agent status (record only) + +`/fork`, `/clone`, and `--fork` run entirely on the legacy `SessionManager` (`createBranchedSession`, `forkFrom`) and are not migrated here. Future mapping when coding-agent adopts `SessionRepo`: `/fork` → `{ scope: "branch", branch: "main", entryId, position: "before" }`; `/clone` → `{ scope: "branch", branch: "main" }`; `--fork` → `{ scope: "tree" }`. + +## 2. Problems in current source + +- `ForkOptions` defaults `scope` to `"branch"` and hard-codes source/destination `main` (`fork.ts`, both backend branch readers). +- No ancestry membership check: `selectForkContents` walks parents from any supplied `entryId` anywhere in the tree and labels the result `main`. +- Every backend materializes the full source: `snapshotEntriesAndValues()` (Memory/JSONL), `readAllScalarValueRows` + `readAllEntryRows`/`scanBranchEntries` arrays (SQLite), all funneled through in-memory `createForkSnapshot`. +- `createForkSnapshot` and its array snapshots force all backends through a materialized in-memory source copy. +- A JSONL fork of a closed source uses `JsonlStorage.open()`, which rewrites the source file on a torn tail — a fork can mutate its source today. +- Lists are never copied; application values are excluded with no declared tree policy (`values.md` explicitly defers it). +- Namespace fork knowledge is repeated across `fork.ts`, `values.md` prose, and conformance assertions; nothing forces a new `pi.*` namespace to declare fork semantics. +- `entriesComplete?: false` exists only to let SQLite branch snapshots skip tip validation for other branches. + +## 3. Required result + +1. `ForkOptions` and validation exactly as §1.1; all rejection paths create no destination file, database rows, or reserved-but-leaked ids. +2. The closed classifier/driver of §1.2, exported from core and consumed by all three backends; unknown reserved namespaces (exact `pi` or undeclared `pi.*`) fail the fork only when current surviving state exists, identically on every backend. +3. Streaming procedures of §1.3 on all three backends; `createForkSnapshot`/`captureForkSource`/`snapshot()` fork plumbing and their exports removed from `session/index.ts` and the sqlite-node import surface. +4. Valid destination-local sequence allocation on every backend; identical logical destination state across backends for identical sources (conformance). +5. JSONL source non-mutation, including torn-tail sources and legacy v3 sources. +6. Documentation: `harness.md` §2.7 (and the §1.7 fork-related sentences), `values.md` "Forks and rewrites" and backend snapshot mentions, `post-wp05-roadmap.md` (retire the SQLite fork-cost item, add/point to this package), `packages/session-backends/sqlite-node/README.md` fork paragraphs, `packages/agent/benchmark/session/README.md` if dataset wording changes. Historical WP docs and released changelogs untouched. Format/storage versions unchanged; no migration. + +## 4. Implementation slices + +### Slice A — contract and classifier (core, Memory reference) + +Files: `session/types.ts`, new `session/fork-policy.ts`, `session/fork.ts` (rewritten or deleted), `session/values.ts` (only if new helpers are needed), `session/index.ts`, `session/memory.ts`, `session/in-memory-storage-state.ts`, conformance `testing/conformance/session-repo.ts`, `test/harness/memory-conformance.test.ts`, `test/harness/memory-session-repo.test.ts`. + +1. Replace `ForkOptions`; implement validation and the classifier/driver. +2. Rewrite Memory fork as direct destination construction at a commit-queue boundary. +3. Rewrite the shared fork conformance for the new contract (§5) and pass it on Memory. + +### Slice B — JSONL streaming + +Files: `session/jsonl/storage.ts`, `session/jsonl/repo.ts`, `session/jsonl/types.ts`, streaming reader support in the `FileSystem` capability (`harness/types.ts`, `harness/env/*`) if required, `test/harness/jsonl-session-repo.test.ts`, `jsonl-session-repo-conformance.test.ts`, `jsonl-storage.test.ts`, `jsonl-v3-migration.test.ts`. + +1. Read-only sequence-boundary capture with in-memory torn-tail discard; the same capture at the source `commitQueue` boundary for open sources; no source writes on any fork path. +2. Two-scan in-memory current-state fold and emission for both scopes; branch ancestry membership through the structural index; atomic destination publish. +3. Open-v3 fork rejection and the closed-v3 read-only parser/normalizer path; update v3 fork tests per §5. + + +### Slice C — SQLite streaming + +Files: `sqlite/repo.ts`, `sqlite/storage.ts`, `sqlite/session/values.ts`, `sqlite/session/entries.ts`, `sqlite/session/branch-entries.ts`, `sqlite/types.ts`, `test/repo.test.ts`, `test/repo-conformance.test.ts`. + +1. Replace `SqliteStorage.snapshot()`/array snapshot helpers with the independent-reader-plus-boundary-callback design and iterator transfer into the temporary staging database, then stage-to-destination publication with `finally` cleanup. +2. Branch ancestry/membership/labels through the branch index; classifier-matching SQL prefilters. +3. Preserve WP07 identity, reservation, no-create, and close coverage; both layouts. + +### Slice D — benchmarks and documentation + +Files: `testing/benchmark/session-repo.ts`, both `session-repo.bench.ts` files, `benchmark/session/README.md`, `harness.md`, `values.md`, `post-wp05-roadmap.md`, sqlite-node `README.md`, changelogs only under normal branch rules. + +Update fork option literals, add large-source fork benchmarks (tree and branch), and land the §3.6 documentation set. + +## 5. Required tests + +### Contract and validation + +- branch scope rejects: unknown branch name; data-only Branch (tip only); `entryId` not on the named Branch's tip ancestry (present elsewhere in the tree); `entryId` unknown; `entryId` with a `null` tip. Each rejection leaves no destination artifact and releases its reserved id. Fork tests do not construct or audit malformed lane records outside the selected Branch. +- branch scope accepts: omitted `entryId` (tip), explicit tip, mid-ancestry entry, `position: "before"` at a mid entry and at the root entry (`null` destination tip), `null` source tip with no `entryId`. +- destination shape: exactly one Branch, same name, copied config, fresh idle lane state, no other lane values, `parentSessionId` set. +- tree scope: unreachable entries copied; every tip copied; configured lanes get config plus fresh idle state; data-only Branches stay data-only. Forking does not audit malformed lane records. +- both scopes: session name copied; labels only for copied entries (branch fork excludes labels of non-ancestry entries); `pi.result`, `pi.op.*`, `pi.pending.*` (pending entries, tool checkpoints, assistant frame lists), and usage rows absent; exact stats expectation — a fork copying N message entries reports `getStats()` = `{ messageCount: N, usage: all-zero }`, matching current conformance; the ledger-completeness invariant ("a fork's ledger starts at zero") retained. + +### Application values and lists + +- tree fork copies every non-`pi.*` scalar and every surviving list element; a list deleted-then-reappended in the source reproduces only survivors. +- branch fork copies no application values/lists (pin the §1.1 scalar and list traces as regression cases: post-fork-point overwrite and delete-destroyed elements must not resurface under any implementation). +- current surviving state in an unknown reserved namespace (exact `pi` or an undeclared `pi.*` scalar or surviving list element) fails the fork on every backend; the same namespace **set then deleted** before the fork fails no backend — including JSONL, where the dead history remains as physical lines and the disk fold must classify it as absent. Construct both cases via raw committed writes. + +### Sequence allocation + +- sequence preservation, source high-water marks, and source list cursors are not part of the fork contract. Each destination assigns valid local sequences and list cursors; its first post-fork commit must allocate without collision. +- JSONL captures source `nextSeq` only to establish the append-only source boundary; a torn incomplete final write does not belong to the fork. + +### Streaming source reads and source non-mutation + +- instrumented source readers (test decorators/spies) assert fork paths never call `snapshotEntriesAndValues`, `captureForkSource`, `readAllScalarValueRows`, `readAllEntryRows`, whole-file `readTextFile`, or source-sized `.all()` arrays on entries/values/lists; deterministic large fixtures (extend the existing benchmark dataset generators) exercise tree and branch forks over multi-thousand-entry sources. +- SQLite staging databases are removed on success and failure; a failed fork leaves neither a stage file nor destination rows. +- JSONL: fork of a closed torn-tail source succeeds, the destination excludes the torn transaction wholly, and the source file bytes are unchanged (byte-compare before/after); fork of a legacy v3 source leaves the source file unchanged; no `.tmp` appears beside the source. +- JSONL sequence-boundary capture: a source append completed after capture is wholly absent from the fork. +- JSONL destination content: the destination file contains only current selected scalar rows and surviving list elements — no delete records, superseded sets, or dead application history. +- legacy v3: forking an **open** v3 source rejects with the clear pre-upgrade error; after one normal non-empty commit upgrades it to format 4, a fork succeeds and preserves the persisted ids; a **closed** v3 tree fork succeeds through the read-only parser and leaves the source byte-identical; a closed v3 branch fork with omitted `entryId` succeeds when a complete configured lane is reconstructed, a caller-supplied id from an earlier open rejects, and a data-only reconstructed main rejects. + +### Coordination and ordering + +- Memory open-source forks keep the queue-boundary conformance case: a commit admitted before the fork appears wholly; one admitted after does not. +- JSONL open-source forks capture the sequence boundary at the `commitQueue` boundary: a commit admitted before the boundary appears wholly; a source append completed after boundary capture proceeds while the fork is still streaming and is wholly absent from that fork. +- SQLite same-repo: the independent reader opens first, the boundary callback on the source `commitQueue` establishes its snapshot and releases the queue, and a subsequent source commit on the writer connection **completes while the reader is still streaming into the stage** and is wholly absent from that fork; a later fork includes it wholly. Assert this in **both** per-file and shared-container layouts (satisfiable in shared containers because stage writes target another file), plus the WP07 live external-worker case unchanged. +- destination-reservation races (create vs fork on one id, both orders) unchanged. + +### Backend equivalence + +- one shared conformance source (entries, unreachable entries, multiple lanes, data-only Branch, labels, app scalars/lists with deletes, open operation with pending/frame/checkpoint state, usage rows) forks to identical logical destination state on Memory, JSONL, and SQLite for both scopes. + +## 6. Validation and review + +After each slice: `npm run check`, then the modified focused tests via the repository Vitest binary from the owning package; `./test.sh` before final review. Grep guards: no remaining `createForkSnapshot|captureForkSource|ForkSourceSnapshot|entriesComplete` outside historical docs and this handoff; no **production fork source path** referencing `"main"` as a literal (this document and the recorded coding-agent mapping in §1.5 intentionally mention main). + +Review checkpoints (delegated reviews use provider `anthropic`, model `claude-fable-5`): + +1. After Slice A: contract, classifier closure, conformance shape. +2. After Slice C: same-repo ordering, reader independence, and streaming-source evidence. +3. Final review of source, tests, docs, and exclusions. + +## 7. Exclusions + +Do not include: + +- compatibility aliases, a default scope, or implicit `main`; +- any `seq <= tip` cutoff or other historical-state reconstruction; +- a fork-policy registry, plugin hook, or DSL; per-address application opt-ins; +- copying usage rows, `pi.result`, or any open-operation state under any option; +- coding-agent `/fork`, `/clone`, `--fork`, or RPC migration; +- storage-version bumps, migrations, or format changes (formats stay WIP-in-place); +- J1 snapshot compaction (the streaming reader may be built shareable, nothing more); +- SQLite branch-segment redesign or the uncompacted-divergence fix; +- `SessionRepo` interface changes beyond `ForkOptions` (repository `close()` belongs to the lifecycle package); +- precise-rewrite tooling; search; edits to WP00–WP07 documents or released changelog sections. + +If implementation requires an excluded item, stop and revise the handoff. + +## 8. Exit condition + +WP08 is complete when: + +- `ForkOptions` is exactly the §1.1 union with the stated validation, on all three backends; +- branch forks require a complete configured source AgentLane and enforce ancestry membership; tree forks copy the complete immutable tree, every tip, configured and data-only Branches; +- tree forks carry all current application values and surviving list elements; branch forks carry none; +- one closed core classifier owns every namespace disposition; unknown reserved namespaces (exact `pi` or undeclared `pi.*`) fail forks exactly when current surviving state exists, equivalently on all backends; +- all supported fork paths — including closed legacy v3 sources — stream source writes without materializing full source snapshot arrays, proven by instrumented-reader tests over large fixtures; JSONL may retain structural indexes and legacy compaction context in memory; forking an open legacy-v3 source is explicitly unsupported (clear rejection) until an ordinary commit upgrades it, not handled by an in-memory exception; +- JSONL forks never mutate their source, including torn-tail and legacy v3 sources, and JSONL destinations contain only current selected rows; +- SQLite forks stage through a temporary on-disk database (source reader → stage while open, stage → one destination `BEGIN IMMEDIATE` after reader close, stage deleted in `finally`), with the independent-reader boundary design and concurrent later writer commits proven in both layouts, preserving all WP07 behavior; +- destinations allocate valid local sequences and backends produce identical logical destinations; +- shared conformance, focused backend tests, benchmarks, `npm run check`, and `./test.sh` pass; +- normative docs and the roadmap reflect the new contract with historical documents untouched; +- final Fable review reports no blocker. + +## 9. TODO + +### Cleanup after invocation cancellation + +Cancellation-safe filesystem cleanup is deferred. Cleanup currently receives the caller's Context, so an aborted invocation can leave a staged file behind even though readers close and the destination id reservation is released. Define the cleanup Context policy before implementing this; do not import Chord's `withoutAbortSignal` directly into JSONL code as a temporary workaround. diff --git a/packages/agent/docs/work-packages/09-lane-snapshot-settled-tools.md b/packages/agent/docs/work-packages/09-lane-snapshot-settled-tools.md new file mode 100644 index 000000000..9c65f1cc5 --- /dev/null +++ b/packages/agent/docs/work-packages/09-lane-snapshot-settled-tools.md @@ -0,0 +1,517 @@ +# Work package 09 — LaneSnapshot settled-but-unplaced tools + +## Status and baseline + +- Repository: `earendil-works/pi` +- Branch at handoff creation: `dev` +- Baseline commit: `d14d6b22327d545d6a253f932165b63e48d7f9c8` +- The user reported the worktree clean immediately before this handoff. +- This document began as an implementation handoff after conversation compaction and now records the implemented design. It is not itself the normative harness specification; `packages/agent/docs/harness.md` remains normative. + +## Goal + +Keep every started or settled-but-unplaced tool call visible in `LaneSnapshot.operation.runningTools` until its immutable `toolResult` entry is placed in the transcript. + +The intended projection is: + +```text +planned → not yet represented in runningTools +effect_pending → runningTools(status: "running") +outcome_ready → runningTools(status: "settled") +completed → transcript toolResult entry +``` + +For each call after it becomes presentation-active, `runningTools` and placed transcript entries must not have a gap or overlap. Placement is a source-prefix flush, not an all-tools barrier. Remove a call from `runningTools` on its own `entry_added`, never on `turn_end`. + +## Original bug + +A call disappeared between real-effect completion and source-ordered tree placement: + +1. `packages/agent/src/harness/runtime/reducer.ts`: `tool_end` spliced the call out of `runningTools`. +2. `packages/agent/src/harness/runtime/lane.ts`: `captureLaneSnapshot()`, `case "tools"`, projected only `effect_pending` calls and skipped `outcome_ready` calls. +3. The finalized result was already durable at `pendingEntry(resultEntryId)`, but a fresh/reconnected snapshot could not display it. +4. It reappeared only after `entry_added` placed the immutable `toolResult` entry. + +For a parallel batch `[A, B, C]`, if B settles while A remains pending, B may stay `outcome_ready` until A is ready. If A is already placed, B can place without waiting for C. Therefore clearing everything at `turn_end` is wrong: early-placed results would temporarily exist in both `transcript` and `runningTools`. + +## Confirmed current architecture + +Mini does **not** replicate structural object deltas. + +- `packages/coding-agent/src/experimental/mini/worker/lane-service.ts` sends one initial full snapshot and then forwards individual `HarnessEvent` objects. +- `packages/coding-agent/src/experimental/mini/tui/session.ts` folds those events through `reduceLaneSnapshot()`. +- Reconnect/rebase fetches another complete snapshot. +- `tool_update` currently carries a complete replacement progress result, not a nested diff. + +Implemented durable tool flow in `packages/agent/src/harness/runtime/drive/tools.ts`: + +```text +prepare +→ before_tool +→ intent commit (effect_pending + effective args), then tool_start +→ execute/update/checkpoint +→ after_tool +→ finalize +→ publishToolOutcome staging commit (pendingEntry + outcome_ready), then tool_end +→ materializeReady prefix placement +→ entry_added +``` + +All real, immediate synthetic, cancellation, and recovery outcomes converge through `publishToolOutcome()`. + +`Lane.settleOperation()` supports commit-bound events. It commits, publishes process-local state, constructs the event batch, and the public operation awaits delivery. In parallel execution, `materializeReady()` is scheduled only after the outcome-completion promise resolves. Consequently, `tool_end` is delivered after staging and before placement. + +## Agreed event contract change + +Do **not** add `tool_result_ready` or `tool_outcome_ready`. + +Instead, redefine harness `tool_start`/`tool_end` as tool-call processing/result lifecycle events rather than exclusively real external-effect lifecycle events. + +### Fresh executed call + +```text +intent commit +→ tool_start +→ tool_update* +→ execute/finalize +→ TX[pendingEntry + outcome_ready + cleanup] +→ tool_end +→ source-ordered placement +→ entry_added +``` + +### Fresh synthetic call + +```text +TX[pendingEntry + outcome_ready] +→ tool_start +→ tool_end +→ source-ordered placement +→ entry_added +``` + +The staging transaction's post-commit event batch contains `tool_start` followed by `tool_end`, so a watcher cannot observe either lifecycle event without the authoritative staged state. + +Fresh synthetic calls include: + +- unknown tool; +- argument preparation or validation failure; +- `before_tool` denial or invalid replacement arguments; +- genuine assistant `length`/truncated call handling; +- cancellation while still `planned` or after intent but before effect admission. + +### Recovery + +Historical lifecycle events are not replayed. + +- A restored `effect_pending` call is already represented by the initial snapshot. +- Safe replay emits recovery-tagged `tool_start` from the checkpoint-clear commit and `tool_end` from the later outcome-staging commit. +- Unsafe interruption synthesis may emit a recovery-tagged `tool_end` without a newly emitted `tool_start`; the initial snapshot supplied the running row. +- A call already restored as `outcome_ready` appears as settled in the initial snapshot and needs no replayed end event before placement. + +### Meaning of `tool_end` + +After this change, `tool_end` means: + +> The complete final tool result is durably staged and the call is `outcome_ready`. + +It becomes the authoritative reducer transition from running to settled. It must be emitted **after** the staging commit, not before it. + +The old distinction between actually executed and synthetic results was encoded by omitting lifecycle events. There is no in-repo runtime consumer requiring that distinction. If preserving it is desired, discuss adding an explicit field such as `execution: "executed" | "synthetic"`; this field was discussed but **not agreed**, so do not add it silently. + +## LaneSnapshot type + +Change `LaneSnapshot.operation.runningTools` in `packages/agent/src/harness/agent-harness.ts` to use one `result` field for both progress and final output. Do not retain a separate `partialResult` field in the snapshot. + +Prefer a discriminated union so invalid combinations are unrepresentable: + +```ts +type SnapshotTool = + | { + status: "running"; + toolCallId: string; + toolName: string; + args: unknown; + result?: AgentToolResult; // latest complete progress snapshot + } + | { + status: "settled"; + toolCallId: string; + toolName: string; + args: unknown; + result: AgentToolResult; // complete finalized result + isError: boolean; + }; +``` + +The user explicitly agreed to the discriminated union. + +`tool_update.partialResult` may remain named `partialResult` in the event API; the reducer assigns it to the snapshot row's unified `result` field. + +The current mini transport sends semantic events, not structural deltas, so `tool_end` still carries the complete final result even if it equals the latest update. Unifying the snapshot field is still the correct state model. + +## Exact reducer behavior + +File: `packages/agent/src/harness/runtime/reducer.ts` + +### `tool_start` + +- Resolve the operation with `matchingOperation(snapshot, event.runId)`. +- Upsert by `toolCallId`; do not blindly push. +- Set `status: "running"`, `toolName`, `args`. +- Clear stale settled-only fields if replacing an existing row. +- Preserve no stale final result. A newly started/replayed call may receive later `tool_update` values. + +Upsert is required because a watch may capture durable `effect_pending` state before the buffered `tool_start` event is delivered. + +### `tool_update` + +- Use `matchingOperation(snapshot, event.runId)`, not `snapshot.operation` directly. +- Find the matching row. +- For a running row, replace `result` with `event.partialResult`. +- Ignore wrong-operation or missing rows. + +### `tool_end` + +- Resolve with `matchingOperation(snapshot, event.runId)`. +- Find the existing row by batch-local `toolCallId`; only one tool batch is presentation-active at a time. +- Replace it with `status: "settled"`, preserving its arguments and using `event.result` and `event.isError`. +- This naturally removes the provisional interpretation of the old `result`; there is no separate `partialResult` to delete. +- The finalized result remains displayed until placement. + +`tool_end` does not carry arguments and cannot create a row. Fresh synthetic `tool_start` and `tool_end` are emitted together after the staging commit, eliminating the old capture/event gap. Unsafe recovery relies on the initial snapshot's running row. + +### `entry_added` + +If `event.entry` is a message whose role is `toolResult`, remove the matching batch-local `toolCallId` from `snapshot.operation?.runningTools`, then apply the transcript update. Harness events are serialized, trusted, emitted exactly once, and not historically replayed, so neither duplicate-entry handling nor cross-batch identity is needed. + +Do not clear tool rows on `turn_end`. + +## Exact authoritative capture behavior + +File: `packages/agent/src/harness/runtime/lane.ts`, `captureLaneSnapshot()`, `case "tools"`. + +The assistant entry is already loaded once. For each batch call: + +### `planned` + +Skip. It has not become presentation-active yet. + +### `completed` + +Skip. Its `toolResult` entry must already be in the captured transcript. + +### `effect_pending` + +- Validate that `assistant.message.content[sourceIndex]` is the matching `toolCall` block. +- Read `operationToolArgs(operationId, turnId, sourceIndex)`; it is required for effect-pending calls. +- Read optional `pendingToolOutput(operationId, resultEntryId)`. +- Project: + +```ts +{ + status: "running", + toolCallId: block.id, + toolName: block.name, + args: persistedArgs, + ...(checkpoint === undefined ? {} : { result: checkpoint }) +} +``` + +### `outcome_ready` + +- Validate the source tool-call block. +- Read `pendingEntry(call.resultEntryId)`. +- Require a message payload with role `toolResult`. +- Validate staged `toolCallId` and `toolName` against the source block. +- Read `operationToolArgs(...)` when present. +- Use `persistedArgs ?? block.arguments`. Immediate synthetic calls may never have written `operationToolArgs`, and this absence is legal only for the outcome-ready projection. +- Reconstruct the canonical `AgentToolResult` from the staged `ToolResultMessage` and the durable call termination flag as needed. +- Project `status: "settled"`, `result`, and `isError`. + +The event and capture representations must normalize the final result identically so folding through `tool_end` equals a later authoritative snapshot. Pay attention to optional `details`, `usage`, and `terminate`; do not rely on incidental object-property presence differences. + +A missing or mismatched staged result for `outcome_ready` is presentation corruption and must fault snapshot capture. + +## Runtime event production changes + +Primary file: `packages/agent/src/harness/runtime/drive/tools.ts` + +Related helpers: `packages/agent/src/harness/execution/tools.ts` and `packages/agent/src/harness/runtime/drive/tool-placement.ts`. + +### Internal outcome shape + +Current: + +```ts +type ToolOutcome = { message: ToolResultMessage; terminate: boolean }; +``` + +Extend/refactor it so post-commit event production has the complete canonical final result and `isError`, without lossy reconstruction. It must retain enough data for: + +- staged `ToolResultMessage`; +- `tool_end.result`; +- `tool_end.isError`; +- durable/effective `terminate` after cancellation normalization. + +Synthetic helpers currently return `ToolResultMessage` directly. Refactor carefully so synthetic outcomes also carry the canonical result data. Do not invent `details` in the transcript: existing unknown/invalid synthetic results deliberately omit message details. + +### Commit-bound `tool_start` + +For fresh execution, `publishToolIntent()` attaches `tool_start` to the commit that persists effective arguments and changes the call to `effect_pending`. The public operation awaits delivery before admitting `executeToolCall()`, preserving `tool_start → tool_update*` without requiring each update callback to await delivery. + +For a fresh synthetic call that never writes effect intent, `publishToolOutcome()` attaches `tool_start` before `tool_end` in the outcome-staging commit's event batch. It reports the source block arguments. + +For safe recovery, the checkpoint-clear commit emits recovery-tagged `tool_start` using persisted effective arguments. Do not emit a fresh start for an already-restored unsafe `effect_pending` call; its initial snapshot is the baseline. + +### Post-commit `tool_end` + +Remove the current pre-staging `tool_end` emission from `performToolInvocation()`. + +`publishToolOutcome()` attaches `tool_end` to the same staging command's `events` callback. The event carries: + +- `runId`, `turnId`, `toolCallId`, `toolName`; +- canonical final `result`; +- `isError`; +- cancellation-normalized durable `terminate`; +- `recovery: true` where applicable. + +Arguments belong to `tool_start` and are not repeated on `tool_end`. Event data describes the state actually committed, especially cancellation forcing `terminate: false`. + +Because `Lane.command()` awaits retained event delivery and `runParallel()` schedules materialization from the outcome-completion promise, the required order is: + +```text +staging commit +→ tool_end delivery +→ source-ready message lifecycle +→ placement commit +→ entry_added +``` + +### Call sites to audit + +Every `publishToolOutcome()` call must supply the source tool call and recovery context correctly: + +- immediate outcome in `startToolInvocation()`; +- cancellation after intent but before execution; +- normal `performToolInvocation()` completion; +- safe replay completion; +- unsafe recovery interruption; +- sequential cancellation of `planned`; +- sequential cancellation of `effect_pending`. + +Also audit the synchronous `AbortRequested` path inside `performToolInvocation()`: a call with durable intent must still have coherent start/end presentation even if effect admission fails immediately. + +## Mini and other presentation consumers + +### Mini + +File: `packages/coding-agent/src/experimental/mini/tui/view.ts`, `MiniTui.apply()`. + +For each `runningTools` row: + +```text +status running: + markExecutionStarted() + if result exists: updateResult({...result, isError:false}, true) + +status settled: + do not call markExecutionStarted() + updateResult({...result, isError}, false) +``` + +The final result remains visible while awaiting placement. After `entry_added`, transcript synchronization supplies the immutable `ToolResultMessage` and the row is no longer in `runningTools`. + +### Other shared consumer + +Apply equivalent handling in: + +- `packages/coding-agent/src/experimental/client-tui-chat.ts` + +Read `packages/coding-agent/src/modes/interactive/components/tool-execution.ts` before editing to confirm `updateResult(result, isPartial)` semantics. + +## Tests + +### Reducer tests + +File: `packages/agent/test/harness/runtime/reducer.test.ts` + +Add a parallel batch event-fold test with calls 0, 1, 2: + +1. Establish all three as running. +2. Call 2 settles before call 0. +3. Call 0 settles and is placed while call 1 still runs. +4. Call 1 settles. +5. Placement flushes calls 1 and 2 in source order. +6. After every settlement and placement event, assert each presentation-active call appears in exactly one of: + - `operation.runningTools`; or + - transcript `toolResult` entries. +7. Assert call 2 remains present with `status:"settled"` and its final result while blocked by earlier calls. +8. Assert each `entry_added` removes only its matching active row. + +Add focused coverage for: + +- `tool_update` from a stale/wrong `runId` does not mutate the current operation; +- `tool_start` upserts rather than duplicates a row captured from durable intent; +- `tool_end` settles the existing batch-local row. + +### Capture/watch tests + +File: `packages/agent/test/harness/runtime/watch.test.ts` + +Construct a durable tools state containing: + +- `planned` (omitted); +- `effect_pending` with checkpoint (`status:"running"`, checkpoint exposed as `result`); +- `effect_pending` without checkpoint; +- real `outcome_ready` with persisted effective args; +- synthetic `outcome_ready` without `operationToolArgs`, falling back to source block arguments; +- completed call represented only by a transcript entry where practical. + +Assert staged settled content, `isError`, arguments, and absence of duplicates. Add missing/mismatched `pendingEntry` corruption assertions. + +### Runtime tool tests + +File: `packages/agent/test/harness/runtime/drive-tools.test.ts` + +Update/add ordering assertions proving: + +- real: `intent commit < tool_start < tool_update* < staging commit < tool_end < entry_added`; +- immediate synthetic: `staging commit < tool_start < tool_end < entry_added`, no tool effect and no `after_tool`; +- planned cancellation gets coherent start/end; +- unsafe recovery uses initial snapshot plus recovery-tagged end without replaying effects; +- B can emit post-commit end and remain settled while A blocks placement; +- source-order placement remains unchanged; +- a crash after staging cannot replay the call. + +The baseline normative tests/documentation required `tool_end` before staging; the implementation reverses those expectations deliberately. + +### Type/event catalog tests + +Audit: + +- `packages/agent/test/harness/types.test.ts` +- `packages/agent/src/harness/telemetry.ts` + +No new event name is added. `tool_end` omits arguments and its semantics change. + +### Mini regression + +After unit tests, run the real mini abort smoke test used previously: + +- execute exactly `sleep 20`; +- send Escape after about two seconds; +- assert `Command aborted` and elapsed time appear; +- assert raw ANSI contains `toolErrorBg` (`48;2;60;40;40`); +- verify the durable session contains an `isError:true` tool result and operation status `aborted`. + +Also exercise a parallel batch where a later tool finishes first and verify its final result remains visible until in-order placement. + +## Documentation changes + +Read both documents completely before editing: + +- `packages/agent/docs/harness.md` (normative) +- `packages/agent/docs/tool-durability.md` + +The implementation updates baseline statements that required: + +- `tool_end` before staging; +- `tool_start`/`tool_end` only for real effects; +- synthetic outcomes emitting no lifecycle; +- `outcome_ready` being omitted from `runningTools`; +- snapshot field `partialResult`. + +Important known locations from the baseline: + +- `harness.md` §3.8 around lines 784–796; +- `harness.md` §5.4 `LaneSnapshot` around lines 1101–1134; +- `harness.md` §5.5 events around lines 1140–1158; +- `harness.md` tool phases around lines 1214–1224; +- `harness.md` conformance requirements around lines 1386–1400; +- `tool-durability.md` finalization around lines 255–280; +- `tool-durability.md` snapshots/events around lines 568–584; +- `tool-durability.md` test requirements around lines 671–679. + +The revised docs must state: + +- `tool_end` is post-staging durability evidence for a final result; +- fresh synthetic outcomes receive start/end lifecycle; +- recovery events are not historically replayed; +- `outcome_ready` remains projected as settled until placement; +- `entry_added` moves settled presentation into transcript; +- the unified snapshot `result` is provisional when running and final when settled. + +Do not modify the separate `response.ts`/`tool-placement.ts` recovery `turn_end` discrepancy unless separately requested. + +## Files to reread completely after compaction + +Core/specification: + +1. `packages/agent/docs/harness.md` +2. `packages/agent/docs/tool-durability.md` +3. `packages/agent/src/harness/agent-harness.ts` +4. `packages/agent/src/harness/runtime/reducer.ts` +5. `packages/agent/src/harness/runtime/lane.ts` +6. `packages/agent/src/harness/runtime/drive/tools.ts` +7. `packages/agent/src/harness/runtime/drive/tool-placement.ts` +8. `packages/agent/src/harness/execution/tools.ts` +9. `packages/agent/src/harness/runtime/types.ts` +10. `packages/agent/src/harness/session/types.ts` +11. `packages/agent/src/harness/events.ts` +12. `packages/agent/src/harness/telemetry.ts` + +Tests: + +13. `packages/agent/test/harness/runtime/reducer.test.ts` +14. `packages/agent/test/harness/runtime/watch.test.ts` +15. `packages/agent/test/harness/runtime/drive-tools.test.ts` +16. `packages/agent/test/harness/types.test.ts` +17. Relevant test helpers imported by those files. + +Mini/presentation: + +18. `packages/coding-agent/src/experimental/mini/tui/session.ts` +19. `packages/coding-agent/src/experimental/mini/worker/lane-service.ts` +20. `packages/coding-agent/src/experimental/mini/shared/protocol.ts` +21. `packages/coding-agent/src/experimental/mini/tui/view.ts` +22. `packages/coding-agent/src/experimental/client-tui-chat.ts` +23. `packages/coding-agent/src/modes/interactive/components/tool-execution.ts` + +Before editing, run `git status --short` and inspect current diffs because other Pi sessions may share the worktree. + +## Validation commands + +From repository root, after changes: + +```bash +cd packages/agent +node "$(git rev-parse --show-toplevel)/node_modules/vitest/dist/cli.js" --run test/harness/runtime/reducer.test.ts +node "$(git rev-parse --show-toplevel)/node_modules/vitest/dist/cli.js" --run test/harness/runtime/watch.test.ts +node "$(git rev-parse --show-toplevel)/node_modules/vitest/dist/cli.js" --run test/harness/runtime/drive-tools.test.ts +node "$(git rev-parse --show-toplevel)/node_modules/vitest/dist/cli.js" --run test/harness/types.test.ts +cd "$(git rev-parse --show-toplevel)" +npm run check +``` + +Do not run `npm test`, the full Vitest suite, or `npm run build` unless requested. + +If a delegated review is used, repository policy requires: + +```text +--provider anthropic --model claude-fable-5 +``` + +Keep extensions enabled. + +## Non-goals + +- No generic structural-delta transport for mini. +- No optimization to avoid the final result crossing the wire once in `tool_end` and again in `entry_added`. +- No change to source-prefix placement semantics. +- No clearing on `turn_end`. +- No change to tool effect replay/durability rules. +- No change to `after_tool`: it still runs only for actual fresh/safely replayed effects under its existing cancellation contract. +- No work on the separate recovery `turn_end` discrepancy between `response.ts` and `tool-placement.ts`. +- No backward-compatibility layer unless the user explicitly asks for one. +- Do not commit unless the user asks. diff --git a/packages/agent/scripts/generate-telemetry-docs.ts b/packages/agent/scripts/generate-telemetry-docs.ts index 93e888d8e..65a32e3b7 100644 --- a/packages/agent/scripts/generate-telemetry-docs.ts +++ b/packages/agent/scripts/generate-telemetry-docs.ts @@ -111,7 +111,7 @@ export function generateTelemetryDocs(outputPath: string, check: boolean): void if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) { generateTelemetryDocs( - resolve(import.meta.dirname, "../telemetry-schema.md"), + resolve(import.meta.dirname, "../docs/telemetry-schema.md"), process.argv.includes("--check"), ); } diff --git a/packages/agent/test/harness/telemetry.test.ts b/packages/agent/test/harness/telemetry.test.ts index 3e0736cbc..8a5c036a0 100644 --- a/packages/agent/test/harness/telemetry.test.ts +++ b/packages/agent/test/harness/telemetry.test.ts @@ -34,7 +34,7 @@ describe("agent telemetry schemas", () => { "pi.harness.event_handler", "pi.session.write", ]); - const actual = readFileSync(resolve(import.meta.dirname, "../../telemetry-schema.md"), "utf8"); + const actual = readFileSync(resolve(import.meta.dirname, "../../docs/telemetry-schema.md"), "utf8"); expect(actual).toBe(renderAgentTelemetrySchemaMarkdown()); });

    (checkpoint and owns from the live projection) + fence; phases[P].run ... repeat until terminal(closure) -> return closure to the normal terminal path +recover: no checkpoint -> fence; initial.run + start checkpoint P -> fence; phases[P].run + inflight checkpoint P -> fence; phases[P].recover + after any committed transition the loop continues with run +``` + +The scheduler calls the generated method once per invocation. `commitStateTransition` is a module-private +symbol on the runtime; it is not exported and grants no author capability. Transition callbacks run on the +line: reads before the first write, then compose same-commit entries/tasks/state; no effects, waits, hooks or +nested commits. + +Why `initial` cannot commit: with no durable phase, a crash would rerun `initial` and duplicate any child it +created. Children are created in the first transition callback, atomically with the first checkpoint. +Checkpoint-backed handlers keep `commit` for same-phase bookkeeping (e.g. create a child and record its ID in +the current phase before waiting); phase changes only through returned transitions. + +`start` means the checkpoint represents no uncertain external effect, so reopen reruns `run`. `inflight` +means an effect may have started, so reopen requires `recover`. + +Fence: immediately before every handler call the adapter synchronously checks the exact invocation object, +ctx, session phase, durable mark and abort signal. If a mark landed first, the handler is never invoked; the +ordinary cancellation unwind returns from the outer method and fresh abort follows. If the transition +persisted first, the handler is admitted and a later mark signals it normally. + +## 17. Test matrix + +Each item is a test, run in both orders where a race is named. + +**Storage / line** +- IDs vs sequences: failed callback burns IDs; reopen reuses uncommitted IDs, never committed ones; + state-only writes advance `Seq` without `nextId`; IDs minted only in a since-retired sidecar are not reused + after reopen (high-water carried by the retiring main record). +- Read before write ok; read after write → `ReadAfterWrite`, nothing persisted. +- Rewindable write after entry in one batch → reject. +- Protected internals: external `conversationValue("pi.inbox")` rejects at construction; a hand-built `pi.inbox` + address rejects `ScopeViolation` on generic read/write/watch/`rootValues`; imported `generationKind.config.model` + works through `c.value`/`c.config`/`rootValues`. +- Concurrent commits serialize; an awaited read holds the line. +- Uncertain storage failure → fail-stop; callback not retried. +- Terminal `task.set` retires scratch and unreferenced output atomically; retired reads reject in all backends. +- Rewind through fork caps; tombstones; list remove/clear at historical positions; nested forks; unrelated + position → `InvalidHistoryPosition`. +- Scan cursors, `through`, `newestHead`, filters; list remove of a foreign element (other definition or + unrelated conversation) rejects; a fork removes an inherited parent element through its own bound address + (local removal, parent unchanged, historical reads before the removal still show it); removing it twice is + idempotent. +- Pre-persistence assertion failures fault: invalid transition, unpaired ownership, output ref to retired output. + +**Admission** +- Idle send: one batch places, creates one generation, records result + receipt. +- Busy send: followUp/steer queue, no task; reject throws and writes nothing. +- Duplicate requestId: same batch, later, other conversation → original handle, nothing written. +- `h.input(requestId)` for a core request-keyed write receipt → `undefined`; a `send` reusing that key → + `RequestKindMismatch`; public `tx.write` has no request key and returns the input id only. +- `h.createConversation({ input, requestId })` / `c.fork({ at, input, requestId })` with an existing receipt for + that key → `RequestKindMismatch`, no conversation and no input created; `h.input(requestId)` still finds the + original; a fresh key creates both atomically. +- No `processHost` → `pi.job` fails `spawn` before any checkpoint. +- Withdraw vs place: exactly one terminal result. +- Hydration: `wait`/`result` return `input: UserEntry` and `answer.{entry, message}` where `message === + entry.model[0]`; a later edit targeting the assistant entry does not change a previously or subsequently + hydrated `message`. Passive writes have no public input outcome. +- Steer vs post_tools; followUp vs final answer: consumed exactly once. `all` vs `one-at-a-time` per mode. + Placement order is by input ID across modes: a steer admitted before a write precedes it at the boundary. +- Queued write with head vs collapse committed meanwhile → `unanswered/stale`, no fault. +- Queued write placed at post-tools boundary lands after the complete exchange. +- Reset before followUp in the inbox (`R` then `B`) at post-tools: old group `unanswered/terminated`, no + continuation, `B` placed after `R` starts a fresh generation whose cutoff context starts at `R`. FollowUp before + reset (`B` then `R`): `B` placed then made invisible by `R` → `unanswered/stale`, no generation for `B`; with + `one-at-a-time` and `B1, B2, R` queued, the unselected `B2` is also removed and `unanswered/stale`, so a later + idle send does not resurrect it; same cases at a final-answer boundary and an idle send; idle `c.reset` with + followUps left queued by a task abort → they resolve `unanswered/stale` in the reset's commit and a later send + starts from the reset alone. +- Several inputs in one group all resolve to the same answer. +- Task abort keeps queue; conversation abort withdraws steer/followUp, keeps write. +- Idle-turn boundary after post_tools with no continuation. +- Owned conversation seed: create + config values + isolation deletes + section seed + first accept + owner + checkpoint in one commit; crash before commit leaves nothing; with `parent`, child reads no rewindable + `CoreConfigBundle` member from the parent; first preparation removes every unseeded inherited section and + every inherited tool outside the isolated loadout; a host fork with `parent` keeps inherited sections and + config and only applies overrides; sync `tx.createConversation` rejects `input`/`copySticky` at the type + level; `SeedWrite` delete cuts an inherited plugin value; first generation reserved in the same drain still + sees the seed; missing seeded model → `no_model` failure, no fault. +- Child hooks: inert creation then `c.hooks.on` then `send` runs child handlers on the first request; atomic + creation with `profile` dispatches pre-registered subtree handlers by profile on the first request; reopen + with code reinstalled before `resume` restores both; unknown profile falls to defaults. +- Collapse does not make busy: idle send during collapse starts generation. +- Yield continuation: same `inputs` carried, results stay `placed`; transcript order is assistant, then placed + writes, then the `pi.user` continuation; queued trigger arriving between hook and closure wins and the yield + decision is discarded; a queued reset/handoff placed at the boundary also discards it: inputs `done(answer)`, + no continuation, no successor. + +**Tasks / scheduler** +- Reserve before effect; reopen runs `recover` with and without checkpoint. +- Closure commits once; throwing closure → nothing persisted, session faults. +- Dependency terminal (any outcome) wakes dependent exactly once. +- Resume vs commit both orders: nothing missed, nothing double-dispatched; drain-exit race. +- Tasks across conversations run concurrently; background continues after foreground idle. +- Self-wait and dependency-path wait reject; `after` cycle rejects. +- Cancelled waiter removes only itself. +- Output: a `mutate` with no tracked change persists nothing and emits nothing (no `Seq` consumed); a `replace` + with a structurally equal value still persists one base flush and emits one event; two tasks write one output + in line order; shared ref keeps output live past first task's terminal; + `initial` not called for a shared ref; final closure gets `tx.output`; `mutate` throw faults; sidecar after + retirement ignored on reopen. +- Output across conversations: task in B adopts A's output → B's watch envelope carries a synthetic full base + before `task_start` and later deltas apply; last local ref ends in B → B drops it while A keeps it; the + adjacent serialized commits in either order stay gapless: delta-before-adoption (the synthetic base already + includes it) and adoption-before-delta (the next `task_output` applies it). + +**Cancellation / lifecycle** +- Mark vs effect return: one invocation; fresh abort after old return. +- Post-mark commit/scratch/output → reject before callback. +- Closure vs mark: committed-first wins; otherwise closure discarded. +- Pending marked before reservation: no execute, abort runs. +- Repeated mark during abort: no effect. +- Close writes nothing; shutdown marks all, preserves queues; crash during shutdown → reopen + resume finishes. +- Close sequence: `close` waits behind the admitted commit, which persists; then phase `closing` on the line, + waiters reject, watches get `closed/session`, queued operations reject `Closed`; invocations joined before + `storage.close`; the Harness object is permanently unusable. Fault: waiters reject `Faulted`, watches get + `closed/session`, nothing written afterwards. +- Conversation abort reaches foreground ordinary tasks and threshold collapse in the subtree; background jobs + and manual collapse survive. +- Broken abort handler: shutdown rejects, task stays durable. +- Orphan reconciliation: missing kind → orphaned, scratch retired, registered descendants marked, queues kept. +- `pi.tool` abort marks children with `abortWithTool: true` only; subagent queues preserved. +- Registry: replace/remove of a kind with live instances rejects. +- Token identity: `tx.task(unregisteredKind)` and `tx.write(unregisteredEntryKind)` reject `StaleDefinition`; after + `h.entryKinds.replace(newNote)`, `tx.write(oldNote, …)` rejects and `tx.write(newNote, …)` works; a task + method that captured the old definition keeps running to completion with it; a structurally identical + redeclared token with the same `kind` string rejects. +- Hooks: `c.hook.` methods exist for exactly the fixed kinds' points and dispatch to the same registry + as `c.hooks.on(kind, kind.hooks.point, ...)`; a duplicate flattened built-in name fails install; two plugin + kinds with same-named points register independently by token. +- Hooks: ordinary kind declares a custom point; `collect`/`first`/`chain` folds; `skip` vs `abort` on throw; + cancellation propagates regardless; harness-wide then outer→inner order; `subtree` reaches owned + conversations only; handler rerun after crash re-enters the phase. + +**Core kinds** (faux provider, faux tools) +- Generation: tool calls → N tools + one post_tools atomically; final answer → boundary + successor. +- Retry sequence: `retrying{attempt: n}` → `requesting{attempt: n+1}` with identical cutoff/system/model/thinking; + step 1 does not rerun; overflow measured on the post-hook request; a hook that adds a tool declaration makes + that name appear in `requesting.tools`. +- Streaming: provider `partial` is never inserted into the tracker (mutating it after `start` changes nothing + in the output); each encoded frame yields exactly one incremental Chord delta (no whole-message replace per + token); after the stream the tracker value equals `reduceAssistantMessageFrames(frames)`; the stored assistant + entry is a validated clone. +- Recover from each generation phase; failed attempts produce `pi.usage` entries, the final attempt's usage is + on `pi.assistant` only; recovery from `requesting` with retries left records `pi.usage{ error: "interrupted" }` + and retries; with retries exhausted or disabled it terminalizes `failed/*` with no assistant entry and inputs + `unanswered/failed`; + `c.config.model` changed between crash and recover → recovered request still uses `Prep.model`; `beforeRequest` + reruns on recovery from `prepared`; `requesting` checkpoint carries post-hook tool names; `retry` policy + honoured (`maxRetries`, `retryDelayMs`, `enabled: false` → no retry); `afterResponse` runs once per attempt + including error/aborted responses and deferred fetch results; deferred: `pollAfterMs` respected, + re-deferred handle refreshes the checkpoint, abort calls `cancelDeferred`. +- Generation terminal error: non-retryable/exhausted error and provider-side `aborted` → one display-only + assistant entry, inputs `unanswered/failed`; overflow with no collapsible prefix → notice + `failed/overflow`, + no collapse task; replacement generation after failed collapse → notice + inputs `unanswered/failed`. +- Threshold: estimate over `threshold` → foreground collapse created, generation completes, conversation abort + reaches it; `through` = boundary before the largest suffix within `keepRecent`; whole context fits → no task + and manual collapse rejects `NothingToCollapse`; overflow via `estimateContextTokens` → `failed/overflow` + + replacement generation. +- Overflow → collapse + replacement generation atomically; replacement handles collapse failure/decline. +- Tool: `beforeTool` output changing `call.name`/`id` → blocked "call identity changed", not invoked; call naming a + tool absent from the originating generation's offered `tools` → "tool not offered" error result, registry not + consulted; a deferred generation that completed with tool calls (last checkpoint `deferred`) passes the evidence + check and proceeds to normal registry lookup; generation record missing → fault; missing tool / bad args (original or after a + `beforeTool` rewrite) / `block` / throw → error result, not fault; `beforeTool` chain merges `call` + rewrites and stops at the first `block` (later handlers not run); crash after reservation but before + `started` → recover reruns validation/hooks and starts (no interruption result), for `safe` and `unsafe` + alike; crash after `started` persists the final rewritten call and never reruns `beforeTool`: current + missing/unsafe/schema-incompatible definition yields an interrupted/unavailable result, while current safe + and valid replays the persisted arguments; parallel completion order vs projection order; output bounds and + truncation diagnostics. +- Tool child (once the mediated capability is settled): keyed creation is idempotent after safe replay; + wait timeout → background result without marking; closure reread finds terminal child → normal result; + `abortWithTool: false` child survives tool abort; sibling resolver rejects fork-only ancestry. +- Tool control: `terminate` ends the turn; `handoff` ends it and appends `pi.handoff` *before* the final + boundary so a queued follow-up's generation starts in the fresh context; `handoff` beats `terminate`; several + `addTools` append to the existing `selectedTools` in call order with stable dedupe (nothing previously + selected is dropped) in one write, and the next baseline/delta shows the change; `addTools` reads + `selectedTools` fresh in the closure, so a config change made during `afterTools` is not lost. +- Post_tools synthesizes results for orphaned/aborted tools; mixed batch (one orphaned tool, another with + `addTools`): the `selectedTools` write precedes the synthesized result entry in the `Write[]`, the fork + invariant holds, and both effects land in one commit. +- Post_tools aborted (task abort and conversation abort): every input in `inputs` resolves `unanswered/aborted`; + `InputHandle.wait` on them settles; no successor generation; queued followUp survives task abort. +- Collapse: stale head → `failed/stale`; edits in between do not stale; summary head placement; `decline` → + `failed/declined`; hook `summary` goes to `prepared` without a provider call; crash in `summarizing` reruns + the summarizer as a failed attempt (`retrying` then `attempt + 1`, or `failed/provider` when exhausted/disabled, + never the same attempt twice); crash in `prepared` finalizes without a call; unset model → `failed/no_model` + before any hook or checkpoint; summarizer request offers + no tools; a summarizer response with tool calls is a provider failure; head moves while `beforeCollapse` + waits → `failed/stale`, hook decision discarded; retry passes through `retrying` with `untilMs`. +- Job: `spawning` before OS effect; fake host `status` → `running`/`exited` continues; `unknown` (on recover or + mid-poll) → `interrupted` unless `rerun`, which calls `start` again with the same key; `start` for a known + key is a no-op; `start` rejecting (ENOENT) → `failed/spawn`, no fault; `status` rejecting → `failed/interrupted` + without rerun, no fault; `kill` rejecting in abort → fault; recover from `waiting` sleeps the stored deadline + then spawns; abort with no key → `{ killed: false }` and no host call; abort with a spawning/running key → + TERM, 5 s, KILL (idempotent if already exited); each schedule + occurrence writes its notice with the next `waiting` checkpoint; dropped byte counts copied from `status`; + abort ends the schedule. +- System: preparation reruns when config, `h.sections`, `h.tools` or the newest head change while handlers run (a collapse landing mid-preparation yields a baseline, not a stale delta); baseline content is `## key\nrendered` blocks joined by blank lines; delta content is `The key section + now reads:` / `no longer applies.` blocks; unchanged draft appends nothing; delta on change with `toolsRemoved` before `toolsAdded`; data-only + change → `model: []`; baseline after a head with omission edits for superseded managed entries; no repeated + baseline; fork sees inherited managed entries; restart without plugin keeps payload and rendered text; + staleness retry when a managed entry lands during preparation; throwing handler's edits rolled back only. + +**Watch** +- Capture vs commit: no gap, no duplicate. +- One boundary commit → one envelope. +- Overflow closes; new watch has fresh base. Listener throw closes only that watch. +- Output delta ordering vs `task_end`; output removed when last reference ends. +- Session watch: conversation created, session values and lists; `closed/session` on close. +- Lists: requested list captured with complete elements; append/remove/clear in one commit → one `list` event + with ops in write order, folded before the listener; element IDs stable; `pi.inbox` and built-in config + rejected in `values`/`lists`; session watch rejects conversation lists. +- Reset: idle → entry appended before `reset` resolves; busy → queued and `reset` resolves on admission, entry + placed at the boundary and visible via watch; never stale. +- Defaults: unset `followUpMode` reads `"one-at-a-time"` via `c.config`, `view.config` and the post_tools + boundary; `delete` restores the default and the `config` event carries the default as `value`, never + `undefined`; initial watch capture has every defaulted key present; unset `model` reads `undefined` and + generation fails `no_model`; historical `c.config.thinkingLevel.get(entryId)` at a position with no write + resolves the default. Compile: `ConfigSnapshot["followUpMode"]` is exactly + `"all" | "one-at-a-time"` and `["model"]` is `ModelRef | undefined`; `c.value(plainToken).get` is `T | + undefined`, `c.value(defaultedToken).get` is `T`; `conversationValue("plan.mode", { rewind: true })` is + `UnboundValue` and with `default: false` it is `& { default: boolean }`; `sessionValue("n", + { default: "x" })` reads as `string`. +- Watch options: the same plugin address requested twice appears once in the view and yields one `value` + event; a built-in config token or `pi.inbox` in `values` rejects rather than deduplicating into `config`. +- `view.config` captured without requesting; `c.config.model.set` → one `config` event `{ type: "config", key: + "model", value, previous }` folded into `view.config.model`; `c.config.set({ model, followUpMode })` → two `config` events in + one envelope, both folded before the listener; a requested plugin value lands in `view.values` via `value`; + built-in definitions never emit `value`; protected addresses cannot be requested. +- Active transcript: capture with no head returns all entries; with a head returns, in transcript order, every + entry from the boundary through current with the head entry at its actual position. + Reducer: plain entry appends; summary entry with `head = k` drops entries with id < k then appends the + summary; reset/handoff (`head = self`) drops all prior active entries then appends itself; both in the same + fold as the rest of that commit; `context` event updates in the same envelope; pre-head entries remain + readable through entry scans; capture after a head equals the incrementally reduced view. + +**State adapter** +- Live transitions through several phases: one scheduler `execute`; each handler gets the applied projection. +- Initial child creation vs crash: no transition → initial reruns with no child; transition → next phase, no + duplicate. +- Reopen dispatch: absent → initial; start → run; inflight → recover. +- Same-phase commit vs abort: both or neither; post-mark commit rejects. +- Transition vs mark both orders: mark-first prevents the next handler. +- Compile: broad string phase rejects; missing/extra phase key rejects; extra transition field rejects; + inflight without recover rejects; a hand-built transition object mixing phase `"a"` with phase `"b"`'s payload + is rejected as a `StateResult` (negative fixture), while `actions.transition("a", ...)` with the right payload + is accepted. + +**Backends** +- JSONL: main through Seq 100, scratch through 150 → reopen at 150; terminal at 151 with unlink failure → + sidecar ignored; torn suffix truncated; malformed complete line fails open; kill -9 during each phase. +- SQLite: same conformance; lock contention fails fast. +- 100k terminal tasks, 2 live: open touches only live seed + ancestry. + +## 18. Work packages + +One commit per package, reviewed before the next starts. Run the package's tests and `npm run check` before +each commit. Any package test exercising providers or tool execution uses faux implementations; no real API. + +| # | package | delivers | +|---|---|---| +| 1 | types | sections 1–3 and 11.2 declarations (`HookPoint`/`defineHookPoint` are referenced by `TaskKind`), `Stored` and its compile assertions (section 1), `defineTask`/`defineCoreTask`/`defineEntry`/address helpers; compile tests | +| 2 | storage | `Write`, `Storage`, `MemoryStorage`, conformance tests | +| 3 | line | Session line, transactions, read-before-write, validation and assertions (5.2), capability-bound builders | +| 4 | context | context derivation, forks, tool-result repair, transcript invariant assertions | +| 5 | admission | inbox/results/receipts, send, boundaries and queue modes, withdraw, wait; test generation kind | +| 6 | task kernel | reservation, runtimes, closures, scratch, output tracker, invocation identity | +| 7 | scheduler | resume/kick/drain, dependencies, waiters, idle | +| 8 | cancellation | mark/revoke/join/fresh abort, conversation abort, close/shutdown/fault, open + orphan reconciliation, Harness object over the test generation kind, registries, hooks registry | +| 9 | system | sections, draft, managed entries, canonical fold, baseline/delta, staleness retry (section 12); depends on pi-ai `SystemMessage` having landed | +| 10 | generation + tool + post_tools | 10.1–10.3, `ToolResult`/`ToolDeclaration`, hooks of 11.2; `pi.tool` runs an internal test executor, no ordinary tool API; faux provider only. Uses pi-ai's landed `SystemMessage`/messages-only adapters; no parallel representation | +| 10a | ordinary tool API | section 11.1 gated block (execution signature, facade, output access, memos, children, resolver), after its shape is decided and appended; then the built-in tools | +| 11 | collapse + job | 10.4, 10.5 | +| 12 | watch | section 14 | +| 13 | jsonl | section 15 | +| 14 | sqlite | section 15 | +| 15 | state adapter | section 16 | + +Package 10 is the design validation point: if generation/tool/post_tools do not fit the kernel from 6–9, +change the kernel before building 12–15. Its acceptance also includes the size check: count production lines +under `src/harness/pico/` excluding `memory-storage`, `jsonl-storage`, `sqlite-storage` and tests; materially +over 4k means stop and simplify before continuing, never add caching or indexing layers to compensate. + +## 19. Repository rules + +- Location: `packages/agent/src/harness/pico/`. No imports from `harness/runtime`, `harness/session`, + `agent-harness.ts` or earlier Pico prototypes. Chord and pi-ai are allowed. Read reusable leaf code before + copying, then own the copy. +- Erasable TypeScript only. No `any` in Pico declarations (imported pi-ai types exempt). +- Check external types in `node_modules`; do not guess. +- Test with `node .../vitest/dist/cli.js --run test/harness/pico/.test.ts` and `npm run check`. Never + the full suite. +- Commit only files of the current package, by explicit path. +- Production size target 3k–4k lines excluding storage backends and tests (intro, WP10); report the count in + each package's commit message from WP6 onward. +- Public exports: `pico/index.ts` exports only the public surface. `defineCoreTask`, `CoreTaskTx`, + `CoreAbortTx`, `CoreTaskRuntime`, `CoreAbortTaskRuntime`, the internal admission helpers (`accept`, + `queueInput`, request-keyed `write`), the core runtime/builder constructors and `commitStateTransition` are + module-internal and never re-exported; no `export *` from a module that declares them. The fixed kind tokens + (`generationKind`, `toolKind`, `postToolsKind`, `collapseKind`, `jobKind`) with their `config` and `hooks` + witnesses, `defineTask`, `defineStateTask`, `defineEntry`, `defineHookPoint`, `defineSystemSection`, address + helpers, `Harness`, storages and all public types are exported. A compile test asserts that + `import { defineCoreTask } from ".../pico"` fails. + +## Appendix A. Type index + +Every identifier used above that is not declared inline. Imported: `Context`, `Op`, `applyImmutable`, `track` +from Chord; `Message`, `UserMessage`, `AssistantMessage`, `ToolResultMessage`, `TextContent`, `ImageContent`, +`ToolCall`, `Tool`, `Models`, `Usage`, `DeferredHandle`, `RetryPolicy`, `isRetryableAssistantError`, `retryDelayMs`, `estimateContextTokens`, `AssistantMessageFrameEncoder` from pi-ai (`reduceAssistantMessageFrames` only in tests, as the oracle); `JsonRepresentation` from Chord (as `Stored`, section 1); `ThinkingLevel` +re-exported from `packages/agent/src/types.ts` (`"off" | "minimal" | "low" | "medium" | "high" | "xhigh" | +"max"`; never redeclared in Pico); `TSchema`, `Static` from TypeBox. + +```ts +declare const taskOutputType: unique symbol; // taskKindBrand is declared in 3.3 + +// identity and payloads +type ModelRef = { readonly provider: string; readonly modelId: string }; // field names match the existing models-provider service +// durable payloads below are exact `type` aliases so they satisfy JsonObject; public non-durable shapes stay interfaces +type AssistantEntryData = { readonly attempt: number }; // usage is already on AssistantMessage.usage; not duplicated +type UsageEntryData = { readonly attempt: number; readonly usage?: Stored; readonly error: string }; +type UsageEntry = EntryBase & { readonly data: UsageEntryData }; // pi.usage, data-only +interface UnboundValue { readonly namespace: string; readonly key?: string; readonly rewind: R; readonly default?: T; readonly [addressType]?: T; bind(conversationId: Id): Value } +// DefaultedUnboundValue: section 2; its bind result retains required default metadata. +type PayloadOf = D extends Address | UnboundValue ? T : never; +interface UnboundList { readonly namespace: string; readonly key?: string; readonly rewind: R; readonly [addressType]?: T; bind(conversationId: Id): List } +// `addressType` is declared in section 2; bound and unbound definitions both carry the witness so inference survives binding +type ConversationValueDefinition = UnboundValue; // the name used for config bundle members +type ConversationListDefinition = UnboundList; +// SeedWrite (set | delete): section 8.1. Unbound addresses are bound to the target conversation (root = 1, or the created child). `RootValueWrite` is the set form: { address, value }. +type RootValueWrite = Extract; +// conversationValue(ns, { rewind: true }) -> UnboundValue; { rewind: false } -> UnboundValue +interface StickyPublicValue { get(ctx: Context): Promise; set(value: T, ctx: Context): Promise; delete(ctx: Context): Promise } // Read is the resolved type (default applied) for defaulted definitions +// CoreConfigs, CoreConfigBundle, BundleValues, Settings, ConfigFacade, DisjointBundles: section 2.1. CoreHooks, CoreHookBundle, HookFacade: section 13.1. +type Acceptance = { readonly kind: "send" | "write"; readonly requestId?: string; readonly conversationId: Id; readonly inputId: Id }; // durable receipt; `kind` keeps write receipts out of Harness.input +interface InternalAcceptOptions { readonly input: UserInput; readonly requestId?: string; readonly whenBusy?: "followUp" | "steer" | "reject" } // public-typed; the helper validates and stores Stored +type StoredEntryDraft = { readonly kind: string; readonly data?: JsonValue; readonly model?: readonly Stored[]; readonly head?: Id | "self"; readonly edits?: readonly ContextEdit[] }; +// StoredInputResult, InputOutcome, TerminalInputOutcome: section 9.1 +// ConversationSeed, ConversationCreateSpec: section 8.1. Report: section 13. +type ToolResultData = { readonly details?: JsonValue; readonly usage?: ToolUsage; readonly diagnostics?: readonly ToolDiagnostic[]; readonly control?: ToolControl; readonly truncated?: { readonly bytes: number; readonly lines: number } }; +type ToolUsage = { readonly [key: string]: number }; +type ToolOutputState = { progress?: string; log: string[]; details?: JsonValue }; // mutable OutputState +type EnvironmentInfo = { readonly cwd: string }; // exact alias; the established minimal built-in; richer environment data is an application section +type SkillInfo = { readonly name: string; readonly description: string }; // exact alias; systemSections.skills renders the list +// Registries and open arrays take the erased default `SystemSection` (= SystemSection): `render(value: T)` is method +// syntax, so every SystemSection is assignable under method bivariance. Seeds pair a section with a value, which the erased +// form cannot check, so seeds are built only through the checked helper: +type SectionSeed = + | { readonly section: SystemSection; readonly value: JsonValue } // erased set pair + | { readonly key: string; readonly remove: true }; // remove an inherited section by stable key (host forks) +declare function sectionSeed(section: SystemSection, value: T): SectionSeed; // checked set +declare function removeSection(key: string): SectionSeed; // removal + +// tasks +type NewTask = Omit & { readonly status: "pending"; readonly owns: readonly []; readonly abort?: never; readonly checkpoint?: never; readonly outcome?: never }; +type RunningTask = Omit, "output"> & TaskOutputField & { readonly status: "running"; readonly outcome?: never }; +type Completion = { readonly status: "completed"; readonly result: R } | { readonly status: "failed"; readonly failure: F }; +type TerminalClosure = (tx: FinalTx, current: RunningTask) => Completion | Promise>; +type CoreTerminalClosure = (tx: CoreFinalTx, current: RunningTask) => Completion | Promise>; +type AbortClosure = (tx: AbortFinalTx, current: RunningTask) => A | Promise; +type CoreAbortClosure = (tx: CoreAbortFinalTx, current: RunningTask) => A | Promise; +interface TaskOutputKind { readonly kind: string; readonly [taskOutputType]?: O } +declare function defineTaskOutput(kind: string): TaskOutputKind; +interface TaskOutputRef { readonly id: Id; readonly kind: string; readonly [taskOutputType]?: O } +type TaskOutputField = [O] extends [never] ? Record : { readonly output: TaskOutputRef }; // no-output tasks have no output key +type TaskOutputDefinition = [O] extends [never] ? { readonly output?: never } : { readonly output: { readonly kind: TaskOutputKind; initial(input: I): O } }; +type FinalOutput = [O] extends [never] ? Record : { readonly output: O }; +// RuntimeOutput, AbortRuntimeOutput, TaskRuntimeBase, CommitRuntime, AbortRuntimeBase: section 5.4 +interface TaskOutput { readonly ref: TaskOutputRef; read(ctx: Context): Promise; mutate(m: (state: O) => undefined, ctx: Context): Promise; replace(value: O, ctx: Context): Promise } +interface ReadonlyTaskOutput { readonly ref: TaskOutputRef; read(ctx: Context): Promise } +type TaskSpec = { readonly conversationId?: Id; readonly input: InputOf; readonly after?: readonly Id[]; readonly background?: true } & ([OutputOf] extends [never] ? { readonly output?: never } : { readonly output?: TaskOutputRef> }); +type OrdinaryKind = { readonly kind: string; readonly [taskKindBrand]: "ordinary"; readonly hooks?: HookPoints; readonly config?: ConfigBundle }; +type CoreKind = { readonly kind: string; readonly [taskKindBrand]: "core"; readonly hooks?: HookPoints; readonly config?: ConfigBundle }; +type AnyDefinedKind = OrdinaryKind | CoreKind; +type PayloadsOf = + K extends TaskKind ? { input: I; checkpoint: C; result: R; failure: F; aborted: A; output: O; hooks: H } : + K extends CoreTaskKind ? { input: I; checkpoint: C; result: R; failure: F; aborted: A; output: O; hooks: H } : never; +type InputOf = PayloadsOf["input"]; type CheckpointOf = PayloadsOf["checkpoint"]; type ResultOf = PayloadsOf["result"]; +type FailureOf = PayloadsOf["failure"]; type AbortedOf = PayloadsOf["aborted"]; type OutputOf = PayloadsOf["output"]; +type TaskOf = DistributiveOmit, CheckpointOf, ResultOf, FailureOf, AbortedOf>, "output"> & TaskOutputField>; +type DistributiveOmit = T extends unknown ? Omit : never; +type NoExtra = Actual & Record, never>; +type ExactJsonInput = Expected extends readonly JsonValue[] ? Actual : Expected extends JsonObject ? NoExtra : Actual; +// CoreTaskKinds carries the exact fixed-kind token types, not the erased CoreKind, so CoreConfigBundle/CoreHookBundle/ConfigEvent/HookFacade derive exact keys: +interface CoreTaskKinds { + readonly generation: typeof generationKind; // CoreTaskKind & { config: typeof generationKind.config } + readonly tool: typeof toolKind; + readonly postTools: typeof postToolsKind; + readonly collapse: typeof collapseKind; + readonly job: typeof jobKind; // ordinary; config {} and hooks {} +} +interface OpenInspection { readonly pending: readonly Task[]; readonly running: readonly Task[]; readonly orphaned: readonly Task[]; readonly unknownEntryKinds: readonly string[] } + +// state adapter exactness helpers +type ExactPhaseHandler = Actual extends { readonly role: infer Role } ? Actual extends Extract ? NoExtra, Actual> : never : never; +type ExactPhaseMap = Actual & Record, never> & { readonly [P in keyof Expected]: ExactPhaseHandler }; +type ExactStateTaskDefinition = NoExtra & { readonly phases: ExactPhaseMap }; + +// state facades +interface PublicValue { get(ctx: Context): Promise; get(at: Id, ctx: Context): Promise; set(value: T, ctx: Context): Promise; delete(ctx: Context): Promise } // Read = resolved type; both current and historical get apply the definition's default when nothing is stored at that position +interface PublicList { read(ctx: Context): Promise[]>; read(at: Id, ctx: Context): Promise[]>; append(value: T, ctx: Context): Promise; remove(id: Id, ctx: Context): Promise; clear(ctx: Context): Promise } +interface ScratchReader { value>(a: D): Pick, ResolvedValue>, "get">; list(a: List): Pick, "read"> } + +// registries and hooks +interface MutableRegistry { // async, on the line + get(name: string, ctx: Context): Promise; register(d: D, ctx: Context): Promise; replace(d: D, ctx: Context): Promise; remove(name: string, ctx: Context): Promise; + readonly revision: number; // monotonically increasing; bumped by register/replace/remove; read by preparation snapshots (12.5) +} +// hooks: HookPoint, HookPoints, HookHandler, HookInfo, HookRunner, HookResult, HarnessHookRegistry, ConversationHookRegistry are declared in 11.2 +type HookIn